Commit 379b73c3c3 for freeswitch.com
commit 379b73c3c31999659f63a48f00e66db72d6eb8b8
Author: Andrey Volk <andywolk@gmail.com>
Date: Mon Nov 16 20:17:39 2026 +0300
[libyuv] Update to hash 79ca9187e540faaaca47b34622e110d07f335d05 from https://chromium.googlesource.com/libyuv/libyuv/
diff --git a/Makefile.am b/Makefile.am
index 71e57cd006..6d1982842d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -179,10 +179,10 @@ libfreeswitch_libyuv_la_SOURCES = \
libs/libyuv/source/compare.cc \
libs/libyuv/source/compare_common.cc \
libs/libyuv/source/compare_gcc.cc \
-libs/libyuv/source/compare_mmi.cc \
-libs/libyuv/source/compare_msa.cc \
+libs/libyuv/source/compare_hvx.cc \
libs/libyuv/source/compare_neon64.cc \
libs/libyuv/source/compare_neon.cc \
+libs/libyuv/source/compare_wasm.cc \
libs/libyuv/source/compare_win.cc \
libs/libyuv/source/convert_argb.cc \
libs/libyuv/source/convert.cc \
@@ -200,28 +200,43 @@ libs/libyuv/source/rotate_argb.cc \
libs/libyuv/source/rotate.cc \
libs/libyuv/source/rotate_common.cc \
libs/libyuv/source/rotate_gcc.cc \
-libs/libyuv/source/rotate_mmi.cc \
-libs/libyuv/source/rotate_msa.cc \
+libs/libyuv/source/rotate_hvx.cc \
+libs/libyuv/source/rotate_lasx.cc \
+libs/libyuv/source/rotate_lsx.cc \
libs/libyuv/source/rotate_neon64.cc \
libs/libyuv/source/rotate_neon.cc \
+libs/libyuv/source/rotate_rvv.cc \
+libs/libyuv/source/rotate_sme.cc \
+libs/libyuv/source/rotate_sve.cc \
+libs/libyuv/source/rotate_wasm.cc \
libs/libyuv/source/rotate_win.cc \
libs/libyuv/source/row_any.cc \
libs/libyuv/source/row_common.cc \
libs/libyuv/source/row_gcc.cc \
-libs/libyuv/source/row_mmi.cc \
-libs/libyuv/source/row_msa.cc \
+libs/libyuv/source/row_hvx.cc \
+libs/libyuv/source/row_lasx.cc \
+libs/libyuv/source/row_lsx.cc \
libs/libyuv/source/row_neon64.cc \
libs/libyuv/source/row_neon.cc \
+libs/libyuv/source/row_rvv.cc \
+libs/libyuv/source/row_sme.cc \
+libs/libyuv/source/row_sve.cc \
+libs/libyuv/source/row_wasm.cc \
libs/libyuv/source/row_win.cc \
libs/libyuv/source/scale_any.cc \
libs/libyuv/source/scale_argb.cc \
libs/libyuv/source/scale.cc \
libs/libyuv/source/scale_common.cc \
libs/libyuv/source/scale_gcc.cc \
-libs/libyuv/source/scale_mmi.cc \
-libs/libyuv/source/scale_msa.cc \
+libs/libyuv/source/scale_hvx.cc \
+libs/libyuv/source/scale_lsx.cc \
libs/libyuv/source/scale_neon64.cc \
libs/libyuv/source/scale_neon.cc \
+libs/libyuv/source/scale_rgb.cc \
+libs/libyuv/source/scale_rvv.cc \
+libs/libyuv/source/scale_sme.cc \
+libs/libyuv/source/scale_uv.cc \
+libs/libyuv/source/scale_wasm.cc \
libs/libyuv/source/scale_win.cc \
libs/libyuv/source/video_common.cc
diff --git a/libs/libyuv/.clang-format b/libs/libyuv/.clang-format
new file mode 100644
index 0000000000..59d487051f
--- /dev/null
+++ b/libs/libyuv/.clang-format
@@ -0,0 +1,6 @@
+# Defines the Chromium style for automatic reformatting.
+# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+BasedOnStyle: Chromium
+---
+Language: Java
+BasedOnStyle: Google
diff --git a/libs/libyuv/.gitignore b/libs/libyuv/.gitignore
new file mode 100644
index 0000000000..20d679b7f2
--- /dev/null
+++ b/libs/libyuv/.gitignore
@@ -0,0 +1,37 @@
+*.pyc
+.landmines
+pin-log.txt
+/base
+/build
+/buildtools
+/google_apis
+/links
+/links.db
+/ios
+/mojo
+/native_client
+/net
+/out
+/unit_test/out
+/source/out
+/sde-avx-sse-transition-out.txt
+/testing
+/third_party
+/tools
+
+# Files generated by CMake build
+cmake_install.cmake
+CMakeCache.txt
+CMakeFiles/
+yuvconvert
+libgtest.a
+libyuv.a
+libyuv_unittest
+
+# Files generated by winarm.mk build
+libyuv_arm.lib
+source/*.o
+
+# Files generated by perf
+perf.data
+perf.data.old
diff --git a/libs/libyuv/.gn b/libs/libyuv/.gn
new file mode 100644
index 0000000000..43a31e13e9
--- /dev/null
+++ b/libs/libyuv/.gn
@@ -0,0 +1,41 @@
+# Copyright 2015 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("//build/dotfile_settings.gni")
+
+# The location of the build configuration file.
+buildconfig = "//build/config/BUILDCONFIG.gn"
+
+# The hermetic Python interpreter to use by default.
+script_executable = "//third_party/cpython3/host/bin/python3"
+
+# The secondary source root is a parallel directory tree where
+# GN build files are placed when they can not be placed directly
+# in the source tree, e.g. for third party source trees.
+secondary_source = "//build/secondary/"
+
+# These are the targets to check headers for by default. The files in targets
+# matching these patterns (see "gn help label_pattern" for format) will have
+# their includes checked for proper dependencies when you run either
+# "gn check" or "gn gen --check".
+check_targets = [ "//libyuv/*" ]
+
+# These are the list of GN files that run exec_script. This allowlist exists
+# to force additional review for new uses of exec_script, which is strongly
+# discouraged except for gypi_to_gn calls.
+exec_script_allowlist = build_dotfile_settings.exec_script_allowlist +
+ [ "//build_overrides/build.gni" ]
+
+default_args = {
+ android_static_analysis = "off"
+ mac_sdk_min = "10.12"
+ ios_deployment_target = "12.0"
+
+ # Use Siso instead of Ninja.
+ use_siso = true
+}
diff --git a/libs/libyuv/AUTHORS b/libs/libyuv/AUTHORS
index 9686ac13eb..42959c1d87 100644
--- a/libs/libyuv/AUTHORS
+++ b/libs/libyuv/AUTHORS
@@ -2,3 +2,7 @@
# Name or Organization <email address>
Google Inc.
+
+Ho Cheung <uioptt24@gmail.com>
+Ivan Pavlotskiy <ivan.pavlotskiy@lgepartner.com>
+Sida Zhu <zhusida@bytedance.com>
diff --git a/libs/libyuv/Android.bp b/libs/libyuv/Android.bp
index ea76f9435b..aa953dadea 100644
--- a/libs/libyuv/Android.bp
+++ b/libs/libyuv/Android.bp
@@ -1,6 +1,31 @@
+package {
+ default_applicable_licenses: ["external_libyuv_license"],
+}
+
+// Added automatically by a large-scale-change
+//
+// large-scale-change included anything that looked like it might be a license
+// text as a license_text. e.g. LICENSE, NOTICE, COPYING etc.
+//
+// Please consider removing redundant or irrelevant files from 'license_text:'.
+// See: http://go/android-license-faq
+license {
+ name: "external_libyuv_license",
+ visibility: [":__subpackages__"],
+ license_kinds: [
+ "SPDX-license-identifier-BSD",
+ ],
+ license_text: [
+ "LICENSE",
+ "PATENTS",
+ ],
+}
+
cc_library {
name: "libyuv",
vendor_available: true,
+ product_available: true,
+ host_supported: true,
vndk: {
enabled: true,
},
@@ -9,47 +34,48 @@ cc_library {
"source/compare.cc",
"source/compare_common.cc",
"source/compare_gcc.cc",
+ "source/compare_hvx.cc",
"source/compare_neon.cc",
"source/compare_neon64.cc",
- "source/compare_mmi.cc",
- "source/compare_msa.cc",
"source/convert.cc",
"source/convert_argb.cc",
"source/convert_from.cc",
"source/convert_from_argb.cc",
+ "source/convert_jpeg.cc",
"source/convert_to_argb.cc",
"source/convert_to_i420.cc",
"source/cpu_id.cc",
+ "source/mjpeg_decoder.cc",
+ "source/mjpeg_validate.cc",
"source/planar_functions.cc",
"source/rotate.cc",
"source/rotate_any.cc",
"source/rotate_argb.cc",
"source/rotate_common.cc",
"source/rotate_gcc.cc",
- "source/rotate_mmi.cc",
- "source/rotate_msa.cc",
+ "source/rotate_hvx.cc",
"source/rotate_neon.cc",
"source/rotate_neon64.cc",
+ "source/rotate_rvv.cc",
"source/row_any.cc",
"source/row_common.cc",
"source/row_gcc.cc",
- "source/row_mmi.cc",
- "source/row_msa.cc",
+ "source/row_hvx.cc",
"source/row_neon.cc",
"source/row_neon64.cc",
+ "source/row_rvv.cc",
"source/scale.cc",
"source/scale_any.cc",
"source/scale_argb.cc",
"source/scale_common.cc",
"source/scale_gcc.cc",
- "source/scale_mmi.cc",
- "source/scale_msa.cc",
+ "source/scale_hvx.cc",
"source/scale_neon.cc",
"source/scale_neon64.cc",
+ "source/scale_rgb.cc",
+ "source/scale_rvv.cc",
+ "source/scale_uv.cc",
"source/video_common.cc",
- "source/convert_jpeg.cc",
- "source/mjpeg_decoder.cc",
- "source/mjpeg_validate.cc",
],
cflags: [
@@ -58,11 +84,30 @@ cc_library {
"-Wno-unused-parameter",
"-fexceptions",
"-DHAVE_JPEG",
+ "-DLIBYUV_UNLIMITED_DATA",
],
+ arch: {
+ arm: {
+ cflags: ["-mfpu=neon"],
+ },
+ },
+
shared_libs: ["libjpeg"],
export_include_dirs: ["include"],
+
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media.swcodec",
+ ],
+ min_sdk_version: "29",
+ sanitize: {
+ cfi: true,
+ config: {
+ cfi_assembly_support: true,
+ },
+ },
}
// compatibilty static library until all uses of libyuv_static are replaced
@@ -71,6 +116,11 @@ cc_library_static {
name: "libyuv_static",
vendor_available: true,
whole_static_libs: ["libyuv"],
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media.swcodec",
+ ],
+ min_sdk_version: "29",
}
cc_test {
@@ -79,10 +129,10 @@ cc_test {
shared_libs: ["libjpeg"],
cflags: ["-Wall", "-Werror"],
srcs: [
- "unit_test/unit_test.cc",
"unit_test/basictypes_test.cc",
"unit_test/color_test.cc",
"unit_test/compare_test.cc",
+ "unit_test/convert_argb_test.cc",
"unit_test/convert_test.cc",
"unit_test/cpu_test.cc",
"unit_test/cpu_thread_test.cc",
@@ -91,7 +141,11 @@ cc_test {
"unit_test/rotate_argb_test.cc",
"unit_test/rotate_test.cc",
"unit_test/scale_argb_test.cc",
+ "unit_test/scale_plane_test.cc",
+ "unit_test/scale_rgb_test.cc",
"unit_test/scale_test.cc",
+ "unit_test/scale_uv_test.cc",
+ "unit_test/unit_test.cc",
"unit_test/video_common_test.cc",
],
}
@@ -114,6 +168,15 @@ cc_test {
static_libs: ["libyuv"],
}
+cc_test {
+ name: "i444tonv12_eg",
+ gtest: false,
+ srcs: [
+ "util/i444tonv12_eg.cc",
+ ],
+ static_libs: ["libyuv"],
+}
+
cc_test {
name: "psnr",
gtest: false,
@@ -125,6 +188,15 @@ cc_test {
static_libs: ["libyuv"],
}
+cc_test {
+ name: "yuvconstants",
+ gtest: false,
+ srcs: [
+ "util/yuvconstants.c",
+ ],
+ static_libs: ["libyuv"],
+}
+
cc_test {
name: "yuvconvert",
gtest: false,
diff --git a/libs/libyuv/Android.mk b/libs/libyuv/Android.mk
index 0975d64a38..8f8d1e442c 100644
--- a/libs/libyuv/Android.mk
+++ b/libs/libyuv/Android.mk
@@ -1,4 +1,7 @@
# This is the Android makefile for libyuv for NDK.
+
+# Ignore this file during non-NDK builds.
+ifdef NDK_ROOT
LOCAL_PATH:= $(call my-dir)
include $(CLEAR_VARS)
@@ -9,10 +12,10 @@ LOCAL_SRC_FILES := \
source/compare.cc \
source/compare_common.cc \
source/compare_gcc.cc \
- source/compare_mmi.cc \
- source/compare_msa.cc \
+ source/compare_hvx.cc \
source/compare_neon.cc \
source/compare_neon64.cc \
+ source/compare_win.cc \
source/convert.cc \
source/convert_argb.cc \
source/convert_from.cc \
@@ -26,26 +29,31 @@ LOCAL_SRC_FILES := \
source/rotate_argb.cc \
source/rotate_common.cc \
source/rotate_gcc.cc \
- source/rotate_mmi.cc \
- source/rotate_msa.cc \
+ source/rotate_hvx.cc \
source/rotate_neon.cc \
source/rotate_neon64.cc \
+ source/rotate_rvv.cc \
+ source/rotate_win.cc \
source/row_any.cc \
source/row_common.cc \
source/row_gcc.cc \
- source/row_mmi.cc \
- source/row_msa.cc \
+ source/row_hvx.cc \
source/row_neon.cc \
source/row_neon64.cc \
+ source/row_rvv.cc \
+ source/row_win.cc \
source/scale.cc \
source/scale_any.cc \
source/scale_argb.cc \
source/scale_common.cc \
source/scale_gcc.cc \
- source/scale_mmi.cc \
- source/scale_msa.cc \
+ source/scale_hvx.cc \
source/scale_neon.cc \
source/scale_neon64.cc \
+ source/scale_rgb.cc \
+ source/scale_rvv.cc \
+ source/scale_uv.cc \
+ source/scale_win.cc \
source/video_common.cc
common_CFLAGS := -Wall -fexceptions
@@ -85,20 +93,25 @@ LOCAL_MODULE_TAGS := tests
LOCAL_CPP_EXTENSION := .cc
LOCAL_C_INCLUDES += $(LOCAL_PATH)/include
LOCAL_SRC_FILES := \
- unit_test/unit_test.cc \
- unit_test/basictypes_test.cc \
- unit_test/color_test.cc \
- unit_test/compare_test.cc \
- unit_test/convert_test.cc \
- unit_test/cpu_test.cc \
- unit_test/cpu_thread_test.cc \
- unit_test/math_test.cc \
- unit_test/planar_test.cc \
- unit_test/rotate_argb_test.cc \
- unit_test/rotate_test.cc \
- unit_test/scale_argb_test.cc \
- unit_test/scale_test.cc \
+ unit_test/basictypes_test.cc \
+ unit_test/color_test.cc \
+ unit_test/compare_test.cc \
+ unit_test/convert_argb_test.cc \
+ unit_test/convert_test.cc \
+ unit_test/cpu_test.cc \
+ unit_test/cpu_thread_test.cc \
+ unit_test/math_test.cc \
+ unit_test/planar_test.cc \
+ unit_test/rotate_argb_test.cc \
+ unit_test/rotate_test.cc \
+ unit_test/scale_argb_test.cc \
+ unit_test/scale_plane_test.cc \
+ unit_test/scale_rgb_test.cc \
+ unit_test/scale_test.cc \
+ unit_test/scale_uv_test.cc \
+ unit_test/unit_test.cc \
unit_test/video_common_test.cc
LOCAL_MODULE := libyuv_unittest
include $(BUILD_NATIVE_TEST)
+endif # NDK_ROOT
diff --git a/libs/libyuv/BUILD.bazel b/libs/libyuv/BUILD.bazel
new file mode 100644
index 0000000000..62bc4cd0e3
--- /dev/null
+++ b/libs/libyuv/BUILD.bazel
@@ -0,0 +1,287 @@
+# Copyright 2026 The LibYuv Project Authors. All rights reserved.
+#
+# Description:
+# The libyuv package provides implementation yuv image conversion, rotation
+# and scaling.
+
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load(":libyuv.bzl", "libyuv_hdrs", "libyuv_neon_srcs", "libyuv_rvv_srcs", "libyuv_srcs", "libyuv_sve_srcs", "libyuv_test_srcs")
+
+# Detect if we are in Google's internal repository (where source files are
+# under files/) or open source (where source files are at the root).
+PREFIX = "files/" if len(native.glob(["files/source/*.cc"])) > 0 else ""
+
+# We support NEON for all arm processors, as we only support ARMv7a+.
+SUPPORTS_NEON_COPTS = [
+ "-DLIBYUV_NEON",
+]
+
+# ARMv7a requires that we explicitly enable NEON. arm64 enables it implicitly.
+ENABLE_NEON_COPTS = [
+ "-mfpu=neon",
+]
+
+ENABLE_NEON64_COPTS = [
+ "-march=armv8-a+dotprod+i8mm",
+]
+
+NEON_COPTS = select({
+ "@platforms//cpu:armv7": ENABLE_NEON_COPTS + SUPPORTS_NEON_COPTS,
+ "@platforms//cpu:aarch64": ENABLE_NEON64_COPTS + SUPPORTS_NEON_COPTS,
+ "//conditions:default": [],
+})
+
+SUPPORTS_SVE_COPTS = [
+ "-DLIBYUV_SVE",
+]
+
+ENABLE_SVE_COPTS = [
+ "-march=armv8.5-a+i8mm+sve2",
+]
+
+SVE_COPTS = select({
+ "@platforms//cpu:aarch64": ENABLE_SVE_COPTS + SUPPORTS_SVE_COPTS,
+ "//conditions:default": [],
+})
+
+SUPPORTS_RVV_COPTS = [
+ "-DLIBYUV_RVV",
+ "-fno-tree-vectorize",
+ "-fno-tree-slp-vectorize",
+]
+
+ENABLE_RVV_COPTS = [
+ "-march=rv32gcv",
+]
+
+ENABLE_RVV64_COPTS = [
+ "-march=rv64gcv",
+]
+
+RVV_COPTS = select({
+ "@platforms//cpu:riscv32": ENABLE_RVV_COPTS + SUPPORTS_RVV_COPTS,
+ "@platforms//cpu:riscv64": ENABLE_RVV64_COPTS + SUPPORTS_RVV_COPTS,
+ "//conditions:default": [],
+})
+
+ARCH_SUPPORT_COPTS = select({
+ "@platforms//cpu:armv7": SUPPORTS_NEON_COPTS,
+ "@platforms//cpu:aarch64": SUPPORTS_NEON_COPTS + SUPPORTS_SVE_COPTS,
+ "@platforms//cpu:riscv32": SUPPORTS_RVV_COPTS,
+ "@platforms//cpu:riscv64": SUPPORTS_RVV_COPTS,
+ "@platforms//cpu:wasm32": ["-msimd128"],
+ "@platforms//cpu:wasm64": ["-msimd128"],
+ "//conditions:default": [],
+})
+
+PLATFORM_COPTS = select({
+ "@platforms//os:android": ["-DHAVE_JPEG"],
+ "@platforms//os:linux": ["-DHAVE_JPEG"],
+ "@platforms//os:windows": ["-DHAVE_JPEG"],
+ "@platforms//os:macos": ["-DHAVE_JPEG"],
+ "//conditions:default": [],
+}) + [
+ "-DLIBYUV_DISABLE_SME",
+ "-DLIBYUV_USE_ABSL_FLAGS",
+ "-DGOOGLE_COMMANDLINEFLAGS_FULL_API",
+]
+
+ALL_COPTS = PLATFORM_COPTS + ARCH_SUPPORT_COPTS
+
+ALL_NEON_COPTS = NEON_COPTS + PLATFORM_COPTS
+
+ALL_SVE_COPTS = SVE_COPTS + PLATFORM_COPTS
+
+ALL_RVV_COPTS = RVV_COPTS + PLATFORM_COPTS
+
+NEON_DEPS = select({
+ "@platforms//cpu:armv7": [":libyuv_neon_impl"],
+ "@platforms//cpu:aarch64": [":libyuv_neon_impl"],
+ "//conditions:default": [],
+})
+
+SVE_DEPS = select({
+ "@platforms//cpu:aarch64": [":libyuv_sve_impl"],
+ "//conditions:default": [],
+})
+
+RVV_DEPS = select({
+ "@platforms//cpu:riscv32": [":libyuv_rvv_impl"],
+ "@platforms//cpu:riscv64": [":libyuv_rvv_impl"],
+ "//conditions:default": [],
+})
+
+PLATFORM_DEPS = select({
+ "@platforms//os:android": ["@libjpeg_turbo//:jpeg"],
+ "@platforms//os:linux": ["@libjpeg_turbo//:jpeg"],
+ "@platforms//os:windows": ["@libjpeg_turbo//:jpeg"],
+ "@platforms//os:macos": ["@libjpeg_turbo//:jpeg"],
+ "//conditions:default": [],
+})
+
+ALL_DEPS = NEON_DEPS + SVE_DEPS + RVV_DEPS + PLATFORM_DEPS
+
+cc_library(
+ name = "libyuv",
+ hdrs = [
+ PREFIX + "include/libyuv.h",
+ # Headers included by libyuv.h.
+ PREFIX + "include/libyuv/basic_types.h",
+ PREFIX + "include/libyuv/compare.h",
+ PREFIX + "include/libyuv/convert.h",
+ PREFIX + "include/libyuv/convert_argb.h",
+ PREFIX + "include/libyuv/convert_from.h",
+ PREFIX + "include/libyuv/convert_from_argb.h",
+ PREFIX + "include/libyuv/cpu_id.h",
+ PREFIX + "include/libyuv/mjpeg_decoder.h",
+ PREFIX + "include/libyuv/planar_functions.h",
+ PREFIX + "include/libyuv/rotate.h",
+ PREFIX + "include/libyuv/rotate_argb.h",
+ PREFIX + "include/libyuv/row.h", # Bug 553148013: Should be private.
+ PREFIX + "include/libyuv/scale.h",
+ PREFIX + "include/libyuv/scale_argb.h",
+ PREFIX + "include/libyuv/scale_rgb.h",
+ PREFIX + "include/libyuv/scale_uv.h",
+ PREFIX + "include/libyuv/version.h",
+ PREFIX + "include/libyuv/video_common.h",
+ # Headers included by libyuv/row.h.
+ PREFIX + "include/libyuv/cpu_support.h", # Bug 553148013: Should be private.
+ ],
+ strip_include_prefix = PREFIX + "include",
+ visibility = ["//visibility:public"],
+ deps = [":libyuv_impl"],
+)
+
+cc_library(
+ name = "libyuv_impl",
+ srcs = libyuv_srcs(PREFIX),
+ hdrs = libyuv_hdrs(PREFIX),
+ copts = ALL_COPTS + selects.with_or({
+ (
+ "@platforms//os:android",
+ "@platforms//os:ios",
+ "@platforms//os:tvos",
+ "@platforms//os:windows",
+ "@platforms//os:macos",
+ ): [],
+ "//conditions:default": [
+ "-fomit-frame-pointer",
+ ],
+ }) + selects.with_or({
+ (
+ "@platforms//os:android",
+ "@platforms//os:ios",
+ "@platforms//os:tvos",
+ ): ["-O2"],
+ "//conditions:default": [],
+ }),
+ includes = [PREFIX + "include"],
+ deps = ALL_DEPS,
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "libyuv_neon_impl",
+ srcs = libyuv_neon_srcs(PREFIX),
+ copts = ALL_NEON_COPTS,
+ includes = [PREFIX + "include"],
+ visibility = ["//visibility:private"],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "libyuv_sve_impl",
+ srcs = libyuv_sve_srcs(PREFIX),
+ copts = ALL_SVE_COPTS,
+ includes = [PREFIX + "include"],
+ visibility = ["//visibility:private"],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "libyuv_rvv_impl",
+ srcs = libyuv_rvv_srcs(PREFIX),
+ copts = ALL_RVV_COPTS,
+ includes = [PREFIX + "include"],
+ visibility = ["//visibility:private"],
+ alwayslink = 1,
+)
+
+cc_test(
+ name = "libyuv_test",
+ srcs = libyuv_test_srcs(PREFIX),
+ copts = ALL_COPTS + [
+ "-Wno-unused-variable",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":libyuv",
+ "@com_google_googletest//:gtest_main",
+ "@com_google_absl//absl/flags:flag",
+ "@com_google_absl//absl/flags:parse",
+ ],
+)
+
+cc_library(
+ name = "psnr_lib",
+ srcs = [PREFIX + "util/psnr.cc"],
+ hdrs = [PREFIX + "util/psnr.h"],
+ copts = PLATFORM_COPTS,
+ visibility = ["//visibility:private"],
+)
+
+cc_library(
+ name = "ssim_lib",
+ srcs = [PREFIX + "util/ssim.cc"],
+ hdrs = [PREFIX + "util/ssim.h"],
+ visibility = ["//visibility:private"],
+)
+
+cc_binary(
+ name = "psnr",
+ srcs = [PREFIX + "util/psnr_main.cc"],
+ copts = PLATFORM_COPTS + [
+ "-Wframe-larger-than=64000",
+ ],
+ deps = [
+ ":libyuv",
+ ":psnr_lib",
+ ":ssim_lib",
+ ],
+)
+
+cc_binary(
+ name = "i444tonv12_eg",
+ srcs = [PREFIX + "util/i444tonv12_eg.cc"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":libyuv",
+ ],
+)
+
+cc_binary(
+ name = "cpuid",
+ srcs = [PREFIX + "util/cpuid.c"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":libyuv",
+ ],
+)
+
+cc_binary(
+ name = "yuvconstants",
+ srcs = [PREFIX + "util/yuvconstants.c"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":libyuv",
+ ],
+)
+
+cc_binary(
+ name = "yuvconvert",
+ srcs = [PREFIX + "util/yuvconvert.cc"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":libyuv",
+ ],
+)
diff --git a/libs/libyuv/BUILD.gn b/libs/libyuv/BUILD.gn
index 8904fd6c69..3c37e05910 100644
--- a/libs/libyuv/BUILD.gn
+++ b/libs/libyuv/BUILD.gn
@@ -6,12 +6,14 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
-import("libyuv.gni")
+import("//build/config/features.gni")
+import("//build/config/sanitizers/sanitizers.gni")
import("//testing/test.gni")
+import("libyuv.gni")
declare_args() {
- # Set to false to disable building with gflags.
- libyuv_use_gflags = true
+ # Set to false to disable building with absl flags.
+ libyuv_use_absl_flags = true
# When building a shared library using a target in WebRTC or
# Chromium projects that depends on libyuv, setting this flag
@@ -21,26 +23,49 @@ declare_args() {
config("libyuv_config") {
include_dirs = [ "include" ]
- if (is_android && current_cpu == "arm64") {
- ldflags = [ "-Wl,--dynamic-linker,/system/bin/linker64" ]
+
+ # Define CHROMIUM to tell cpu_id to avoid sandbox unsafe system calls.
+ defines = [ "CHROMIUM" ]
+ if (libyuv_enable_rowwin) {
+ defines += [ "LIBYUV_ENABLE_ROWWIN=1" ]
+ }
+ if (!libyuv_use_neon) {
+ defines += [ "LIBYUV_DISABLE_NEON" ]
+ }
+ if (!libyuv_use_sve) {
+ defines += [ "LIBYUV_DISABLE_SVE" ]
+ }
+ if (!libyuv_use_sme) {
+ defines += [ "LIBYUV_DISABLE_SME" ]
+ }
+ if (libyuv_disable_rvv) {
+ defines += [ "LIBYUV_DISABLE_RVV" ]
+ } else if (libyuv_use_rvv) {
+ defines += [ "LIBYUV_RVV" ]
}
- if (is_android && current_cpu != "arm64") {
- ldflags = [ "-Wl,--dynamic-linker,/system/bin/linker" ]
+ if (!libyuv_use_lsx) {
+ defines += [ "LIBYUV_DISABLE_LSX" ]
+ }
+ if (!libyuv_use_lasx) {
+ defines += [ "LIBYUV_DISABLE_LASX" ]
+ }
+ if (current_cpu == "wasm") {
+ cflags_cc = [ "-msimd128" ]
}
}
# This target is built when no specific target is specified on the command line.
group("default") {
testonly = true
- deps = [
- ":libyuv",
- ]
+ deps = [ ":libyuv" ]
if (libyuv_include_tests) {
deps += [
":compare",
":cpuid",
+ ":i444tonv12_eg",
":libyuv_unittest",
":psnr",
+ ":yuvconstants",
":yuvconvert",
]
}
@@ -50,30 +75,38 @@ group("libyuv") {
all_dependent_configs = [ ":libyuv_config" ]
deps = []
- if (is_win && target_cpu == "x64") {
+ if (is_win && target_cpu == "x64" && current_toolchain == default_toolchain) {
# Compile with clang in order to get inline assembly
- public_deps = [
- ":libyuv_internal(//build/toolchain/win:win_clang_x64)",
- ]
+ public_deps = [ ":libyuv_internal(//build/toolchain/win:win_clang_x64)" ]
} else {
- public_deps = [
- ":libyuv_internal",
- ]
+ public_deps = [ ":libyuv_internal" ]
}
if (libyuv_use_neon) {
deps += [ ":libyuv_neon" ]
}
- if (libyuv_use_msa) {
- deps += [ ":libyuv_msa" ]
+ if (libyuv_use_sve) {
+ deps += [ ":libyuv_sve" ]
}
- if (libyuv_use_mmi) {
- deps += [ ":libyuv_mmi" ]
+ if (libyuv_use_sme) {
+ deps += [ ":libyuv_sme" ]
}
- if (!is_ios) {
+ if (libyuv_use_rvv && !libyuv_disable_rvv) {
+ deps += [ ":libyuv_rvv" ]
+ }
+
+ if (libyuv_use_lsx) {
+ deps += [ ":libyuv_lsx" ]
+ }
+
+ if (libyuv_use_lasx) {
+ deps += [ ":libyuv_lasx" ]
+ }
+
+ if (!is_ios && !libyuv_disable_jpeg) {
# Make sure that clients of libyuv link with libjpeg. This can't go in
# libyuv_internal because in Windows x64 builds that will generate a clang
# build of libjpeg, and we don't want two copies.
@@ -89,20 +122,26 @@ static_library("libyuv_internal") {
"include/libyuv.h",
"include/libyuv/basic_types.h",
"include/libyuv/compare.h",
+ "include/libyuv/compare_row.h",
"include/libyuv/convert.h",
"include/libyuv/convert_argb.h",
"include/libyuv/convert_from.h",
"include/libyuv/convert_from_argb.h",
"include/libyuv/cpu_id.h",
+ "include/libyuv/cpu_support.h",
+ "include/libyuv/loongson_intrinsics.h",
"include/libyuv/mjpeg_decoder.h",
"include/libyuv/planar_functions.h",
"include/libyuv/rotate.h",
"include/libyuv/rotate_argb.h",
"include/libyuv/rotate_row.h",
"include/libyuv/row.h",
+ "include/libyuv/row_sve.h",
"include/libyuv/scale.h",
"include/libyuv/scale_argb.h",
+ "include/libyuv/scale_rgb.h",
"include/libyuv/scale_row.h",
+ "include/libyuv/scale_uv.h",
"include/libyuv/version.h",
"include/libyuv/video_common.h",
@@ -110,6 +149,8 @@ static_library("libyuv_internal") {
"source/compare.cc",
"source/compare_common.cc",
"source/compare_gcc.cc",
+ "source/compare_hvx.cc",
+ "source/compare_wasm.cc",
"source/compare_win.cc",
"source/convert.cc",
"source/convert_argb.cc",
@@ -127,16 +168,24 @@ static_library("libyuv_internal") {
"source/rotate_argb.cc",
"source/rotate_common.cc",
"source/rotate_gcc.cc",
+ "source/rotate_hvx.cc",
+ "source/rotate_wasm.cc",
"source/rotate_win.cc",
"source/row_any.cc",
"source/row_common.cc",
"source/row_gcc.cc",
+ "source/row_hvx.cc",
+ "source/row_wasm.cc",
"source/row_win.cc",
"source/scale.cc",
"source/scale_any.cc",
"source/scale_argb.cc",
"source/scale_common.cc",
"source/scale_gcc.cc",
+ "source/scale_hvx.cc",
+ "source/scale_rgb.cc",
+ "source/scale_uv.cc",
+ "source/scale_wasm.cc",
"source/scale_win.cc",
"source/video_common.cc",
]
@@ -150,7 +199,7 @@ static_library("libyuv_internal") {
configs += [ "//build/config/gcc:symbol_visibility_default" ]
}
- if (!is_ios) {
+ if ((!is_ios || use_blink) && !libyuv_disable_jpeg) {
defines += [ "HAVE_JPEG" ]
# Needed to pull in libjpeg headers. Can't add //third_party:jpeg to deps
@@ -158,9 +207,7 @@ static_library("libyuv_internal") {
deps += [ "//third_party:jpeg_includes" ]
}
- # Always enable optimization for Release and NaCl builds (to workaround
- # crbug.com/538243).
- if (!is_debug || is_nacl) {
+ if (!is_debug) {
configs -= [ "//build/config/compiler:default_optimization" ]
# Enable optimize for speed (-O2) over size (-Os).
@@ -192,61 +239,126 @@ if (libyuv_use_neon) {
"source/scale_neon64.cc",
]
- deps = [
- ":libyuv_internal",
- ]
+ deps = [ ":libyuv_internal" ]
public_configs = [ ":libyuv_config" ]
- # Always enable optimization for Release and NaCl builds (to workaround
- # crbug.com/538243).
- if (!is_debug) {
- configs -= [ "//build/config/compiler:default_optimization" ]
+ if (current_cpu == "arm64") {
+ # The -march arguments below are incompatible with libc++ modules.
+ use_libcxx_modules = false
- # Enable optimize for speed (-O2) over size (-Os).
- # TODO(fbarchard): Consider optimize_speed which is O3.
- configs += [ "//build/config/compiler:optimize_max" ]
+ # TODO(thakis): Do we get this from global flags already? Is this needed?
+ cflags = [ "-march=armv8-a+dotprod+i8mm" ]
}
+ }
+}
+
+if (libyuv_use_sve) {
+ static_library("libyuv_sve") {
+ sources = [
+ "source/rotate_sve.cc",
+ "source/row_sve.cc",
+ ]
- if (current_cpu != "arm64") {
- configs -= [ "//build/config/compiler:compiler_arm_fpu" ]
- cflags = [ "-mfpu=neon" ]
+ deps = [ ":libyuv_internal" ]
+
+ public_configs = [ ":libyuv_config" ]
+
+ # The -march arguments below are incompatible with libc++ modules.
+ use_libcxx_modules = false
+
+ # SVE2 is an Armv9-A feature, but use Armv8.5-A for compatibility.
+ cflags = [ "-march=armv8.5-a+i8mm+sve2" ]
+ }
+}
+
+if (libyuv_use_sme) {
+ static_library("libyuv_sme") {
+ sources = [
+ "source/rotate_sme.cc",
+ "source/row_sme.cc",
+ "source/scale_sme.cc",
+ ]
+
+ deps = [ ":libyuv_internal" ]
+
+ public_configs = [ ":libyuv_config" ]
+
+ # The -march arguments below are incompatible with libc++ modules.
+ use_libcxx_modules = false
+
+ # SME is an Armv9-A feature, but use Armv8.5-A for compatibility with
+ # processors like Apple M4/M5 that support SME without SVE.
+ cflags = [ "-march=armv8.5-a+i8mm+sme" ]
+ }
+}
+
+if (libyuv_use_rvv && !libyuv_disable_rvv) {
+ static_library("libyuv_rvv") {
+ sources = [
+ "source/rotate_rvv.cc",
+ "source/row_rvv.cc",
+ "source/scale_rvv.cc",
+ ]
+
+ deps = [ ":libyuv_internal" ]
+
+ public_configs = [ ":libyuv_config" ]
+
+ # The -march arguments below are incompatible with libc++ modules.
+ use_libcxx_modules = false
+
+ if (current_cpu == "riscv64") {
+ cflags = [
+ "-march=rv64gcv",
+ "-fno-tree-vectorize",
+ "-fno-tree-slp-vectorize",
+ ]
+ } else {
+ cflags = [
+ "-march=rv32gcv",
+ "-fno-tree-vectorize",
+ "-fno-tree-slp-vectorize",
+ ]
}
}
}
-if (libyuv_use_msa) {
- static_library("libyuv_msa") {
+if (libyuv_use_lsx) {
+ static_library("libyuv_lsx") {
sources = [
- # MSA Source Files
- "source/compare_msa.cc",
- "source/rotate_msa.cc",
- "source/row_msa.cc",
- "source/scale_msa.cc",
+ # LSX Source Files
+ "source/rotate_lsx.cc",
+ "source/row_lsx.cc",
+ "source/scale_lsx.cc",
]
- deps = [
- ":libyuv_internal",
+ cflags_cc = [
+ "-mlsx",
+ "-Wno-c++11-narrowing",
]
+ deps = [ ":libyuv_internal" ]
+
public_configs = [ ":libyuv_config" ]
}
}
-if (libyuv_use_mmi) {
- static_library("libyuv_mmi") {
+if (libyuv_use_lasx) {
+ static_library("libyuv_lasx") {
sources = [
- # MMI Source Files
- "source/compare_mmi.cc",
- "source/rotate_mmi.cc",
- "source/row_mmi.cc",
- "source/scale_mmi.cc",
+ # LASX Source Files
+ "source/rotate_lasx.cc",
+ "source/row_lasx.cc",
]
- deps = [
- ":libyuv_internal",
+ cflags_cc = [
+ "-mlasx",
+ "-Wno-c++11-narrowing",
]
+ deps = [ ":libyuv_internal" ]
+
public_configs = [ ":libyuv_config" ]
}
}
@@ -267,19 +379,36 @@ if (libyuv_include_tests) {
]
}
}
+
config("libyuv_unittest_config") {
defines = [ "GTEST_RELATIVE_PATH" ]
}
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ # Override the relative PT_INTERP from //third_party/instrumented_libs so
+ # standalone test binaries can be launched from any working directory
+ # (such as the builder root on LUCI Linux MSan).
+ config("libyuv_msan_ldflags") {
+ if (msan_track_origins == 0) {
+ _msan_archive = "msan-no-origins-noble-lib"
+ } else {
+ _msan_archive = "msan-chained-origins-noble-lib"
+ }
+ _ld_linux = rebase_path(
+ "//third_party/instrumented_libs/binaries/$_msan_archive/lib/ld-linux-x86-64.so.2")
+ ldflags = [ "-Wl,--dynamic-linker=$_ld_linux" ]
+ }
+ }
+
test("libyuv_unittest") {
testonly = true
sources = [
- # sources
- # headers
"unit_test/basictypes_test.cc",
"unit_test/color_test.cc",
"unit_test/compare_test.cc",
+ "unit_test/convert_argb_test.cc",
"unit_test/convert_test.cc",
"unit_test/cpu_test.cc",
"unit_test/cpu_thread_test.cc",
@@ -288,7 +417,10 @@ if (libyuv_include_tests) {
"unit_test/rotate_argb_test.cc",
"unit_test/rotate_test.cc",
"unit_test/scale_argb_test.cc",
+ "unit_test/scale_plane_test.cc",
+ "unit_test/scale_rgb_test.cc",
"unit_test/scale_test.cc",
+ "unit_test/scale_uv_test.cc",
"unit_test/unit_test.cc",
"unit_test/unit_test.h",
"unit_test/video_common_test.cc",
@@ -300,20 +432,28 @@ if (libyuv_include_tests) {
]
defines = []
- if (libyuv_use_gflags) {
- defines += [ "LIBYUV_USE_GFLAGS" ]
- deps += [ "//third_party/gflags" ]
+ if (libyuv_use_absl_flags) {
+ defines += [ "LIBYUV_USE_ABSL_FLAGS" ]
+ deps += [
+ "//third_party/abseil-cpp/absl/flags:flag",
+ "//third_party/abseil-cpp/absl/flags:parse",
+ ]
}
configs += [ ":libyuv_unittest_warnings_config" ]
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
- public_deps = [
- "//testing/gtest",
- ]
+ public_deps = [ "//testing/gtest" ]
public_configs = [ ":libyuv_unittest_config" ]
- if (is_linux) {
+ if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
+
+ # For enabling ASLR.
+ ldflags = [ "-pie" ]
}
if (is_ios) {
configs -= [ "//build/config/compiler:default_symbols" ]
@@ -323,15 +463,10 @@ if (libyuv_include_tests) {
if (!is_ios && !libyuv_disable_jpeg) {
defines += [ "HAVE_JPEG" ]
}
- if (is_android) {
- deps += [ "//testing/android/native_test:native_test_native_code" ]
- }
# TODO(YangZhang): These lines can be removed when high accuracy
# YUV to RGB to Neon is ported.
- if ((target_cpu == "armv7" || target_cpu == "armv7s" ||
- (target_cpu == "arm" && arm_version >= 7) || target_cpu == "arm64") &&
- (arm_use_neon || arm_optionally_use_neon)) {
+ if (target_cpu == "arm" || target_cpu == "arm64") {
defines += [ "LIBYUV_NEON" ]
}
@@ -349,12 +484,14 @@ if (libyuv_include_tests) {
# sources
"util/compare.cc",
]
- deps = [
- ":libyuv",
- ]
- if (is_linux) {
+ deps = [ ":libyuv" ]
+ if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
}
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
}
executable("yuvconvert") {
@@ -362,12 +499,29 @@ if (libyuv_include_tests) {
# sources
"util/yuvconvert.cc",
]
- deps = [
- ":libyuv",
+ deps = [ ":libyuv" ]
+ if (is_linux || is_chromeos) {
+ cflags = [ "-fexceptions" ]
+ }
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
+ }
+
+ executable("yuvconstants") {
+ sources = [
+ # sources
+ "util/yuvconstants.c",
]
- if (is_linux) {
+ deps = [ ":libyuv" ]
+ if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
}
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
}
executable("psnr") {
@@ -377,13 +531,27 @@ if (libyuv_include_tests) {
"util/psnr_main.cc",
"util/ssim.cc",
]
- deps = [
- ":libyuv",
- ]
+ deps = [ ":libyuv" ]
if (!is_ios && !libyuv_disable_jpeg) {
defines = [ "HAVE_JPEG" ]
}
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
+ }
+
+ executable("i444tonv12_eg") {
+ sources = [
+ # sources
+ "util/i444tonv12_eg.cc",
+ ]
+ deps = [ ":libyuv" ]
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
}
executable("cpuid") {
@@ -391,8 +559,25 @@ if (libyuv_include_tests) {
# sources
"util/cpuid.c",
]
- deps = [
- ":libyuv",
- ]
+ deps = [ ":libyuv" ]
+ if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
+ is_msan) {
+ configs += [ ":libyuv_msan_ldflags" ]
+ }
+ }
+}
+
+if (!build_with_chromium) {
+ # GN evaluates each .gn file once per toolchain, so restricting to default
+ # toolchain will ensure write_file() is called only once.
+ if (current_toolchain == default_toolchain) {
+ # Write debug logs to gn_logs.txt.
+ # This is also required for Siso builds.
+ import("//build/gn_logs.gni")
+ lines = [
+ "Generated during 'gn gen' by //BUILD.gn.",
+ "",
+ ] + build_gn_logs
+ write_file("$root_build_dir/gn_logs.txt", lines)
}
}
diff --git a/libs/libyuv/CM_linux_packages.cmake b/libs/libyuv/CM_linux_packages.cmake
index 5f676f8998..4caf15662e 100644
--- a/libs/libyuv/CM_linux_packages.cmake
+++ b/libs/libyuv/CM_linux_packages.cmake
@@ -1,6 +1,7 @@
# determine the version number from the #define in libyuv/version.h
EXECUTE_PROCESS (
- COMMAND grep --perl-regex --only-matching "(?<=LIBYUV_VERSION )[0-9]+" include/libyuv/version.h
+ COMMAND grep -Eo "LIBYUV_VERSION\ [0-9]+" include/libyuv/version.h
+ COMMAND grep -Eo "[0-9]+"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE YUV_VERSION_NUMBER
OUTPUT_STRIP_TRAILING_WHITESPACE )
@@ -8,7 +9,7 @@ SET ( YUV_VER_MAJOR 0 )
SET ( YUV_VER_MINOR 0 )
SET ( YUV_VER_PATCH ${YUV_VERSION_NUMBER} )
SET ( YUV_VERSION ${YUV_VER_MAJOR}.${YUV_VER_MINOR}.${YUV_VER_PATCH} )
-MESSAGE ( "Building ver.: ${YUV_VERSION}" )
+MESSAGE ( VERBOSE "Building ver.: ${YUV_VERSION}" )
# is this a 32-bit or 64-bit build?
IF ( CMAKE_SIZEOF_VOID_P EQUAL 8 )
@@ -45,7 +46,7 @@ ELSE ()
SET ( YUV_SYSTEM_NAME "amd-${YUV_BIT_SIZE}" )
ENDIF ()
ENDIF ()
-MESSAGE ( "Packaging for: ${YUV_SYSTEM_NAME}" )
+MESSAGE ( VERBOSE "Packaging for: ${YUV_SYSTEM_NAME}" )
# define all the variables needed by CPack to create .deb and .rpm packages
SET ( CPACK_PACKAGE_VENDOR "Frank Barchard" )
diff --git a/libs/libyuv/CMakeLists.txt b/libs/libyuv/CMakeLists.txt
index ed4948f066..83211a2f29 100644
--- a/libs/libyuv/CMakeLists.txt
+++ b/libs/libyuv/CMakeLists.txt
@@ -1,51 +1,276 @@
# CMakeLists for libyuv
# Originally created for "roxlu build system" to compile libyuv on windows
-# Run with -DTEST=ON to build unit tests
+# Run with -DUNIT_TEST=ON to build unit tests
-PROJECT ( YUV C CXX ) # "C" is required even for C++ projects
-CMAKE_MINIMUM_REQUIRED( VERSION 2.8 )
-OPTION( TEST "Built unit tests" OFF )
+cmake_minimum_required( VERSION 3.16 )
+project ( YUV C CXX ) # "C" is required even for C++ projects
+option( UNIT_TEST "Built unit tests" OFF )
+option( LIBYUV_DISABLE_JPEG "Disable JPEG support" OFF )
-SET ( ly_base_dir ${PROJECT_SOURCE_DIR} )
-SET ( ly_src_dir ${ly_base_dir}/source )
-SET ( ly_inc_dir ${ly_base_dir}/include )
-SET ( ly_tst_dir ${ly_base_dir}/unit_test )
-SET ( ly_lib_name yuv )
-SET ( ly_lib_static ${ly_lib_name} )
-SET ( ly_lib_shared ${ly_lib_name}_shared )
+include(CheckCSourceCompiles)
-FILE ( GLOB_RECURSE ly_source_files ${ly_src_dir}/*.cc )
-LIST ( SORT ly_source_files )
+set ( ly_base_dir ${PROJECT_SOURCE_DIR} )
+set ( ly_src_dir ${ly_base_dir}/source )
+set ( ly_inc_dir ${ly_base_dir}/include )
+set ( ly_tst_dir ${ly_base_dir}/unit_test )
+set ( ly_lib_name yuv )
+set ( ly_lib_static ${ly_lib_name} )
+set ( ly_lib_shared ${ly_lib_name}_shared )
-FILE ( GLOB_RECURSE ly_unittest_sources ${ly_tst_dir}/*.cc )
-LIST ( SORT ly_unittest_sources )
+# We cannot use GLOB here since we want to be able to separate out files that
+# need particular flags to enable architecture extensions like AArch64's SVE.
+# TODO: More of these files could be separated out for different architectures.
+set ( ly_common_source_files
+ ${ly_src_dir}/compare.cc
+ ${ly_src_dir}/compare_common.cc
+ ${ly_src_dir}/compare_gcc.cc
+ ${ly_src_dir}/compare_hvx.cc
+ ${ly_src_dir}/compare_wasm.cc
+ ${ly_src_dir}/compare_win.cc
+ ${ly_src_dir}/convert.cc
+ ${ly_src_dir}/convert_argb.cc
+ ${ly_src_dir}/convert_from.cc
+ ${ly_src_dir}/convert_from_argb.cc
+ ${ly_src_dir}/convert_jpeg.cc
+ ${ly_src_dir}/convert_to_argb.cc
+ ${ly_src_dir}/convert_to_i420.cc
+ ${ly_src_dir}/cpu_id.cc
+ ${ly_src_dir}/mjpeg_decoder.cc
+ ${ly_src_dir}/mjpeg_validate.cc
+ ${ly_src_dir}/planar_functions.cc
+ ${ly_src_dir}/rotate.cc
+ ${ly_src_dir}/rotate_any.cc
+ ${ly_src_dir}/rotate_argb.cc
+ ${ly_src_dir}/rotate_common.cc
+ ${ly_src_dir}/rotate_gcc.cc
+ ${ly_src_dir}/rotate_hvx.cc
+ ${ly_src_dir}/rotate_rvv.cc
+ ${ly_src_dir}/rotate_wasm.cc
+ ${ly_src_dir}/rotate_win.cc
+ ${ly_src_dir}/row_any.cc
+ ${ly_src_dir}/row_common.cc
+ ${ly_src_dir}/row_gcc.cc
+ ${ly_src_dir}/row_hvx.cc
+ ${ly_src_dir}/row_rvv.cc
+ ${ly_src_dir}/row_wasm.cc
+ ${ly_src_dir}/row_win.cc
+ ${ly_src_dir}/scale.cc
+ ${ly_src_dir}/scale_any.cc
+ ${ly_src_dir}/scale_argb.cc
+ ${ly_src_dir}/scale_common.cc
+ ${ly_src_dir}/scale_gcc.cc
+ ${ly_src_dir}/scale_hvx.cc
+ ${ly_src_dir}/scale_rgb.cc
+ ${ly_src_dir}/scale_rvv.cc
+ ${ly_src_dir}/scale_uv.cc
+ ${ly_src_dir}/scale_wasm.cc
+ ${ly_src_dir}/scale_win.cc
+ ${ly_src_dir}/video_common.cc)
-INCLUDE_DIRECTORIES( BEFORE ${ly_inc_dir} )
+file ( GLOB_RECURSE ly_unittest_sources ${ly_tst_dir}/*.cc )
+list ( SORT ly_unittest_sources )
+
+include_directories( BEFORE ${ly_inc_dir} )
+
+if(MSVC)
+ add_definitions ( -D_CRT_SECURE_NO_WARNINGS )
+endif()
+
+# Need to set PIC to allow creating shared libraries from object file libraries.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# Build the set of objects that do not need to be compiled with flags to enable
+# particular architecture features.
+add_library( ${ly_lib_name}_common_objects OBJECT ${ly_common_source_files} )
+set(ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_common_objects>)
+
+string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSPROC)
+set(LOONGARCH64_ALIASES loongarch64)
+list(FIND LOONGARCH64_ALIASES "${SYSPROC}" LOONGARCH64MATCH)
+
+if(LOONGARCH64MATCH GREATER "-1")
+ set(LOONGARCH64 1)
+endif()
+
+if(NOT MSVC)
+ string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" arch_lowercase)
+
+ if(arch_lowercase MATCHES "^arm" AND NOT arch_lowercase STREQUAL "arm64")
+ # Enable Arm Neon kernels.
+ add_definitions(-DLIBYUV_NEON=1)
+ add_library(${ly_lib_name}_neon OBJECT
+ ${ly_src_dir}/compare_neon.cc
+ ${ly_src_dir}/rotate_neon.cc
+ ${ly_src_dir}/row_neon.cc
+ ${ly_src_dir}/scale_neon.cc)
+ target_compile_options(${ly_lib_name}_neon PRIVATE -mfpu=neon)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_neon>)
+ endif()
+
+ if(arch_lowercase STREQUAL "aarch64" OR arch_lowercase STREQUAL "arm64")
+ # Enable AArch64 Neon dot-product and i8mm kernels.
+ add_library(${ly_lib_name}_neon64 OBJECT
+ ${ly_src_dir}/compare_neon64.cc
+ ${ly_src_dir}/rotate_neon64.cc
+ ${ly_src_dir}/row_neon64.cc
+ ${ly_src_dir}/scale_neon64.cc)
+ target_compile_options(${ly_lib_name}_neon64 PRIVATE -march=armv8.2-a+dotprod+i8mm)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_neon64>)
+
+ # Enable AArch64 SVE kernels.
+ add_library(${ly_lib_name}_sve OBJECT
+ ${ly_src_dir}/rotate_sve.cc
+ ${ly_src_dir}/row_sve.cc)
+ target_compile_options(${ly_lib_name}_sve PRIVATE -march=armv8.5-a+i8mm+sve2)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sve>)
+
+ # Check whether the compiler can compile SME functions; this fails
+ # with Clang for Windows.
+ set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+ set(OLD_CMAKE_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE})
+ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=armv8.5-a+i8mm+sme")
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+ check_c_source_compiles("
+__arm_locally_streaming void func(void) { }
+int main(void) { return 0; }
+ " CAN_COMPILE_SME)
+ set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE ${OLD_CMAKE_TRY_COMPILE_TARGET_TYPE})
+
+ if (CAN_COMPILE_SME)
+ # Enable AArch64 SME kernels.
+ add_library(${ly_lib_name}_sme OBJECT
+ ${ly_src_dir}/rotate_sme.cc
+ ${ly_src_dir}/row_sme.cc
+ ${ly_src_dir}/scale_sme.cc)
+ target_compile_options(${ly_lib_name}_sme PRIVATE -march=armv8.5-a+i8mm+sme)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sme>)
+ else()
+ add_definitions(-DLIBYUV_DISABLE_SME)
+ endif()
+ endif()
+
+ if(arch_lowercase MATCHES "^riscv")
+ option(USE_RVV "Enable RISC-V Vector" ON)
+ if(USE_RVV)
+ if(NOT RISCV_COMPILER_FLAGS)
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8 OR arch_lowercase STREQUAL "riscv64")
+ set(RISCV_COMPILER_FLAGS "-march=rv64gcv")
+ else()
+ set(RISCV_COMPILER_FLAGS "-march=rv32gcv")
+ endif()
+ endif()
+
+ include(CheckCSourceCompiles)
+ set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+ set(OLD_CMAKE_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE})
+ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${RISCV_COMPILER_FLAGS} -fno-tree-vectorize -fno-tree-slp-vectorize")
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+ check_c_source_compiles("
+int main(void) { return 0; }
+ " CAN_COMPILE_RVV)
+ set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE ${OLD_CMAKE_TRY_COMPILE_TARGET_TYPE})
+
+ if(CAN_COMPILE_RVV)
+ if(NOT CMAKE_C_FLAGS MATCHES "-march=" AND NOT CMAKE_C_FLAGS MATCHES "-mcpu=")
+ set(CMAKE_C_FLAGS "${RISCV_COMPILER_FLAGS} -fno-tree-vectorize -fno-tree-slp-vectorize ${CMAKE_C_FLAGS}")
+ set(CMAKE_CXX_FLAGS "${RISCV_COMPILER_FLAGS} -fno-tree-vectorize -fno-tree-slp-vectorize ${CMAKE_CXX_FLAGS}")
+ endif()
+ else()
+ add_definitions(-DLIBYUV_DISABLE_RVV)
+ endif()
+ else()
+ add_definitions(-DLIBYUV_DISABLE_RVV)
+ endif()
+ endif()
+
+ if(arch_lowercase MATCHES "^wasm" OR EMSCRIPTEN)
+ set(CMAKE_C_FLAGS "-msimd128 ${CMAKE_C_FLAGS}")
+ set(CMAKE_CXX_FLAGS "-msimd128 ${CMAKE_CXX_FLAGS}")
+ endif()
+endif()
+
+if(LOONGARCH64)
+ include(CheckCXXSourceCompiles)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-narrowing")
+ check_cxx_source_compiles("
+ int main(int argc, char **argv) {
+ __asm__ volatile (
+ \"vadd.w $vr0, $vr1, $vr1\"
+ );
+ return 0; }" SUPPORTS_LSX)
+
+ check_cxx_source_compiles("
+ int main(int argc, char **argv) {
+ __asm__ volatile (
+ \"xvadd.w $xr0, $xr1, $xr1\"
+ );
+ return 0; }" SUPPORTS_LASX)
+
+ if(SUPPORTS_LSX)
+ # These kernels use only LSX (128-bit) instructions and must not be
+ # compiled with -mlasx, which would let the compiler emit 256-bit LASX
+ # code (e.g. for stack-array copies) and crash on LSX-only CPUs.
+ add_library(${ly_lib_name}_lsx OBJECT
+ ${ly_src_dir}/rotate_lsx.cc
+ ${ly_src_dir}/row_lsx.cc
+ ${ly_src_dir}/scale_lsx.cc)
+ target_compile_options(${ly_lib_name}_lsx PRIVATE -mlsx)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_lsx>)
+ endif()
+ if(SUPPORTS_LASX)
+ add_library(${ly_lib_name}_lasx OBJECT
+ ${ly_src_dir}/rotate_lasx.cc
+ ${ly_src_dir}/row_lasx.cc)
+ target_compile_options(${ly_lib_name}_lasx PRIVATE -mlasx)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_lasx>)
+ endif()
+endif()
# this creates the static library (.a)
-ADD_LIBRARY ( ${ly_lib_static} STATIC ${ly_source_files} )
+add_library( ${ly_lib_static} STATIC ${ly_lib_parts})
# this creates the shared library (.so)
-ADD_LIBRARY ( ${ly_lib_shared} SHARED ${ly_source_files} )
-SET_TARGET_PROPERTIES ( ${ly_lib_shared} PROPERTIES OUTPUT_NAME "${ly_lib_name}" )
-SET_TARGET_PROPERTIES ( ${ly_lib_shared} PROPERTIES PREFIX "lib" )
+add_library( ${ly_lib_shared} SHARED ${ly_lib_parts})
+set_target_properties( ${ly_lib_shared} PROPERTIES OUTPUT_NAME "${ly_lib_name}" )
+set_target_properties( ${ly_lib_shared} PROPERTIES PREFIX "lib" )
+if(WIN32)
+ set_target_properties( ${ly_lib_shared} PROPERTIES IMPORT_PREFIX "lib" )
+endif()
+
+# this creates the cpuid tool
+add_executable ( cpuid ${ly_base_dir}/util/cpuid.c )
+target_link_libraries ( cpuid ${ly_lib_static} )
# this creates the conversion tool
-ADD_EXECUTABLE ( yuvconvert ${ly_base_dir}/util/yuvconvert.cc )
-TARGET_LINK_LIBRARIES ( yuvconvert ${ly_lib_static} )
+add_executable ( yuvconvert ${ly_base_dir}/util/yuvconvert.cc )
+target_link_libraries ( yuvconvert ${ly_lib_static} )
+# this creates the yuvconstants tool
+add_executable ( yuvconstants ${ly_base_dir}/util/yuvconstants.c )
+target_link_libraries ( yuvconstants ${ly_lib_static} )
+if(NOT MSVC)
+ target_link_libraries ( yuvconstants m )
+endif()
-INCLUDE ( FindJPEG )
-if (JPEG_FOUND)
- include_directories( ${JPEG_INCLUDE_DIR} )
- target_link_libraries( yuvconvert ${JPEG_LIBRARY} )
- add_definitions( -DHAVE_JPEG )
+if (NOT LIBYUV_DISABLE_JPEG)
+ find_package ( JPEG )
+ if (JPEG_FOUND)
+ include_directories( ${JPEG_INCLUDE_DIR} )
+ target_link_libraries( ${ly_lib_shared} ${JPEG_LIBRARY} )
+ add_definitions( -DHAVE_JPEG )
+ endif()
endif()
-if(TEST)
+if(UNIT_TEST)
find_library(GTEST_LIBRARY gtest)
if(GTEST_LIBRARY STREQUAL "GTEST_LIBRARY-NOTFOUND")
- set(GTEST_SRC_DIR /usr/src/gtest CACHE STRING "Location of gtest sources")
+ set(GTEST_SRC_DIR_DEFAULT /usr/src/gtest)
+ if (CMAKE_CROSSCOMPILING)
+ set(GTEST_SRC_DIR_DEFAULT ${CMAKE_SOURCE_DIR}/third_party/googletest/src/googletest)
+ endif()
+ set(GTEST_SRC_DIR ${GTEST_SRC_DIR_DEFAULT} CACHE STRING "Location of gtest sources")
if(EXISTS ${GTEST_SRC_DIR}/src/gtest-all.cc)
message(STATUS "building gtest from sources in ${GTEST_SRC_DIR}")
set(gtest_sources ${GTEST_SRC_DIR}/src/gtest-all.cc)
@@ -54,12 +279,15 @@ if(TEST)
include_directories(${GTEST_SRC_DIR}/include)
set(GTEST_LIBRARY gtest)
else()
- message(FATAL_ERROR "TEST is set but unable to find gtest library")
+ message(FATAL_ERROR "UNIT_TEST is set but unable to find gtest library")
endif()
endif()
add_executable(libyuv_unittest ${ly_unittest_sources})
target_link_libraries(libyuv_unittest ${ly_lib_name} ${GTEST_LIBRARY})
+ if(NOT MSVC)
+ target_link_libraries(libyuv_unittest m)
+ endif()
find_library(PTHREAD_LIBRARY pthread)
if(NOT PTHREAD_LIBRARY STREQUAL "PTHREAD_LIBRARY-NOTFOUND")
target_link_libraries(libyuv_unittest pthread)
@@ -81,11 +309,11 @@ endif()
# install the conversion tool, .so, .a, and all the header files
-INSTALL ( PROGRAMS ${CMAKE_BINARY_DIR}/yuvconvert DESTINATION bin )
-INSTALL ( TARGETS ${ly_lib_static} DESTINATION lib )
-INSTALL ( TARGETS ${ly_lib_shared} LIBRARY DESTINATION lib RUNTIME DESTINATION bin )
-INSTALL ( DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include )
+install ( TARGETS yuvconvert DESTINATION bin )
+install ( TARGETS ${ly_lib_static} DESTINATION lib )
+install ( TARGETS ${ly_lib_shared} LIBRARY DESTINATION lib RUNTIME DESTINATION bin ARCHIVE DESTINATION lib )
+install ( DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include )
# create the .deb and .rpm packages using cpack
-INCLUDE ( CM_linux_packages.cmake )
+include ( CM_linux_packages.cmake )
diff --git a/libs/libyuv/DEPS b/libs/libyuv/DEPS
index 758848138d..fb25ec79c9 100644
--- a/libs/libyuv/DEPS
+++ b/libs/libyuv/DEPS
@@ -1,1400 +1,745 @@
+gclient_gn_args_file = 'src/build/config/gclient_args.gni'
+gclient_gn_args = [
+ 'generate_location_tags',
+]
+
vars = {
'chromium_git': 'https://chromium.googlesource.com',
- 'chromium_revision': '08ed8f830153eebaf021fabcf92aed05a4cdc2a8',
+ 'chromium_revision': '2fec1883d8c89392206801daa91f95eb2f8ab3e1',
+ 'gn_version': 'git_revision:4f6a76b64b8279e98004f541f8e136307efe5e01',
+ # ninja CIPD package version.
+ # https://chrome-infra-packages.appspot.com/p/infra/3pp/tools/ninja
+ 'ninja_version': 'version:3@1.12.1.chromium.4',
+ # reclient CIPD package version
+ 'reclient_version': 're_client_version:0.185.0.db415f21-gomaip',
+ # siso CIPD package version.
+ 'siso_version': 'git_revision:9863d88c26094a24fb848f8994da81e005810c76',
+ # CPython 3 CIPD package version for Siso hermetic toolchain.
+ 'cpython3_version': 'version:3@3.11.9.chromium.38',
+ # Fetch configuration files required for the 'use_remoteexec' gn arg
+ 'download_remoteexec_cfg': False,
+ # RBE instance to use for running remote builds
+ 'rbe_instance': Str('projects/rbe-webrtc-developer/instances/default_instance'),
+ # RBE project to download rewrapper config files for. Only needed if
+ # different from the project used in 'rbe_instance'
+ 'rewrapper_cfg_project': Str(''),
+
+ # Keep the Chromium default of generating location tags.
+ 'generate_location_tags': True,
+
+ # By default, download the fuchsia sdk from the public sdk directory.
+ 'fuchsia_sdk_cipd_prefix': 'fuchsia/sdk/core/',
+ 'fuchsia_version': 'version:31.20260204.7.1',
+ # By default, download the fuchsia images from the fuchsia GCS bucket.
+ 'fuchsia_images_bucket': 'fuchsia',
+ 'checkout_fuchsia': False,
+ # Since the images are hundreds of MB, default to only downloading the image
+ # most commonly useful for developers. Bots and developers that need to use
+ # other images can override this with additional images.
+ 'checkout_fuchsia_boot_images': "terminal.x64",
+ 'checkout_fuchsia_product_bundles': '"{checkout_fuchsia_boot_images}" != ""',
+
+ # condition to allowlist deps for non-git-source processing.
+ 'non_git_source': 'True',
+
+ 'android_sdk_build-tools_version': '-jLl4Ibk_WmgTsZaP-ueQwZDhBwkWf5BsQ4UNrkzXF0C',
+ 'android_sdk_emulator_version': '9lGp8nTUCRRWGMnI_96HcKfzjnxEJKUcfvfwmA3wXNkC',
+ 'android_sdk_platform-tools_version': 'qTD9QdBlBf3dyHsN1lJ0RH6AhHxR42Hmg2Ih-Vj4zIEC',
+ 'android_sdk_platforms_version': 'gxwLT70eR_ObwZJzKK8UIS-N549yAocNTmc0JHgO7gUC',
}
deps = {
'src/build':
- Var('chromium_git') + '/chromium/src/build' + '@' + '2ee8adf846a4b485fae5ea87290f82447c638c1a',
+ Var('chromium_git') + '/chromium/src/build' + '@' + 'bc3e93b3c459cfa0bb6bef8944b6398bbd9a7be8',
'src/buildtools':
- Var('chromium_git') + '/chromium/src/buildtools' + '@' + '74cfb57006f83cfe050817526db359d5c8a11628',
+ Var('chromium_git') + '/chromium/src/buildtools' + '@' + '136da69a1267b8db487354b96d44d0cc8add5aeb',
'src/testing':
- Var('chromium_git') + '/chromium/src/testing' + '@' + 'ae52062d9c6d7e38df1c17b53caad1289a4742a1',
+ Var('chromium_git') + '/chromium/src/testing' + '@' + 'd274a4df83dc1781e42bade23902491faf30a0d2',
'src/third_party':
- Var('chromium_git') + '/chromium/src/third_party' + '@' + '3a7aa3a24ca6db31d7afbee24f1c61dffc6d21f4',
+ Var('chromium_git') + '/chromium/src/third_party' + '@' + '51ac942dd6e95df61e747ad812f410a3c23947a0',
'src/buildtools/linux64': {
'packages': [
{
- 'package': 'gn/gn/linux-amd64',
- 'version': 'git_revision:972ed755f8e6d31cae9ba15fcd08136ae1a7886f',
+ 'package': 'gn/gn/linux-${{arch}}',
+ 'version': Var('gn_version'),
}
],
'dep_type': 'cipd',
'condition': 'checkout_linux',
},
+
'src/buildtools/mac': {
'packages': [
{
- 'package': 'gn/gn/mac-amd64',
- 'version': 'git_revision:972ed755f8e6d31cae9ba15fcd08136ae1a7886f',
+ 'package': 'gn/gn/mac-${{arch}}',
+ 'version': Var('gn_version'),
}
],
'dep_type': 'cipd',
'condition': 'checkout_mac',
},
+
'src/buildtools/win': {
'packages': [
{
'package': 'gn/gn/windows-amd64',
- 'version': 'git_revision:972ed755f8e6d31cae9ba15fcd08136ae1a7886f',
+ 'version': Var('gn_version'),
}
],
'dep_type': 'cipd',
'condition': 'checkout_win',
},
- 'src/buildtools/clang_format/script':
- Var('chromium_git') + '/chromium/llvm-project/cfe/tools/clang-format.git' + '@' + '96636aa0e9f047f17447f2d45a094d0b59ed7917',
- 'src/buildtools/third_party/libc++/trunk':
- Var('chromium_git') + '/chromium/llvm-project/libcxx.git' + '@' + '5938e0582bac570a41edb3d6a2217c299adc1bc6',
- 'src/buildtools/third_party/libc++abi/trunk':
- Var('chromium_git') + '/chromium/llvm-project/libcxxabi.git' + '@' + '0d529660e32d77d9111912d73f2c74fc5fa2a858',
- 'src/buildtools/third_party/libunwind/trunk':
- Var('chromium_git') + '/external/llvm.org/libunwind.git' + '@' + '69d9b84cca8354117b9fe9705a4430d789ee599b',
-
- 'src/third_party/catapult':
- Var('chromium_git') + '/catapult.git' + '@' + '2e1d9ff85e0a54a39f55d656086ff00a830cb7a9',
- 'src/third_party/colorama/src':
- Var('chromium_git') + '/external/colorama.git' + '@' + '799604a1041e9b3bc5d2789ecbd7e8db2e18e6b8',
- 'src/third_party/freetype/src':
- Var('chromium_git') + '/chromium/src/third_party/freetype2.git' + '@' + '3fa35aa420ee88856c60d3c0b7fedd43801953cc',
- 'src/third_party/googletest/src':
- Var('chromium_git') + '/external/github.com/google/googletest.git' + '@' + '565f1b848215b77c3732bca345fe76a0431d8b34',
- 'src/third_party/harfbuzz-ng/src':
- Var('chromium_git') + '/external/github.com/harfbuzz/harfbuzz.git' + '@' + 'bbad1b8298125d78c159ed7fdd7bde6a3f3fff56',
- 'src/third_party/libjpeg_turbo':
- Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git' + '@' + 'd460d6b1cb965c3363f36f7ed716f13d60cdb65d',
- 'src/third_party/yasm/source/patched-yasm':
- Var('chromium_git') + '/chromium/deps/yasm/patched-yasm.git' + '@' + '720b70524a4424b15fc57e82263568c8ba0496ad',
- 'src/tools':
- Var('chromium_git') + '/chromium/src/tools' + '@' + '716cf8f74e9df7f53f474bae612b0b69bff8ed4e',
- 'src/tools/swarming_client':
- Var('chromium_git') + '/infra/luci/client-py.git' + '@' + '96f125709acfd0b48fc1e5dae7d6ea42291726ac',
-
- # libyuv-only dependencies (not present in Chromium).
- 'src/third_party/gflags':
- Var('chromium_git') + '/external/webrtc/deps/third_party/gflags' + '@' + '892576179b45861b53e04a112996a738309cf364',
- 'src/third_party/gflags/src':
- Var('chromium_git') + '/external/github.com/gflags/gflags' + '@' + '03bebcb065c83beff83d50ae025a55a4bf94dfca',
- 'src/third_party/gtest-parallel':
- Var('chromium_git') + '/external/webrtc/deps/third_party/gtest-parallel' + '@' + '1dad0e9f6d82ff994130b529d7d814b40eb32b0e',
-
- 'src/third_party/lss': {
- 'url': Var('chromium_git') + '/linux-syscall-support.git' + '@' + '8048ece6c16c91acfe0d36d1d3cc0890ab6e945c',
- 'condition': 'checkout_android or checkout_linux',
- },
-
- # Android deps:
- 'src/third_party/accessibility_test_framework': {
- 'packages': [
- {
- 'package': 'chromium/third_party/accessibility-test-framework',
- 'version': 'b5ec1e56e58e56bc1a0c77d43111c37f9b512c8a',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/auto/src': {
- 'url': Var('chromium_git') + '/external/github.com/google/auto.git' + '@' + '8a81a858ae7b78a1aef71ac3905fade0bbd64e82',
- 'condition': 'checkout_android',
- },
- 'src/third_party/boringssl/src':
- 'https://boringssl.googlesource.com/boringssl.git' + '@' + '05cd93068b0a553afc48f69acbceae10c6a17593',
- 'src/base': {
- 'url': Var('chromium_git') + '/chromium/src/base' + '@' + '3bd361995da8b0cc4e32789e3490f17e4c2997b4',
- 'condition': 'checkout_android',
- },
- 'src/third_party/bazel': {
- 'packages': [
- {
- 'package': 'chromium/third_party/bazel',
- 'version': '1794576f65a721eb0af320a0701e48d31f1b2415',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/bouncycastle': {
- 'packages': [
- {
- 'package': 'chromium/third_party/bouncycastle',
- 'version': 'c078e87552ba26e776566fdaf0f22cd8712743d0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/android_ndk': {
- 'url': Var('chromium_git') + '/android_ndk.git' + '@' + '62582753e869484bf0cc7f7e8d184ce0077033c2',
- 'condition': 'checkout_android',
- },
- 'src/third_party/android_support_test_runner': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_support_test_runner',
- 'version': '96d4bf848cd210fdcbca6bcc8c1b4b39cbd93141',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/android_sdk/public': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_sdk/public/build-tools',
- 'version': '5DL7LQQjVMLClXLzLgmGysccPGsGcjJdvH9z5-uetiIC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/emulator',
- 'version': 'xhyuoquVvBTcJelgRjMKZeoBVSQRjB7pLVJPt5C9saIC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/extras',
- 'version': 'ppQ4TnqDvBHQ3lXx5KPq97egzF5X2FFyOrVHkGmiTMQC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/patcher',
- 'version': 'I6FNMhrXlpB-E1lOhMlvld7xt9lBVNOO83KIluXDyA0C',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/platform-tools',
- 'version': 'MSnxgXN7IurL-MQs1RrTkSFSb8Xd1UtZjLArI8Ty1FgC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/platforms',
- 'version': 'Kg2t9p0YnQk8bldUv4VA3o156uPXLUfIFAmVZ-Gm5ewC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/sources',
- 'version': 'K9uEn3JvNELEVjjVK_GQD3ZQD3rqAnJSxCWxjmUmRkgC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/tools',
- 'version': 'wYcRQC2WHsw2dKWs4EA7fw9Qsyzu1ds1_fRjKmGxe5QC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/tools-lint',
- 'version': '89hXqZYzCum3delB5RV7J_QyWkaRodqdtQS0s3LMh3wC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/tools/clang/dsymutil': {
+ 'src/buildtools/reclient': {
'packages': [
{
- 'package': 'chromium/llvm-build-tools/dsymutil',
- 'version': 'OWlhXkmj18li3yhJk59Kmjbc5KdgLh56TwCd1qBdzlIC',
+ 'package': 'infra/rbe/client/${{platform}}',
+ 'version': Var('reclient_version'),
}
],
- 'condition': 'checkout_mac',
'dep_type': 'cipd',
},
- 'src/third_party/android_build_tools/aapt2': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_build_tools/aapt2',
- 'version': 'version:3.6.0-alpha03-5516695-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/byte_buddy': {
- 'packages': [
- {
- 'package': 'chromium/third_party/byte_buddy',
- 'version': 'c9b53316603fc2d997c899c7ca1707f809b918cd',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/ced/src': {
- 'url': Var('chromium_git') + '/external/github.com/google/compact_enc_det.git' + '@' + 'ba412eaaacd3186085babcd901679a48863c7dd5',
- 'condition': 'checkout_android',
- },
- 'src/third_party/errorprone/lib': {
- 'url': Var('chromium_git') + '/chromium/third_party/errorprone.git' + '@' + '980d49e839aa4984015efed34b0134d4b2c9b6d7',
- 'condition': 'checkout_android',
- },
- 'src/third_party/findbugs': {
- 'url': Var('chromium_git') + '/chromium/deps/findbugs.git' + '@' + '4275d9ac8610db6b1bc9a5e887f97e41b33fac67',
- 'condition': 'checkout_android',
- },
- 'src/third_party/gson': {
- 'packages': [
- {
- 'package': 'chromium/third_party/gson',
- 'version': '681931c9778045903a0ed59856ce2dd8dd7bf7ca',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/guava': {
- 'packages': [
- {
- 'package': 'chromium/third_party/guava',
- 'version': 'a6fba501f3a0de88b9be1daa2052632de5b96a46',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/hamcrest': {
- 'packages': [
- {
- 'package': 'chromium/third_party/hamcrest',
- 'version': '37eccfc658fe79695d6abb6dd497463c4372032f',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/icu': {
- 'url': Var('chromium_git') + '/chromium/deps/icu.git' + '@' + '53f6b233a41ec982d8445996247093f7aaf41639',
- },
- 'src/third_party/icu4j': {
- 'packages': [
- {
- 'package': 'chromium/third_party/icu4j',
- 'version': 'e87e5bed2b4935913ee26a3ebd0b723ee2344354',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/intellij': {
- 'packages': [
- {
- 'package': 'chromium/third_party/intellij',
- 'version': '77c2721b024b36ee073402c08e6d8428c0295336',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/jsr-305/src': {
- 'url': Var('chromium_git') + '/external/jsr-305.git' + '@' + '642c508235471f7220af6d5df2d3210e3bfc0919',
- 'condition': 'checkout_android',
- },
- 'src/third_party/junit/src': {
- 'url': Var('chromium_git') + '/external/junit.git' + '@' + '64155f8a9babcfcf4263cf4d08253a1556e75481',
- 'condition': 'checkout_android',
- },
- 'src/third_party/mockito/src': {
- 'url': Var('chromium_git') + '/external/mockito/mockito.git' + '@' + '04a2a289a4222f80ad20717c25144981210d2eac',
- 'condition': 'checkout_android',
- },
- 'src/third_party/objenesis': {
- 'packages': [
- {
- 'package': 'chromium/third_party/objenesis',
- 'version': '9e367f55e5a65781ee77bfcbaa88fb82b30e75c0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/ow2_asm': {
- 'packages': [
- {
- 'package': 'chromium/third_party/ow2_asm',
- 'version': 'NNAhdJzMdnutUVqfSJm5v0tVazA9l3Dd6CRwH6N4Q5kC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/r8': {
- 'packages': [
- {
- 'package': 'chromium/third_party/r8',
- 'version': '6xVKWv-ssICwyU5FC1osaRpeZio2kM4Tko33I_SIK-EC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/proguard': {
- 'packages': [
- {
- 'package': 'chromium/third_party/proguard',
- 'version': '3bd778c422ea5496de2ef25c007a517dbb5ce5ca',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/requests/src': {
- 'url': Var('chromium_git') + '/external/github.com/kennethreitz/requests.git' + '@' + 'f172b30356d821d180fa4ecfa3e71c7274a32de4',
- 'condition': 'checkout_android',
- },
- 'src/third_party/robolectric': {
- 'packages': [
- {
- 'package': 'chromium/third_party/robolectric',
- 'version': '1KXoOiNP1a_uZNdM2ybWKwAQNow1dHTXTig-ZK4Xgq8C',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/robolectric/robolectric': {
- 'url': Var('chromium_git') + '/external/robolectric.git' + '@' + '3d6dcabf5521e028c8efc2778ab6bd8c7b6d923c',
- 'condition': 'checkout_android',
- },
- 'src/third_party/sqlite4java': {
- 'packages': [
- {
- 'package': 'chromium/third_party/sqlite4java',
- 'version': '889660698187baa7c8b0d79f7bf58563125fbd66',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/ub-uiautomator/lib': {
- 'url': Var('chromium_git') + '/chromium/third_party/ub-uiautomator.git' + '@' + '00270549ce3161ae72ceb24712618ea28b4f9434',
+ 'src/third_party/catapult':
+ Var('chromium_git') + '/catapult.git' + '@' + '520359912034c843ec1903864f88182b1afb88d1',
+ 'src/third_party/clang-format/script':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/clang/tools/clang-format.git' + '@' + 'c2725e0622e1a86d55f14514f2177a39efea4a0e',
+ 'src/third_party/compiler-rt/src':
+ 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/compiler-rt.git@996704b467befd58a41cda583dfb5d6f75d446e6',
+ 'src/third_party/colorama/src':
+ Var('chromium_git') + '/external/colorama.git' + '@' + '3de9f013df4b470069d03d250224062e8cf15c49',
+ 'src/third_party/cpu_features/src': {
+ 'url': Var('chromium_git') + '/external/github.com/google/cpu_features.git' + '@' + '936b9ab5515dead115606559502e3864958f7f6e',
'condition': 'checkout_android',
},
- 'src/third_party/xstream': {
- 'packages': [
- {
- 'package': 'chromium/third_party/xstream',
- 'version': '4278b1b78b86ab7a1a29e64d5aec9a47a9aab0fe',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- # iOS deps:
- 'src/ios': {
- 'url': Var('chromium_git') + '/chromium/src/ios' + '@' + '2f3f83ac15360abe4307e5092647d6dd3601e861',
- 'condition': 'checkout_ios'
- },
-
- # Win deps:
- # Dependencies used by libjpeg-turbo
- 'src/third_party/yasm/binaries': {
- 'url': Var('chromium_git') + '/chromium/deps/yasm/binaries.git' + '@' + '52f9b3f4b0aa06da24ef8b123058bb61ee468881',
- 'condition': 'checkout_win',
- },
-
- # === ANDROID_DEPS Generated Code Start ===
- # Generated by //tools/android/roll/android_deps/fetch_all.py
- 'src/third_party/android_deps/libs/android_arch_core_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_core_common',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ 'src/third_party/depot_tools':
+ Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + 'ccffb57777dca15a09c9a1029acdd776d4ae9a06',
+ 'third_party/fuchsia-gn-sdk': {
+ 'url': Var('chromium_git') + '/chromium/src/third_party/fuchsia-gn-sdk.git' + '@' + '0d6902558d92fe3d49ba9a8f638ddea829be595b',
+ 'condition': 'checkout_fuchsia',
},
+ 'src/third_party/googletest/src':
+ Var('chromium_git') + '/external/github.com/google/googletest.git' + '@' + '4fe3307fb2d9f86d19777c7eb0e4809e9694dde7',
+ 'src/third_party/harfbuzz-ng/src':
+ Var('chromium_git') + '/external/github.com/harfbuzz/harfbuzz.git' + '@' + 'fa2908bf16d2ccd6623f4d575455fea72a1a722b',
+ 'src/third_party/instrumented_libs': {
+ 'url': Var('chromium_git') + '/chromium/third_party/instrumented_libraries.git' + '@' + '69015643b3f68dbd438c010439c59adc52cac808',
+ 'condition': 'checkout_instrumented_libraries',
+ },
+ 'src/third_party/libc++/src':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxx.git' + '@' + '7ab65651aed6802d2599dcb7a73b1f82d5179d05',
+ 'src/third_party/libc++abi/src':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxxabi.git' + '@' + '8f11bb1d4438d0239d0dfc1bd9456a9f31629dda',
+ 'src/third_party/llvm-libc/src':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libc.git' + '@' + 'd38523b674e26b7c8d61ed2e48d6cfe248b12da0',
+
+ 'src/third_party/libunwind/src':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libunwind.git' + '@' + '17ccf7d110c5526cb77e93cfd8330f491fb2bf18',
+ 'src/third_party/libjpeg_turbo':
+ Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git' + '@' + '6bb85251a8382b5e07f635a981ac685cc5ab5053',
+ 'src/third_party/nasm':
+ Var('chromium_git') + '/chromium/deps/nasm.git' + '@' + 'af5eeeb054bebadfbb79c7bcd100a95e2ad4525f',
+ 'src/tools':
+ Var('chromium_git') + '/chromium/src/tools' + '@' + '73a128cf7335a71785ed9324d6a15b5559ca8050',
- 'src/third_party/android_deps/libs/android_arch_lifecycle_livedata': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_livedata',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_livedata_core': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_livedata_core',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_core_runtime': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_core_runtime',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_viewmodel': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_viewmodel',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_localbroadcastmanager': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_localbroadcastmanager',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_print': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_print',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_common',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_common_java8': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_common_java8',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_runtime': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_runtime',
- 'version': 'version:1.1.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_annotation_annotation': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_annotation_annotation',
- 'version': 'version:1.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_lifecycle_lifecycle_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_lifecycle_lifecycle_common',
- 'version': 'version:2.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_test_core': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_test_core',
- 'version': 'version:1.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_test_ext_junit': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_test_ext_junit',
- 'version': 'version:1.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_test_monitor': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_test_monitor',
- 'version': 'version:1.1.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_test_rules': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_test_rules',
- 'version': 'version:1.1.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/androidx_test_runner': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/androidx_test_runner',
- 'version': 'version:1.1.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_animated_vector_drawable': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_animated_vector_drawable',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_appcompat_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_appcompat_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_asynclayoutinflater': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_asynclayoutinflater',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_coordinatorlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_coordinatorlayout',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_customview': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_customview',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_drawerlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_drawerlayout',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_slidingpanelayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_slidingpanelayout',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_loader': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_loader',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_cardview_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_cardview_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_design': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_design',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_gridlayout_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_gridlayout_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_leanback_v17': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_leanback_v17',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_mediarouter_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_mediarouter_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_multidex': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_multidex',
- 'version': 'version:1.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_palette_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_palette_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_preference_leanback_v17': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_preference_leanback_v17',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_preference_v14': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_preference_v14',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_preference_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_preference_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_recyclerview_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_recyclerview_v7',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_annotations': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_annotations',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_compat': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_compat',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_core_ui': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_core_ui',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_core_utils': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_core_utils',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_fragment': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_fragment',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_media_compat': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_media_compat',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
+ # libyuv-only dependencies (not present in Chromium).
+ 'src/third_party/gtest-parallel':
+ Var('chromium_git') + '/external/webrtc/deps/third_party/gtest-parallel' + '@' + '1dad0e9f6d82ff994130b529d7d814b40eb32b0e',
- 'src/third_party/android_deps/libs/com_android_support_support_v13': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_v13',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ 'src/third_party/lss': {
+ 'url': Var('chromium_git') + '/linux-syscall-support.git' + '@' + '29164a80da4d41134950d76d55199ea33fbb9613',
+ 'condition': 'checkout_android or checkout_linux',
},
- 'src/third_party/android_deps/libs/com_android_support_support_v4': {
+ 'src/third_party/re2/src':
+ Var('chromium_git') + '/external/github.com/google/re2.git' + '@' + '972a15cedd008d846f1a39b2e88ce48d7f166cbd',
+
+ # Android deps:
+ 'src/third_party/kotlin_stdlib/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_v4',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/kotlin_stdlib',
+ 'version': 'FfVJUc4q9S6JB4uqVGPJQ3anrc1Y_W0O-oGK_2MJA5gC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_support_vector_drawable': {
+ 'src/third_party/kotlinc/current': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_vector_drawable',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/kotlinc',
+ 'version': 'KZWh3uhuenRgsoGqNYgSo82FO3sMwZjvmuR9TBvanTcC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_transition': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_transition',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ 'src/third_party/android_toolchain/ndk': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_toolchain/android_toolchain',
+ 'version': 'KXOia11cm9lVdUdPlbGLu8sCz6Y4ey_HV2s8_8qeqhgC',
+ },
+ ],
+ 'condition': 'checkout_android',
+ 'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_swiperefreshlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_swiperefreshlayout',
- 'version': 'version:28.0.0-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ 'src/third_party/androidx/cipd': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/androidx',
+ 'version': 'CL9NOUuKPO8ESaAQIKtBbN6MV0jndo4M-vT-Fjf3ZGoC',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_viewpager': {
+ 'src/third_party/android_support_test_runner': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_viewpager',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_support_test_runner',
+ 'version': '96d4bf848cd210fdcbca6bcc8c1b4b39cbd93141',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_collections': {
+ 'src/third_party/android_sdk/public': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_collections',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_sdk/public/build-tools/36.1.0',
+ 'version': Var('android_sdk_build-tools_version'),
},
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_versionedparcelable': {
- 'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_versionedparcelable',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_sdk/public/emulator',
+ 'version': Var('android_sdk_emulator_version'),
},
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_cursoradapter': {
- 'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_cursoradapter',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_sdk/public/platform-tools',
+ 'version': Var('android_sdk_platform-tools_version'),
},
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_interpolator': {
- 'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_interpolator',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_sdk/public/platforms/android-36.1',
+ 'version': Var('android_sdk_platforms_version'),
},
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_documentfile': {
- 'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_documentfile',
- 'version': 'version:28.0.0-cr0',
+ 'package': 'chromium/third_party/android_sdk/public/cmdline-tools/linux',
+ 'version': 'LZa8CWNVWS6UUQgQ7IJdFCqRV1Bmx2-alTNqEDJpJkcC',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth': {
+ 'src/third_party/android_build_tools/aapt2/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/android_build_tools/aapt2',
+ 'version': 'vUP1cs7krVsNHbUJpwSDR7rvWuYLGzINVa9xopPpt74C',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth_api_phone': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_api_phone',
- 'version': 'version:15.0.1-cr0',
- },
- ],
+ 'src/third_party/ced/src': {
+ 'url': Var('chromium_git') + '/external/github.com/google/compact_enc_det.git' + '@' + 'ba412eaaacd3186085babcd901679a48863c7dd5',
+ 'condition': 'checkout_android',
+ },
+ 'src/third_party/errorprone/lib': {
+ 'url': Var('chromium_git') + '/chromium/third_party/errorprone.git' + '@' + '980d49e839aa4984015efed34b0134d4b2c9b6d7',
'condition': 'checkout_android',
- 'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth_base': {
+ 'src/third_party/findbugs': {
+ 'url': Var('chromium_git') + '/chromium/deps/findbugs.git' + '@' + '4275d9ac8610db6b1bc9a5e887f97e41b33fac67',
+ 'condition': 'checkout_android',
+ },
+ 'src/third_party/gson': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_base',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/gson',
+ 'version': '681931c9778045903a0ed59856ce2dd8dd7bf7ca',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_base': {
+ 'src/third_party/hamcrest/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_base',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/hamcrest',
+ 'version': 'dBioOAmFJjqAr_DY7dipbXdVfAxUQwjOBNibMPtX8lQC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_basement': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_basement',
- 'version': 'version:15.0.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+
+ 'src/third_party/icu': {
+ 'url': Var('chromium_git') + '/chromium/deps/icu.git' + '@' + 'a86a32e67b8d1384b33f8fa48c83a6079b86f8cd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_cast': {
+ 'src/third_party/icu4j/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_cast',
- 'version': 'version:16.0.1-cr0',
+ 'package': 'chromium/third_party/icu4j',
+ 'version': '8dV7WRVX0tTaNNqkLEnCA_dMofr2MJXFK400E7gOFygC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_cast_framework': {
+ 'src/third_party/intellij': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_cast_framework',
- 'version': 'version:16.0.1-cr0',
+ 'package': 'chromium/third_party/intellij',
+ 'version': '77c2721b024b36ee073402c08e6d8428c0295336',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_clearcut': {
+ 'src/third_party/jdk/current': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_clearcut',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/jdk/linux-amd64',
+ 'version': '2iiuF-nKDH3moTImx2op4WTRetbfhzKoZhH7Xo44zGsC',
},
],
- 'condition': 'checkout_android',
+ # Needed on Linux for use on chromium_presubmit (for checkstyle).
+ 'condition': '(checkout_android or checkout_linux) and non_git_source',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_fido': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_fido',
- 'version': 'version:15.0.1-cr0',
- },
- ],
+ 'src/third_party/jsr-305/src': {
+ 'url': Var('chromium_git') + '/external/jsr-305.git' + '@' + '642c508235471f7220af6d5df2d3210e3bfc0919',
+ 'condition': 'checkout_android',
+ },
+ 'src/third_party/junit/src': {
+ 'url': Var('chromium_git') + '/external/junit.git' + '@' + '0eb5ce72848d730da5bd6d42902fdd6a8a42055d',
+ 'condition': 'checkout_android',
+ },
+ 'src/third_party/libunwindstack': {
+ 'url': Var('chromium_git') + '/chromium/src/third_party/libunwindstack.git' + '@' + '0928ad0d25e4af07c8be5ab06d0ca584f9f4ceb5',
'condition': 'checkout_android',
- 'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_flags': {
+ 'src/third_party/ninja': {
+ 'packages': [
+ {
+ 'package': 'infra/3pp/tools/ninja/${{platform}}',
+ 'version': Var('ninja_version'),
+ }
+ ],
+ 'dep_type': 'cipd',
+ },
+ 'src/third_party/siso/cipd': {
+ 'packages': [
+ {
+ 'package': 'build/siso/${{platform}}',
+ 'version': Var('siso_version'),
+ }
+ ],
+ 'dep_type': 'cipd',
+ },
+ # Always download Linux x64 package regardless of host OS for RBE workers.
+ 'src/third_party/cpython3/linux-amd64': {
'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_flags',
- 'version': 'version:15.0.1-cr0',
- },
+ {
+ 'package': 'infra/3pp/tools/cpython3/linux-amd64',
+ 'version': Var('cpython3_version'),
+ },
],
- 'condition': 'checkout_android',
+ 'condition': 'non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_gcm': {
+ # Host platform package.
+ 'src/third_party/cpython3/host': {
'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_gcm',
- 'version': 'version:15.0.1-cr0',
- },
+ {
+ 'package': 'infra/3pp/tools/cpython3/${{platform}}',
+ 'version': Var('cpython3_version'),
+ },
],
- 'condition': 'checkout_android',
+ 'condition': 'non_git_source',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_iid': {
+ 'src/third_party/mockito/src': {
+ 'url': Var('chromium_git') + '/external/mockito/mockito.git' + '@' + '7c3641bcef717ffa7d765f2c86b847d0aab1aac9',
+ 'condition': 'checkout_android',
+ },
+ 'src/third_party/ow2_asm': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_iid',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/ow2_asm',
+ 'version': 'NNAhdJzMdnutUVqfSJm5v0tVazA9l3Dd6CRwH6N4Q5kC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_instantapps': {
+ 'src/third_party/r8/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_instantapps',
- 'version': 'version:16.0.0-cr0',
+ 'package': 'chromium/third_party/r8',
+ 'version': 'MW7AMY0Q5SzgubYVtiVVg4LMjsFBcLKwKyc2dqcArAMC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_location': {
+ # This duplication is intentional, so we avoid updating the r8.jar used by
+ # dexing unless necessary, since each update invalidates all incremental
+ # dexing and unnecessarily slows down all bots.
+ 'src/third_party/r8/d8/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_location',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/r8',
+ 'version': 'MW7AMY0Q5SzgubYVtiVVg4LMjsFBcLKwKyc2dqcArAMC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_phenotype': {
+ 'src/third_party/proguard': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_phenotype',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/proguard',
+ 'version': 'Fd91BJFVlmiO6c46YMTsdy7n2f5Sk2hVVGlzPLvqZPsC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
+ 'src/third_party/requests/src': {
+ 'url': Var('chromium_git') + '/external/github.com/kennethreitz/requests.git' + '@' + 'c7e0fc087ceeadb8b4c84a0953a422c474093d6d',
+ 'condition': 'checkout_android',
+ },
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_places_placereport': {
+ 'src/third_party/robolectric/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_places_placereport',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/robolectric',
+ 'version': 'dr-aJxRAPYDTBJXnjfht-bdxyywD6BP1lrcjZZPnRG0C',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_stats': {
+ 'src/third_party/sqlite4java/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_stats',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/sqlite4java',
+ 'version': 'LofjKH9dgXIAJhRYCPQlMFywSwxYimrfDeBmaHc-Z5EC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_tasks': {
+ 'src/third_party/turbine/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_tasks',
- 'version': 'version:15.0.1-cr0',
+ 'package': 'chromium/third_party/turbine',
+ 'version': 'BMHNhxMhr7uGz1rh_Od_JE4kAdP9K5MXr6GN2R9tQkAC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
+ 'src/third_party/ub-uiautomator/lib': {
+ 'url': Var('chromium_git') + '/chromium/third_party/ub-uiautomator.git' + '@' + '00270549ce3161ae72ceb24712618ea28b4f9434',
+ 'condition': 'checkout_android',
+ },
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_vision': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_vision',
- 'version': 'version:15.0.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ # iOS deps:
+ 'src/ios': {
+ 'url': Var('chromium_git') + '/chromium/src/ios' + '@' + '5d7abd83379e442938c885daa681ad8c48c21f63',
+ 'condition': 'checkout_ios'
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_vision_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_vision_common',
- 'version': 'version:15.0.1-cr0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
+ 'src/third_party/llvm-build/Release+Asserts': {
+ 'dep_type': 'gcs',
+ 'bucket': 'chromium-browser-clang',
+ 'objects': [
+ {
+ # The Android libclang_rt.builtins libraries are currently only included in the Linux clang package.
+ 'object_name': 'Linux_x64/clang-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '9a5dfd23f6d37b21709327aa39562225c01a058cc209d1c629448d6d6d874ab2',
+ 'size_bytes': 57775784,
+ 'generation': 1770870606052091,
+ 'condition': 'host_os == "linux" or checkout_android',
+ },
+ {
+ 'object_name': 'Linux_x64/llvmobjdump-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '103ba738fb6ebcd290e38458fdec8fa7b95bc3216af1e60ad260bc065b49d447',
+ 'size_bytes': 5798432,
+ 'generation': 1770870606263889,
+ 'condition': '((checkout_linux or checkout_mac or checkout_android) and host_os == "linux")',
+ },
+ {
+ 'object_name': 'Mac/clang-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '0230c7c9a625ceb2d2c650d995fb9cb5cdb6ab7c0c05042f560cc0cda441a2d0',
+ 'size_bytes': 54682532,
+ 'generation': 1770870608103936,
+ 'condition': 'host_os == "mac" and host_cpu == "x64"',
+ },
+ {
+ 'object_name': 'Mac/clang-mac-runtime-library-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': 'a7af5d119637f67bff7f390923a78a712b1067157c019aebf2dcddbc4ed7b640',
+ 'size_bytes': 1015272,
+ 'generation': 1770870616072374,
+ 'condition': 'checkout_mac and not host_os == "mac"',
+ },
+ {
+ 'object_name': 'Mac/llvmobjdump-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '556b8abf3ec08c42769986ea8940ac7f09fa2cfe49328a2ccccee4641efce6c4',
+ 'size_bytes': 5703228,
+ 'generation': 1770870608115133,
+ 'condition': 'host_os == "mac" and host_cpu == "x64"',
+ },
+ {
+ 'object_name': 'Mac_arm64/clang-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '34102ff942cb0d7e2163fada5aeb53f5c2438d07d8caf5ed79970b133adc04be',
+ 'size_bytes': 45849908,
+ 'generation': 1770870617609650,
+ 'condition': 'host_os == "mac" and host_cpu == "arm64"',
+ },
+ {
+ 'object_name': 'Mac_arm64/llvmobjdump-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': 'aea68efe7aeab8eb4d2371e79e7dcb021e1448d8edd0d4851fa34907b93205a6',
+ 'size_bytes': 5446112,
+ 'generation': 1770870617921094,
+ 'condition': 'host_os == "mac" and host_cpu == "arm64"',
+ },
+ {
+ 'object_name': 'Win/clang-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': 'cca114f624db433c2ba6458a6de34e6e474bea379f483817dd5dc0a02c59df3c',
+ 'size_bytes': 49244312,
+ 'generation': 1770870628650736,
+ 'condition': 'host_os == "win"',
+ },
+ {
+ 'object_name': 'Win/clang-win-runtime-library-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '3f2ad3f2adb92384ecdcb33385aa8beacd7f38d27bed6179e39b2e971469d028',
+ 'size_bytes': 2595252,
+ 'generation': 1770870636929948,
+ 'condition': 'checkout_win and not host_os == "win"',
+ },
+ {
+ 'object_name': 'Win/llvmobjdump-llvmorg-23-init-3706-gfc648683-1.tar.xz',
+ 'sha256sum': '7baba8d463b92d16c510d67be60eed89897ba76898121ea709264cbb41066af1',
+ 'size_bytes': 5859968,
+ 'generation': 1770870628750202,
+ 'condition': '(checkout_linux or checkout_mac or checkout_android) and host_os == "win"',
+ },
+ ]
},
- 'src/third_party/android_deps/libs/com_google_code_findbugs_jsr305': {
+ # Everything coming after this is automatically updated by the auto-roller.
+ # === ANDROID_DEPS Generated Code Start ===
+ # Generated by //third_party/android_deps/fetch_all.py
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_common': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_code_findbugs_jsr305',
- 'version': 'version:1.3.9-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_android_tools_common',
+ 'version': 'version:2@30.2.0-beta01.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger': {
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_layoutlib_layoutlib_api': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger',
- 'version': 'version:2.17-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_android_tools_layoutlib_layoutlib_api',
+ 'version': 'version:2@30.2.0-beta01.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_compiler': {
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_sdk_common': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_compiler',
- 'version': 'version:2.17-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_android_tools_sdk_common',
+ 'version': 'version:2@30.2.0-beta01.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_producers': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_producers',
- 'version': 'version:2.17-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework',
+ 'version': 'version:2@4.0.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_spi': {
+ 'src/third_party/android_deps/cipd/libs/com_googlecode_java_diff_utils_diffutils': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_spi',
- 'version': 'version:2.17-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_googlecode_java_diff_utils_diffutils',
+ 'version': 'version:2@1.3.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_squareup_javapoet': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_annotations',
- 'version': 'version:2.1.3-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/com_squareup_javapoet',
+ 'version': 'version:2@1.13.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_errorprone_javac_shaded': {
+ 'src/third_party/android_deps/cipd/libs/net_bytebuddy_byte_buddy': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_javac_shaded',
- 'version': 'version:9-dev-r4023-3-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/net_bytebuddy_byte_buddy',
+ 'version': 'version:2@1.17.6.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_googlejavaformat_google_java_format': {
+ 'src/third_party/android_deps/cipd/libs/net_bytebuddy_byte_buddy_agent': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_googlejavaformat_google_java_format',
- 'version': 'version:1.5-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/net_bytebuddy_byte_buddy_agent',
+ 'version': 'version:2@1.17.6.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_guava_guava': {
+ 'src/third_party/android_deps/cipd/libs/org_ccil_cowan_tagsoup_tagsoup': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_guava_guava',
- 'version': 'version:25.0-jre-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_ccil_cowan_tagsoup_tagsoup',
+ 'version': 'version:2@1.2.1.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_j2objc_j2objc_annotations': {
+ 'src/third_party/android_deps/cipd/libs/org_checkerframework_checker_compat_qual': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_j2objc_j2objc_annotations',
- 'version': 'version:1.1-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_checkerframework_checker_compat_qual',
+ 'version': 'version:2@2.5.5.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_protobuf_protobuf_lite': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlin_kotlin_android_extensions_runtime': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_protobuf_protobuf_lite',
- 'version': 'version:3.0.1-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_android_extensions_runtime',
+ 'version': 'version:2@1.9.22.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_squareup_javapoet': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlin_kotlin_parcelize_runtime': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_squareup_javapoet',
- 'version': 'version:1.11.0-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_parcelize_runtime',
+ 'version': 'version:2@1.9.22.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/javax_annotation_jsr250_api': {
+ 'src/third_party/android_deps/cipd/libs/org_jsoup_jsoup': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/javax_annotation_jsr250_api',
- 'version': 'version:1.0-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_jsoup_jsoup',
+ 'version': 'version:2@1.15.1.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/javax_inject_javax_inject': {
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_android': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/javax_inject_javax_inject',
- 'version': 'version:1-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_android',
+ 'version': 'version:2@5.19.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/net_sf_kxml_kxml2': {
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_core': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/net_sf_kxml_kxml2',
- 'version': 'version:2.3.0-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_core',
+ 'version': 'version:2@5.19.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_checkerframework_checker_compat_qual': {
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_subclass': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/org_checkerframework_checker_compat_qual',
- 'version': 'version:2.3.0-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_subclass',
+ 'version': 'version:2@5.19.0.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_codehaus_mojo_animal_sniffer_annotations': {
+ 'src/third_party/android_deps/cipd/libs/org_objenesis_objenesis': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/org_codehaus_mojo_animal_sniffer_annotations',
- 'version': 'version:1.14-cr0',
+ 'package': 'chromium/third_party/android_deps/libs/org_objenesis_objenesis',
+ 'version': 'version:2@3.3.cr2',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
# === ANDROID_DEPS Generated Code End ===
}
-# Define rules for which include paths are allowed in our source.
-include_rules = [ '+gflags' ]
-
-pre_deps_hooks = [
- {
- # Remove any symlinks from before 177567c518b121731e507e9b9c4049c4dc96e4c8.
- # TODO(kjellander): Remove this in March 2017.
- 'name': 'cleanup_links',
- 'pattern': '.',
- 'action': ['python', 'src/cleanup_links.py'],
- },
-]
-
hooks = [
{
# This clobbers when necessary (based on get_landmines.py). It should be
@@ -1403,7 +748,7 @@ hooks = [
'name': 'landmines',
'pattern': '.',
'action': [
- 'python',
+ 'python3',
'src/build/landmines.py',
'--landmine-scripts',
'src/tools_libyuv/get_landmines.py',
@@ -1416,155 +761,167 @@ hooks = [
'name': 'sysroot_arm',
'pattern': '.',
'condition': 'checkout_linux and checkout_arm',
- 'action': ['python', 'src/build/linux/sysroot_scripts/install-sysroot.py',
+ 'action': ['python3', 'src/build/linux/sysroot_scripts/install-sysroot.py',
'--arch=arm'],
},
{
'name': 'sysroot_arm64',
'pattern': '.',
'condition': 'checkout_linux and checkout_arm64',
- 'action': ['python', 'src/build/linux/sysroot_scripts/install-sysroot.py',
+ 'action': ['python3', 'src/build/linux/sysroot_scripts/install-sysroot.py',
'--arch=arm64'],
},
{
'name': 'sysroot_x86',
'pattern': '.',
'condition': 'checkout_linux and (checkout_x86 or checkout_x64)',
- 'action': ['python', 'src/build/linux/sysroot_scripts/install-sysroot.py',
+ 'action': ['python3', 'src/build/linux/sysroot_scripts/install-sysroot.py',
'--arch=x86'],
},
- {
- 'name': 'sysroot_mips',
- 'pattern': '.',
- 'condition': 'checkout_linux and checkout_mips',
- 'action': ['python', 'src/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=mips'],
- },
{
'name': 'sysroot_x64',
'pattern': '.',
'condition': 'checkout_linux and checkout_x64',
- 'action': ['python', 'src/build/linux/sysroot_scripts/install-sysroot.py',
+ 'action': ['python3', 'src/build/linux/sysroot_scripts/install-sysroot.py',
'--arch=x64'],
},
{
# Update the Windows toolchain if necessary.
'name': 'win_toolchain',
'pattern': '.',
- 'action': ['python', 'src/build/vs_toolchain.py', 'update'],
+ 'action': ['python3', 'src/build/vs_toolchain.py', 'update'],
},
{
# Update the Mac toolchain if necessary.
'name': 'mac_toolchain',
'pattern': '.',
- 'action': ['python', 'src/build/mac_toolchain.py'],
+ 'action': ['python3', 'src/build/mac_toolchain.py'],
+ 'condition': 'checkout_mac',
},
- # Pull binutils for linux, enabled debug fission for faster linking /
- # debugging when used with clang on Ubuntu Precise.
- # https://code.google.com/p/chromium/issues/detail?id=352046
{
- 'name': 'binutils',
- 'pattern': 'src/third_party/binutils',
+ 'name': 'Download Fuchsia SDK from GCS',
+ 'pattern': '.',
+ 'condition': 'checkout_fuchsia',
'action': [
- 'python',
- 'src/third_party/binutils/download.py',
+ 'python3',
+ 'src/build/fuchsia/update_sdk.py',
+ '--cipd-prefix={fuchsia_sdk_cipd_prefix}',
+ '--version={fuchsia_version}',
],
},
{
- # Pull clang if needed or requested via GYP_DEFINES.
- # Note: On Win, this should run after win_toolchain, as it may use it.
- 'name': 'clang',
+ 'name': 'Download Fuchsia system images',
'pattern': '.',
- 'action': ['python', 'src/tools/clang/scripts/update.py'],
+ 'condition': 'checkout_fuchsia and checkout_fuchsia_product_bundles',
+ 'action': [
+ 'python3',
+ 'src/build/fuchsia/update_product_bundles.py',
+ '{checkout_fuchsia_boot_images}',
+ ],
},
{
# Update LASTCHANGE.
'name': 'lastchange',
'pattern': '.',
- 'action': ['python', 'src/build/util/lastchange.py',
+ 'action': ['python3', 'src/build/util/lastchange.py',
'-o', 'src/build/util/LASTCHANGE'],
},
- # Pull clang-format binaries using checked-in hashes.
+ # Pull luci-go binaries (isolate, swarming) using checked-in hashes.
{
- 'name': 'clang_format_win',
+ 'name': 'luci-go_win',
'pattern': '.',
'action': [ 'download_from_google_storage',
'--no_resume',
'--platform=win32',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/win/clang-format.exe.sha1',
+ '--bucket', 'chromium-luci',
+ '-d', 'src/tools/luci-go/win64',
],
},
{
- 'name': 'clang_format_mac',
+ 'name': 'luci-go_mac',
'pattern': '.',
'action': [ 'download_from_google_storage',
'--no_resume',
'--platform=darwin',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/mac/clang-format.sha1',
+ '--bucket', 'chromium-luci',
+ '-d', 'src/tools/luci-go/mac64',
],
},
{
- 'name': 'clang_format_linux',
+ 'name': 'luci-go_linux',
'pattern': '.',
- 'condition': 'host_os == "linux"',
'action': [ 'download_from_google_storage',
'--no_resume',
'--platform=linux*',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/linux64/clang-format.sha1',
+ '--bucket', 'chromium-luci',
+ '-d', 'src/tools/luci-go/linux64',
],
},
- # Pull luci-go binaries (isolate, swarming) using checked-in hashes.
{
- 'name': 'luci-go_win',
+ 'name': 'Generate component metadata for tests',
'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=win32',
- '--no_auth',
- '--bucket', 'chromium-luci',
- '-d', 'src/tools/luci-go/win64',
+ 'action': [
+ 'vpython3',
+ 'src/testing/generate_location_tags.py',
+ '--out',
+ 'src/testing/location_tags.json',
],
},
+ # Download remote exec cfg files
{
- 'name': 'luci-go_mac',
+ 'name': 'configure_reclient_cfgs',
'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=darwin',
- '--no_auth',
- '--bucket', 'chromium-luci',
- '-d', 'src/tools/luci-go/mac64',
- ],
+ 'condition': 'download_remoteexec_cfg',
+ 'action': ['python3',
+ 'src/buildtools/reclient_cfgs/configure_reclient_cfgs.py',
+ '--rbe_instance',
+ Var('rbe_instance'),
+ '--reproxy_cfg_template',
+ 'reproxy.cfg.template',
+ '--rewrapper_cfg_project',
+ Var('rewrapper_cfg_project'),
+ '--quiet',
+ ],
+ },
+ # Configure Siso for developer builds.
+ {
+ 'name': 'configure_siso',
+ 'pattern': '.',
+ 'action': ['python3',
+ 'src/build/config/siso/configure_siso.py',
+ '--rbe_instance',
+ Var('rbe_instance'),
+ ],
},
{
- 'name': 'luci-go_linux',
+ 'name': 'dsymutil_mac_arm64',
'pattern': '.',
- 'action': [ 'download_from_google_storage',
+ 'condition': 'host_os == "mac" and host_cpu == "arm64"',
+ 'action': [ 'python3',
+ 'src/third_party/depot_tools/download_from_google_storage.py',
'--no_resume',
- '--platform=linux*',
'--no_auth',
- '--bucket', 'chromium-luci',
- '-d', 'src/tools/luci-go/linux64',
+ '--bucket', 'chromium-browser-clang',
+ '-s', 'src/tools/clang/dsymutil/bin/dsymutil.arm64.sha1',
+ '-o', 'src/tools/clang/dsymutil/bin/dsymutil',
],
},
{
- # We used to use src as a CIPD root. We moved it to a different directory
- # in crrev.com/c/930178 but left the clobber here to ensure that that CL
- # could be reverted safely. This can be safely removed once crbug.com/794764
- # is resolved.
- 'name': 'Android Clobber Deprecated CIPD Root',
+ 'name': 'dsymutil_mac_x64',
'pattern': '.',
- 'condition': 'checkout_android',
- 'action': ['src/build/cipd/clobber_cipd_root.py',
- '--root', 'src',
+ 'condition': 'host_os == "mac" and host_cpu == "x64"',
+ 'action': [ 'python3',
+ 'src/third_party/depot_tools/download_from_google_storage.py',
+ '--no_resume',
+ '--no_auth',
+ '--bucket', 'chromium-browser-clang',
+ '-s', 'src/tools/clang/dsymutil/bin/dsymutil.x64.sha1',
+ '-o', 'src/tools/clang/dsymutil/bin/dsymutil',
],
},
]
-recursedeps = []
+recursedeps = [
+ 'src/buildtools',
+ 'src/third_party/instrumented_libs',
+]
diff --git a/libs/libyuv/DIR_METADATA b/libs/libyuv/DIR_METADATA
new file mode 100644
index 0000000000..8bc04f159c
--- /dev/null
+++ b/libs/libyuv/DIR_METADATA
@@ -0,0 +1,3 @@
+monorail {
+ component: "Internals>Images>Codecs"
+}
diff --git a/libs/libyuv/GEMINI.md b/libs/libyuv/GEMINI.md
new file mode 100644
index 0000000000..d9cfd977a1
--- /dev/null
+++ b/libs/libyuv/GEMINI.md
@@ -0,0 +1,177 @@
+# Gemini Project Context: libyuv Row Functions
+
+This file provides context for the core row-processing architecture of
+libyuv. Use these guidelines when refactoring, reviewing, or generating
+code within the `row_*.cc` files.
+
+## Architectural Overview
+
+Libyuv uses a dispatch system where high-level conversion functions call
+optimized "Row" functions. These functions are categorized by SIMD architecture
+and compiler compatibility.
+
+## Source File Map
+
+### x86 Architectures (32-bit and 64-bit)
+
+* **row_gcc.cc**: **Master copy.** Contains inline assembly in GCC syntax for
+ GCC and Clang. Supports AVX, and AVX512. AVX512 implementations are strictly
+ for 64-bit targets.
+* **row_win.cc**: Derivative of `row_gcc.cc`. Contains C++ intrinsics
+ specifically for Visual C++ (MSVC). Can be tested with Clang using
+ `-DLIBYUV_ENABLE_ROWWIN`.
+* **Note**: Use either `row_gcc` or `row_win`, never both.
+
+### Arm Architecture (32-bit)
+
+* **row_neon.cc**: 32-bit Arm. Written entirely in inline assembly for
+ GCC/Clang.
+
+### Arm Architecture (AArch64) (64-bit)
+
+* **row_neon64.cc**: 64-bit Arm (AArch64) Neon. Written entirely in inline
+ assembly for GCC/Clang.
+* **row_sve.cc**: Armv9 Scalable Vector Extension (SVE2).
+* **row_sme.cc**: Armv9 Scalable Matrix Extension (SME) and Streaming SVE
+ (SSVE).
+* **row_sve.h**: Streaming-compatible implementations shared across
+ **row_sve.cc** and **row_sme.cc**.
+
+### Other Architectures
+
+* **row_rvv.cc**: RISC-V Vector (RVV). Implemented using intrinsics. Optimized
+ for SiFive X280.
+* **row_hvx.cc**: Qualcomm Hexagon Vector Extensions (HVX). Implemented using
+ intrinsics.
+* **row_lsx.cc / row_lasx.cc**: Loongarch MIPS-like extensions.
+
+### Utility and Fallbacks
+
+* **row_common.cc**: Portable C/C++ versions and multi-step `ARGB` wrapper
+ row functions. This is the reference implementation.
+* **row_any.cc**: Handles "remainder" pixels for widths not multiples of SIMD
+ register size. Used for SSE/AVX2, Neon, and LoongArch (LSX/LASX). Not
+ required for AVX-512, SVE2, SME, RVV, or HVX, which handle tail masking
+ directly inside the Row function.
+
+## Deprecated and Lower-Priority Formats
+
+The following datatypes and functions are deprecated or lower priority and are
+not always fully hand-optimized across every SIMD extension:
+
+* **16-bit RGB formats** (deprecated): `RGB565` (`FOURCC_RGBP`,
+ `FOURCC_L565`), `ARGB1555` (`FOURCC_RGBO`, `FOURCC_L555`, `FOURCC_5551`),
+ and `ARGB4444` (`FOURCC_R444`).
+* **Legacy fixed-matrix RGB-to-YUV row functions** (deprecated in favor of
+ `*MatrixRow` variants): `RGB565ToYRow`, `RGB565ToUVRow`, `ARGB1555ToYRow`,
+ `ARGB1555ToUVRow`, `ARGB4444ToYRow`, `ARGB4444ToUVRow`.
+* **Row-biplanar YUV** (deprecated): `M420` (`FOURCC_M420`).
+
+When adding SIMD coverage for deprecated or less relevant formats, prefer
+two-step wrapper implementations in `source/row_common.cc` that convert to or
+from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing
+`*MatrixRow` functions) rather than writing large bespoke assembly kernels.
+
+## Coding Guidelines
+
+1. **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to
+ enable or disable specific instruction set extensions.
+2. **Inline asm clobbers (all architectures)**: Every register written by an
+ inline asm block must be listed in its clobber list: vector registers,
+ general purpose registers, mask/predicate registers, `"cc"` when flags
+ are set (`subs`, `cmp`, `whilelt`, ...) and `"memory"` when the block
+ stores. Declare them even if the ABI treats them as caller-saved: with
+ LTO/LTCG the compiler may keep values live in any register across the
+ asm, and other OS ABIs (e.g. Windows) differ in which registers are
+ callee-saved. Never remove a clobber because it "is not needed by the
+ ABI". Use real register names, not numbers (`"v22"`, not `"22"`). Order
+ the list `"memory"` first (when stores occur), then `"cc"` (on
+ architectures with flags), then registers.
+
+### x86 Architectures (32-bit and 64-bit)
+
+1. **AVX512 Logic**: AVX512 row functions are strictly enabled for **64-bit x86
+ only**. Do not create `_Any_` wrappers for AVX512 row functions; handle
+ arbitrary tail widths directly inside the Row function using `k1`-`k7`
+ opmask registers.
+2. **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to
+ `row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8`
+ intrinsic.
+3. **Clobber names**: Always declare vector clobbers as `"xmmN"`, never
+ `"ymmN"` or `"zmmN"`, including for AVX2 and AVX512 code. Clang for
+ Windows must preserve xmm6-xmm15 and does not save them when they are
+ declared as ymm or zmm.
+4. **MSan Intrinsics (`*_win.cc`)**: Clang builds with MemorySanitizer
+ (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and compile `*_win.cc`
+ (`row_win.cc`, `scale_win.cc`, `compare_win.cc`, `rotate_win.cc`) instead of
+ inline assembly (`*_gcc.cc`). When porting `*_gcc.cc` to `*_win.cc`, never
+ use AVX-512 intrinsics that LLVM's MSan pass does not model:
+ * **Down-converting masked stores**: Never use
+ `_mm512_mask_cvtepi32_storeu_epi8`, `_mm512_mask_cvtepi64_storeu_epi8`,
+ `_mm512_mask_cvtepi16_storeu_epi8`, or
+ `_mm512_mask_cvtepi32_storeu_epi16`. They lower to target-specific
+ `@llvm.x86.avx512.mask.pmov.*.mem.*` intrinsics that LLVM MSan does not
+ model, leaving the destination shadow memory poisoned (uninitialized).
+ Instead, convert to a narrower vector first and use a standard masked
+ store (e.g. `_mm_mask_storeu_epi8(dst, mask, _mm512_cvtepi32_epi8(v))`),
+ which lowers to `@llvm.masked.store` for MSan and still compiles to a
+ single `vpmovdb` instruction at `-O2`.
+ * **Ternary logic (`_mm*_ternarylogic_epi32` / `epi64`)**: Never use
+ `_mm512_ternarylogic_epi32`/`epi64` (or `_mm256_`/`_mm_` variants)
+ anywhere in `*_win.cc` when porting `vpternlogd`/`vpternlogq` from
+ `*_gcc.cc`. They lower to `@llvm.x86.avx512.pternlog.*`, which LLVM MSan
+ does not model bitwise—`handleUnknownIntrinsic` traps immediately with
+ `use-of-uninitialized-value` if *any* bit of any of the three inputs is
+ uninitialized (such as the unused alpha byte of an `RGBX` buffer or tail
+ padding), even if masked off by `imm8` or merely copied to memory.
+ * For boolean blends and inversions (e.g. `ARGBGrayRow_AVX512VNNI`,
+ `I422ToAR30Row_AVX512BW`, `ARGBToUV444MatrixRow_AVX512VNNIVBMI`),
+ use standard bitwise intrinsics (`_mm512_and_si512`,
+ `_mm512_andnot_si512`, `_mm512_or_si512`, `_mm512_xor_si512`), which
+ LLVM MSan models bit-by-bit and Clang `-O2` folds back into
+ `vpternlog` in non-MSan builds.
+ * For write-only partial channel updates into `dst` (e.g. writing
+ alpha in `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores
+ (`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending.
+ * **Single-source variable permutes (`_mm512_permutexvar_epi*` /
+ `_mm512_mask_permutexvar_epi*`)**: Never use
+ `_mm512_permutexvar_epi{8,16,32,64}` or
+ `_mm512_mask_permutexvar_epi{8,16,32,64}` directly. They lower to
+ `@llvm.x86.avx512.permvar.*`, which LLVM MSan does not model
+ lane-by-lane—instead it ORs the input shadows without permuting them,
+ poisoning initialized output lanes whenever *any* lane of the input
+ vector is uninitialized (e.g. `_mm512_castsi128_si512`,
+ `_mm512_castsi256_si512`, unused `RGBX` alpha bytes, or uninitialized
+ tail lanes). Use the `PermutexvarEpi*` and `MaskPermutexvarEpi*`
+ helpers in `row_win.cc` / `scale_win.cc`, which use portable
+ lane-by-lane C loops under `LIBYUV_MSAN` and the native intrinsics
+ otherwise.
+ * **512-bit float-to-int truncation (`_mm512_cvttps_epi32`)**: Never use
+ `_mm512_cvttps_epi32` directly; it lowers to
+ `@llvm.x86.avx512.mask.cvttps2dq.512`, which LLVM MSan does not model
+ lane-by-lane. Use the `CvttpsEpi32` helper in `row_win.cc`, which uses
+ a portable lane-by-lane C loop under `LIBYUV_MSAN` and
+ `_mm512_cvttps_epi32` otherwise.
+
+### AArch64
+
+1. For SVE2/SME, aim to use full predicates (`ptrue`) and full vectors for the
+ main loop body and use predication only to mask the tail loop as needed.
+ Pointer updates should be done with `inc[bhwd]` or `add`, not by `incp`.
+
+## Changelist (CL) Format Guidelines
+
+When adding new code, remove trailing spaces.
+When generating descriptions, follow the Chromium standard format. Wrap
+commit message text at 72 characters
+
+### Changelist (CL) Description Example
+
+\[libyuv] Optimized ARGBToRGB24 for AVX2
+
+Detailed technical explanation of the change. Describe the SIMD
+optimization logic and any platform-specific constraints. Mention
+if this impacts specific row files like row_gcc.cc.
+
+Test: libyuv_unittest --gunit_filter=*ARGBToRGB24*
+Bug: libyuv:12345, b/67890
diff --git a/libs/libyuv/OWNERS b/libs/libyuv/OWNERS
index 7b21adfe6c..7345e8767b 100644
--- a/libs/libyuv/OWNERS
+++ b/libs/libyuv/OWNERS
@@ -1,8 +1,10 @@
fbarchard@chromium.org
-magjed@chromium.org
+wtc@google.com
+mbonadei@chromium.org
+jansson@google.com
-per-file *.gn=phoglund@chromium.org
+per-file *.gn=mbonadei@chromium.org,jansson@google.com
per-file .gitignore=*
per-file AUTHORS=*
per-file DEPS=*
-per-file PRESUBMIT.py=phoglund@chromium.org
+per-file PRESUBMIT.py=mbonadei@chromium.org,jansson@google.com
diff --git a/libs/libyuv/PRESUBMIT.py b/libs/libyuv/PRESUBMIT.py
index 2cf1542ff1..37df28a106 100644
--- a/libs/libyuv/PRESUBMIT.py
+++ b/libs/libyuv/PRESUBMIT.py
@@ -6,67 +6,60 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
-import os
-
-
-def _RunPythonTests(input_api, output_api):
- def join(*args):
- return input_api.os_path.join(input_api.PresubmitLocalPath(), *args)
-
- test_directories = [
- root for root, _, files in os.walk(join('tools_libyuv'))
- if any(f.endswith('_test.py') for f in files)
- ]
-
- tests = []
- for directory in test_directories:
- tests.extend(
- input_api.canned_checks.GetUnitTestsInDirectory(
- input_api,
- output_api,
- directory,
- whitelist=[r'.+_test\.py$']))
- return input_api.RunTests(tests, parallel=True)
+# Runs PRESUBMIT.py in py3 mode by git cl presubmit.
+USE_PYTHON3 = True
def _CommonChecks(input_api, output_api):
- """Checks common to both upload and commit."""
- results = []
- results.extend(input_api.canned_checks.RunPylint(input_api, output_api,
- black_list=(r'^base[\\\/].*\.py$',
- r'^build[\\\/].*\.py$',
- r'^buildtools[\\\/].*\.py$',
- r'^ios[\\\/].*\.py$',
- r'^out.*[\\\/].*\.py$',
- r'^testing[\\\/].*\.py$',
- r'^third_party[\\\/].*\.py$',
- r'^tools[\\\/].*\.py$',
- # TODO(kjellander): should arguably be checked.
- r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$',
- r'^xcodebuild.*[\\\/].*\.py$',),
- disabled_warnings=['F0401', # Failed to import x
- 'E0611', # No package y in x
- 'W0232', # Class has no __init__ method
- ],
- pylintrc='pylintrc'))
- results.extend(_RunPythonTests(input_api, output_api))
- return results
+ """Checks common to both upload and commit."""
+ results = []
+ results.extend(
+ input_api.canned_checks.RunPylint(
+ input_api,
+ output_api,
+ files_to_skip=(
+ r'^base[\\\/].*\.py$',
+ r'^build[\\\/].*\.py$',
+ r'^buildtools[\\\/].*\.py$',
+ r'^ios[\\\/].*\.py$',
+ r'^out.*[\\\/].*\.py$',
+ r'^testing[\\\/].*\.py$',
+ r'^third_party[\\\/].*\.py$',
+ r'^tools[\\\/].*\.py$',
+ # TODO(kjellander): should arguably be checked.
+ r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$',
+ r'^xcodebuild.*[\\\/].*\.py$',
+ ),
+ disabled_warnings=[
+ 'F0401', # Failed to import x
+ 'E0611', # No package y in x
+ 'W0232', # Class has no __init__ method
+ ],
+ pylintrc='pylintrc',
+ version='2.7',
+ )
+ )
+ return results
def CheckChangeOnUpload(input_api, output_api):
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- results.extend(
- input_api.canned_checks.CheckGNFormatted(input_api, output_api))
- return results
+ results = []
+ results.extend(_CommonChecks(input_api, output_api))
+ results.extend(
+ input_api.canned_checks.CheckGNFormatted(input_api, output_api)
+ )
+ return results
def CheckChangeOnCommit(input_api, output_api):
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
- results.extend(input_api.canned_checks.CheckChangeWasUploaded(
- input_api, output_api))
- results.extend(input_api.canned_checks.CheckChangeHasDescription(
- input_api, output_api))
- return results
+ results = []
+ results.extend(_CommonChecks(input_api, output_api))
+ results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
+ results.extend(
+ input_api.canned_checks.CheckChangeWasUploaded(input_api, output_api)
+ )
+ results.extend(
+ input_api.canned_checks.CheckChangeHasDescription(input_api,
+ output_api)
+ )
+ return results
diff --git a/libs/libyuv/README.chromium b/libs/libyuv/README.chromium
index dadffc9c3b..0bac66a43c 100644
--- a/libs/libyuv/README.chromium
+++ b/libs/libyuv/README.chromium
@@ -1,8 +1,13 @@
Name: libyuv
-URL: http://code.google.com/p/libyuv/
-Version: 1735
-License: BSD
+URL: https://chromium.googlesource.com/libyuv/libyuv/
+Version: 1977
+Revision: DEPS
+License: BSD-3-Clause
License File: LICENSE
+Shipped: yes
+Security Critical: yes
+Update Mechanism: Manual
Description:
libyuv is an open source project that includes YUV conversion and scaling functionality.
+
diff --git a/libs/libyuv/README.md b/libs/libyuv/README.md
index db70b7f08d..4baa69cd5e 100644
--- a/libs/libyuv/README.md
+++ b/libs/libyuv/README.md
@@ -5,8 +5,9 @@
* Convert to RGB formats for rendering/effects.
* Rotate by 90/180/270 degrees to adjust for mobile devices in portrait mode.
* Optimized for SSSE3/AVX2 on x86/x64.
-* Optimized for Neon on Arm.
+* Optimized for Neon/SVE2/SME on Arm.
* Optimized for MSA on Mips.
+* Optimized for RVV on RISC-V.
### Development
diff --git a/libs/libyuv/WORKSPACE.bazel b/libs/libyuv/WORKSPACE.bazel
new file mode 100644
index 0000000000..0b288ac64c
--- /dev/null
+++ b/libs/libyuv/WORKSPACE.bazel
@@ -0,0 +1,14 @@
+workspace(name = "libyuv")
+
+android_sdk_repository(
+ name = "androidsdk",
+ # Bazel will automatically use the ANDROID_HOME environment variable.
+)
+
+android_ndk_repository(
+ name = "androidndk",
+ # Bazel will automatically use the ANDROID_NDK_HOME environment variable.
+)
+
+# Optional: If you need to fetch specific dependencies (e.g., GoogleTest, libjpeg-turbo)
+# you can define them here using http_archive.
diff --git a/libs/libyuv/build_overrides/build.gni b/libs/libyuv/build_overrides/build.gni
index 6d8319b965..2a7051d075 100644
--- a/libs/libyuv/build_overrides/build.gni
+++ b/libs/libyuv/build_overrides/build.gni
@@ -6,9 +6,6 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
-# Some non-Chromium builds don't use Chromium's third_party/binutils.
-linux_use_bundled_binutils_override = true
-
# Variable that can be used to support multiple build scenarios, like having
# Chromium specific targets in a client project's GN file etc.
build_with_chromium = false
@@ -16,6 +13,9 @@ build_with_chromium = false
# Some non-Chromium builds don't support building java targets.
enable_java_templates = true
+# Enables assertions on safety checks in libc++.
+enable_safe_libcxx = true
+
# Allow using custom suppressions files (currently not used by libyuv).
asan_suppressions_file = "//build/sanitizers/asan_suppressions.cc"
lsan_suppressions_file = "//build/sanitizers/lsan_suppressions.cc"
@@ -35,12 +35,31 @@ ignore_elf32_limitations = true
# Use bundled hermetic Xcode installation maintained by Chromium,
# except for local iOS builds where it is unsupported.
-if (host_os == "mac") {
+# Allow for mac cross compile on linux machines.
+if (host_os == "mac" || host_os == "linux") {
_result = exec_script("//build/mac/should_use_hermetic_xcode.py",
[ target_os ],
"value")
assert(_result != 2,
- "Do not allow building targets with the default" +
+ "Do not allow building targets with the default " +
"hermetic toolchain if the minimum OS version is not met.")
use_system_xcode = _result == 0
}
+
+declare_args() {
+ # Limits the defined //third_party/android_deps targets to only "buildCompile"
+ # and "buildCompileNoDeps" targets. This is useful for third-party
+ # repositories which do not use JUnit tests. For instance,
+ # limit_android_deps == true removes "gn gen" requirement for
+ # //third_party/robolectric .
+ limit_android_deps = false
+
+ # Allows googletest to pretty-print various absl types.
+ # Defined here rather than in gtest.gni to match chromium.
+ gtest_enable_absl_printers = true
+}
+
+vpython_spec_files = [
+ "//vpython.toml",
+ "//vpython.toml.uv.lock",
+]
diff --git a/libs/libyuv/build_overrides/partition_alloc.gni b/libs/libyuv/build_overrides/partition_alloc.gni
new file mode 100644
index 0000000000..dcf8ac2d25
--- /dev/null
+++ b/libs/libyuv/build_overrides/partition_alloc.gni
@@ -0,0 +1,17 @@
+# Copyright 2022 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+# Use default values for PartitionAlloc as standalone library from
+# base/allocator/partition_allocator/build_overrides/partition_alloc.gni
+use_partition_alloc_as_malloc_default = false
+use_allocator_shim_default = false
+enable_backup_ref_ptr_support_default = false
+enable_mte_checked_ptr_support_default = false
+put_ref_count_in_previous_slot_default = false
+enable_backup_ref_ptr_slow_checks_default = false
+enable_dangling_raw_ptr_checks_default = false
diff --git a/libs/libyuv/cleanup_links.py b/libs/libyuv/cleanup_links.py
deleted file mode 100755
index ba2907892f..0000000000
--- a/libs/libyuv/cleanup_links.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# This is a copy of the file from WebRTC in:
-# https://chromium.googlesource.com/external/webrtc/+/master/cleanup_links.py
-
-"""Script to cleanup symlinks created from setup_links.py.
-
-Before 177567c518b121731e507e9b9c4049c4dc96e4c8 (#15754) we had a Chromium
-checkout which we created symlinks into. In order to do clean syncs after
-landing that change, this script cleans up any old symlinks, avoiding annoying
-manual cleanup needed in order to complete gclient sync.
-"""
-
-import logging
-import optparse
-import os
-import shelve
-import subprocess
-import sys
-
-
-ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-LINKS_DB = 'links'
-
-# Version management to make future upgrades/downgrades easier to support.
-SCHEMA_VERSION = 1
-
-class WebRTCLinkSetup(object):
- def __init__(self, links_db, dry_run=False):
- self._dry_run = dry_run
- self._links_db = links_db
-
- def CleanupLinks(self):
- logging.debug('CleanupLinks')
- for source, link_path in self._links_db.iteritems():
- if source == 'SCHEMA_VERSION':
- continue
- if os.path.islink(link_path) or sys.platform.startswith('win'):
- # os.path.islink() always returns false on Windows
- # See http://bugs.python.org/issue13143.
- logging.debug('Removing link to %s at %s', source, link_path)
- if not self._dry_run:
- if os.path.exists(link_path):
- if sys.platform.startswith('win') and os.path.isdir(link_path):
- subprocess.check_call(['rmdir', '/q', '/s', link_path],
- shell=True)
- else:
- os.remove(link_path)
- del self._links_db[source]
-
-
-def _initialize_database(filename):
- links_database = shelve.open(filename)
- # Wipe the database if this version of the script ends up looking at a
- # newer (future) version of the links db, just to be sure.
- version = links_database.get('SCHEMA_VERSION')
- if version and version != SCHEMA_VERSION:
- logging.info('Found database with schema version %s while this script only '
- 'supports %s. Wiping previous database contents.', version,
- SCHEMA_VERSION)
- links_database.clear()
- links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
- return links_database
-
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option('-d', '--dry-run', action='store_true', default=False,
- help='Print what would be done, but don\'t perform any '
- 'operations. This will automatically set logging to '
- 'verbose.')
- parser.add_option('-v', '--verbose', action='store_const',
- const=logging.DEBUG, default=logging.INFO,
- help='Print verbose output for debugging.')
- options, _ = parser.parse_args()
-
- if options.dry_run:
- options.verbose = logging.DEBUG
- logging.basicConfig(format='%(message)s', level=options.verbose)
-
- # Work from the root directory of the checkout.
- script_dir = os.path.dirname(os.path.abspath(__file__))
- os.chdir(script_dir)
-
- # The database file gets .db appended on some platforms.
- db_filenames = [LINKS_DB, LINKS_DB + '.db']
- if any(os.path.isfile(f) for f in db_filenames):
- links_database = _initialize_database(LINKS_DB)
- try:
- symlink_creator = WebRTCLinkSetup(links_database, options.dry_run)
- symlink_creator.CleanupLinks()
- finally:
- for f in db_filenames:
- if os.path.isfile(f):
- os.remove(f)
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/libs/libyuv/codereview.settings b/libs/libyuv/codereview.settings
index 00ba1d37d0..b226fae57a 100644
--- a/libs/libyuv/codereview.settings
+++ b/libs/libyuv/codereview.settings
@@ -1,6 +1,5 @@
-# This file is used by git cl to get repository specific information.
+# This file is used by `git cl` to get repository specific information.
CODE_REVIEW_SERVER: codereview.chromium.org
GERRIT_HOST: True
PROJECT: libyuv
-TRY_ON_UPLOAD: False
VIEW_VC: https://chromium.googlesource.com/libyuv/libyuv/+/
diff --git a/libs/libyuv/docs/deprecated_builds.md b/libs/libyuv/docs/deprecated_builds.md
index 29e0bf9bc3..747260ee8e 100644
--- a/libs/libyuv/docs/deprecated_builds.md
+++ b/libs/libyuv/docs/deprecated_builds.md
@@ -11,23 +11,7 @@ Refer to chromium instructions for each platform for other prerequisites.
Create a working directory, enter it, and run:
- gclient config https://chromium.googlesource.com/libyuv/libyuv
- gclient sync
-
-
-Then you'll get a .gclient file like:
-
- solutions = [
- { "name" : "libyuv",
- "url" : "https://chromium.googlesource.com/libyuv/libyuv",
- "deps_file" : "DEPS",
- "managed" : True,
- "custom_deps" : {
- },
- "safesync_url": "",
- },
- ];
-
+ fetch libyuv
For iOS add `;target_os=['ios'];` to your OSX .gclient and run `GYP_DEFINES="OS=ios" gclient sync.`
@@ -157,19 +141,13 @@ ia32
GYP_DEFINES="OS=android target_arch=ia32 android_full_debug=1" GYP_CROSSCOMPILE=1 ./gyp_libyuv
ninja -j7 -C out/Debug yuv_unittest_apk
-mipsel
-
- GYP_DEFINES="OS=android target_arch=mipsel" GYP_CROSSCOMPILE=1 ./gyp_libyuv
- ninja -j7 -C out/Debug yuv_unittest_apk
- ninja -j7 -C out/Release yuv_unittest_apk
-
arm32 disassembly:
- third_party/android_ndk/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin/arm-linux-androideabi-objdump -d out/Release/obj/source/libyuv.row_neon.o
+ llvm-objdump -d out/Release/obj/source/libyuv.row_neon.o
arm64 disassembly:
- third_party/android_ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-objdump -d out/Release/obj/source/libyuv.row_neon64.o
+ llvm-objdump -d out/Release/obj/source/libyuv.row_neon64.o
Running tests:
@@ -195,16 +173,6 @@ Running test with C code:
gn gen out/Official "--args=is_debug=false is_official_build=true is_chrome_branded=true"
ninja -C out/Official
-#### Building mips with GN
-
-mipsel
- gn gen out/Default "--args=is_debug=false target_cpu=\"mipsel\" target_os = \"android\" mips_arch_variant = \"r6\" mips_use_msa = true is_component_build = true is_clang = false"
- ninja -C out/Default
-
-mips64el
- gn gen out/Default "--args=is_debug=false target_cpu=\"mips64el\" target_os = \"android\" mips_arch_variant = \"r6\" mips_use_msa = true is_component_build = true is_clang = false"
- ninja -C out/Default
-
### Linux
GYP_DEFINES="target_arch=x64" ./gyp_libyuv
@@ -239,6 +207,7 @@ If you get a compile error for atlthunk.lib on Windows, read http://www.chromium
ninja -C out/Debug libyuv_unittest
ninja -C out/Debug compare
ninja -C out/Debug yuvconvert
+ ninja -C out/Debug yuvconstants
ninja -C out/Debug psnr
ninja -C out/Debug cpuid
diff --git a/libs/libyuv/docs/environment_variables.md b/libs/libyuv/docs/environment_variables.md
index cd8159ad5a..a46744e45b 100644
--- a/libs/libyuv/docs/environment_variables.md
+++ b/libs/libyuv/docs/environment_variables.md
@@ -7,9 +7,11 @@ For test purposes, environment variables can be set to control libyuv behavior.
By default the cpu is detected and the most advanced form of SIMD is used. But you can disable instruction sets selectively, or completely, falling back on C code. Set the variable to 1 to disable the specified instruction set.
## All CPUs
+
LIBYUV_DISABLE_ASM
## Intel CPUs
+
LIBYUV_DISABLE_X86
LIBYUV_DISABLE_SSE2
LIBYUV_DISABLE_SSSE3
@@ -22,19 +24,39 @@ By default the cpu is detected and the most advanced form of SIMD is used. But
LIBYUV_DISABLE_F16C
LIBYUV_DISABLE_AVX512BW
LIBYUV_DISABLE_AVX512VL
+ LIBYUV_DISABLE_AVX512VNNI
LIBYUV_DISABLE_AVX512VBMI
LIBYUV_DISABLE_AVX512VBMI2
LIBYUV_DISABLE_AVX512VBITALG
- LIBYUV_DISABLE_AVX512VPOPCNTDQ
- LIBYUV_DISABLE_GFNI
+ LIBYUV_DISABLE_AVX10
+ LIBYUV_DISABLE_AVX10_2
+ LIBYUV_DISABLE_AVXVNNI
+ LIBYUV_DISABLE_AVXVNNIINT8
+ LIBYUV_DISABLE_AMXINT8
+ LIBYUV_DISABLE_AVX512BMM
+ LIBYUV_DISABLE_AMXTF32
-## ARM CPUs
+## Arm CPUs
LIBYUV_DISABLE_NEON
+ LIBYUV_DISABLE_NEON_DOTPROD
+ LIBYUV_DISABLE_NEON_I8MM
+ LIBYUV_DISABLE_SVE
+ LIBYUV_DISABLE_SVE2
+ LIBYUV_DISABLE_SME
+
+## LOONGARCH CPUs
+
+ LIBYUV_DISABLE_LSX
+ LIBYUV_DISABLE_LASX
+
+## RISCV CPUs
+
+ LIBYUV_DISABLE_RVV
+
+## Hexagon CPUs
-## MIPS CPUs
- LIBYUV_DISABLE_MSA
- LIBYUV_DISABLE_MMI
+ LIBYUV_DISABLE_HVX
# Test Width/Height/Repeat
diff --git a/libs/libyuv/docs/feature_detection.md b/libs/libyuv/docs/feature_detection.md
new file mode 100644
index 0000000000..d32e84bfed
--- /dev/null
+++ b/libs/libyuv/docs/feature_detection.md
@@ -0,0 +1,108 @@
+# Introduction
+
+Several routines in libyuv have multiple implementations specialized for a
+variety of CPU architecture extensions. Libyuv will automatically detect and
+use the latest architecture extension present on a machine for which a kernel
+implementation is available.
+
+# Feature detection on AArch64
+
+## Architecture extensions of interest
+
+The Arm 64-bit A-class architecture has a number of vector extensions which can
+be used to accelerate libyuv kernels.
+
+### Neon extensions
+
+Neon is available and mandatory in AArch64 from the base Armv8.0-A
+architecture. Neon can be used even if later extensions like the Scalable
+Vector Extension (SVE) are also present. The exception to this is if the CPU is
+currently operating in streaming mode as introduced by the Scalable Matrix
+Extension, described later.
+
+There are also a couple of architecture extensions present for Neon that we can
+take advantage of in libyuv:
+
+* The Neon DotProd extension is architecturally available from Armv8.1-A and
+ becomes mandatory from Armv8.4-A. This extension provides instructions to
+ perform a pairwise widening multiply of groups of four bytes from two source
+ vectors, taking the sum of the four widened multiply results within each
+ group to give a 32-bit result, accumulating into a destination vector.
+
+* The Neon I8MM extension extends the DotProd extension with support for
+ mixed-sign DotProds. The I8MM extension is architecturally available from
+ Armv8.1-A and becomes mandatory from Armv8.6-A. It does not strictly depend
+ on the DotProd extension being implemented, however at time of writing there
+ is no known micro-architecture implementation where I8MM is implemented
+ without the DotProd extension also being implemented.
+
+### The Scalable Vector Extension (SVE)
+
+The two Scalable Vector extensions (SVE and SVE2) provides equivalent
+functionality to most existing Neon instructions but with the ability to
+efficiently operate on vector registers with a run-time-determined vector
+length.
+
+The original version of SVE is architecturally available from Armv8.2-A and is
+primarily targeted at HPC applications. This focus means it does not include
+most of the DSP-style operations that are necessary for most libyuv
+color-conversion kernels, though it can still be used for many scaling or
+rotation kernels.
+
+SVE does not strictly depend on either of the Neon DotProd or I8MM extensions
+being implemented. The only micro-architecture at time of writing where SVE is
+implemented without these two extensions both also being implemented is the
+Fujitsu A64FX, which is not a CPU of interest for libyuv.
+
+SVE2 extends the base SVE extension with the remaining instructions from Neon,
+porting these instructions to operate on scalable vectors. SVE2 is
+architecturally available from Armv9.0-A. If SVE2 is implemented then SVE must
+also be implemented. Since Armv9.0-A is based on Armv8.5-A this implies that
+the Neon DotProd extension is also implemented. Interestingly this means that
+the I8MM extension is not mandatory since it only becomes mandatory from
+Armv8.6-A or Armv9.1-A, however there is no micro-architecture at time of
+writing where SVE2 is implemented without all previously-mentioned features
+also being implemented.
+
+### The Scalable Matrix Extension (SME)
+
+The Scalable Matrix Extension (SME) is an optional feature introduced from
+Armv9.2-A. SME exists alongside SVE and introduces new execution modes for
+applications performing extended periods of data processing. In particular SME
+introduces a few new components of interest:
+
+* Access to a scalable two-dimensional ZA tile register and new instructions to
+ interact with rows and columns of the ZA tiles. This can be useful for data
+ transformations like transposes.
+
+* A streaming SVE (SSVE) mode, during which the SVE vector length matches the
+ ZA tile register width. In typical systems where the ZA tile register width
+ is longer than the core SVE vector length, SSVE processing allows for faster
+ data processing, even if the ZA tile register is unused. While the CPU is
+ executing in streaming mode, Neon instructions are unavailable.
+
+* When both SSVE and the ZA tile registers are enabled there are additional
+ outer-product instructions accumulating into a whole ZA tile, suitable for
+ accelerating matrix arithmetic. This is likely less useful in libyuv.
+
+## Linux and Android
+
+On AArch64 running under Linux and Android, features are detected by inspecting
+the CPU auxiliary vector via `getauxval(AT_HWCAP)` and `getauxval(AT_HWCAP2)`,
+inspecting the returned bitmask.
+
+## Windows
+
+On Windows we detect features using the `IsProcessorFeaturePresent` interface
+and passing an enum parameter for the feature we want to check. More
+information on this can be found here:
+
+ https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent#parameters
+
+## Apple Silicon
+
+On Apple Silicon we detect features using the `sysctlbyname` interface and
+passing a string representing the feature we want to detect. More information
+on this can be found here:
+
+ https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
diff --git a/libs/libyuv/docs/formats.md b/libs/libyuv/docs/formats.md
index 97e8ce05f4..b0a81a94bd 100644
--- a/libs/libyuv/docs/formats.md
+++ b/libs/libyuv/docs/formats.md
@@ -4,7 +4,9 @@ Formats (FOURCC) supported by libyuv are detailed here.
# Core Formats
-There are 2 core formats supported by libyuv - I420 and ARGB. All YUV formats can be converted to/from I420. All RGB formats can be converted to/from ARGB.
+There are 2 core formats supported by libyuv - I420 and ARGB.
+ All YUV formats can be converted to/from I420.
+ All RGB formats can be converted to/from ARGB.
Filtering functions such as scaling and planar functions work on I420 and/or ARGB.
@@ -36,26 +38,34 @@ This is how OSX formats map to libyuv
The following is extracted from video_common.h as a complete list of formats supported by libyuv.
enum FourCC {
- // 9 Primary YUV formats: 5 planar, 2 biplanar, 2 packed.
+ // 15 Primary YUV formats: 6 planar, 7 biplanar, 2 packed.
FOURCC_I420 = FOURCC('I', '4', '2', '0'),
FOURCC_I422 = FOURCC('I', '4', '2', '2'),
FOURCC_I444 = FOURCC('I', '4', '4', '4'),
FOURCC_I400 = FOURCC('I', '4', '0', '0'),
FOURCC_NV21 = FOURCC('N', 'V', '2', '1'),
FOURCC_NV12 = FOURCC('N', 'V', '1', '2'),
+ FOURCC_NV16 = FOURCC('N', 'V', '1', '6'),
+ FOURCC_NV24 = FOURCC('N', 'V', '2', '4'),
+ FOURCC_P010 = FOURCC('P', '0', '1', '0'),
+ FOURCC_P210 = FOURCC('P', '2', '1', '0'),
+ FOURCC_P410 = FOURCC('P', '4', '1', '0'),
FOURCC_YUY2 = FOURCC('Y', 'U', 'Y', '2'),
FOURCC_UYVY = FOURCC('U', 'Y', 'V', 'Y'),
- FOURCC_H010 = FOURCC('H', '0', '1', '0'), // unofficial fourcc. 10 bit lsb
+ FOURCC_I010 = FOURCC('I', '0', '1', '0'), // bt.601 10 bit 420
+ FOURCC_I210 = FOURCC('I', '2', '1', '0'), // bt.601 10 bit 422
// 1 Secondary YUV format: row biplanar.
- FOURCC_M420 = FOURCC('M', '4', '2', '0'),
+ FOURCC_M420 = FOURCC('M', '4', '2', '0'), // deprecated.
- // 11 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp, 1 10 bpc
+ // 13 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp, 1 10 bpc, 2 64 bpp
FOURCC_ARGB = FOURCC('A', 'R', 'G', 'B'),
FOURCC_BGRA = FOURCC('B', 'G', 'R', 'A'),
FOURCC_ABGR = FOURCC('A', 'B', 'G', 'R'),
FOURCC_AR30 = FOURCC('A', 'R', '3', '0'), // 10 bit per channel. 2101010.
FOURCC_AB30 = FOURCC('A', 'B', '3', '0'), // ABGR version of 10 bit
+ FOURCC_AR64 = FOURCC('A', 'R', '6', '4'), // 16 bit per channel.
+ FOURCC_AB64 = FOURCC('A', 'B', '6', '4'), // ABGR version of 16 bit
FOURCC_24BG = FOURCC('2', '4', 'B', 'G'),
FOURCC_RAW = FOURCC('r', 'a', 'w', ' '),
FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'),
@@ -66,15 +76,34 @@ The following is extracted from video_common.h as a complete list of formats sup
// 1 Primary Compressed YUV format.
FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'),
- // 8 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias.
+ // 23 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias.
FOURCC_YV12 = FOURCC('Y', 'V', '1', '2'),
FOURCC_YV16 = FOURCC('Y', 'V', '1', '6'),
FOURCC_YV24 = FOURCC('Y', 'V', '2', '4'),
FOURCC_YU12 = FOURCC('Y', 'U', '1', '2'), // Linux version of I420.
- FOURCC_J420 = FOURCC('J', '4', '2', '0'),
- FOURCC_J400 = FOURCC('J', '4', '0', '0'), // unofficial fourcc
- FOURCC_H420 = FOURCC('H', '4', '2', '0'), // unofficial fourcc
- FOURCC_H422 = FOURCC('H', '4', '2', '2'), // unofficial fourcc
+ FOURCC_J420 =
+ FOURCC('J', '4', '2', '0'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J422 =
+ FOURCC('J', '4', '2', '2'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J444 =
+ FOURCC('J', '4', '4', '4'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J400 =
+ FOURCC('J', '4', '0', '0'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_F420 = FOURCC('F', '4', '2', '0'), // bt.709 full, unofficial fourcc
+ FOURCC_F422 = FOURCC('F', '4', '2', '2'), // bt.709 full, unofficial fourcc
+ FOURCC_F444 = FOURCC('F', '4', '4', '4'), // bt.709 full, unofficial fourcc
+ FOURCC_H420 = FOURCC('H', '4', '2', '0'), // bt.709, unofficial fourcc
+ FOURCC_H422 = FOURCC('H', '4', '2', '2'), // bt.709, unofficial fourcc
+ FOURCC_H444 = FOURCC('H', '4', '4', '4'), // bt.709, unofficial fourcc
+ FOURCC_U420 = FOURCC('U', '4', '2', '0'), // bt.2020, unofficial fourcc
+ FOURCC_U422 = FOURCC('U', '4', '2', '2'), // bt.2020, unofficial fourcc
+ FOURCC_U444 = FOURCC('U', '4', '4', '4'), // bt.2020, unofficial fourcc
+ FOURCC_F010 = FOURCC('F', '0', '1', '0'), // bt.709 full range 10 bit 420
+ FOURCC_H010 = FOURCC('H', '0', '1', '0'), // bt.709 10 bit 420
+ FOURCC_U010 = FOURCC('U', '0', '1', '0'), // bt.2020 10 bit 420
+ FOURCC_F210 = FOURCC('F', '2', '1', '0'), // bt.709 full range 10 bit 422
+ FOURCC_H210 = FOURCC('H', '2', '1', '0'), // bt.709 10 bit 422
+ FOURCC_U210 = FOURCC('U', '2', '1', '0'), // bt.2020 10 bit 422
// 14 Auxiliary aliases. CanonicalFourCC() maps these to canonical fourcc.
FOURCC_IYUV = FOURCC('I', 'Y', 'U', 'V'), // Alias for I420.
@@ -97,13 +126,35 @@ The following is extracted from video_common.h as a complete list of formats sup
# Planar YUV
The following formats contains a full size Y plane followed by 1 or 2
- planes for UV: I420, I422, I444, I400, NV21, NV12, I400
+ planes for UV: I420, I422, I444, I400, NV21, NV12, NV16, NV24,
+ P010, P210, P410
The size (subsampling) of the UV varies.
- I420, NV12 and NV21 are half width, half height
- I422, NV16 and NV61 are half width, full height
- I444, NV24 and NV42 are full width, full height
+ I420, NV12, NV21 and P010 are half width, half height
+ I422, NV16, NV61 and P210 are half width, full height
+ I444, NV24, NV42 and P410 are full width, full height
I400 and J400 have no chroma channel.
+# Color space
+ The YUV formats start with a letter to specify the color space. e.g. I420
+ I = BT.601 limited range
+ J = BT.601 full range (J = JPEG that uses this)
+ H = BT.709 limited range (H for HD)
+ F = BT.709 full range (F for Full range)
+ U = BT.2020 limited range (U for UHD)
+ V = BT.2020 full range
+ For YUV to RGB conversions, a matrix can be passed. See also convert_argh.h
+
+# HDR formats
+ Planar formats with 10 or 12 bits use the following fourcc:
+ I010, I012, P010, P012 are half width, half height
+ I210, I212, P210, P212 are half width, full height
+ I410, I412, P410, P412 are full width, full height
+ where
+ I is the color space (see above) and 3 planes: Y, U and V.
+ P is a biplanar format, similar to NV12 but 16 bits, with the valid bits in the high bits. There is a Y plane and a UV plane.
+ 0, 2 or 4 is the last digit of subsampling: 4:2:0, 4:2:2, or 4:4:4
+ 10 or 12 is the bits per channel. The bits are in the low bits of a 16 bit channel.
+
# The ARGB FOURCC
There are 4 ARGB layouts - ARGB, BGRA, ABGR and RGBA. ARGB is most common by far, used for screen formats, and windows webcam drivers.
@@ -114,13 +165,13 @@ A fourcc provided by capturer, can be thought of string, e.g. "ARGB".
On little endian machines, as an int, this would have 'A' in the lowest byte. The FOURCC macro reverses the order:
- #define FOURCC(a, b, c, d) (((uint32)(a)) | ((uint32)(b) << 8) | ((uint32)(c) << 16) | ((uint32)(d) << 24))
+ #define FOURCC(a, b, c, d) (((uint32_t)(a)) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
-So the "ARGB" string, read as an uint32, is
+So the "ARGB" string, read as an uint32_t, is
FOURCC_ARGB = FOURCC('A', 'R', 'G', 'B')
-If you were to read ARGB pixels as uint32's, the alpha would be in the high byte, and the blue in the lowest byte. In memory, these are stored little endian, so 'B' is first, then 'G', 'R' and 'A' last.
+If you were to read ARGB pixels as uint32_t's, the alpha would be in the high byte, and the blue in the lowest byte. In memory, these are stored little endian, so 'B' is first, then 'G', 'R' and 'A' last.
When calling conversion functions, the names match the FOURCC, so in this case it would be I420ToARGB().
@@ -152,6 +203,13 @@ The 2 bit alpha has 4 values. Here are the comparable 8 bit alpha values.
The 10 bit RGB values range from 0 to 1023.
XR30 is the same as AR30 but with no alpha channel.
+# AB64 and AR64
+
+AB64 is similar to ABGR, with 16 bit (2 bytes) per channel. Each channel stores an unsigned short.
+In memory R is the lowest and A is the highest.
+Each channel has value ranges from 0 to 65535.
+AR64 is similar to ARGB.
+
# NV12 and NV21
NV12 is a biplanar format with a full sized Y plane followed by a single
@@ -161,3 +219,11 @@ The 12 in NV12 refers to 12 bits per pixel. NV12 has a half width and half
height chroma channel, and therefore is a 420 subsampling.
NV16 is 16 bits per pixel, with half width and full height. aka 422.
NV24 is 24 bits per pixel with full sized chroma channel. aka 444.
+Most NV12 functions allow the destination Y pointer to be NULL.
+
+# YUY2 and UYVY
+
+YUY2 is a packed YUV format with half width, full height.
+
+YUY2 is YUYV in memory
+UYVY is UYVY in memory
diff --git a/libs/libyuv/docs/getting_started.md b/libs/libyuv/docs/getting_started.md
index 4426b606e0..d9807ae40e 100644
--- a/libs/libyuv/docs/getting_started.md
+++ b/libs/libyuv/docs/getting_started.md
@@ -11,21 +11,7 @@ Refer to chromium instructions for each platform for other prerequisites.
Create a working directory, enter it, and run:
- gclient config --name src https://chromium.googlesource.com/libyuv/libyuv
- gclient sync
-
-Then you'll get a .gclient file like:
-
- solutions = [
- { "name" : "src",
- "url" : "https://chromium.googlesource.com/libyuv/libyuv",
- "deps_file" : "DEPS",
- "managed" : True,
- "custom_deps" : {
- },
- "safesync_url": "",
- },
- ];
+ fetch libyuv
For iOS add `;target_os=['ios'];` to your OSX .gclient and run `gclient sync.`
@@ -57,15 +43,47 @@ To get just the source (not buildable):
## Building the Library and Unittests
+### Bazel
+
+Libyuv can be built using [Bazel](https://bazel.build/).
+
+#### Android Prerequisites
+To build for Android using Bazel, you must have the Android SDK and NDK installed. Bazel will look for the following environment variables to locate them:
+* `ANDROID_HOME`: Set this to the path of your Android SDK.
+* `ANDROID_NDK_HOME`: Set this to the path of your Android NDK.
+
+Ensure these variables are set before running the Bazel Android build commands.
+
+**Android arm64:**
+
+ bazel build -c opt --config=android_arm64 //:libyuv_test
+
+ # Or, specifying standard open-source flags (if NDK is set up in workspace):
+ bazel build -c opt --cpu=arm64-v8a --crosstool_top=//external:android/crosstool //:libyuv_test
+
+**Linux x86_64:**
+
+ bazel build -c opt //:libyuv_test
+
+ # Or, specifying a specific CPU architecture:
+ bazel build -c opt --cpu=haswell //:libyuv_test
+
+Additional commonly used compiler options can be passed to Bazel via `--copt`:
+
+ bazel build -c opt --config=android_arm64 \
+ --copt=-DLIBYUV_UNLIMITED_DATA \
+ --copt=-DENABLE_ROW_TESTS \
+ //:libyuv_test
+
### Windows
- call gn gen out\Release "--args=is_debug=false target_cpu=\"x64\""
- call gn gen out\Debug "--args=is_debug=true target_cpu=\"x64\""
+ gn gen out\Release "--args=is_debug=false target_cpu=\"x64\""
+ gn gen out\Debug "--args=is_debug=true target_cpu=\"x64\""
ninja -v -C out\Release
ninja -v -C out\Debug
- call gn gen out\Release "--args=is_debug=false target_cpu=\"x86\""
- call gn gen out\Debug "--args=is_debug=true target_cpu=\"x86\""
+ gn gen out\Release "--args=is_debug=false target_cpu=\"x86\""
+ gn gen out\Debug "--args=is_debug=true target_cpu=\"x86\""
ninja -v -C out\Release
ninja -v -C out\Debug
@@ -130,20 +148,13 @@ ia32
ninja -v -C out/Debug libyuv_unittest
ninja -v -C out/Release libyuv_unittest
-mips
-
- gn gen out/Release "--args=is_debug=false target_os=\"android\" target_cpu=\"mips64el\" mips_arch_variant=\"r6\" mips_use_msa=true is_component_build=true"
- gn gen out/Debug "--args=is_debug=true target_os=\"android\" target_cpu=\"mips64el\" mips_arch_variant=\"r6\" mips_use_msa=true is_component_build=true"
- ninja -v -C out/Debug libyuv_unittest
- ninja -v -C out/Release libyuv_unittest
-
arm disassembly:
- third_party/android_ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-objdump -d ./out/Release/obj/libyuv/row_common.o >row_common.txt
+ llvm-objdump -d ./out/Release/obj/libyuv/row_common.o >row_common.txt
- third_party/android_ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-objdump -d ./out/Release/obj/libyuv_neon/row_neon.o >row_neon.txt
+ llvm-objdump -d ./out/Release/obj/libyuv_neon/row_neon.o >row_neon.txt
- third_party/android_ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-objdump -d ./out/Release/obj/libyuv_neon/row_neon64.o >row_neon64.txt
+ llvm-objdump -d ./out/Release/obj/libyuv_neon/row_neon64.o >row_neon64.txt
Caveat: Disassembly may require optimize_max be disabled in BUILD.gn
@@ -165,6 +176,7 @@ Running test with C code:
ninja -C out/Debug libyuv_unittest
ninja -C out/Debug compare
ninja -C out/Debug yuvconvert
+ ninja -C out/Debug yuvconstants
ninja -C out/Debug psnr
ninja -C out/Debug cpuid
@@ -175,22 +187,13 @@ Running test with C code:
ninja -v -C out/Debug libyuv_unittest
ninja -v -C out/Release libyuv_unittest
-### MIPS Linux
-
-mips
-
- gn gen out/Release "--args=is_debug=false target_os=\"linux\" target_cpu=\"mips64el\" mips_arch_variant=\"loongson3\" mips_use_mmi=true is_component_build=false use_sysroot=false use_gold=false"
- gn gen out/Debug "--args=is_debug=true target_os=\"linux\" target_cpu=\"mips64el\" mips_arch_variant=\"loongson3\" mips_use_mmi=true is_component_build=false use_sysroot=false use_gold=false"
- ninja -v -C out/Debug libyuv_unittest
- ninja -v -C out/Release libyuv_unittest
-
## Building the Library with make
### Linux
make V=1 -f linux.mk
make V=1 -f linux.mk clean
- make V=1 -f linux.mk CXX=clang++
+ make V=1 -f linux.mk CXX=clang++ CC=clang
## Building the library with cmake
@@ -219,6 +222,88 @@ Install cmake: http://www.cmake.org/
make -j4
make package
+## Building RISC-V target with cmake
+
+### Native build on RISC-V SBC (e.g. SpacemiT K1/K3)
+
+On a 64-bit RISC-V system, CMake automatically detects `riscv64` and enables
+RVV with `-march=rv64gcv`:
+
+ cmake -B out/Release -DCMAKE_BUILD_TYPE=Release -DUNIT_TEST=ON
+ cmake --build out/Release
+
+To build for 32-bit RISC-V targets:
+
+ cmake -B out/Release -DCMAKE_BUILD_TYPE=Release -DRISCV_COMPILER_FLAGS="-march=rv32gcv"
+ cmake --build out/Release
+
+#### Customized Compiler Flags
+
+Customized compiler flags can be passed with `-DRISCV_COMPILER_FLAGS="xxx"`.
+When assigned, the specified flags are used instead of the default `-march=rv64gcv`:
+
+Targeting SiFive X280:
+
+ cmake -B out/Release -DUNIT_TEST=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DRISCV_COMPILER_FLAGS="-mcpu=sifive-x280"
+
+Targeting Coral (32-bit without floating point, Zve32x):
+
+ cmake -B out/Release -DUNIT_TEST=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DRISCV_COMPILER_FLAGS="-march=rv32imac_zve32x"
+
+To disable RVV:
+
+ cmake -B out/Release -DCMAKE_BUILD_TYPE=Release -DUSE_RVV=OFF
+
+### Building RISC-V with Bazel and Blaze
+
+To build with Bazel for 64-bit or 32-bit RISC-V targets:
+
+ bazel test --platforms=@platforms//cpu:riscv64 //...
+ bazel test --platforms=@platforms//cpu:riscv32 //...
+
+For custom targets such as SiFive X280 or Coral:
+
+ bazel test --platforms=@platforms//cpu:riscv64 --copt=-mcpu=sifive-x280 //...
+ bazel test --platforms=@platforms//cpu:riscv32 --copt=-march=rv32imac_zve32x //...
+
+In google3 using Blaze:
+
+ blaze test --config=mpu64 //third_party/libyuv:libyuv_test
+ blaze run --config=mpu64 //third_party/libyuv:cpuid
+ blaze run --config=mpu64 //third_party/libyuv:yuvconvert
+
+### Cross-compile for RISC-V target using QEMU
+
+#### Prerequisite: build risc-v clang toolchain and qemu
+
+If you don't have prebuilt clang and riscv64 qemu, run the script to download source and build them.
+
+ ./riscv_script/prepare_toolchain_qemu.sh
+
+After running script, clang & qemu are built in `build-toolchain-qemu/riscv-clang/` & `build-toolchain-qemu/riscv-qemu/`.
+
+#### Cross-compile for RISC-V target
+ cmake -B out/Release/ -DUNIT_TEST=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_TOOLCHAIN_FILE="./riscv_script/riscv-clang.cmake" \
+ -DTOOLCHAIN_PATH={TOOLCHAIN_PATH} \
+ -DUSE_RVV=ON .
+ cmake --build out/Release/
+
+### Run on QEMU
+
+#### Run libyuv_unittest on QEMU
+ cd out/Release/
+ USE_RVV=ON \
+ TOOLCHAIN_PATH={TOOLCHAIN_PATH} \
+ QEMU_PREFIX_PATH={QEMU_PREFIX_PATH} \
+ ../../riscv_script/run_qemu.sh libyuv_unittest
+
+
## Setup for Arm Cross compile
See also https://www.ccoderun.ca/programming/2015-12-20_CrossCompiling/index.html
@@ -239,6 +324,20 @@ See also https://www.ccoderun.ca/programming/2015-12-20_CrossCompiling/index.htm
## Running Unittests
+### Bazel
+
+You can run the tests using Bazel's `test` command. This will build and run the test in an isolated environment:
+
+ bazel test -c opt //:libyuv_test
+
+To pass specific arguments to the test binary (like a gtest filter), use `--test_arg`:
+
+ bazel test -c opt //:libyuv_test --test_arg=--gtest_filter="*" --test_output=all
+
+Alternatively, you can run the compiled binary directly from the `bazel-bin` directory:
+
+ ./bazel-bin/libyuv_test --gtest_filter="*"
+
### Windows
out\Release\libyuv_unittest.exe --gtest_catch_exceptions=0 --gtest_filter="*"
diff --git a/libs/libyuv/download_vs_toolchain.py b/libs/libyuv/download_vs_toolchain.py
index 4b3457899f..9fc89affe5 100644
--- a/libs/libyuv/download_vs_toolchain.py
+++ b/libs/libyuv/download_vs_toolchain.py
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-#
+#!/usr/bin/env vpython3
+
# Copyright 2014 The LibYuv Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style license
@@ -22,8 +22,8 @@ sys.path.insert(0, os.path.join(checkout_root, 'build'))
sys.path.insert(0, os.path.join(checkout_root, 'tools', 'find_depot_tools'))
-import vs_toolchain
+import vs_toolchain # pylint: disable=wrong-import-position
if __name__ == '__main__':
- sys.exit(vs_toolchain.main())
+ sys.exit(vs_toolchain.main())
diff --git a/libs/libyuv/include/libyuv.h b/libs/libyuv/include/libyuv.h
index aeffd5ef7a..692742d261 100644
--- a/libs/libyuv/include/libyuv.h
+++ b/libs/libyuv/include/libyuv.h
@@ -25,7 +25,8 @@
#include "libyuv/row.h"
#include "libyuv/scale.h"
#include "libyuv/scale_argb.h"
-#include "libyuv/scale_row.h"
+#include "libyuv/scale_rgb.h"
+#include "libyuv/scale_uv.h"
#include "libyuv/version.h"
#include "libyuv/video_common.h"
diff --git a/libs/libyuv/include/libyuv/basic_types.h b/libs/libyuv/include/libyuv/basic_types.h
index 1bea67f2f2..c015b493bc 100644
--- a/libs/libyuv/include/libyuv/basic_types.h
+++ b/libs/libyuv/include/libyuv/basic_types.h
@@ -11,37 +11,6 @@
#ifndef INCLUDE_LIBYUV_BASIC_TYPES_H_
#define INCLUDE_LIBYUV_BASIC_TYPES_H_
-#include <stddef.h> // For size_t and NULL
-
-#if !defined(INT_TYPES_DEFINED) && !defined(GG_LONGLONG)
-#define INT_TYPES_DEFINED
-
-#if defined(_MSC_VER) && (_MSC_VER < 1600)
-#include <sys/types.h> // for uintptr_t on x86
-typedef unsigned __int64 uint64_t;
-typedef __int64 int64_t;
-typedef unsigned int uint32_t;
-typedef int int32_t;
-typedef unsigned short uint16_t;
-typedef short int16_t;
-typedef unsigned char uint8_t;
-typedef signed char int8_t;
-#else
-#include <stdint.h> // for uintptr_t and C99 types
-#endif // defined(_MSC_VER) && (_MSC_VER < 1600)
-// Types are deprecated. Enable this macro for legacy types.
-#ifdef LIBYUV_LEGACY_TYPES
-typedef uint64_t uint64;
-typedef int64_t int64;
-typedef uint32_t uint32;
-typedef int32_t int32;
-typedef uint16_t uint16;
-typedef int16_t int16;
-typedef uint8_t uint8;
-typedef int8_t int8;
-#endif // LIBYUV_LEGACY_TYPES
-#endif // INT_TYPES_DEFINED
-
#if !defined(LIBYUV_API)
#if defined(_WIN32) || defined(__CYGWIN__)
#if defined(LIBYUV_BUILDING_SHARED_LIBRARY)
@@ -60,9 +29,4 @@ typedef int8_t int8;
#endif // __GNUC__
#endif // LIBYUV_API
-// TODO(fbarchard): Remove bool macros.
-#define LIBYUV_BOOL int
-#define LIBYUV_FALSE 0
-#define LIBYUV_TRUE 1
-
#endif // INCLUDE_LIBYUV_BASIC_TYPES_H_
diff --git a/libs/libyuv/include/libyuv/compare.h b/libs/libyuv/include/libyuv/compare.h
index 3353ad71c6..a8196d792c 100644
--- a/libs/libyuv/include/libyuv/compare.h
+++ b/libs/libyuv/include/libyuv/compare.h
@@ -11,6 +11,9 @@
#ifndef INCLUDE_LIBYUV_COMPARE_H_
#define INCLUDE_LIBYUV_COMPARE_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
diff --git a/libs/libyuv/include/libyuv/compare_row.h b/libs/libyuv/include/libyuv/compare_row.h
index e95b9d93eb..cb5cbc827b 100644
--- a/libs/libyuv/include/libyuv/compare_row.h
+++ b/libs/libyuv/include/libyuv/compare_row.h
@@ -11,82 +11,164 @@
#ifndef INCLUDE_LIBYUV_COMPARE_ROW_H_
#define INCLUDE_LIBYUV_COMPARE_ROW_H_
-#include "libyuv/basic_types.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
+// The following are available for Visual C 32/64 bit, and for clang with
+// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are
+// intrinsics in compare_win.cc.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+#define HAS_HAMMINGDISTANCE_SSE42
+#define HAS_SUMSQUAREERROR_SSE2
+#define HAS_HASHDJB2_SSE41
+#define HAS_SSIM4X4ROW_SSE2
+#define HAS_SSIMWINDOWROW_SSE2
+#if defined(VISUALC_HAS_AVX2)
+#define HAS_HAMMINGDISTANCE_AVX2
+#define HAS_SUMSQUAREERROR_AVX2
+#define HAS_HASHDJB2_AVX2
+#define HAS_SSIM4X4ROW_AVX2
+#define HAS_SSIMWINDOWROW_AVX2
+#define HAS_ARGBDETECTROW_AVX2
#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer)
-#define LIBYUV_DISABLE_X86
+#if defined(VISUALC_HAS_AVX512)
+#define HAS_HAMMINGDISTANCE_AVX512BW
+#define HAS_HAMMINGDISTANCE_AVX512VNNI
+#define HAS_SUMSQUAREERROR_AVX512BW
+#define HAS_SUMSQUAREERROR_AVX512VNNI
+#define HAS_HASHDJB2_AVX512BW
+#define HAS_SSIM4X4ROW_AVX512BW
+#define HAS_SSIMWINDOWROW_AVX512BW
#endif
#endif
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-// The following are available for Visual C and GCC:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_IX86))
-#define HAS_HASHDJB2_SSE41
-#define HAS_SUMSQUAREERROR_SSE2
+// The following are available for 32/64 bit GCC and clang, as inline assembly
+// in compare_gcc.cc:
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (!defined(_MSC_VER) || defined(__clang__))
+#if defined(__x86_64__)
#define HAS_HAMMINGDISTANCE_SSE42
#endif
-
-// The following are available for Visual C and clangcl 32 bit:
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER) && \
- (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2))
-#define HAS_HASHDJB2_AVX2
+#define HAS_HAMMINGDISTANCE_SSSE3
+#define HAS_SUMSQUAREERROR_SSE2
+#define HAS_HASHDJB2_SSE41
+#define HAS_SSIM4X4ROW_SSE2
+#define HAS_SSIMWINDOWROW_SSE2
+#if defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)
+#define HAS_HAMMINGDISTANCE_AVX2
#define HAS_SUMSQUAREERROR_AVX2
+#define HAS_HASHDJB2_AVX2
+#define HAS_SSIM4X4ROW_AVX2
+#define HAS_SSIMWINDOWROW_AVX2
+#define HAS_ARGBDETECTROW_AVX2
#endif
-
-// The following are available for GCC and clangcl 64 bit:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
-#define HAS_HAMMINGDISTANCE_SSSE3
+#if defined(__x86_64__) && \
+ (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512))
+#define HAS_HAMMINGDISTANCE_AVX512BW
+#define HAS_HAMMINGDISTANCE_AVX512VNNI
+#define HAS_SUMSQUAREERROR_AVX512BW
+#define HAS_SUMSQUAREERROR_AVX512VNNI
+#define HAS_HASHDJB2_AVX512BW
+#define HAS_SSIM4X4ROW_AVX512BW
+#define HAS_SSIMWINDOWROW_AVX512BW
#endif
-
-// The following are available for GCC and clangcl 64 bit:
-#if !defined(LIBYUV_DISABLE_X86) && defined(CLANG_HAS_AVX2) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
-#define HAS_HAMMINGDISTANCE_AVX2
#endif
// The following are available for Neon:
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
-#define HAS_SUMSQUAREERROR_NEON
#define HAS_HAMMINGDISTANCE_NEON
+#define HAS_SUMSQUAREERROR_NEON
+#define HAS_HASHDJB2_NEON
+#define HAS_SSIM4X4ROW_NEON
+#define HAS_SSIMWINDOWROW_NEON
+#define HAS_ARGBDETECTROW_NEON
#endif
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#define HAS_HAMMINGDISTANCE_MSA
-#define HAS_SUMSQUAREERROR_MSA
+// The following are available for AArch64 Neon:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_HAMMINGDISTANCE_NEON_DOTPROD
+#define HAS_SUMSQUAREERROR_NEON_DOTPROD
+#define HAS_SSIM4X4ROW_NEON_DOTPROD
#endif
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-#define HAS_HAMMINGDISTANCE_MMI
-#define HAS_SUMSQUAREERROR_MMI
+// The following are available for AArch64 SVE:
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+#define HAS_SSIM4X4ROW_SVE2
+#define HAS_SSIMWINDOWROW_SVE2
+#endif
+
+// The following are available for AArch64 SME:
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_SSIM4X4ROW_SME
+#define HAS_SSIMWINDOWROW_SME
+#endif
+
+// The following are available for RISC-V Vector (RVV):
+#if !defined(LIBYUV_DISABLE_RVV) && \
+ (defined(__riscv_vector) || defined(LIBYUV_RVV))
+#define HAS_HAMMINGDISTANCE_RVV
+#define HAS_SUMSQUAREERROR_RVV
+#define HAS_HASHDJB2_RVV
+#define HAS_SSIM4X4ROW_RVV
+#if defined(__riscv_f) || defined(__riscv_zve32f)
+#define HAS_SSIMWINDOWROW_RVV
+#endif
+#define HAS_ARGBDETECTROW_RVV
+#endif
+
+// The following are available for LoongArch LSX:
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+#define HAS_HAMMINGDISTANCE_LSX
+#define HAS_SUMSQUAREERROR_LSX
+#define HAS_HASHDJB2_LSX
+#define HAS_SSIM4X4ROW_LSX
+#define HAS_SSIMWINDOWROW_LSX
+#define HAS_ARGBDETECTROW_LSX
+#endif
+
+// The following are available for LoongArch LASX:
+#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
+#define HAS_HAMMINGDISTANCE_LASX
+#define HAS_SUMSQUAREERROR_LASX
+#define HAS_HASHDJB2_LASX
+#define HAS_SSIM4X4ROW_LASX
+#define HAS_SSIMWINDOWROW_LASX
+#define HAS_ARGBDETECTROW_LASX
+#endif
+
+// The following are available for WebAssembly SIMD128:
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#define HAS_HAMMINGDISTANCE_WASMSIMD
+#define HAS_SUMSQUAREERROR_WASMSIMD
+#define HAS_HASHDJB2_WASMSIMD
+#define HAS_SSIM4X4ROW_WASMSIMD
+#define HAS_SSIMWINDOWROW_WASMSIMD
+#define HAS_ARGBDETECTROW_WASMSIMD
+#endif
+
+// The following are available for Hexagon HVX:
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#define HAS_HAMMINGDISTANCE_HVX
+#define HAS_SUMSQUAREERROR_HVX
+#define HAS_HASHDJB2_HVX
+#define HAS_SSIM4X4ROW_HVX
+#define HAS_SSIMWINDOWROW_HVX
+#define HAS_ARGBDETECTROW_HVX
#endif
uint32_t HammingDistance_C(const uint8_t* src_a,
@@ -101,13 +183,31 @@ uint32_t HammingDistance_SSSE3(const uint8_t* src_a,
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count);
+uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
uint32_t HammingDistance_NEON(const uint8_t* src_a,
const uint8_t* src_b,
int count);
-uint32_t HammingDistance_MSA(const uint8_t* src_a,
+uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t HammingDistance_RVV(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t HammingDistance_LSX(const uint8_t* src_a,
const uint8_t* src_b,
int count);
-uint32_t HammingDistance_MMI(const uint8_t* src_a,
+uint32_t HammingDistance_LASX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t HammingDistance_HVX(const uint8_t* src_a,
const uint8_t* src_b,
int count);
uint32_t SumSquareError_C(const uint8_t* src_a,
@@ -119,19 +219,168 @@ uint32_t SumSquareError_SSE2(const uint8_t* src_a,
uint32_t SumSquareError_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count);
+uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
uint32_t SumSquareError_NEON(const uint8_t* src_a,
const uint8_t* src_b,
int count);
-uint32_t SumSquareError_MSA(const uint8_t* src_a,
+uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t SumSquareError_RVV(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t SumSquareError_LSX(const uint8_t* src_a,
const uint8_t* src_b,
int count);
-uint32_t SumSquareError_MMI(const uint8_t* src_a,
+uint32_t SumSquareError_LASX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
+uint32_t SumSquareError_HVX(const uint8_t* src_a,
const uint8_t* src_b,
int count);
uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_RVV(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_HVX(const uint8_t* src, int count, uint32_t seed);
+
+void Ssim4x4Row_C(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_SSE2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_AVX2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_NEON(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_SVE2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_SME(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_RVV(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_LSX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_LASX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+void Ssim4x4Row_HVX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width);
+float SsimWindowRow_C(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_SSE2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_AVX2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_AVX512BW(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_NEON(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_SVE2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_SME(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_RVV(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_LSX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_LASX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_WASMSIMD(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+float SsimWindowRow_HVX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows);
+
+uint32_t ARGBDetectRow_C(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_RVV(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_WASMSIMD(const uint8_t* argb, int width);
+uint32_t ARGBDetectRow_HVX(const uint8_t* argb, int width);
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/include/libyuv/convert.h b/libs/libyuv/include/libyuv/convert.h
index f571142fab..fdaec5eaf5 100644
--- a/libs/libyuv/include/libyuv/convert.h
+++ b/libs/libyuv/include/libyuv/convert.h
@@ -11,6 +11,9 @@
#ifndef INCLUDE_LIBYUV_CONVERT_H_
#define INCLUDE_LIBYUV_CONVERT_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#include "libyuv/rotate.h" // For enum RotationMode.
@@ -25,116 +28,763 @@ namespace libyuv {
extern "C" {
#endif
-// Convert I444 to I420.
+struct ArgbConstants;
+
+// Convert I444 to I420.
+LIBYUV_API
+int I444ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I444 to NV12.
+LIBYUV_API
+int I444ToNV12(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I444 to NV21.
+LIBYUV_API
+int I444ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
+// Convert I422 to I420.
+LIBYUV_API
+int I422ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I422 to I444.
+LIBYUV_API
+int I422ToI444(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I422 to I210.
+LIBYUV_API
+int I422ToI210(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert MM21 to NV12.
+LIBYUV_API
+int MM21ToNV12(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert MM21 to I420.
+LIBYUV_API
+int MM21ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert MM21 to YUY2
+LIBYUV_API
+int MM21ToYUY2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ int width,
+ int height);
+
+// Convert MT2T to P010
+// Note that src_y and src_uv point to packed 10-bit values, so the Y plane will
+// be 10 / 8 times the dimensions of the image. Also for this reason,
+// src_stride_y and src_stride_uv are given in bytes.
+LIBYUV_API
+int MT2TToP010(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I422 to NV21.
+LIBYUV_API
+int I422ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
+// Copy I420 to I420.
+#define I420ToI420 I420Copy
+LIBYUV_API
+int I420Copy(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I420 to I444.
+LIBYUV_API
+int I420ToI444(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Copy I010 to I010
+#define I010ToI010 I010Copy
+#define H010ToH010 I010Copy
+LIBYUV_API
+int I010Copy(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert 10 bit YUV to 8 bit
+#define H010ToH420 I010ToI420
+LIBYUV_API
+int I010ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H210ToH420 I210ToI420
+LIBYUV_API
+int I210ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H210ToH422 I210ToI422
+LIBYUV_API
+int I210ToI422(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H410ToH420 I410ToI420
+LIBYUV_API
+int I410ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H410ToH444 I410ToI444
+LIBYUV_API
+int I410ToI444(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H012ToH420 I012ToI420
+LIBYUV_API
+int I012ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H212ToH422 I212ToI422
+LIBYUV_API
+int I212ToI422(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H212ToH420 I212ToI420
+LIBYUV_API
+int I212ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H412ToH444 I412ToI444
+LIBYUV_API
+int I412ToI444(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define H412ToH420 I412ToI420
+LIBYUV_API
+int I412ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert 10 bit P010 to 8 bit NV12.
+// dst_y can be NULL
+LIBYUV_API
+int P010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+#define I412ToI012 I410ToI010
+#define H410ToH010 I410ToI010
+#define H412ToH012 I410ToI010
+LIBYUV_API
+int I410ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+#define I212ToI012 I210ToI010
+#define H210ToH010 I210ToI010
+#define H212ToH012 I210ToI010
+LIBYUV_API
+int I210ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I010 to I410
+LIBYUV_API
+int I010ToI410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I012 to I412
+#define I012ToI412 I010ToI410
+
+// Convert I210 to I410
+LIBYUV_API
+int I210ToI410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I212 to I412
+#define I212ToI412 I210ToI410
+
+// Convert I010 to P010
+LIBYUV_API
+int I010ToP010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert 10 bit YUV I010 to NV12
+LIBYUV_API
+int I010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I210 to P210
+LIBYUV_API
+int I210ToP210(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I410 to P410
+LIBYUV_API
+int I410ToP410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I012 to P012
+LIBYUV_API
+int I012ToP012(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I212 to P212
+LIBYUV_API
+int I212ToP212(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I412 to P412
+LIBYUV_API
+int I412ToP412(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert I400 (grey) to I420.
+LIBYUV_API
+int I400ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert J420 to I420.
+LIBYUV_API
+int J420ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert I400 (grey) to NV21.
+LIBYUV_API
+int I400ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
+#define J400ToJ420 I400ToI420
+
+// Convert NV12 to I420.
+LIBYUV_API
+int NV12ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert NV21 to I420.
+LIBYUV_API
+int NV21ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert NV12 to NV24.
+LIBYUV_API
+int NV12ToNV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert NV16 to NV24.
+LIBYUV_API
+int NV16ToNV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Convert P010 to I010.
LIBYUV_API
-int I444ToI420(const uint8_t* src_y,
+int P010ToI010(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_u,
+ uint16_t* dst_u,
int dst_stride_u,
- uint8_t* dst_v,
+ uint16_t* dst_v,
int dst_stride_v,
int width,
int height);
-// Convert I444 to NV21.
+// Convert P210 to I210.
LIBYUV_API
-int I444ToNV21(const uint8_t* src_y,
+int P210ToI210(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
int width,
int height);
-// Convert I422 to I420.
+// Convert P410 to I410.
LIBYUV_API
-int I422ToI420(const uint8_t* src_y,
+int P410ToI410(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_u,
+ uint16_t* dst_u,
int dst_stride_u,
- uint8_t* dst_v,
+ uint16_t* dst_v,
int dst_stride_v,
int width,
int height);
-// Convert I422 to NV21.
+// Convert P010 to I420.
LIBYUV_API
-int I422ToNV21(const uint8_t* src_y,
+int P010ToI420(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
+ const uint16_t* src_uv,
+ int src_stride_uv,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
int width,
int height);
-// Copy I420 to I420.
-#define I420ToI420 I420Copy
-LIBYUV_API
-int I420Copy(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height);
-
-// Copy I010 to I010
-#define I010ToI010 I010Copy
-#define H010ToH010 I010Copy
-LIBYUV_API
-int I010Copy(const uint16_t* src_y,
- int src_stride_y,
- const uint16_t* src_u,
- int src_stride_u,
- const uint16_t* src_v,
- int src_stride_v,
- uint16_t* dst_y,
- int dst_stride_y,
- uint16_t* dst_u,
- int dst_stride_u,
- uint16_t* dst_v,
- int dst_stride_v,
- int width,
- int height);
-
-// Convert 10 bit YUV to 8 bit
-#define H010ToH420 I010ToI420
+// Convert P210 to I420.
LIBYUV_API
-int I010ToI420(const uint16_t* src_y,
+int P210ToI420(const uint16_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
- int src_stride_u,
- const uint16_t* src_v,
- int src_stride_v,
+ const uint16_t* src_uv,
+ int src_stride_uv,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -144,10 +794,12 @@ int I010ToI420(const uint16_t* src_y,
int width,
int height);
-// Convert I400 (grey) to I420.
+// Convert P410 to I420.
LIBYUV_API
-int I400ToI420(const uint8_t* src_y,
+int P410ToI420(const uint16_t* src_y,
int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -157,49 +809,59 @@ int I400ToI420(const uint8_t* src_y,
int width,
int height);
-// Convert I400 (grey) to NV21.
+// Convert P012 to I012.
LIBYUV_API
-int I400ToNV21(const uint8_t* src_y,
+int P012ToI012(const uint16_t* src_y,
int src_stride_y,
- uint8_t* dst_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
int width,
int height);
-#define J400ToJ420 I400ToI420
-
-// Convert NV12 to I420.
+// Convert P010 to P410.
LIBYUV_API
-int NV12ToI420(const uint8_t* src_y,
+int P010ToP410(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_uv,
+ const uint16_t* src_uv,
int src_stride_uv,
- uint8_t* dst_y,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
-// Convert NV21 to I420.
+// Convert P012 to P412.
+#define P012ToP412 P010ToP410
+
+// Convert P016 to P416.
+#define P016ToP416 P010ToP410
+
+// Convert P210 to P410.
LIBYUV_API
-int NV21ToI420(const uint8_t* src_y,
+int P210ToP410(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
+// Convert P212 to P412.
+#define P212ToP412 P210ToP410
+
+// Convert P216 to P416.
+#define P216ToP416 P210ToP410
+
// Convert YUY2 to I420.
LIBYUV_API
int YUY2ToI420(const uint8_t* src_yuy2,
@@ -248,19 +910,6 @@ int AYUVToNV21(const uint8_t* src_ayuv,
int width,
int height);
-// Convert M420 to I420.
-LIBYUV_API
-int M420ToI420(const uint8_t* src_m420,
- int src_stride_m420,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height);
-
// Convert Android420 to I420.
LIBYUV_API
int Android420ToI420(const uint8_t* src_y,
@@ -292,6 +941,35 @@ int ARGBToI420(const uint8_t* src_argb,
int width,
int height);
+// RGB to I420 with matrix. See ArgbConstants in convert_from_argb.h for usage.
+LIBYUV_API
+int ARGBToI420Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB to I420 with Alpha
+LIBYUV_API
+int ARGBToI420Alpha(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height);
+
// BGRA little endian (argb in memory) to I420.
LIBYUV_API
int BGRAToI420(const uint8_t* src_bgra,
@@ -318,6 +996,71 @@ int ABGRToI420(const uint8_t* src_abgr,
int width,
int height);
+// Convert AR30 to I420.
+LIBYUV_API
+int AR30ToI420(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Convert AB30 to I420.
+LIBYUV_API
+int AB30ToI420(const uint8_t* src_ab30,
+ int src_stride_ab30,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// BGRA little endian (argb in memory) to I422.
+LIBYUV_API
+int BGRAToI422(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// ABGR little endian (rgba in memory) to I422.
+LIBYUV_API
+int ABGRToI422(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// RGBA little endian (abgr in memory) to I422.
+LIBYUV_API
+int RGBAToI422(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
// RGBA little endian (abgr in memory) to I420.
LIBYUV_API
int RGBAToI420(const uint8_t* src_rgba,
@@ -370,7 +1113,46 @@ int RAWToI420(const uint8_t* src_raw,
int width,
int height);
-// RGB16 (RGBP fourcc) little endian to I420.
+// RGB big endian (rgb in memory) to I444.
+LIBYUV_API
+int RAWToI444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// RGB big endian (rgb in memory) to J420.
+LIBYUV_API
+int RAWToJ420(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// RGB big endian (rgb in memory) to J444.
+LIBYUV_API
+int RAWToJ444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// RGB16 (RGBP fourcc) little endian to I420. Deprecated.
LIBYUV_API
int RGB565ToI420(const uint8_t* src_rgb565,
int src_stride_rgb565,
@@ -383,7 +1165,7 @@ int RGB565ToI420(const uint8_t* src_rgb565,
int width,
int height);
-// RGB15 (RGBO fourcc) little endian to I420.
+// RGB15 (RGBO fourcc) little endian to I420. Deprecated.
LIBYUV_API
int ARGB1555ToI420(const uint8_t* src_argb1555,
int src_stride_argb1555,
@@ -396,7 +1178,7 @@ int ARGB1555ToI420(const uint8_t* src_argb1555,
int width,
int height);
-// RGB12 (R444 fourcc) little endian to I420.
+// RGB12 (R444 fourcc) little endian to I420. Deprecated.
LIBYUV_API
int ARGB4444ToI420(const uint8_t* src_argb4444,
int src_stride_argb4444,
@@ -418,7 +1200,15 @@ int RGB24ToJ400(const uint8_t* src_rgb24,
int width,
int height);
-#ifdef HAVE_JPEG
+// RGB big endian (rgb in memory) to J400.
+LIBYUV_API
+int RAWToJ400(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height);
+
// src_width/height provided by capture.
// dst_width/height for clipping determine final size.
LIBYUV_API
@@ -448,13 +1238,25 @@ int MJPGToNV21(const uint8_t* sample,
int dst_width,
int dst_height);
+// JPEG to NV12
+LIBYUV_API
+int MJPGToNV12(const uint8_t* sample,
+ size_t sample_size,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height);
+
// Query size of MJPG in pixels.
LIBYUV_API
int MJPGSize(const uint8_t* sample,
size_t sample_size,
int* width,
int* height);
-#endif
// Convert camera sample to I420 with cropping, rotation and vertical flip.
// "src_size" is needed to parse MJPG.
diff --git a/libs/libyuv/include/libyuv/convert_argb.h b/libs/libyuv/include/libyuv/convert_argb.h
index e8ed1f59f3..59b690f3a8 100644
--- a/libs/libyuv/include/libyuv/convert_argb.h
+++ b/libs/libyuv/include/libyuv/convert_argb.h
@@ -11,20 +11,80 @@
#ifndef INCLUDE_LIBYUV_CONVERT_ARGB_H_
#define INCLUDE_LIBYUV_CONVERT_ARGB_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#include "libyuv/rotate.h" // For enum RotationMode.
-
-// TODO(fbarchard): This set of functions should exactly match convert.h
-// TODO(fbarchard): Add tests. Create random content of right size and convert
-// with C vs Opt and or to I420 and compare.
-// TODO(fbarchard): Some of these functions lack parameter setting.
+#include "libyuv/scale.h" // For enum FilterMode.
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Conversion matrix for YUV to RGB
+LIBYUV_API extern const struct YuvConstants kYuvI601Constants; // BT.601
+LIBYUV_API extern const struct YuvConstants kYuvJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct YuvConstants kYuvH709Constants; // BT.709
+LIBYUV_API extern const struct YuvConstants kYuvF709Constants; // BT.709 full
+LIBYUV_API extern const struct YuvConstants kYuv2020Constants; // BT.2020
+LIBYUV_API extern const struct YuvConstants kYuvV2020Constants; // BT.2020 full
+
+// Conversion matrix for YVU to BGR
+LIBYUV_API extern const struct YuvConstants kYvuI601Constants; // BT.601
+LIBYUV_API extern const struct YuvConstants kYvuJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct YuvConstants kYvuH709Constants; // BT.709
+LIBYUV_API extern const struct YuvConstants kYvuF709Constants; // BT.709 full
+LIBYUV_API extern const struct YuvConstants kYvu2020Constants; // BT.2020
+LIBYUV_API extern const struct YuvConstants kYvuV2020Constants; // BT.2020 full
+
+// Macros for end swapped destination Matrix conversions.
+// Swap UV and pass mirrored kYvuJPEGConstants matrix.
+// TODO(fbarchard): Add macro for each Matrix function.
+#define kYuvI601ConstantsVU kYvuI601Constants
+#define kYuvJPEGConstantsVU kYvuJPEGConstants
+#define kYuvH709ConstantsVU kYvuH709Constants
+#define kYuvF709ConstantsVU kYvuF709Constants
+#define kYuv2020ConstantsVU kYvu2020Constants
+#define kYuvV2020ConstantsVU kYvuV2020Constants
+
+#define NV12ToABGRMatrix(a, b, c, d, e, f, g, h, i) \
+ NV21ToARGBMatrix(a, b, c, d, e, f, g##VU, h, i)
+#define NV21ToABGRMatrix(a, b, c, d, e, f, g, h, i) \
+ NV12ToARGBMatrix(a, b, c, d, e, f, g##VU, h, i)
+#define NV12ToRAWMatrix(a, b, c, d, e, f, g, h, i) \
+ NV21ToRGB24Matrix(a, b, c, d, e, f, g##VU, h, i)
+#define NV21ToRAWMatrix(a, b, c, d, e, f, g, h, i) \
+ NV12ToRGB24Matrix(a, b, c, d, e, f, g##VU, h, i)
+#define I010ToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I010ToARGBMatrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I210ToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I210ToARGBMatrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I410ToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I410ToARGBMatrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I010ToAB30Matrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I010ToAR30Matrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I210ToAB30Matrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I210ToAR30Matrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I410ToAB30Matrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I410ToAR30Matrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I012ToAB30Matrix(a, b, c, d, e, f, g, h, i, j, k) \
+ I012ToAR30Matrix(a, b, e, f, c, d, g, h, i##VU, j, k)
+#define I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I420AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+#define I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I422AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+#define I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I444AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+#define I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I010AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+#define I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I210AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+#define I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, k, l, m, n) \
+ I410AlphaToARGBMatrix(a, b, e, f, c, d, g, h, i, j, k##VU, l, m, n)
+
// Alias.
#define ARGBToARGB ARGBCopy
@@ -50,7 +110,7 @@ int I420ToARGB(const uint8_t* src_y,
int width,
int height);
-// Duplicate prototype for function in convert_from.h for remoting.
+// Convert I420 to ABGR.
LIBYUV_API
int I420ToABGR(const uint8_t* src_y,
int src_stride_y,
@@ -63,65 +123,78 @@ int I420ToABGR(const uint8_t* src_y,
int width,
int height);
-// Convert I010 to ARGB.
+// Convert J420 to ARGB.
LIBYUV_API
-int I010ToARGB(const uint16_t* src_y,
+int J420ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert I010 to ARGB.
+// Convert J420 to ABGR.
LIBYUV_API
-int I010ToARGB(const uint16_t* src_y,
+int J420ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert H420 to ARGB.
+LIBYUV_API
+int H420ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert I010 to ABGR.
+// Convert H420 to ABGR.
LIBYUV_API
-int I010ToABGR(const uint16_t* src_y,
+int H420ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height);
-// Convert H010 to ARGB.
+// Convert U420 to ARGB.
LIBYUV_API
-int H010ToARGB(const uint16_t* src_y,
+int U420ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert H010 to ABGR.
+// Convert U420 to ABGR.
LIBYUV_API
-int H010ToABGR(const uint16_t* src_y,
+int U420ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
@@ -141,22 +214,22 @@ int I422ToARGB(const uint8_t* src_y,
int width,
int height);
-// Convert I444 to ARGB.
+// Convert I422 to ABGR.
LIBYUV_API
-int I444ToARGB(const uint8_t* src_y,
+int I422ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert J444 to ARGB.
+// Convert J422 to ARGB.
LIBYUV_API
-int J444ToARGB(const uint8_t* src_y,
+int J422ToARGB(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
@@ -167,9 +240,9 @@ int J444ToARGB(const uint8_t* src_y,
int width,
int height);
-// Convert I444 to ABGR.
+// Convert J422 to ABGR.
LIBYUV_API
-int I444ToABGR(const uint8_t* src_y,
+int J422ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
@@ -180,201 +253,113 @@ int I444ToABGR(const uint8_t* src_y,
int width,
int height);
-// Convert I420 with Alpha to preattenuated ARGB.
-LIBYUV_API
-int I420AlphaToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- const uint8_t* src_a,
- int src_stride_a,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height,
- int attenuate);
-
-// Convert I420 with Alpha to preattenuated ABGR.
-LIBYUV_API
-int I420AlphaToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- const uint8_t* src_a,
- int src_stride_a,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height,
- int attenuate);
-
-// Convert I400 (grey) to ARGB. Reverse of ARGBToI400.
-LIBYUV_API
-int I400ToARGB(const uint8_t* src_y,
- int src_stride_y,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
-
-// Convert J400 (jpeg grey) to ARGB.
+// Convert H422 to ARGB.
LIBYUV_API
-int J400ToARGB(const uint8_t* src_y,
+int H422ToARGB(const uint8_t* src_y,
int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Alias.
-#define YToARGB I400ToARGB
-
-// Convert NV12 to ARGB.
+// Convert H422 to ABGR.
LIBYUV_API
-int NV12ToARGB(const uint8_t* src_y,
+int H422ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert NV21 to ARGB.
+// Convert U422 to ARGB.
LIBYUV_API
-int NV21ToARGB(const uint8_t* src_y,
+int U422ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert NV12 to ABGR.
+// Convert U422 to ABGR.
LIBYUV_API
-int NV12ToABGR(const uint8_t* src_y,
+int U422ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height);
-// Convert NV21 to ABGR.
+// Convert I444 to ARGB.
LIBYUV_API
-int NV21ToABGR(const uint8_t* src_y,
+int I444ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height);
-
-// Convert NV12 to RGB24.
-LIBYUV_API
-int NV12ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height);
-
-// Convert NV21 to RGB24.
-LIBYUV_API
-int NV21ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height);
-
-// Convert NV21 to YUV24.
-LIBYUV_API
-int NV21ToYUV24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_yuv24,
- int dst_stride_yuv24,
- int width,
- int height);
-
-// Convert NV12 to RAW.
-LIBYUV_API
-int NV12ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height);
-
-// Convert NV21 to RAW.
-LIBYUV_API
-int NV21ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height);
-
-// Convert M420 to ARGB.
-LIBYUV_API
-int M420ToARGB(const uint8_t* src_m420,
- int src_stride_m420,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert YUY2 to ARGB.
+// Convert I444 to ABGR.
LIBYUV_API
-int YUY2ToARGB(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int I444ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert UYVY to ARGB.
+// Convert J444 to ARGB.
LIBYUV_API
-int UYVYToARGB(const uint8_t* src_uyvy,
- int src_stride_uyvy,
+int J444ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert J420 to ARGB.
+// Convert J444 to ABGR.
LIBYUV_API
-int J420ToARGB(const uint8_t* src_y,
+int J444ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert J422 to ARGB.
+// Convert H444 to ARGB.
LIBYUV_API
-int J422ToARGB(const uint8_t* src_y,
+int H444ToARGB(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
@@ -385,9 +370,9 @@ int J422ToARGB(const uint8_t* src_y,
int width,
int height);
-// Convert J420 to ABGR.
+// Convert H444 to ABGR.
LIBYUV_API
-int J420ToABGR(const uint8_t* src_y,
+int H444ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
@@ -398,74 +383,113 @@ int J420ToABGR(const uint8_t* src_y,
int width,
int height);
-// Convert J422 to ABGR.
+// Convert U444 to ARGB.
LIBYUV_API
-int J422ToABGR(const uint8_t* src_y,
+int U444ToARGB(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height);
-// Convert H420 to ARGB.
+// Convert U444 to ABGR.
LIBYUV_API
-int H420ToARGB(const uint8_t* src_y,
+int U444ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert H422 to ARGB.
+// Convert I444 to RGB24.
LIBYUV_API
-int H422ToARGB(const uint8_t* src_y,
+int I444ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+// Convert I444 to RAW.
+LIBYUV_API
+int I444ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+// Convert I010 to ARGB.
+LIBYUV_API
+int I010ToARGB(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
+ const uint16_t* src_u,
int src_stride_u,
- const uint8_t* src_v,
+ const uint16_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert H420 to ABGR.
+// Convert I010 to ABGR.
LIBYUV_API
-int H420ToABGR(const uint8_t* src_y,
+int I010ToABGR(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
+ const uint16_t* src_u,
int src_stride_u,
- const uint8_t* src_v,
+ const uint16_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height);
-// Convert H422 to ABGR.
+// Convert H010 to ARGB.
LIBYUV_API
-int H422ToABGR(const uint8_t* src_y,
+int H010ToARGB(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
+ const uint16_t* src_u,
int src_stride_u,
- const uint8_t* src_v,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert H010 to ABGR.
+LIBYUV_API
+int H010ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height);
-// Convert H010 to ARGB.
+// Convert U010 to ARGB.
LIBYUV_API
-int H010ToARGB(const uint16_t* src_y,
+int U010ToARGB(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
int src_stride_u,
@@ -476,207 +500,1838 @@ int H010ToARGB(const uint16_t* src_y,
int width,
int height);
-// Convert I010 to AR30.
+// Convert U010 to ABGR.
LIBYUV_API
-int I010ToAR30(const uint16_t* src_y,
+int U010ToABGR(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
int src_stride_u,
const uint16_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert H010 to AR30.
+// Convert I210 to ARGB.
LIBYUV_API
-int H010ToAR30(const uint16_t* src_y,
+int I210ToARGB(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
int src_stride_u,
const uint16_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height);
-// Convert I010 to AB30.
+// Convert I210 to ABGR.
LIBYUV_API
-int I010ToAB30(const uint16_t* src_y,
+int I210ToABGR(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
int src_stride_u,
const uint16_t* src_v,
int src_stride_v,
- uint8_t* dst_ab30,
- int dst_stride_ab30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Convert H010 to AB30.
+// Convert H210 to ARGB.
LIBYUV_API
-int H010ToAB30(const uint16_t* src_y,
+int H210ToARGB(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
int src_stride_u,
const uint16_t* src_v,
int src_stride_v,
- uint8_t* dst_ab30,
- int dst_stride_ab30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height);
-// BGRA little endian (argb in memory) to ARGB.
+// Convert H210 to ABGR.
LIBYUV_API
-int BGRAToARGB(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int H210ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// ABGR little endian (rgba in memory) to ARGB.
+// Convert U210 to ARGB.
LIBYUV_API
-int ABGRToARGB(const uint8_t* src_abgr,
- int src_stride_abgr,
+int U210ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// RGBA little endian (abgr in memory) to ARGB.
+// Convert U210 to ABGR.
LIBYUV_API
-int RGBAToARGB(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int U210ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
-// Deprecated function name.
-#define BG24ToARGB RGB24ToARGB
-
-// RGB little endian (bgr in memory) to ARGB.
+// Convert I420 with Alpha to preattenuated ARGB.
LIBYUV_API
-int RGB24ToARGB(const uint8_t* src_rgb24,
- int src_stride_rgb24,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
+int I420AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate);
-// RGB big endian (rgb in memory) to ARGB.
+// Convert I420 with Alpha to preattenuated ABGR.
LIBYUV_API
-int RAWToARGB(const uint8_t* src_raw,
- int src_stride_raw,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
+int I420AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate);
-// RGB16 (RGBP fourcc) little endian to ARGB.
+// Convert I422 with Alpha to preattenuated ARGB.
LIBYUV_API
-int RGB565ToARGB(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
+int I422AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate);
-// RGB15 (RGBO fourcc) little endian to ARGB.
+// Convert I422 with Alpha to preattenuated ABGR.
LIBYUV_API
-int ARGB1555ToARGB(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
+int I422AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate);
-// RGB12 (R444 fourcc) little endian to ARGB.
+// Convert I444 with Alpha to preattenuated ARGB.
LIBYUV_API
-int ARGB4444ToARGB(const uint8_t* src_argb4444,
- int src_stride_argb4444,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height);
+int I444AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate);
-// Aliases
-#define AB30ToARGB AR30ToABGR
-#define AB30ToABGR AR30ToARGB
-#define AB30ToAR30 AR30ToAB30
+// Convert I444 with Alpha to preattenuated ABGR.
+LIBYUV_API
+int I444AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate);
-// Convert AR30 To ARGB.
+// Convert I400 (grey) to ARGB. Reverse of ARGBToI400.
LIBYUV_API
-int AR30ToARGB(const uint8_t* src_ar30,
- int src_stride_ar30,
+int I400ToARGB(const uint8_t* src_y,
+ int src_stride_y,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height);
-// Convert AR30 To ABGR.
+// Convert J400 (jpeg grey) to ARGB.
LIBYUV_API
-int AR30ToABGR(const uint8_t* src_ar30,
- int src_stride_ar30,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
+int J400ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height);
-// Convert AR30 To AB30.
+// Alias.
+#define YToARGB I400ToARGB
+
+// Convert NV12 to ARGB.
LIBYUV_API
-int AR30ToAB30(const uint8_t* src_ar30,
+int NV12ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert NV21 to ARGB.
+LIBYUV_API
+int NV21ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert NV12 to ABGR.
+LIBYUV_API
+int NV12ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert NV21 to ABGR.
+LIBYUV_API
+int NV21ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert NV12 to RGB24.
+LIBYUV_API
+int NV12ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+// Convert NV21 to RGB24.
+LIBYUV_API
+int NV21ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+// Convert NV21 to YUV24.
+LIBYUV_API
+int NV21ToYUV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_yuv24,
+ int dst_stride_yuv24,
+ int width,
+ int height);
+
+// Convert NV12 to RAW.
+LIBYUV_API
+int NV12ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+// Convert NV21 to RAW.
+LIBYUV_API
+int NV21ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+// Convert YUY2 to ARGB.
+LIBYUV_API
+int YUY2ToARGB(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert UYVY to ARGB.
+LIBYUV_API
+int UYVYToARGB(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert I010 to AR30.
+LIBYUV_API
+int I010ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert H010 to AR30.
+LIBYUV_API
+int H010ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert I010 to AB30.
+LIBYUV_API
+int I010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert H010 to AB30.
+LIBYUV_API
+int H010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert U010 to AR30.
+LIBYUV_API
+int U010ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert U010 to AB30.
+LIBYUV_API
+int U010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert I210 to AR30.
+LIBYUV_API
+int I210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert I210 to AB30.
+LIBYUV_API
+int I210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert H210 to AR30.
+LIBYUV_API
+int H210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert H210 to AB30.
+LIBYUV_API
+int H210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert U210 to AR30.
+LIBYUV_API
+int U210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert U210 to AB30.
+LIBYUV_API
+int U210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// BGRA little endian (argb in memory) to ARGB.
+LIBYUV_API
+int BGRAToARGB(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// ABGR little endian (rgba in memory) to ARGB.
+LIBYUV_API
+int ABGRToARGB(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// RGBA little endian (abgr in memory) to ARGB.
+LIBYUV_API
+int RGBAToARGB(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Deprecated function name.
+#define BG24ToARGB RGB24ToARGB
+
+// RGB little endian (bgr in memory) to ARGB.
+LIBYUV_API
+int RGB24ToARGB(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// RGB big endian (rgb in memory) to ARGB.
+LIBYUV_API
+int RAWToARGB(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// RGB big endian (rgb in memory) to RGBA.
+LIBYUV_API
+int RAWToRGBA(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height);
+
+// RGB16 (RGBP fourcc) little endian to ARGB. Deprecated.
+LIBYUV_API
+int RGB565ToARGB(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// RGB15 (RGBO fourcc) little endian to ARGB. Deprecated.
+LIBYUV_API
+int ARGB1555ToARGB(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// RGB12 (R444 fourcc) little endian to ARGB. Deprecated.
+LIBYUV_API
+int ARGB4444ToARGB(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Aliases
+#define AB30ToARGB AR30ToABGR
+#define AB30ToABGR AR30ToARGB
+#define AB30ToAR30 AR30ToAB30
+
+// Convert AR30 To ARGB.
+LIBYUV_API
+int AR30ToARGB(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert AR30 To ABGR.
+LIBYUV_API
+int AR30ToABGR(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert AR30 To AB30.
+LIBYUV_API
+int AR30ToAB30(const uint8_t* src_ar30,
int src_stride_ar30,
uint8_t* dst_ab30,
int dst_stride_ab30,
int width,
int height);
-#ifdef HAVE_JPEG
-// src_width/height provided by capture
-// dst_width/height for clipping determine final size.
+// Convert AR64 to ARGB.
+LIBYUV_API
+int AR64ToARGB(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert AB64 to ABGR.
+#define AB64ToABGR AR64ToARGB
+
+// Convert AB64 to ARGB.
+LIBYUV_API
+int AB64ToARGB(const uint16_t* src_ab64,
+ int src_stride_ab64,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert AR64 to ABGR.
+#define AR64ToABGR AB64ToARGB
+
+// Convert AR64 To AB64.
+LIBYUV_API
+int AR64ToAB64(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint16_t* dst_ab64,
+ int dst_stride_ab64,
+ int width,
+ int height);
+
+// Convert AB64 To AR64.
+#define AB64ToAR64 AR64ToAB64
+
+// src_width/height provided by capture
+// dst_width/height for clipping determine final size.
+LIBYUV_API
+int MJPGToARGB(const uint8_t* sample,
+ size_t sample_size,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height);
+
+// Convert Android420 to ARGB.
+LIBYUV_API
+int Android420ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Convert Android420 to ABGR.
+LIBYUV_API
+int Android420ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert NV12 to RGB565. Deprecated.
+LIBYUV_API
+int NV12ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height);
+
+// Convert I422 to BGRA.
+LIBYUV_API
+int I422ToBGRA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height);
+
+// Convert I422 to ABGR.
+LIBYUV_API
+int I422ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+// Convert I422 to RGBA.
+LIBYUV_API
+int I422ToRGBA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToBGRA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToRGBA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+LIBYUV_API
+int I420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+LIBYUV_API
+int H420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+LIBYUV_API
+int H420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+LIBYUV_API
+int J420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+LIBYUV_API
+int J420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+// Convert I422 to RGB24.
+LIBYUV_API
+int I422ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+// Convert I422 to RAW.
+LIBYUV_API
+int I422ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height);
+
+// Convert I420 to RGB565. Deprecated.
+LIBYUV_API
+int I420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height);
+
+// Convert J420 to RGB565. Deprecated.
+LIBYUV_API
+int J420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height);
+
+// Convert H420 to RGB565. Deprecated.
+LIBYUV_API
+int H420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height);
+
+// Convert I422 to RGB565. Deprecated.
+LIBYUV_API
+int I422ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height);
+
+// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). Deprecated.
+// Values in dither matrix from 0 to 7 recommended.
+// The order of the dither matrix is first byte is upper left.
+
+LIBYUV_API
+int I420ToRGB565Dither(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const uint8_t* dither4x4,
+ int width,
+ int height);
+
+// Convert I420 to ARGB1555. Deprecated.
+LIBYUV_API
+int I420ToARGB1555(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb1555,
+ int dst_stride_argb1555,
+ int width,
+ int height);
+
+// Convert I420 to ARGB4444. Deprecated.
+LIBYUV_API
+int I420ToARGB4444(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb4444,
+ int dst_stride_argb4444,
+ int width,
+ int height);
+
+// Convert I420 to AR30.
+LIBYUV_API
+int I420ToAR30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
+
+// Convert I420 to AB30.
+LIBYUV_API
+int I420ToAB30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert H420 to AR30.
LIBYUV_API
-int MJPGToARGB(const uint8_t* sample,
- size_t sample_size,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int src_width,
- int src_height,
- int dst_width,
- int dst_height);
-#endif
+int H420ToAR30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height);
-// Convert Android420 to ARGB.
+// Convert H420 to AB30.
LIBYUV_API
-int Android420ToARGB(const uint8_t* src_y,
+int H420ToAB30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height);
+
+// Convert I420 to ARGB with matrix.
+LIBYUV_API
+int I420ToARGBMatrix(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- int src_pixel_stride_uv,
uint8_t* dst_argb,
int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
int width,
int height);
-// Convert Android420 to ABGR.
+// Convert I422 to ARGB with matrix.
LIBYUV_API
-int Android420ToABGR(const uint8_t* src_y,
+int I422ToARGBMatrix(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- int src_pixel_stride_uv,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I444 to ARGB with matrix.
+LIBYUV_API
+int I444ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I444 to RGB24 with matrix.
+LIBYUV_API
+int I444ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit 420 YUV to ARGB with matrix.
+LIBYUV_API
+int I010ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit 420 YUV to ARGB with matrix.
+LIBYUV_API
+int I210ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit 444 YUV to ARGB with matrix.
+LIBYUV_API
+int I410ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit YUV to ARGB with matrix.
+LIBYUV_API
+int I010ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// multiply 12 bit yuv into high bits to allow any number of bits.
+LIBYUV_API
+int I012ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 12 bit YUV to ARGB with matrix.
+LIBYUV_API
+int I012ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit 422 YUV to ARGB with matrix.
+LIBYUV_API
+int I210ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert 10 bit 444 YUV to ARGB with matrix.
+LIBYUV_API
+int I410ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
int width,
int height);
+// Convert P010 to ARGB with matrix.
+LIBYUV_API
+int P010ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert P210 to ARGB with matrix.
+LIBYUV_API
+int P210ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert P410 to ARGB with matrix.
+LIBYUV_API
+int P410ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert P010 to AR30 with matrix.
+LIBYUV_API
+int P010ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert P210 to AR30 with matrix.
+LIBYUV_API
+int P210ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert P410 to AR30 with matrix.
+LIBYUV_API
+int P410ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// P012 and P010 use most significant bits so the conversion is the same.
+// Convert P012 to ARGB with matrix.
+#define P012ToARGBMatrix P010ToARGBMatrix
+// Convert P012 to AR30 with matrix.
+#define P012ToAR30Matrix P010ToAR30Matrix
+// Convert P212 to ARGB with matrix.
+#define P212ToARGBMatrix P210ToARGBMatrix
+// Convert P212 to AR30 with matrix.
+#define P212ToAR30Matrix P210ToAR30Matrix
+// Convert P412 to ARGB with matrix.
+#define P412ToARGBMatrix P410ToARGBMatrix
+// Convert P412 to AR30 with matrix.
+#define P412ToAR30Matrix P410ToAR30Matrix
+
+// Convert P016 to ARGB with matrix.
+#define P016ToARGBMatrix P010ToARGBMatrix
+// Convert P016 to AR30 with matrix.
+#define P016ToAR30Matrix P010ToAR30Matrix
+// Convert P216 to ARGB with matrix.
+#define P216ToARGBMatrix P210ToARGBMatrix
+// Convert P216 to AR30 with matrix.
+#define P216ToAR30Matrix P210ToAR30Matrix
+// Convert P416 to ARGB with matrix.
+#define P416ToARGBMatrix P410ToARGBMatrix
+// Convert P416 to AR30 with matrix.
+#define P416ToAR30Matrix P410ToAR30Matrix
+
+// Convert I420 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I420AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert I422 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I422AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert I444 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I444AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert I010 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I010AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert I210 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I210AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert I410 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I410AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate);
+
+// Convert NV12 to ARGB with matrix.
+LIBYUV_API
+int NV12ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert NV21 to ARGB with matrix.
+LIBYUV_API
+int NV21ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert NV12 to RGB565 with matrix. Deprecated.
+LIBYUV_API
+int NV12ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert NV12 to RGB24 with matrix.
+LIBYUV_API
+int NV12ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert NV21 to RGB24 with matrix.
+LIBYUV_API
+int NV21ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert YUY2 to ARGB with matrix.
+LIBYUV_API
+int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert UYVY to ARGB with matrix.
+LIBYUV_API
+int UYVYToARGBMatrix(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert Android420 to ARGB with matrix.
+LIBYUV_API
+int Android420ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I422 to RGBA with matrix.
+LIBYUV_API
+int I422ToRGBAMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I420 to RGBA with matrix.
+LIBYUV_API
+int I420ToRGBAMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I420 to RGB24 with matrix.
+LIBYUV_API
+int I420ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I422 to RGB24 with matrix.
+LIBYUV_API
+int I422ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I420 to RGB565 with specified color matrix. Deprecated.
+LIBYUV_API
+int I420ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I422 to RGB565 with specified color matrix. Deprecated.
+LIBYUV_API
+int I422ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I420 to AR30 with matrix.
+LIBYUV_API
+int I420ToAR30Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I400 (grey) to ARGB. Reverse of ARGBToI400.
+LIBYUV_API
+int I400ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert I420 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I420ToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I422 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I422ToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I422 to RGB24 with matrix and UV filter mode.
+LIBYUV_API
+int I422ToRGB24MatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I420 to RGB24 with matrix and UV filter mode.
+LIBYUV_API
+int I420ToRGB24MatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I010 to AR30 with matrix and UV filter mode.
+LIBYUV_API
+int I010ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I210 to AR30 with matrix and UV filter mode.
+LIBYUV_API
+int I210ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I010 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I010ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I210 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I210ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert I420 with Alpha to attenuated ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I420AlphaToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter);
+
+// Convert I422 with Alpha to attenuated ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I422AlphaToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter);
+
+// Convert I010 with Alpha to attenuated ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I010AlphaToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter);
+
+// Convert I210 with Alpha to attenuated ARGB with matrix and UV filter mode.
+LIBYUV_API
+int I210AlphaToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter);
+
+// Convert P010 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int P010ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert P210 to ARGB with matrix and UV filter mode.
+LIBYUV_API
+int P210ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert P010 to AR30 with matrix and UV filter mode.
+LIBYUV_API
+int P010ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
+// Convert P210 to AR30 with matrix and UV filter mode.
+LIBYUV_API
+int P210ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter);
+
// Convert camera sample to ARGB with cropping, rotation and vertical flip.
// "sample_size" is needed to parse MJPG.
// "dst_stride_argb" number of bytes in a row of the dst_argb plane.
diff --git a/libs/libyuv/include/libyuv/convert_from.h b/libs/libyuv/include/libyuv/convert_from.h
index 861418d07b..a746629afd 100644
--- a/libs/libyuv/include/libyuv/convert_from.h
+++ b/libs/libyuv/include/libyuv/convert_from.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_CONVERT_FROM_H_
#define INCLUDE_LIBYUV_CONVERT_FROM_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#include "libyuv/rotate.h"
@@ -23,6 +25,7 @@ extern "C" {
// Convert 8 bit YUV to 10 bit.
#define H420ToH010 I420ToI010
+LIBYUV_API
int I420ToI010(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -38,6 +41,24 @@ int I420ToI010(const uint8_t* src_y,
int width,
int height);
+// Convert 8 bit YUV to 12 bit.
+#define H420ToH012 I420ToI012
+LIBYUV_API
+int I420ToI012(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
LIBYUV_API
int I420ToI422(const uint8_t* src_y,
int src_stride_y,
@@ -107,238 +128,131 @@ int I420ToNV21(const uint8_t* src_y,
int width,
int height);
+// Convert I420 to P010.
LIBYUV_API
-int I420ToYUY2(const uint8_t* src_y,
+int I420ToP010(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_yuy2,
- int dst_stride_yuy2,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
+// Convert I422 to P210.
LIBYUV_API
-int I420ToUYVY(const uint8_t* src_y,
+int I422ToP210(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_uyvy,
- int dst_stride_uyvy,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
+// Convert I444 to P410.
LIBYUV_API
-int I420ToARGB(const uint8_t* src_y,
+int I444ToP410(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
+// Convert I420 to P210.
LIBYUV_API
-int I420ToBGRA(const uint8_t* src_y,
+int I420ToP210(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_bgra,
- int dst_stride_bgra,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
+// Convert I420 to P410.
LIBYUV_API
-int I420ToABGR(const uint8_t* src_y,
+int I420ToP410(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height);
LIBYUV_API
-int I420ToRGBA(const uint8_t* src_y,
+int I420ToYUY2(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
int width,
int height);
LIBYUV_API
-int I420ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height);
-
-LIBYUV_API
-int I420ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height);
-
-LIBYUV_API
-int H420ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height);
-
-LIBYUV_API
-int H420ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height);
-
-LIBYUV_API
-int I420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height);
-
-LIBYUV_API
-int J420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height);
-
-LIBYUV_API
-int H420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height);
-
-LIBYUV_API
-int I422ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height);
-
-// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes).
-// Values in dither matrix from 0 to 7 recommended.
-// The order of the dither matrix is first byte is upper left.
-
-LIBYUV_API
-int I420ToRGB565Dither(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- const uint8_t* dither4x4,
- int width,
- int height);
-
-LIBYUV_API
-int I420ToARGB1555(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb1555,
- int dst_stride_argb1555,
- int width,
- int height);
+int I420ToUYVY(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uyvy,
+ int dst_stride_uyvy,
+ int width,
+ int height);
-LIBYUV_API
-int I420ToARGB4444(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb4444,
- int dst_stride_argb4444,
- int width,
- int height);
+// The following are from convert_argb.h
+// DEPRECATED: The prototypes will be removed in future. Use convert_argb.h
-// Convert I420 to AR30.
+// Convert I420 to ARGB.
LIBYUV_API
-int I420ToAR30(const uint8_t* src_y,
+int I420ToARGB(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height);
-// Convert H420 to AR30.
+// Convert I420 to ABGR.
LIBYUV_API
-int H420ToAR30(const uint8_t* src_y,
+int I420ToABGR(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height);
diff --git a/libs/libyuv/include/libyuv/convert_from_argb.h b/libs/libyuv/include/libyuv/convert_from_argb.h
index 08c6aa5649..817df6f9ea 100644
--- a/libs/libyuv/include/libyuv/convert_from_argb.h
+++ b/libs/libyuv/include/libyuv/convert_from_argb.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_CONVERT_FROM_ARGB_H_
#define INCLUDE_LIBYUV_CONVERT_FROM_ARGB_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -18,6 +20,47 @@ namespace libyuv {
extern "C" {
#endif
+// The ArgbConstants below can be used with the ARGBTo*Matrix() functions to
+// process different RGB formats. E.g., if your input is ARGB little endian
+// (bgra in memory) you'll want to use the kArgb* constants. Alternatively, if
+// your input is ABGR little endian (rgba in memory) you'd use the kAbgr* ones.
+//
+// Conversion matrix for xRGB to YUV.
+LIBYUV_API extern const struct ArgbConstants kArgbI601Constants; // BT.601
+LIBYUV_API extern const struct ArgbConstants kArgbJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct ArgbConstants kArgbH709Constants; // BT.709
+LIBYUV_API extern const struct ArgbConstants kArgbF709Constants; // BT.709 full
+LIBYUV_API extern const struct ArgbConstants kArgbU2020Constants; // BT.2020
+LIBYUV_API extern const struct ArgbConstants
+ kArgbV2020Constants; // BT.2020 full
+
+// Conversion matrix for xBGR to YUV.
+LIBYUV_API extern const struct ArgbConstants kAbgrI601Constants; // BT.601
+LIBYUV_API extern const struct ArgbConstants kAbgrJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct ArgbConstants kAbgrH709Constants; // BT.709
+LIBYUV_API extern const struct ArgbConstants kAbgrF709Constants; // BT.709 full
+LIBYUV_API extern const struct ArgbConstants kAbgrU2020Constants; // BT.2020
+LIBYUV_API extern const struct ArgbConstants
+ kAbgrV2020Constants; // BT.2020 full
+
+// Conversion matrix for RGBx to YUV.
+LIBYUV_API extern const struct ArgbConstants kRgbaI601Constants; // BT.601
+LIBYUV_API extern const struct ArgbConstants kRgbaJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct ArgbConstants kRgbaH709Constants; // BT.709
+LIBYUV_API extern const struct ArgbConstants kRgbaF709Constants; // BT.709 full
+LIBYUV_API extern const struct ArgbConstants kRgbaU2020Constants; // BT.2020
+LIBYUV_API extern const struct ArgbConstants
+ kRgbaV2020Constants; // BT.2020 full
+
+// Conversion matrix from BGRx to YUV.
+LIBYUV_API extern const struct ArgbConstants kBgraI601Constants; // BT.601
+LIBYUV_API extern const struct ArgbConstants kBgraJPEGConstants; // BT.601 full
+LIBYUV_API extern const struct ArgbConstants kBgraH709Constants; // BT.709
+LIBYUV_API extern const struct ArgbConstants kBgraF709Constants; // BT.709 full
+LIBYUV_API extern const struct ArgbConstants kBgraU2020Constants; // BT.2020
+LIBYUV_API extern const struct ArgbConstants
+ kBgraV2020Constants; // BT.2020 full
+
// Copy ARGB to ARGB.
#define ARGBToARGB ARGBCopy
LIBYUV_API
@@ -77,6 +120,10 @@ int ARGBToAR30(const uint8_t* src_argb,
int width,
int height);
+// Aliases
+#define ABGRToRGB24 ARGBToRAW
+#define ABGRToRAW ARGBToRGB24
+
// Convert ARGB To RGB24.
LIBYUV_API
int ARGBToRGB24(const uint8_t* src_argb,
@@ -95,7 +142,7 @@ int ARGBToRAW(const uint8_t* src_argb,
int width,
int height);
-// Convert ARGB To RGB565.
+// Convert ARGB To RGB565. Deprecated.
LIBYUV_API
int ARGBToRGB565(const uint8_t* src_argb,
int src_stride_argb,
@@ -104,7 +151,7 @@ int ARGBToRGB565(const uint8_t* src_argb,
int width,
int height);
-// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes).
+// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). Deprecated.
// Values in dither matrix from 0 to 7 recommended.
// The order of the dither matrix is first byte is upper left.
// TODO(fbarchard): Consider pointer to 2d array for dither4x4.
@@ -118,7 +165,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb,
int width,
int height);
-// Convert ARGB To ARGB1555.
+// Convert ARGB To ARGB1555. Deprecated.
LIBYUV_API
int ARGBToARGB1555(const uint8_t* src_argb,
int src_stride_argb,
@@ -127,7 +174,7 @@ int ARGBToARGB1555(const uint8_t* src_argb,
int width,
int height);
-// Convert ARGB To ARGB4444.
+// Convert ARGB To ARGB4444. Deprecated.
LIBYUV_API
int ARGBToARGB4444(const uint8_t* src_argb,
int src_stride_argb,
@@ -149,6 +196,45 @@ int ARGBToI444(const uint8_t* src_argb,
int width,
int height);
+// ARGB to I444 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToI444Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB to AR64.
+LIBYUV_API
+int ARGBToAR64(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height);
+
+// Convert ABGR to AB64.
+#define ABGRToAB64 ARGBToAR64
+
+// Convert ARGB to AB64.
+LIBYUV_API
+int ARGBToAB64(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_ab64,
+ int dst_stride_ab64,
+ int width,
+ int height);
+
+// Convert ABGR to AR64.
+#define ABGRToAR64 ARGBToAB64
+
// Convert ARGB To I422.
LIBYUV_API
int ARGBToI422(const uint8_t* src_argb,
@@ -162,6 +248,34 @@ int ARGBToI422(const uint8_t* src_argb,
int width,
int height);
+// Convert ABGR To I422.
+LIBYUV_API
+int ABGRToI422(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// ARGB to I422 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToI422Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
// Convert ARGB To I420. (also in convert.h)
LIBYUV_API
int ARGBToI420(const uint8_t* src_argb,
@@ -181,10 +295,10 @@ int ARGBToJ420(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_yj,
int dst_stride_yj,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
int width,
int height);
@@ -194,10 +308,23 @@ int ARGBToJ422(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_yj,
int dst_stride_yj,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
+ int width,
+ int height);
+
+// Convert ARGB to J444.
+LIBYUV_API
+int ARGBToJ444(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
int width,
int height);
@@ -210,6 +337,50 @@ int ARGBToJ400(const uint8_t* src_argb,
int width,
int height);
+// Convert ABGR to J420. (JPeg full range I420).
+LIBYUV_API
+int ABGRToJ420(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
+ int width,
+ int height);
+
+// Convert ABGR to J422.
+LIBYUV_API
+int ABGRToJ422(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
+ int width,
+ int height);
+
+// Convert ABGR to J400. (JPeg full range).
+LIBYUV_API
+int ABGRToJ400(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height);
+
+// Convert RGBA to J400. (JPeg full range).
+LIBYUV_API
+int RGBAToJ400(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height);
+
// Convert ARGB to I400.
LIBYUV_API
int ARGBToI400(const uint8_t* src_argb,
@@ -219,15 +390,6 @@ int ARGBToI400(const uint8_t* src_argb,
int width,
int height);
-// Convert ARGB to G. (Reverse of J400toARGB, which replicates G back to ARGB)
-LIBYUV_API
-int ARGBToG(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_g,
- int dst_stride_g,
- int width,
- int height);
-
// Convert ARGB To NV12.
LIBYUV_API
int ARGBToNV12(const uint8_t* src_argb,
@@ -239,6 +401,80 @@ int ARGBToNV12(const uint8_t* src_argb,
int width,
int height);
+// ARGB to NV12 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToNV12Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB To NV16.
+LIBYUV_API
+int ARGBToNV16(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// ARGB to NV16 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToNV16Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB To NV24.
+LIBYUV_API
+int ARGBToNV24(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// ARGB to NV24 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToNV24Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// ARGB to NV21 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToNV21Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
// Convert ARGB To NV21.
LIBYUV_API
int ARGBToNV21(const uint8_t* src_argb,
@@ -250,6 +486,78 @@ int ARGBToNV21(const uint8_t* src_argb,
int width,
int height);
+// Convert ARGB To P010.
+LIBYUV_API
+int ARGBToP010(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// ARGB to P010 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToP010Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB To P210.
+LIBYUV_API
+int ARGBToP210(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// ARGB to P210 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToP210Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
+// Convert ARGB To P410.
+LIBYUV_API
+int ARGBToP410(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// ARGB to P410 with matrix. See ArgbConstants at the top of this file for
+// usage.
+LIBYUV_API
+int ARGBToP410Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height);
+
// Convert ABGR To NV12.
LIBYUV_API
int ABGRToNV12(const uint8_t* src_abgr,
@@ -272,17 +580,6 @@ int ABGRToNV21(const uint8_t* src_abgr,
int width,
int height);
-// Convert ARGB To NV21.
-LIBYUV_API
-int ARGBToNV21(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height);
-
// Convert ARGB To YUY2.
LIBYUV_API
int ARGBToYUY2(const uint8_t* src_argb,
@@ -301,6 +598,39 @@ int ARGBToUYVY(const uint8_t* src_argb,
int width,
int height);
+// RAW to NV21
+LIBYUV_API
+int RAWToNV21(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
+// RGB24 to NV12
+LIBYUV_API
+int RGB24ToNV12(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// RAW to JNV21 full range NV21
+LIBYUV_API
+int RAWToJNV21(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/include/libyuv/cpu_id.h b/libs/libyuv/include/libyuv/cpu_id.h
index b01cd25c57..54512b766d 100644
--- a/libs/libyuv/include/libyuv/cpu_id.h
+++ b/libs/libyuv/include/libyuv/cpu_id.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_CPU_ID_H_
#define INCLUDE_LIBYUV_CPU_ID_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -21,34 +23,60 @@ extern "C" {
// Internal flag to indicate cpuid requires initialization.
static const int kCpuInitialized = 0x1;
-// These flags are only valid on ARM processors.
+// These flags are only valid on Arm processors.
static const int kCpuHasARM = 0x2;
-static const int kCpuHasNEON = 0x4;
-// 0x8 reserved for future ARM flag.
+static const int kCpuHasNEON = 0x100;
+static const int kCpuHasNeonDotProd = 0x200;
+static const int kCpuHasNeonI8MM = 0x400;
+static const int kCpuHasSVE = 0x800;
+static const int kCpuHasSVE2 = 0x1000;
+static const int kCpuHasSME = 0x2000;
+static const int kCpuHasSME2 = 0x4000;
+static const int kCpuHasSVEF32MM = 0x8000;
+
+// These flags are only valid on RISCV processors.
+static const int kCpuHasRISCV = 0x4;
+static const int kCpuHasRVV = 0x100;
+static const int kCpuHasRVVZVFH = 0x200;
// These flags are only valid on x86 processors.
-static const int kCpuHasX86 = 0x10;
-static const int kCpuHasSSE2 = 0x20;
-static const int kCpuHasSSSE3 = 0x40;
-static const int kCpuHasSSE41 = 0x80;
-static const int kCpuHasSSE42 = 0x100; // unused at this time.
-static const int kCpuHasAVX = 0x200;
-static const int kCpuHasAVX2 = 0x400;
-static const int kCpuHasERMS = 0x800;
-static const int kCpuHasFMA3 = 0x1000;
-static const int kCpuHasF16C = 0x2000;
-static const int kCpuHasGFNI = 0x4000;
-static const int kCpuHasAVX512BW = 0x8000;
-static const int kCpuHasAVX512VL = 0x10000;
-static const int kCpuHasAVX512VBMI = 0x20000;
-static const int kCpuHasAVX512VBMI2 = 0x40000;
-static const int kCpuHasAVX512VBITALG = 0x80000;
-static const int kCpuHasAVX512VPOPCNTDQ = 0x100000;
-
-// These flags are only valid on MIPS processors.
-static const int kCpuHasMIPS = 0x200000;
-static const int kCpuHasMSA = 0x400000;
-static const int kCpuHasMMI = 0x800000;
+static const int kCpuHasX86 = 0x8;
+static const int kCpuHasSSE2 = 0x100;
+static const int kCpuHasSSSE3 = 0x200;
+static const int kCpuHasSSE41 = 0x400;
+static const int kCpuHasSSE42 = 0x800;
+static const int kCpuHasAVX = 0x1000;
+static const int kCpuHasAVX2 = 0x2000;
+static const int kCpuHasERMS = 0x4000;
+static const int kCpuHasFSMR = 0x8000;
+static const int kCpuHasFMA3 = 0x10000;
+static const int kCpuHasF16C = 0x20000;
+static const int kCpuHasAVX512BW = 0x40000;
+static const int kCpuHasAVX512VL = 0x80000;
+static const int kCpuHasAVX512VNNI = 0x100000;
+static const int kCpuHasAVX512VBMI = 0x200000;
+static const int kCpuHasAVX512VBMI2 = 0x400000;
+static const int kCpuHasAVX512VBITALG = 0x800000;
+static const int kCpuHasAVX10 = 0x1000000;
+static const int kCpuHasAVX10_2 = 0x2000000;
+static const int kCpuHasAVXVNNI = 0x4000000;
+static const int kCpuHasAVXVNNIINT8 = 0x8000000;
+static const int kCpuHasAMXINT8 = 0x10000000;
+static const int kCpuHasAVX512BMM = 0x20000000;
+static const int kCpuHasAMXTF32 = 0x40000000;
+
+// These flags are only valid on LOONGARCH processors.
+static const int kCpuHasLOONGARCH = 0x20;
+static const int kCpuHasLSX = 0x100;
+static const int kCpuHasLASX = 0x200;
+
+// These flags are only valid on WASM processors.
+static const int kCpuHasWASM = 0x40;
+static const int kCpuHasWASMSIMD = 0x100;
+
+// These flags are only valid on Hexagon processors.
+static const int kCpuHasHexagon = 0x80;
+static const int kCpuHasHVX = 0x100;
// Optional init function. TestCpuFlag does an auto-init.
// Returns cpu_info flags.
@@ -71,6 +99,17 @@ static __inline int TestCpuFlag(int test_flag) {
// Internal function for parsing /proc/cpuinfo.
LIBYUV_API
int ArmCpuCaps(const char* cpuinfo_name);
+LIBYUV_API
+int RiscvCpuCaps(const char* cpuinfo_name);
+
+#ifdef __linux__
+// On Linux, parse AArch64 features from getauxval(AT_HWCAP{,2}).
+LIBYUV_API
+int AArch64CpuCaps(unsigned long hwcap, unsigned long hwcap2);
+#else
+LIBYUV_API
+int AArch64CpuCaps();
+#endif
// For testing, allow CPU flags to be disabled.
// ie MaskCpuFlags(~kCpuHasSSSE3) to disable SSSE3.
diff --git a/libs/libyuv/include/libyuv/cpu_support.h b/libs/libyuv/include/libyuv/cpu_support.h
new file mode 100644
index 0000000000..d007f3d011
--- /dev/null
+++ b/libs/libyuv/include/libyuv/cpu_support.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_CPU_SUPPORT_H_
+#define INCLUDE_LIBYUV_CPU_SUPPORT_H_
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if defined(__pnacl__) || defined(__CLR_VER) || \
+ (defined(__native_client__) && defined(__x86_64__)) || \
+ (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
+#define LIBYUV_DISABLE_X86
+#endif
+
+#if defined(__native_client__)
+#define LIBYUV_DISABLE_NEON
+#endif
+
+// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#if !defined(LIBYUV_MSAN)
+#define LIBYUV_MSAN 1
+#endif
+#if !defined(LIBYUV_DISABLE_NEON)
+#define LIBYUV_DISABLE_NEON
+#endif
+#if !defined(LIBYUV_DISABLE_SME)
+#define LIBYUV_DISABLE_SME
+#endif
+#if !defined(LIBYUV_DISABLE_SVE)
+#define LIBYUV_DISABLE_SVE
+#endif
+#if !defined(LIBYUV_ENABLE_ROWWIN)
+#define LIBYUV_ENABLE_ROWWIN
+#endif
+#endif // __has_feature(memory_sanitizer)
+#endif // defined(__has_feature)
+
+// clang >= 3.5.0 required for Arm64.
+#if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON)
+#if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5))
+#define LIBYUV_DISABLE_NEON
+#endif // clang >= 3.5
+#endif // __clang__
+
+// GCC >= 4.7.0 required for AVX2.
+#if defined(__GNUC__) && !defined(__clang__) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
+#define GCC_HAS_AVX2 1
+#endif // GNUC >= 4.7
+#endif // __GNUC__
+
+// GCC >= 9.0.0 required for AVX512 (including VNNI and VBMI).
+#if defined(__GNUC__) && !defined(__clang__) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && defined(__x86_64__)
+#if (__GNUC__ >= 9)
+#define GCC_HAS_AVX512 1
+#endif // GNUC >= 9
+#endif // __GNUC__
+
+// Function target attributes so GCC enables zmm16..zmm31 and k1..k7 clobbers
+// inside AVX512 functions without emitting AVX512 in non-AVX512 functions.
+#if defined(GCC_HAS_AVX512)
+#define LIBYUV_TARGET_AVX512BW \
+ __attribute__((target("avx512bw,avx512vl,avx512f,bmi2")))
+#define LIBYUV_TARGET_AVX512VBMI \
+ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f,bmi2")))
+#define LIBYUV_TARGET_AVX512VNNI \
+ __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f,bmi2")))
+#define LIBYUV_TARGET_AVX512VNNIVBMI \
+ __attribute__(( \
+ target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f,bmi2")))
+#elif !defined(LIBYUV_ENABLE_ROWWIN) && \
+ !(defined(_MSC_VER) && !defined(__clang__))
+#define LIBYUV_TARGET_AVX512BW
+#define LIBYUV_TARGET_AVX512VBMI
+#define LIBYUV_TARGET_AVX512VNNI
+#define LIBYUV_TARGET_AVX512VNNIVBMI
+#endif
+
+// clang >= 3.4.0 required for AVX2.
+#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
+#define CLANG_HAS_AVX2 1
+#endif // clang >= 3.4
+#endif // __clang__
+
+// clang >= 7.0.0 required for AVX512.
+#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+// clang in xcode follows a different versioning scheme.
+// TODO(fbarchard): fix xcode 9 ios b/789.
+#if (__clang_major__ >= 7) && !defined(__APPLE__)
+#define CLANG_HAS_AVX512 1
+#endif // clang >= 7
+#endif // __clang__
+
+// Visual C 2012 (or clang >= 3.4 with LIBYUV_ENABLE_ROWWIN) required for AVX2.
+#if ((defined(_M_IX86) || defined(_M_X64)) && !defined(__clang__) && \
+ defined(_MSC_VER) && _MSC_VER >= 1700) || \
+ (defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((__clang_major__ > 3) || \
+ (__clang_major__ == 3 && (__clang_minor__ >= 4))))
+#define VISUALC_HAS_AVX2 1
+#endif // VisualStudio >= 2012
+
+// Visual C 2019 (or clang >= 7 with LIBYUV_ENABLE_ROWWIN) required for AVX512.
+#if (defined(_M_X64) || defined(__x86_64__)) && \
+ ((!defined(__clang__) && defined(_MSC_VER) && _MSC_VER >= 1920) || \
+ (defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \
+ (__clang_major__ >= 7) && !defined(__APPLE__)))
+#define VISUALC_HAS_AVX512 1
+#endif // VisualStudio >= 2019
+
+#if defined(LIBYUV_ENABLE_SME)
+#undef LIBYUV_DISABLE_SME
+#endif
+
+// Clang 18+ required for SME due to needing __arm_tpidr2_save from compiler-rt,
+// only enabled on Linux and Android (both define __linux__) for now.
+#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \
+ defined(__linux__) && defined(__clang__) && (__clang_major__ >= 18)
+#define CLANG_HAS_SME 1
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_CPU_SUPPORT_H_
diff --git a/libs/libyuv/include/libyuv/loongson_intrinsics.h b/libs/libyuv/include/libyuv/loongson_intrinsics.h
new file mode 100644
index 0000000000..7dd44c5759
--- /dev/null
+++ b/libs/libyuv/include/libyuv/loongson_intrinsics.h
@@ -0,0 +1,14 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_
+#define INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_
+
+#endif // INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_
diff --git a/libs/libyuv/include/libyuv/macros_msa.h b/libs/libyuv/include/libyuv/macros_msa.h
deleted file mode 100644
index 29997ce11f..0000000000
--- a/libs/libyuv/include/libyuv/macros_msa.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Copyright 2016 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef INCLUDE_LIBYUV_MACROS_MSA_H_
-#define INCLUDE_LIBYUV_MACROS_MSA_H_
-
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#include <msa.h>
-#include <stdint.h>
-
-#if (__mips_isa_rev >= 6)
-#define LW(psrc) \
- ({ \
- const uint8_t* psrc_lw_m = (const uint8_t*)(psrc); \
- uint32_t val_m; \
- asm volatile("lw %[val_m], %[psrc_lw_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_lw_m] "m"(*psrc_lw_m)); \
- val_m; \
- })
-
-#if (__mips == 64)
-#define LD(psrc) \
- ({ \
- const uint8_t* psrc_ld_m = (const uint8_t*)(psrc); \
- uint64_t val_m = 0; \
- asm volatile("ld %[val_m], %[psrc_ld_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_ld_m] "m"(*psrc_ld_m)); \
- val_m; \
- })
-#else // !(__mips == 64)
-#define LD(psrc) \
- ({ \
- const uint8_t* psrc_ld_m = (const uint8_t*)(psrc); \
- uint32_t val0_m, val1_m; \
- uint64_t val_m = 0; \
- val0_m = LW(psrc_ld_m); \
- val1_m = LW(psrc_ld_m + 4); \
- val_m = (uint64_t)(val1_m); /* NOLINT */ \
- val_m = (uint64_t)((val_m << 32) & 0xFFFFFFFF00000000); /* NOLINT */ \
- val_m = (uint64_t)(val_m | (uint64_t)val0_m); /* NOLINT */ \
- val_m; \
- })
-#endif // (__mips == 64)
-
-#define SW(val, pdst) \
- ({ \
- uint8_t* pdst_sw_m = (uint8_t*)(pdst); /* NOLINT */ \
- uint32_t val_m = (val); \
- asm volatile("sw %[val_m], %[pdst_sw_m] \n" \
- : [pdst_sw_m] "=m"(*pdst_sw_m) \
- : [val_m] "r"(val_m)); \
- })
-
-#if (__mips == 64)
-#define SD(val, pdst) \
- ({ \
- uint8_t* pdst_sd_m = (uint8_t*)(pdst); /* NOLINT */ \
- uint64_t val_m = (val); \
- asm volatile("sd %[val_m], %[pdst_sd_m] \n" \
- : [pdst_sd_m] "=m"(*pdst_sd_m) \
- : [val_m] "r"(val_m)); \
- })
-#else // !(__mips == 64)
-#define SD(val, pdst) \
- ({ \
- uint8_t* pdst_sd_m = (uint8_t*)(pdst); /* NOLINT */ \
- uint32_t val0_m, val1_m; \
- val0_m = (uint32_t)((val)&0x00000000FFFFFFFF); \
- val1_m = (uint32_t)(((val) >> 32) & 0x00000000FFFFFFFF); \
- SW(val0_m, pdst_sd_m); \
- SW(val1_m, pdst_sd_m + 4); \
- })
-#endif // !(__mips == 64)
-#else // !(__mips_isa_rev >= 6)
-#define LW(psrc) \
- ({ \
- const uint8_t* psrc_lw_m = (const uint8_t*)(psrc); \
- uint32_t val_m; \
- asm volatile("ulw %[val_m], %[psrc_lw_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_lw_m] "m"(*psrc_lw_m)); \
- val_m; \
- })
-
-#if (__mips == 64)
-#define LD(psrc) \
- ({ \
- const uint8_t* psrc_ld_m = (const uint8_t*)(psrc); \
- uint64_t val_m = 0; \
- asm volatile("uld %[val_m], %[psrc_ld_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_ld_m] "m"(*psrc_ld_m)); \
- val_m; \
- })
-#else // !(__mips == 64)
-#define LD(psrc) \
- ({ \
- const uint8_t* psrc_ld_m = (const uint8_t*)(psrc); \
- uint32_t val0_m, val1_m; \
- uint64_t val_m = 0; \
- val0_m = LW(psrc_ld_m); \
- val1_m = LW(psrc_ld_m + 4); \
- val_m = (uint64_t)(val1_m); /* NOLINT */ \
- val_m = (uint64_t)((val_m << 32) & 0xFFFFFFFF00000000); /* NOLINT */ \
- val_m = (uint64_t)(val_m | (uint64_t)val0_m); /* NOLINT */ \
- val_m; \
- })
-#endif // (__mips == 64)
-
-#define SW(val, pdst) \
- ({ \
- uint8_t* pdst_sw_m = (uint8_t*)(pdst); /* NOLINT */ \
- uint32_t val_m = (val); \
- asm volatile("usw %[val_m], %[pdst_sw_m] \n" \
- : [pdst_sw_m] "=m"(*pdst_sw_m) \
- : [val_m] "r"(val_m)); \
- })
-
-#define SD(val, pdst) \
- ({ \
- uint8_t* pdst_sd_m = (uint8_t*)(pdst); /* NOLINT */ \
- uint32_t val0_m, val1_m; \
- val0_m = (uint32_t)((val)&0x00000000FFFFFFFF); \
- val1_m = (uint32_t)(((val) >> 32) & 0x00000000FFFFFFFF); \
- SW(val0_m, pdst_sd_m); \
- SW(val1_m, pdst_sd_m + 4); \
- })
-#endif // (__mips_isa_rev >= 6)
-
-// TODO(fbarchard): Consider removing __VAR_ARGS versions.
-#define LD_B(RTYPE, psrc) *((RTYPE*)(psrc)) /* NOLINT */
-#define LD_UB(...) LD_B(const v16u8, __VA_ARGS__)
-
-#define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = (in) /* NOLINT */
-#define ST_UB(...) ST_B(v16u8, __VA_ARGS__)
-
-#define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = (in) /* NOLINT */
-#define ST_UH(...) ST_H(v8u16, __VA_ARGS__)
-
-/* Description : Load two vectors with 16 'byte' sized elements
- Arguments : Inputs - psrc, stride
- Outputs - out0, out1
- Return Type - as per RTYPE
- Details : Load 16 byte elements in 'out0' from (psrc)
- Load 16 byte elements in 'out1' from (psrc + stride)
-*/
-#define LD_B2(RTYPE, psrc, stride, out0, out1) \
- { \
- out0 = LD_B(RTYPE, (psrc)); \
- out1 = LD_B(RTYPE, (psrc) + stride); \
- }
-#define LD_UB2(...) LD_B2(const v16u8, __VA_ARGS__)
-
-#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) \
- { \
- LD_B2(RTYPE, (psrc), stride, out0, out1); \
- LD_B2(RTYPE, (psrc) + 2 * stride, stride, out2, out3); \
- }
-#define LD_UB4(...) LD_B4(const v16u8, __VA_ARGS__)
-
-/* Description : Store two vectors with stride each having 16 'byte' sized
- elements
- Arguments : Inputs - in0, in1, pdst, stride
- Details : Store 16 byte elements from 'in0' to (pdst)
- Store 16 byte elements from 'in1' to (pdst + stride)
-*/
-#define ST_B2(RTYPE, in0, in1, pdst, stride) \
- { \
- ST_B(RTYPE, in0, (pdst)); \
- ST_B(RTYPE, in1, (pdst) + stride); \
- }
-#define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__)
-
-#define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) \
- { \
- ST_B2(RTYPE, in0, in1, (pdst), stride); \
- ST_B2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \
- }
-#define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__)
-
-/* Description : Store vectors of 8 halfword elements with stride
- Arguments : Inputs - in0, in1, pdst, stride
- Details : Store 8 halfword elements from 'in0' to (pdst)
- Store 8 halfword elements from 'in1' to (pdst + stride)
-*/
-#define ST_H2(RTYPE, in0, in1, pdst, stride) \
- { \
- ST_H(RTYPE, in0, (pdst)); \
- ST_H(RTYPE, in1, (pdst) + stride); \
- }
-#define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__)
-
-// TODO(fbarchard): Consider using __msa_vshf_b and __msa_ilvr_b directly.
-/* Description : Shuffle byte vector elements as per mask vector
- Arguments : Inputs - in0, in1, in2, in3, mask0, mask1
- Outputs - out0, out1
- Return Type - as per RTYPE
- Details : Byte elements from 'in0' & 'in1' are copied selectively to
- 'out0' as per control vector 'mask0'
-*/
-#define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \
- { \
- out0 = (RTYPE)__msa_vshf_b((v16i8)mask0, (v16i8)in1, (v16i8)in0); \
- out1 = (RTYPE)__msa_vshf_b((v16i8)mask1, (v16i8)in3, (v16i8)in2); \
- }
-#define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__)
-
-/* Description : Interleave both left and right half of input vectors
- Arguments : Inputs - in0, in1
- Outputs - out0, out1
- Return Type - as per RTYPE
- Details : Right half of byte elements from 'in0' and 'in1' are
- interleaved and written to 'out0'
-*/
-#define ILVRL_B2(RTYPE, in0, in1, out0, out1) \
- { \
- out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
- out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \
- }
-#define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__)
-
-#endif /* !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa) */
-
-#endif // INCLUDE_LIBYUV_MACROS_MSA_H_
diff --git a/libs/libyuv/include/libyuv/mjpeg_decoder.h b/libs/libyuv/include/libyuv/mjpeg_decoder.h
index 275f8d4c18..2dc60a2751 100644
--- a/libs/libyuv/include/libyuv/mjpeg_decoder.h
+++ b/libs/libyuv/include/libyuv/mjpeg_decoder.h
@@ -11,6 +11,9 @@
#ifndef INCLUDE_LIBYUV_MJPEG_DECODER_H_
#define INCLUDE_LIBYUV_MJPEG_DECODER_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -26,7 +29,7 @@ namespace libyuv {
extern "C" {
#endif
-LIBYUV_BOOL ValidateJpeg(const uint8_t* sample, size_t sample_size);
+int ValidateJpeg(const uint8_t* sample, size_t sample_size);
#ifdef __cplusplus
} // extern "C"
@@ -81,11 +84,11 @@ class LIBYUV_API MJpegDecoder {
// Loads a new frame, reads its headers, and determines the uncompressed
// image format.
- // Returns LIBYUV_TRUE if image looks valid and format is supported.
- // If return value is LIBYUV_TRUE, then the values for all the following
+ // Returns true if image looks valid and format is supported.
+ // If return value is true, then the values for all the following
// getters are populated.
// src_len is the size of the compressed mjpeg frame in bytes.
- LIBYUV_BOOL LoadFrame(const uint8_t* src, size_t src_len);
+ bool LoadFrame(const uint8_t* src, size_t src_len);
// Returns width of the last loaded frame in pixels.
int GetWidth();
@@ -129,7 +132,7 @@ class LIBYUV_API MJpegDecoder {
// Call this after LoadFrame() if you decide you don't want to decode it
// after all.
- LIBYUV_BOOL UnloadFrame();
+ bool UnloadFrame();
// Decodes the entire image into a one-buffer-per-color-component format.
// dst_width must match exactly. dst_height must be <= to image height; if
@@ -138,16 +141,16 @@ class LIBYUV_API MJpegDecoder {
// at least GetComponentSize(i). The pointers in planes are incremented
// to point to after the end of the written data.
// TODO(fbarchard): Add dst_x, dst_y to allow specific rect to be decoded.
- LIBYUV_BOOL DecodeToBuffers(uint8_t** planes, int dst_width, int dst_height);
+ bool DecodeToBuffers(uint8_t** planes, int dst_width, int dst_height);
// Decodes the entire image and passes the data via repeated calls to a
// callback function. Each call will get the data for a whole number of
// image scanlines.
// TODO(fbarchard): Add dst_x, dst_y to allow specific rect to be decoded.
- LIBYUV_BOOL DecodeToCallback(CallbackFunction fn,
- void* opaque,
- int dst_width,
- int dst_height);
+ bool DecodeToCallback(CallbackFunction fn,
+ void* opaque,
+ int dst_width,
+ int dst_height);
// The helper function which recognizes the jpeg sub-sampling type.
static JpegSubsamplingType JpegSubsamplingTypeHelper(
@@ -159,13 +162,11 @@ class LIBYUV_API MJpegDecoder {
void AllocOutputBuffers(int num_outbufs);
void DestroyOutputBuffers();
- LIBYUV_BOOL StartDecode();
- LIBYUV_BOOL FinishDecode();
+ bool StartDecode();
+ bool FinishDecode();
void SetScanlinePointers(uint8_t** data);
- LIBYUV_BOOL DecodeImcuRow();
-
- int GetComponentScanlinePadding(int component);
+ bool DecodeImcuRow();
// A buffer holding the input data for a frame.
Buffer buf_;
@@ -175,9 +176,9 @@ class LIBYUV_API MJpegDecoder {
jpeg_source_mgr* source_mgr_;
SetJmpErrorMgr* error_mgr_;
- // LIBYUV_TRUE iff at least one component has scanline padding. (i.e.,
+ // true iff at least one component has scanline padding. (i.e.,
// GetComponentScanlinePadding() != 0.)
- LIBYUV_BOOL has_scanline_padding_;
+ bool has_scanline_padding_;
// Temporaries used to point to scanline outputs.
int num_outbufs_; // Outermost size of all arrays below.
diff --git a/libs/libyuv/include/libyuv/planar_functions.h b/libs/libyuv/include/libyuv/planar_functions.h
index 50fe5f681d..6306cd75e3 100644
--- a/libs/libyuv/include/libyuv/planar_functions.h
+++ b/libs/libyuv/include/libyuv/planar_functions.h
@@ -11,6 +11,9 @@
#ifndef INCLUDE_LIBYUV_PLANAR_FUNCTIONS_H_
#define INCLUDE_LIBYUV_PLANAR_FUNCTIONS_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
// TODO(fbarchard): Remove the following headers includes.
@@ -22,24 +25,6 @@ namespace libyuv {
extern "C" {
#endif
-// TODO(fbarchard): Move cpu macros to row.h
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
-// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
-#define HAS_ARGBAFFINEROW_SSE2
-#endif
-
// Copy a plane of data.
LIBYUV_API
void CopyPlane(const uint8_t* src_y,
@@ -66,14 +51,28 @@ void Convert16To8Plane(const uint16_t* src_y,
int width,
int height);
-LIBYUV_API
-void Convert8To16Plane(const uint8_t* src_y,
- int src_stride_y,
- uint16_t* dst_y,
- int dst_stride_y,
- int scale, // 1024 for 10 bits
- int width,
- int height);
+// Convert 8 bit Y to 10/12/16 bit.
+// "bits" accepts bit depths (10, 12, 16) or legacy scale values (1024, 4096,
+// 65536).
+LIBYUV_API
+void Convert8To16Plane(
+ const uint8_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale)
+ int width,
+ int height);
+
+LIBYUV_API
+void Convert8To8Plane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int scale, // 220 for Y, 225 for U,V
+ int bias, // 16
+ int width,
+ int height);
// Set a plane of data to a 32 bit value.
LIBYUV_API
@@ -83,6 +82,50 @@ void SetPlane(uint8_t* dst_y,
int height,
uint32_t value);
+// Convert a plane of tiles of 16 x H to linear.
+LIBYUV_API
+int DetilePlane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int tile_height);
+
+// Convert a plane of 16 bit tiles of 16 x H to linear.
+LIBYUV_API
+int DetilePlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int tile_height);
+
+// Convert a UV plane of tiles of 16 x H into linear U and V planes.
+LIBYUV_API
+void DetileSplitUVPlane(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int tile_height);
+
+// Convert a Y and UV plane of tiles into interlaced YUY2.
+LIBYUV_API
+void DetileToYUY2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ int width,
+ int height,
+ int tile_height);
+
// Split interleaved UV plane into separate U and V planes.
LIBYUV_API
void SplitUVPlane(const uint8_t* src_uv,
@@ -105,6 +148,63 @@ void MergeUVPlane(const uint8_t* src_u,
int width,
int height);
+// Split interleaved msb UV plane into separate lsb U and V planes.
+LIBYUV_API
+void SplitUVPlane_16(const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int depth);
+
+// Merge separate lsb U and V planes into one interleaved msb UV plane.
+LIBYUV_API
+void MergeUVPlane_16(const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int depth);
+
+// Convert lsb plane to msb plane
+LIBYUV_API
+void ConvertToMSBPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int depth);
+
+// Convert msb plane to lsb plane
+LIBYUV_API
+void ConvertToLSBPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int depth);
+
+// Scale U and V to half width and height and merge into interleaved UV plane.
+// width and height are source size, allowing odd sizes.
+// Use for converting I444 or I422 to NV12.
+LIBYUV_API
+void HalfMergeUVPlane(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
// Swap U and V channels in interleaved UV plane.
LIBYUV_API
void SwapUVPlane(const uint8_t* src_uv,
@@ -140,6 +240,92 @@ void MergeRGBPlane(const uint8_t* src_r,
int width,
int height);
+// Split interleaved ARGB plane into separate R, G, B and A planes.
+// dst_a can be NULL to discard alpha plane.
+LIBYUV_API
+void SplitARGBPlane(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_r,
+ int dst_stride_r,
+ uint8_t* dst_g,
+ int dst_stride_g,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height);
+
+// Merge separate R, G, B and A planes into one interleaved ARGB plane.
+// src_a can be NULL to fill opaque value to alpha.
+LIBYUV_API
+void MergeARGBPlane(const uint8_t* src_r,
+ int src_stride_r,
+ const uint8_t* src_g,
+ int src_stride_g,
+ const uint8_t* src_b,
+ int src_stride_b,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
+// Merge separate 'depth' bit R, G and B planes stored in lsb
+// into one interleaved XR30 plane.
+// depth should in range [10, 16]
+LIBYUV_API
+void MergeXR30Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height,
+ int depth);
+
+// Merge separate 'depth' bit R, G, B and A planes stored in lsb
+// into one interleaved AR64 plane.
+// src_a can be NULL to fill opaque value to alpha.
+// depth should in range [1, 16]
+LIBYUV_API
+void MergeAR64Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height,
+ int depth);
+
+// Merge separate 'depth' bit R, G, B and A planes stored in lsb
+// into one interleaved ARGB plane.
+// src_a can be NULL to fill opaque value to alpha.
+// depth should in range [8, 16]
+LIBYUV_API
+void MergeARGB16To8Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int depth);
+
// Copy I400. Supports inverting.
LIBYUV_API
int I400ToI400(const uint8_t* src_y,
@@ -187,6 +373,68 @@ int I444Copy(const uint8_t* src_y,
int width,
int height);
+// Copy I210 to I210.
+#define I210ToI210 I210Copy
+LIBYUV_API
+int I210Copy(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Copy I410 to I410.
+#define I410ToI410 I410Copy
+LIBYUV_API
+int I410Copy(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
+// Copy NV12. Supports inverting.
+LIBYUV_API
+int NV12Copy(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
+// Copy NV21. Supports inverting.
+LIBYUV_API
+int NV21Copy(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height);
+
// Convert YUY2 to I422.
LIBYUV_API
int YUY2ToI422(const uint8_t* src_yuy2,
@@ -233,19 +481,22 @@ int UYVYToNV12(const uint8_t* src_uyvy,
int width,
int height);
-// Convert NV21 to NV12.
+// Convert NV12 to NV21.
LIBYUV_API
-int NV21ToNV12(const uint8_t* src_y,
+int NV12ToNV21(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
+ const uint8_t* src_uv,
+ int src_stride_uv,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_uv,
- int dst_stride_uv,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
int width,
int height);
+// Alias
+#define NV21ToNV12 NV12ToNV21
+
LIBYUV_API
int YUY2ToY(const uint8_t* src_yuy2,
int src_stride_yuy2,
@@ -254,6 +505,14 @@ int YUY2ToY(const uint8_t* src_yuy2,
int width,
int height);
+LIBYUV_API
+int UYVYToY(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height);
+
// Convert I420 to I400. (calls CopyPlane ignoring u/v).
LIBYUV_API
int I420ToI400(const uint8_t* src_y,
@@ -301,6 +560,22 @@ int I400Mirror(const uint8_t* src_y,
int width,
int height);
+// Alias
+#define NV12ToNV12Mirror NV12Mirror
+
+// NV12 mirror.
+LIBYUV_API
+int NV12Mirror(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
// Alias
#define ARGBToARGBMirror ARGBMirror
@@ -313,56 +588,44 @@ int ARGBMirror(const uint8_t* src_argb,
int width,
int height);
-// Convert NV12 to RGB565.
+// Alias
+#define RGB24ToRGB24Mirror RGB24Mirror
+
+// RGB24 mirror.
LIBYUV_API
-int NV12ToRGB565(const uint8_t* src_y,
+int RGB24Mirror(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height);
+
+// Mirror a plane of data.
+LIBYUV_API
+void MirrorPlane(const uint8_t* src_y,
int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
+ uint8_t* dst_y,
+ int dst_stride_y,
int width,
int height);
-// I422ToARGB is in convert_argb.h
-// Convert I422 to BGRA.
+// Mirror a plane of 16-bit data.
LIBYUV_API
-int I422ToBGRA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_bgra,
- int dst_stride_bgra,
- int width,
- int height);
-
-// Convert I422 to ABGR.
-LIBYUV_API
-int I422ToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height);
+void MirrorPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height);
-// Convert I422 to RGBA.
+// Mirror a plane of UV data.
LIBYUV_API
-int I422ToRGBA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- int width,
- int height);
+void MirrorUVPlane(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
// Alias
#define RGB24ToRAW RAWToRGB24
@@ -511,6 +774,10 @@ int ARGBPolynomial(const uint8_t* src_argb,
// Convert plane of 16 bit shorts to half floats.
// Source values are multiplied by scale before storing as half float.
+//
+// Note: Unlike other libyuv functions that operate on uint16_t buffers, the
+// src_stride_y and dst_stride_y parameters of HalfFloatPlane() are in bytes,
+// not in units of uint16_t.
LIBYUV_API
int HalfFloatPlane(const uint16_t* src_y,
int src_stride_y,
@@ -575,15 +842,6 @@ int ARGBCopyYToAlpha(const uint8_t* src_y,
int width,
int height);
-typedef void (*ARGBBlendRow)(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width);
-
-// Get function to Alpha Blend ARGB pixels and store to destination.
-LIBYUV_API
-ARGBBlendRow GetARGBBlend(void);
-
// Alpha Blend ARGB images and store to destination.
// Source is pre-multiplied by alpha using ARGBAttenuate.
// Alpha of destination is set to 255.
@@ -597,6 +855,18 @@ int ARGBBlend(const uint8_t* src_argb0,
int width,
int height);
+// Alpha Blend unattenuated ARGB images and store to destination.
+// Source and destination are not pre-multiplied by alpha.
+LIBYUV_API
+int ARGBAlphaBlend(const uint8_t* src_argb0,
+ int src_stride_argb0,
+ const uint8_t* src_argb1,
+ int src_stride_argb1,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height);
+
// Alpha Blend plane and store to destination.
// Source is not pre-multiplied by alpha.
LIBYUV_API
@@ -671,6 +941,17 @@ int ARGBSubtract(const uint8_t* src_argb0,
int width,
int height);
+// Bitwise OR ARGB image with a specified ARGB value (e.g. 0xff000000 to force
+// opaque alpha).
+LIBYUV_API
+int ARGBOr(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ uint32_t value);
+
// Convert I422 to YUY2.
LIBYUV_API
int I422ToYUY2(const uint8_t* src_y,
@@ -743,6 +1024,19 @@ int ARGBBlur(const uint8_t* src_argb,
int height,
int radius);
+// Gaussian 5x5 blur a float plane.
+// Coefficients of 1, 4, 6, 4, 1.
+// Each destination pixel is a blur of the 5x5
+// pixels from the source.
+// Source edges are clamped.
+LIBYUV_API
+int GaussPlane_F32(const float* src,
+ int src_stride,
+ float* dst,
+ int dst_stride,
+ int width,
+ int height);
+
// Multiply ARGB image by ARGB value.
LIBYUV_API
int ARGBShade(const uint8_t* src_argb,
@@ -768,6 +1062,21 @@ int InterpolatePlane(const uint8_t* src0,
int height,
int interpolation);
+// Interpolate between two images using specified amount of interpolation
+// (0 to 255) and store to destination.
+// 'interpolation' is specified as 8 bit fraction where 0 means 100% src0
+// and 255 means 1% src0 and 99% src1.
+LIBYUV_API
+int InterpolatePlane_16(const uint16_t* src0,
+ int src_stride0, // measured in 16 bit pixels
+ const uint16_t* src1,
+ int src_stride1,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height,
+ int interpolation);
+
// Interpolate between two ARGB images using specified amount of interpolation
// Internally calls InterpolatePlane with width * 4 (bpp).
LIBYUV_API
@@ -807,33 +1116,28 @@ int I420Interpolate(const uint8_t* src0_y,
int height,
int interpolation);
-// Row function for copying pixels from a source with a slope to a row
-// of destination. Useful for scaling, rotation, mirror, texture mapping.
-LIBYUV_API
-void ARGBAffineRow_C(const uint8_t* src_argb,
- int src_argb_stride,
- uint8_t* dst_argb,
- const float* uv_dudv,
- int width);
-// TODO(fbarchard): Move ARGBAffineRow_SSE2 to row.h
-LIBYUV_API
-void ARGBAffineRow_SSE2(const uint8_t* src_argb,
- int src_argb_stride,
- uint8_t* dst_argb,
- const float* uv_dudv,
- int width);
-
// Shuffle ARGB channel order. e.g. BGRA to ARGB.
-// shuffler is 16 bytes and must be aligned.
+// shuffler is 16 bytes.
LIBYUV_API
-int ARGBShuffle(const uint8_t* src_bgra,
- int src_stride_bgra,
+int ARGBShuffle(const uint8_t* src_argb,
+ int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
const uint8_t* shuffler,
int width,
int height);
+// Shuffle AR64 channel order. e.g. AR64 to AB64.
+// shuffler is 16 bytes.
+LIBYUV_API
+int AR64Shuffle(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ const uint8_t* shuffler,
+ int width,
+ int height);
+
// Sobel ARGB effect with planar output.
LIBYUV_API
int ARGBSobelToPlane(const uint8_t* src_argb,
diff --git a/libs/libyuv/include/libyuv/rotate.h b/libs/libyuv/include/libyuv/rotate.h
index c64e0216d5..4b1278d4b1 100644
--- a/libs/libyuv/include/libyuv/rotate.h
+++ b/libs/libyuv/include/libyuv/rotate.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_ROTATE_H_
#define INCLUDE_LIBYUV_ROTATE_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -49,6 +51,24 @@ int I420Rotate(const uint8_t* src_y,
int height,
enum RotationMode mode);
+// Rotate I422 frame.
+LIBYUV_API
+int I422Rotate(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode);
+
// Rotate I444 frame.
LIBYUV_API
int I444Rotate(const uint8_t* src_y,
@@ -67,6 +87,60 @@ int I444Rotate(const uint8_t* src_y,
int height,
enum RotationMode mode);
+// Rotate I010 frame.
+LIBYUV_API
+int I010Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode);
+
+// Rotate I210 frame.
+LIBYUV_API
+int I210Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode);
+
+// Rotate I410 frame.
+LIBYUV_API
+int I410Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode);
+
// Rotate NV12 input and store in I420.
LIBYUV_API
int NV12ToI420Rotate(const uint8_t* src_y,
@@ -83,6 +157,26 @@ int NV12ToI420Rotate(const uint8_t* src_y,
int height,
enum RotationMode mode);
+// Convert Android420 to I420 with rotation.
+// "rotation" can be 0, 90, 180 or 270.
+LIBYUV_API
+int Android420ToI420Rotate(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode rotation);
+
// Rotate a plane by 0, 90, 180, or 270.
LIBYUV_API
int RotatePlane(const uint8_t* src,
@@ -118,39 +212,61 @@ void RotatePlane270(const uint8_t* src,
int width,
int height);
+// Rotate a plane by 0, 90, 180, or 270.
LIBYUV_API
-void RotateUV90(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height);
+int RotatePlane_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height,
+ enum RotationMode mode);
// Rotations for when U and V are interleaved.
-// These functions take one input pointer and
+// These functions take one UV input pointer and
// split the data into two buffers while
-// rotating them. Deprecated.
-LIBYUV_API
-void RotateUV180(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height);
-
-LIBYUV_API
-void RotateUV270(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height);
+// rotating them.
+// width and height expected to be half size for NV12.
+LIBYUV_API
+int SplitRotateUV(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode);
+
+LIBYUV_API
+void SplitRotateUV90(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
+
+LIBYUV_API
+void SplitRotateUV180(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
+
+LIBYUV_API
+void SplitRotateUV270(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
// The 90 and 270 functions are based on transposes.
// Doing a transpose with reversing the read/write
@@ -165,14 +281,14 @@ void TransposePlane(const uint8_t* src,
int height);
LIBYUV_API
-void TransposeUV(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height);
+void SplitTransposeUV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/include/libyuv/rotate_argb.h b/libs/libyuv/include/libyuv/rotate_argb.h
index 20432949ab..00f6c833ce 100644
--- a/libs/libyuv/include/libyuv/rotate_argb.h
+++ b/libs/libyuv/include/libyuv/rotate_argb.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_ROTATE_ARGB_H_
#define INCLUDE_LIBYUV_ROTATE_ARGB_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#include "libyuv/rotate.h" // For RotationMode.
diff --git a/libs/libyuv/include/libyuv/rotate_row.h b/libs/libyuv/include/libyuv/rotate_row.h
index 022293eef2..5caa3bdc9b 100644
--- a/libs/libyuv/include/libyuv/rotate_row.h
+++ b/libs/libyuv/include/libyuv/rotate_row.h
@@ -11,66 +11,133 @@
#ifndef INCLUDE_LIBYUV_ROTATE_ROW_H_
#define INCLUDE_LIBYUV_ROTATE_ROW_H_
-#include "libyuv/basic_types.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
+// The following are available for Visual C 32/64 bit, and for clang with
+// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are
+// intrinsics in rotate_win.cc.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+#if defined(VISUALC_HAS_AVX2)
+#define HAS_TRANSPOSEWX8_AVX2
+#define HAS_TRANSPOSEUVWX8_AVX2
+#define HAS_TRANSPOSEWX8_16_AVX2
+#define HAS_TRANSPOSEWX16_16_AVX2
+#define HAS_TRANSPOSEWX4_32_AVX2
+#define HAS_TRANSPOSEWX8_32_AVX2
+#define HAS_TRANSPOSEWXH_32_AVX2
+#define HAS_TRANSPOSEWX16_AVX2
+#define HAS_TRANSPOSEUVWX16_AVX2
#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer)
-#define LIBYUV_DISABLE_X86
+#if defined(VISUALC_HAS_AVX512)
+#define HAS_TRANSPOSEWX16_AVX512BW
+#define HAS_TRANSPOSEUVWX16_AVX512BW
+#define HAS_TRANSPOSEWX32_16_AVX512BW
+#define HAS_TRANSPOSEWX16_32_AVX512BW
#endif
#endif
-// The following are available for Visual C and clangcl 32 bit:
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
-#define HAS_TRANSPOSEWX8_SSSE3
-#define HAS_TRANSPOSEUVWX8_SSE2
-#endif
-// The following are available for GCC 32 or 64 bit:
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__i386__) || defined(__x86_64__))
-#define HAS_TRANSPOSEWX8_SSSE3
+// The following are available for 32/64 bit GCC and clang, as inline assembly
+// in rotate_gcc.cc:
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (!defined(_MSC_VER) || defined(__clang__))
+#define HAS_TRANSPOSEWX8_AVX2
+#define HAS_TRANSPOSEUVWX8_AVX2
+#define HAS_TRANSPOSEWX8_16_AVX2
+#define HAS_TRANSPOSEWX16_16_AVX2
+#define HAS_TRANSPOSEWX4_32_AVX2
+#define HAS_TRANSPOSEWX8_32_AVX2
+#define HAS_TRANSPOSEWXH_32_AVX2
+#define HAS_TRANSPOSEWX16_AVX2
+#define HAS_TRANSPOSEUVWX16_AVX2
#endif
-// The following are available for 64 bit GCC:
-#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__)
-#define HAS_TRANSPOSEWX8_FAST_SSSE3
-#define HAS_TRANSPOSEUVWX8_SSE2
+// The following are available for AVX512 GCC/clang x64 platforms:
+#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
+ (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (!defined(_MSC_VER) || defined(__clang__))
+#define HAS_TRANSPOSEWX16_AVX512BW
+#define HAS_TRANSPOSEUVWX16_AVX512BW
+#define HAS_TRANSPOSEWX32_16_AVX512BW
+#define HAS_TRANSPOSEWX16_32_AVX512BW
#endif
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
+#if defined(__aarch64__)
+#define HAS_TRANSPOSEWX16_NEON
+#else
#define HAS_TRANSPOSEWX8_NEON
+#endif
#define HAS_TRANSPOSEUVWX8_NEON
+#define HAS_TRANSPOSEWX8_16_NEON
+#define HAS_TRANSPOSEWX4_32_NEON
+#define HAS_TRANSPOSEWXH_32_NEON
#endif
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#define HAS_TRANSPOSEWX16_MSA
-#define HAS_TRANSPOSEUVWX16_MSA
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+#define HAS_TRANSPOSEWX8_16_SVE2
+#define HAS_TRANSPOSEWX4_32_SVE2
#endif
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-#define HAS_TRANSPOSEWX8_MMI
-#define HAS_TRANSPOSEUVWX8_MMI
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_TRANSPOSEWXH_SME
+#define HAS_TRANSPOSEUVWXH_SME
+#define HAS_TRANSPOSEWXH_16_SME
+#define HAS_TRANSPOSEWXH_32_SME
#endif
-void TransposeWxH_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width,
- int height);
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+#define HAS_TRANSPOSEWX8_LSX
+#define HAS_TRANSPOSEUVWX8_LSX
+#define HAS_TRANSPOSEWX8_16_LSX
+#define HAS_TRANSPOSEWX4_32_LSX
+#endif
+
+#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
+#define HAS_TRANSPOSEWX16_LASX
+#define HAS_TRANSPOSEUVWX16_LASX
+#define HAS_TRANSPOSEWX16_16_LASX
+#define HAS_TRANSPOSEWX4_32_LASX
+#endif
+
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
+#define HAS_TRANSPOSEWXH_16_RVV
+#define HAS_TRANSPOSEWX4_32_RVV
+#define HAS_TRANSPOSEWXH_32_RVV
+#endif
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#define HAS_TRANSPOSEWX8_WASMSIMD
+#define HAS_TRANSPOSEUVWX8_WASMSIMD
+#define HAS_TRANSPOSEWX16_WASMSIMD
+#define HAS_TRANSPOSEUVWX16_WASMSIMD
+#define HAS_TRANSPOSEWX8_16_WASMSIMD
+#define HAS_TRANSPOSEWX4_32_WASMSIMD
+#endif
+
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#define HAS_TRANSPOSEWXH_HVX
+#define HAS_TRANSPOSEUVWXH_HVX
+#define HAS_TRANSPOSEWXH_16_HVX
+#define HAS_TRANSPOSEWXH_32_HVX
+#endif
void TransposeWx8_C(const uint8_t* src,
int src_stride,
@@ -87,61 +154,99 @@ void TransposeWx8_NEON(const uint8_t* src,
uint8_t* dst,
int dst_stride,
int width);
-void TransposeWx8_SSSE3(const uint8_t* src,
+void TransposeWx16_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
-void TransposeWx8_MMI(const uint8_t* src,
+void TransposeWxH_SME(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
- int width);
-void TransposeWx8_Fast_SSSE3(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width);
-void TransposeWx16_MSA(const uint8_t* src,
+ int width,
+ int height);
+void TransposeWxH_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+void TransposeWx8_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
+void TransposeWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
void TransposeWx8_Any_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
-void TransposeWx8_Any_SSSE3(const uint8_t* src,
+void TransposeWx16_Any_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
-void TransposeWx8_Any_MMI(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width);
-void TransposeWx8_Fast_Any_SSSE3(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width);
-void TransposeWx16_Any_MSA(const uint8_t* src,
+void TransposeWx8_Any_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
-
-void TransposeUVWxH_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height);
+void TransposeWx16_Any_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_Any_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_Any_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_Any_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_Any_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
void TransposeUVWx8_C(const uint8_t* src,
int src_stride,
@@ -157,13 +262,27 @@ void TransposeUVWx16_C(const uint8_t* src,
uint8_t* dst_b,
int dst_stride_b,
int width);
-void TransposeUVWx8_SSE2(const uint8_t* src,
+void TransposeUVWx8_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width);
+void TransposeUVWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@@ -171,22 +290,52 @@ void TransposeUVWx8_NEON(const uint8_t* src,
uint8_t* dst_b,
int dst_stride_b,
int width);
-void TransposeUVWx8_MMI(const uint8_t* src,
+void TransposeUVWxH_SME(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
+void TransposeUVWxH_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
+void TransposeUVWx8_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width);
-void TransposeUVWx16_MSA(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width);
+void TransposeUVWx16_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx8_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx16_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
-void TransposeUVWx8_Any_SSE2(const uint8_t* src,
+void TransposeUVWx8_Any_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
@@ -200,20 +349,266 @@ void TransposeUVWx8_Any_NEON(const uint8_t* src,
uint8_t* dst_b,
int dst_stride_b,
int width);
-void TransposeUVWx8_Any_MMI(const uint8_t* src,
+void TransposeUVWx16_Any_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx8_Any_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width);
-void TransposeUVWx16_Any_MSA(const uint8_t* src,
+void TransposeUVWx16_Any_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx8_Any_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+void TransposeUVWx16_Any_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width);
+
+void TransposeWx8_16_C(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx32_16_AVX512BW(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_NEON(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_SVE2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWxH_16_SME(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+void TransposeWxH_16_RVV(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+void TransposeWxH_16_HVX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+void TransposeWx8_16_LSX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_16_LASX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_WASMSIMD(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_Any_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx16_16_Any_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_Any_NEON(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_Any_LSX(const uint16_t* src,
int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
+ uint16_t* dst,
+ int dst_stride,
int width);
+void TransposeWx16_16_Any_LASX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWx8_16_Any_WASMSIMD(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWxH_32_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+
+void TransposeWxH_32_SME(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+
+void TransposeWx4_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWxH_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+
+void TransposeWx4_32_SVE2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx8_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWxH_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+
+void TransposeWx16_32_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_RVV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWxH_32_RVV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+void TransposeWxH_32_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
+
+void TransposeWx4_32_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_Any_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx8_32_Any_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_Any_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_Any_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+
+void TransposeWx4_32_Any_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/include/libyuv/row.h b/libs/libyuv/include/libyuv/row.h
index 890766ff3f..8f920f202a 100644
--- a/libs/libyuv/include/libyuv/row.h
+++ b/libs/libyuv/include/libyuv/row.h
@@ -11,108 +11,51 @@
#ifndef INCLUDE_LIBYUV_ROW_H_
#define INCLUDE_LIBYUV_ROW_H_
-#include <stdlib.h> // For malloc.
+#include <stddef.h> // For NULL
+#include <stdint.h>
+#include <stdlib.h> // For malloc
#include "libyuv/basic_types.h"
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer)
-#define LIBYUV_DISABLE_X86
+// This module is for Visual C 32/64 bit
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86))
+#if ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+#define USE_ROW_WIN
+#else
+#define USE_ROW_GCC
#endif
#endif
-// clang >= 3.5.0 required for Arm64.
-#if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON)
-#if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5))
-#define LIBYUV_DISABLE_NEON
-#endif // clang >= 3.5
-#endif // __clang__
-
-// GCC >= 4.7.0 required for AVX2.
-#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
-#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
-#define GCC_HAS_AVX2 1
-#endif // GNUC >= 4.7
-#endif // __GNUC__
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-
-// clang >= 6.0.0 required for AVX512.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-// clang in xcode follows a different versioning scheme.
-// TODO(fbarchard): fix xcode 9 ios b/789.
-#if (__clang_major__ >= 7) && !defined(__APPLE__)
-#define CLANG_HAS_AVX512 1
-#endif // clang >= 7
-#endif // __clang__
-
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
-// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
+
+// The following are available on clang x86 platforms:
+#if defined(USE_ROW_GCC)
// Conversions:
-#define HAS_ABGRTOUVROW_SSSE3
-#define HAS_ABGRTOYROW_SSSE3
-#define HAS_ARGB1555TOARGBROW_SSE2
-#define HAS_ARGB4444TOARGBROW_SSE2
#define HAS_ARGBEXTRACTALPHAROW_SSE2
-#define HAS_ARGBSETROW_X86
#define HAS_ARGBSHUFFLEROW_SSSE3
-#define HAS_ARGBTOARGB1555ROW_SSE2
-#define HAS_ARGBTOARGB4444ROW_SSE2
#define HAS_ARGBTORAWROW_SSSE3
#define HAS_ARGBTORGB24ROW_SSSE3
-#define HAS_ARGBTORGB565DITHERROW_SSE2
-#define HAS_ARGBTORGB565ROW_SSE2
-#define HAS_ARGBTOUV444ROW_SSSE3
-#define HAS_ARGBTOUVJROW_SSSE3
-#define HAS_ARGBTOUVROW_SSSE3
-#define HAS_ARGBTOYJROW_SSSE3
-#define HAS_ARGBTOYROW_SSSE3
-#define HAS_BGRATOUVROW_SSSE3
-#define HAS_BGRATOYROW_SSSE3
#define HAS_COPYROW_ERMS
#define HAS_COPYROW_SSE2
-#define HAS_H422TOARGBROW_SSSE3
-#define HAS_HALFFLOATROW_SSE2
-#define HAS_I400TOARGBROW_SSE2
-#define HAS_I422TOARGB1555ROW_SSSE3
-#define HAS_I422TOARGB4444ROW_SSSE3
#define HAS_I422TOARGBROW_SSSE3
#define HAS_I422TORGB24ROW_SSSE3
-#define HAS_I422TORGB565ROW_SSSE3
#define HAS_I422TORGBAROW_SSSE3
+#define HAS_I422TORGB565ROW_SSSE3
+#define HAS_I422TOARGB1555ROW_SSSE3
+#define HAS_I422TOARGB4444ROW_SSSE3
#define HAS_I422TOUYVYROW_SSE2
#define HAS_I422TOYUY2ROW_SSE2
#define HAS_I444TOARGBROW_SSSE3
-#define HAS_J400TOARGBROW_SSE2
-#define HAS_J422TOARGBROW_SSSE3
+#define HAS_I444TORGB24ROW_SSSE3
#define HAS_MERGEUVROW_SSE2
#define HAS_MIRRORROW_SSSE3
-#define HAS_MIRRORUVROW_SSSE3
#define HAS_NV12TOARGBROW_SSSE3
#define HAS_NV12TORGB24ROW_SSSE3
#define HAS_NV12TORGB565ROW_SSSE3
@@ -120,18 +63,10 @@ extern "C" {
#define HAS_NV21TORGB24ROW_SSSE3
#define HAS_RAWTOARGBROW_SSSE3
#define HAS_RAWTORGB24ROW_SSSE3
-#define HAS_RAWTOYROW_SSSE3
#define HAS_RGB24TOARGBROW_SSSE3
-#define HAS_RGB24TOYROW_SSSE3
-#define HAS_RGB565TOARGBROW_SSE2
-#define HAS_RGBATOUVROW_SSSE3
-#define HAS_RGBATOYROW_SSSE3
#define HAS_SETROW_ERMS
-#define HAS_SETROW_X86
#define HAS_SPLITUVROW_SSE2
#define HAS_UYVYTOARGBROW_SSSE3
-#define HAS_UYVYTOUV422ROW_SSE2
-#define HAS_UYVYTOUVROW_SSE2
#define HAS_UYVYTOYROW_SSE2
#define HAS_YUY2TOARGBROW_SSSE3
#define HAS_YUY2TOUV422ROW_SSE2
@@ -140,42 +75,17 @@ extern "C" {
// Effects:
#define HAS_ARGBADDROW_SSE2
-#define HAS_ARGBAFFINEROW_SSE2
-#define HAS_ARGBATTENUATEROW_SSSE3
-#define HAS_ARGBBLENDROW_SSSE3
-#define HAS_ARGBCOLORMATRIXROW_SSSE3
-#define HAS_ARGBCOLORTABLEROW_X86
+#define HAS_ARGBALPHABLENDROW_SSSE3
#define HAS_ARGBCOPYALPHAROW_SSE2
#define HAS_ARGBCOPYYTOALPHAROW_SSE2
-#define HAS_ARGBGRAYROW_SSSE3
-#define HAS_ARGBLUMACOLORTABLEROW_SSSE3
#define HAS_ARGBMIRRORROW_SSE2
#define HAS_ARGBMULTIPLYROW_SSE2
+#define HAS_ARGBORROW_SSE2
#define HAS_ARGBPOLYNOMIALROW_SSE2
-#define HAS_ARGBQUANTIZEROW_SSE2
-#define HAS_ARGBSEPIAROW_SSSE3
-#define HAS_ARGBSHADEROW_SSE2
#define HAS_ARGBSUBTRACTROW_SSE2
-#define HAS_ARGBUNATTENUATEROW_SSE2
#define HAS_BLENDPLANEROW_SSSE3
-#define HAS_COMPUTECUMULATIVESUMROW_SSE2
-#define HAS_CUMULATIVESUMTOAVERAGEROW_SSE2
-#define HAS_INTERPOLATEROW_SSSE3
-#define HAS_RGBCOLORTABLEROW_X86
-#define HAS_SOBELROW_SSE2
-#define HAS_SOBELTOPLANEROW_SSE2
-#define HAS_SOBELXROW_SSE2
-#define HAS_SOBELXYROW_SSE2
-#define HAS_SOBELYROW_SSE2
-
-// The following functions fail on gcc/clang 32 bit with fpic and framepointer.
-// caveat: clangcl uses row_win.cc which works.
-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
- defined(_MSC_VER)
-// TODO(fbarchard): fix build error on android_full_debug=1
-// https://code.google.com/p/libyuv/issues/detail?id=517
#define HAS_I422ALPHATOARGBROW_SSSE3
-#endif
+#define HAS_I444ALPHATOARGBROW_SSSE3
#endif
// The following are available on all x86 platforms, but
@@ -183,167 +93,658 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
defined(GCC_HAS_AVX2))
+#define HAS_ARGBMIRRORROW_AVX2
+#define HAS_RGB24MIRRORROW_AVX2
+#define HAS_ARGBTOUVMATRIXROW_AVX2
+#define HAS_RGBTOUVMATRIXROW_AVX2
+#define HAS_RGB565TOUVMATRIXROW_AVX2
+#define HAS_ARGB1555TOUVMATRIXROW_AVX2
+#define HAS_ARGB4444TOUVMATRIXROW_AVX2
+#define HAS_MERGEUVROW_AVX2
+#define HAS_MIRRORROW_AVX2
+#define HAS_MIRRORSPLITUVROW_AVX2
+#define HAS_MIRRORUVROW_AVX2
+#define HAS_INTERPOLATEROW_16_AVX2
+#define HAS_INTERPOLATEROW_AVX2
+#define HAS_I422TOARGBROW_AVX2
+#define HAS_NV12TOARGBROW_AVX2
+#define HAS_NV21TOARGBROW_AVX2
+#endif
+
+#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
+ (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
+ defined(GCC_HAS_AVX2))
#define HAS_ARGBCOPYALPHAROW_AVX2
#define HAS_ARGBCOPYYTOALPHAROW_AVX2
#define HAS_ARGBEXTRACTALPHAROW_AVX2
-#define HAS_ARGBMIRRORROW_AVX2
#define HAS_ARGBPOLYNOMIALROW_AVX2
#define HAS_ARGBSHUFFLEROW_AVX2
#define HAS_ARGBTORGB565DITHERROW_AVX2
-#define HAS_ARGBTOUVJROW_AVX2
-#define HAS_ARGBTOUVROW_AVX2
-#define HAS_ARGBTOYJROW_AVX2
-#define HAS_ARGBTOYROW_AVX2
#define HAS_COPYROW_AVX
-#define HAS_H422TOARGBROW_AVX2
#define HAS_HALFFLOATROW_AVX2
-// #define HAS_HALFFLOATROW_F16C // Enable to test halffloat cast
-#define HAS_I400TOARGBROW_AVX2
-#define HAS_I422TOARGB1555ROW_AVX2
-#define HAS_I422TOARGB4444ROW_AVX2
-#define HAS_I422TOARGBROW_AVX2
+#define HAS_I422ALPHATOARGBROW_AVX2
#define HAS_I422TORGB24ROW_AVX2
-#define HAS_I422TORGB565ROW_AVX2
#define HAS_I422TORGBAROW_AVX2
+#define HAS_I422TORGB565ROW_AVX2
+#define HAS_I422TOARGB1555ROW_AVX2
+#define HAS_I422TOARGB4444ROW_AVX2
+#define HAS_I444ALPHATOARGBROW_AVX2
#define HAS_I444TOARGBROW_AVX2
-#define HAS_INTERPOLATEROW_AVX2
-#define HAS_J422TOARGBROW_AVX2
-#define HAS_MERGEUVROW_AVX2
-#define HAS_MIRRORROW_AVX2
-#define HAS_NV12TOARGBROW_AVX2
+#define HAS_I444TORGB24ROW_AVX2
+#define HAS_J400TOARGBROW_AVX2
#define HAS_NV12TORGB24ROW_AVX2
#define HAS_NV12TORGB565ROW_AVX2
-#define HAS_NV21TOARGBROW_AVX2
#define HAS_NV21TORGB24ROW_AVX2
#define HAS_SPLITUVROW_AVX2
#define HAS_UYVYTOARGBROW_AVX2
-#define HAS_UYVYTOUV422ROW_AVX2
-#define HAS_UYVYTOUVROW_AVX2
#define HAS_UYVYTOYROW_AVX2
#define HAS_YUY2TOARGBROW_AVX2
#define HAS_YUY2TOUV422ROW_AVX2
#define HAS_YUY2TOUVROW_AVX2
#define HAS_YUY2TOYROW_AVX2
+// #define HAS_HALFFLOATROW_F16C // Enable to test half float cast
// Effects:
#define HAS_ARGBADDROW_AVX2
-#define HAS_ARGBATTENUATEROW_AVX2
+#define HAS_ARGBALPHABLENDROW_AVX2
#define HAS_ARGBMULTIPLYROW_AVX2
+#define HAS_ARGBORROW_AVX2
#define HAS_ARGBSUBTRACTROW_AVX2
-#define HAS_ARGBUNATTENUATEROW_AVX2
#define HAS_BLENDPLANEROW_AVX2
-
-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
- defined(_MSC_VER)
-// TODO(fbarchard): fix build error on android_full_debug=1
-// https://code.google.com/p/libyuv/issues/detail?id=517
-#define HAS_I422ALPHATOARGBROW_AVX2
-#endif
-#endif
-
-// The following are available for AVX2 Visual C and clangcl 32 bit:
-// TODO(fbarchard): Port to gcc.
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER) && \
- (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2))
-#define HAS_ARGB1555TOARGBROW_AVX2
-#define HAS_ARGB4444TOARGBROW_AVX2
-#define HAS_ARGBTOARGB1555ROW_AVX2
-#define HAS_ARGBTOARGB4444ROW_AVX2
-#define HAS_ARGBTORGB565ROW_AVX2
-#define HAS_J400TOARGBROW_AVX2
-#define HAS_RGB565TOARGBROW_AVX2
-#endif
-
-// The following are also available on x64 Visual C.
-#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_X64) && \
- (!defined(__clang__) || defined(__SSSE3__))
-#define HAS_I422ALPHATOARGBROW_SSSE3
-#define HAS_I422TOARGBROW_SSSE3
#endif
// The following are available for gcc/clang x86 platforms:
// TODO(fbarchard): Port to Visual C
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_AB64TOARGBROW_SSSE3
#define HAS_ABGRTOAR30ROW_SSSE3
+#define HAS_AR64TOARGBROW_SSSE3
+#define HAS_ARGBATTENUATEROW_SSSE3
+#define HAS_ARGBTOAB64ROW_SSSE3
#define HAS_ARGBTOAR30ROW_SSSE3
+#define HAS_ARGBTOAR64ROW_SSSE3
+#define HAS_ARGBUNATTENUATEROW_SSE2
#define HAS_CONVERT16TO8ROW_SSSE3
#define HAS_CONVERT8TO16ROW_SSE2
-// I210 is for H010. 2 = 422. I for 601 vs H for 709.
+#define HAS_HALFMERGEUVROW_SSSE3
+#define HAS_HALFROW_16TO8_SSSE3
+#define HAS_HALFWIDTHROW_16TO8_SSSE3
#define HAS_I210TOAR30ROW_SSSE3
#define HAS_I210TOARGBROW_SSSE3
+#define HAS_I212TOAR30ROW_SSSE3
+#define HAS_I212TOARGBROW_SSSE3
+#define HAS_I400TOARGBROW_SSE2
+#define HAS_I410TOAR30ROW_SSSE3
+#define HAS_I410TOARGBROW_SSSE3
#define HAS_I422TOAR30ROW_SSSE3
-#define HAS_MERGERGBROW_SSSE3
+#define HAS_MERGEARGBROW_SSE2
+#define HAS_MERGEXRGBROW_SSE2
+#define HAS_MIRRORSPLITUVROW_SSSE3
+#define HAS_MIRRORUVROW_SSSE3
+#define HAS_NV21TOYUV24ROW_SSSE3
+#define HAS_P210TOAR30ROW_SSSE3
+#define HAS_RGB24MIRRORROW_SSSE3
+#define HAS_P210TOARGBROW_SSSE3
+#define HAS_P410TOAR30ROW_SSSE3
+#define HAS_P410TOARGBROW_SSSE3
+#define HAS_SPLITARGBROW_SSE2
+#define HAS_SPLITARGBROW_SSSE3
+#define HAS_SPLITRGBROW_SSE41
#define HAS_SPLITRGBROW_SSSE3
+#define HAS_SPLITXRGBROW_SSE2
+#define HAS_SPLITXRGBROW_SSSE3
#define HAS_SWAPUVROW_SSSE3
+#define HAS_YUY2TONVUVROW_SSE2
+// TODO: port row_win to use 8 bit coefficients.
+#define HAS_ARGBTOYMATRIXROW_SSSE3
+#define HAS_RGBTOYMATRIXROW_SSSE3
+#define HAS_ARGBTOUVMATRIXROW_SSSE3
+#define HAS_RGBTOUVMATRIXROW_SSSE3
+#define HAS_ARGBTOUV444MATRIXROW_SSSE3
+#define HAS_RGBTOUV444MATRIXROW_SSSE3
+#define HAS_I210ALPHATOARGBROW_SSSE3
+#define HAS_I410ALPHATOARGBROW_SSSE3
#endif
// The following are available for AVX2 gcc/clang x86 platforms:
// TODO(fbarchard): Port to Visual C
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER))) && \
- (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
+#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_AB64TOARGBROW_AVX2
#define HAS_ABGRTOAR30ROW_AVX2
-#define HAS_ABGRTOUVROW_AVX2
-#define HAS_ABGRTOYROW_AVX2
+#define HAS_AR30TOAB30ROW_AVX2
+#define HAS_AR30TOABGRROW_AVX2
+#define HAS_AR30TOARGBROW_AVX2
+#define HAS_AR64TOARGBROW_AVX2
+#define HAS_ARGBATTENUATEROW_AVX2
+#define HAS_ARGBTOAB64ROW_AVX2
#define HAS_ARGBTOAR30ROW_AVX2
+#define HAS_ARGBTOAR64ROW_AVX2
#define HAS_ARGBTORAWROW_AVX2
#define HAS_ARGBTORGB24ROW_AVX2
+#define HAS_ARGBTORGB565ROW_AVX2
+#define HAS_ARGBTOARGB1555ROW_AVX2
+#define HAS_ARGBTOARGB4444ROW_AVX2
+#define HAS_RAWTOARGBROW_AVX2
+#define HAS_RGB24TOARGBROW_AVX2
+#define HAS_RGB565TOARGBROW_AVX2
+#define HAS_ARGB1555TOARGBROW_AVX2
+#define HAS_ARGB4444TOARGBROW_AVX2
+#define HAS_ARGBTOUV444MATRIXROW_AVX2
+#define HAS_RGBTOUV444MATRIXROW_AVX2
+#define HAS_ARGBTOYMATRIXROW_AVX2
+#define HAS_RGBTOYMATRIXROW_AVX2
+#define HAS_RGB565TOYMATRIXROW_AVX2
+#define HAS_ARGB1555TOYMATRIXROW_AVX2
+#define HAS_ARGB4444TOYMATRIXROW_AVX2
+#define HAS_ARGB1555TOUVROW_AVX2
+#define HAS_ARGB1555TOYROW_AVX2
+#define HAS_ARGB4444TOUVROW_AVX2
+#define HAS_ARGB4444TOYROW_AVX2
+#define HAS_ARGBAFFINEROW_AVX2
+#define HAS_ARGBCOLORMATRIXROW_AVX2
+#define HAS_ARGBCOLORTABLEROW_AVX2
+#define HAS_ARGBLUMACOLORTABLEROW_AVX2
+#define HAS_ARGBQUANTIZEROW_AVX2
+#define HAS_ARGBSEPIAROW_AVX2
+#define HAS_ARGBSETROW_AVX2
+#define HAS_ARGBSHADEROW_AVX2
+#define HAS_ARGBUNATTENUATEROW_AVX2
+#define HAS_ARGBBLENDROW_AVX2
+#define HAS_ARGBGRAYROW_AVX2
+#define HAS_AYUVTOUVROW_AVX2
+#define HAS_AYUVTOVUROW_AVX2
+#define HAS_AYUVTOYROW_AVX2
+#define HAS_BYTETOFLOATROW_AVX2
+#define HAS_COMPUTECUMULATIVESUMROW_AVX2
#define HAS_CONVERT16TO8ROW_AVX2
#define HAS_CONVERT8TO16ROW_AVX2
+#define HAS_CONVERT8TO8ROW_AVX2
+#define HAS_COPYROW_16_AVX2
+#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+#define HAS_DETILEROW_16_AVX2
+#define HAS_DETILEROW_AVX2
+#define HAS_DETILESPLITUVROW_AVX2
+#define HAS_DETILETOYUY2_AVX2
+#define HAS_DIVIDEROW_16_AVX2
+#define HAS_GAUSSCOL_AVX2
+#define HAS_GAUSSCOL_F32_AVX2
+#define HAS_GAUSSROW_AVX2
+#define HAS_GAUSSROW_F32_AVX2
+#define HAS_RGB565TOUVROW_AVX2
+#define HAS_RGB565TOYROW_AVX2
+#define HAS_RGBCOLORTABLEROW_AVX2
+#define HAS_SCALEMAXSAMPLES_AVX2
+#define HAS_SCALESAMPLES_AVX2
+#define HAS_SCALESUMSAMPLES_AVX2
+#define HAS_SETROW_AVX2
+#define HAS_SOBELROW_AVX2
+#define HAS_SOBELTOPLANEROW_AVX2
+#define HAS_SOBELXROW_AVX2
+#define HAS_SOBELXYROW_AVX2
+#define HAS_SOBELYROW_AVX2
+#define HAS_HALFMERGEUVROW_AVX2
+#define HAS_HALFROW_16_AVX2
+#define HAS_HALFROW_16TO8_AVX2
+#define HAS_HALFROW_AVX2
+#define HAS_HALFWIDTHROW_16TO8_AVX2
+#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2
#define HAS_I210TOAR30ROW_AVX2
#define HAS_I210TOARGBROW_AVX2
+#define HAS_I212TOAR30ROW_AVX2
+#define HAS_I212TOARGBROW_AVX2
+#define HAS_I400TOARGBROW_AVX2
+#define HAS_I410TOAR30ROW_AVX2
+#define HAS_I410TOARGBROW_AVX2
#define HAS_I422TOAR30ROW_AVX2
+#define HAS_RAWTORGB24ROW_AVX2
+#define HAS_RAWTORGBAROW_AVX2
#define HAS_I422TOUYVYROW_AVX2
#define HAS_I422TOYUY2ROW_AVX2
+#define HAS_MERGEAR64ROW_AVX2
+#define HAS_MERGEARGB16TO8ROW_AVX2
+#define HAS_MERGEARGBROW_AVX2
+#define HAS_MERGERGBROW_AVX2
#define HAS_MERGEUVROW_16_AVX2
+#define HAS_MERGEXR30ROW_AVX2
+#define HAS_MERGEXR64ROW_AVX2
+#define HAS_MERGEXRGB16TO8ROW_AVX2
+#define HAS_MERGEXRGBROW_AVX2
+#define HAS_MULTIPLYROW_16_AVX2
+#define HAS_NV21TOYUV24ROW_AVX2
+#define HAS_P210TOAR30ROW_AVX2
+#define HAS_P210TOARGBROW_AVX2
+#define HAS_P410TOAR30ROW_AVX2
+#define HAS_P410TOARGBROW_AVX2
+#define HAS_SPLITARGBROW_AVX2
+#define HAS_SPLITRGBROW_AVX2
+#define HAS_SPLITUVROW_16_AVX2
+#define HAS_SPLITXRGBROW_AVX2
+#define HAS_SWAPUVROW_AVX2
+#define HAS_UNPACKMT2T_AVX2
+#define HAS_YUY2TONVUVROW_AVX2
+#define HAS_I210ALPHATOARGBROW_AVX2
+#define HAS_I410ALPHATOARGBROW_AVX2
+#endif
+
+// This module is for Visual C 32/64 bit
+#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_WIN) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+#define HAS_RAWTORGB24ROW_SSSE3
+#define HAS_ARGBALPHABLENDROW_SSSE3
+#define HAS_ARGBORROW_SSE2
+#define HAS_MIRRORSPLITUVROW_SSSE3
+#if defined(VISUALC_HAS_AVX2)
+#define HAS_RAWTORGB24ROW_AVX2
+#define HAS_RAWTOARGBROW_AVX2
+#define HAS_I422TORGB24ROW_AVX2
+#define HAS_RGB24TOARGBROW_AVX2
+#define HAS_RGB565TOARGBROW_AVX2
+#define HAS_ARGB1555TOARGBROW_AVX2
+#define HAS_ARGB4444TOARGBROW_AVX2
+#define HAS_ARGBSHUFFLEROW_AVX2
+#define HAS_ARGBTORGB565ROW_AVX2
+#define HAS_ARGBTORGB565DITHERROW_AVX2
+#define HAS_ARGBTOARGB1555ROW_AVX2
+#define HAS_ARGBTOARGB4444ROW_AVX2
+#define HAS_I422TORGB565ROW_AVX2
+#define HAS_I422TOARGB1555ROW_AVX2
+#define HAS_I422TOARGB4444ROW_AVX2
+#define HAS_NV12TORGB565ROW_AVX2
+#define HAS_CONVERT8TO16ROW_AVX2
+#define HAS_CONVERT8TO8ROW_AVX2
+#define HAS_I422TOAR30ROW_AVX2
+#define HAS_AR30TOAB30ROW_AVX2
+#define HAS_AR30TOABGRROW_AVX2
+#define HAS_AR30TOARGBROW_AVX2
+#define HAS_AYUVTOUVROW_AVX2
+#define HAS_AYUVTOVUROW_AVX2
+#define HAS_AYUVTOYROW_AVX2
+#define HAS_ARGB1555TOUVROW_AVX2
+#define HAS_ARGB1555TOYROW_AVX2
+#define HAS_ARGB4444TOUVROW_AVX2
+#define HAS_ARGB4444TOYROW_AVX2
+#define HAS_ARGBAFFINEROW_AVX2
+#define HAS_ARGBCOLORMATRIXROW_AVX2
+#define HAS_ARGBCOLORTABLEROW_AVX2
+#define HAS_ARGBLUMACOLORTABLEROW_AVX2
+#define HAS_ARGBQUANTIZEROW_AVX2
+#define HAS_ARGBSEPIAROW_AVX2
+#define HAS_ARGBSETROW_AVX2
+#define HAS_ARGBSHADEROW_AVX2
+#define HAS_BYTETOFLOATROW_AVX2
+#define HAS_COPYROW_16_AVX2
+#define HAS_DETILEROW_16_AVX2
+#define HAS_DETILEROW_AVX2
+#define HAS_DETILESPLITUVROW_AVX2
+#define HAS_GAUSSROW_AVX2
+#define HAS_GAUSSROW_F32_AVX2
+#define HAS_RGB565TOUVROW_AVX2
+#define HAS_RGB565TOYROW_AVX2
+#define HAS_RGBCOLORTABLEROW_AVX2
+#define HAS_SETROW_AVX2
+#define HAS_SOBELROW_AVX2
+#define HAS_SOBELTOPLANEROW_AVX2
+#define HAS_SOBELXROW_AVX2
+#define HAS_SOBELXYROW_AVX2
+#define HAS_SOBELYROW_AVX2
+#define HAS_ARGBALPHABLENDROW_AVX2
+#define HAS_ARGBORROW_AVX2
+#define HAS_ARGBTOYMATRIXROW_AVX2
+#define HAS_RGBTOYMATRIXROW_AVX2
+#define HAS_RGB565TOYMATRIXROW_AVX2
+#define HAS_ARGB1555TOYMATRIXROW_AVX2
+#define HAS_ARGB4444TOYMATRIXROW_AVX2
+#define HAS_ARGBTOUV444MATRIXROW_AVX2
+#define HAS_RGBTOUV444MATRIXROW_AVX2
+#define HAS_J400TOARGBROW_AVX2
#define HAS_MULTIPLYROW_16_AVX2
#define HAS_SWAPUVROW_AVX2
-// TODO(fbarchard): Fix AVX2 version of YUV24
-// #define HAS_NV21TOYUV24ROW_AVX2
+#define HAS_AB64TOARGBROW_AVX2
+#define HAS_AR64TOARGBROW_AVX2
+#define HAS_ARGBADDROW_AVX2
+#define HAS_ARGBBLENDROW_AVX2
+#define HAS_ARGBCOPYALPHAROW_AVX2
+#define HAS_ARGBGRAYROW_AVX2
+#define HAS_ARGBPOLYNOMIALROW_AVX2
+#define HAS_ARGBSUBTRACTROW_AVX2
+#define HAS_ARGBTOAB64ROW_AVX2
+#define HAS_ARGBTOAR64ROW_AVX2
+#define HAS_ARGBTORAWROW_AVX2
+#define HAS_ARGBTORGB24ROW_AVX2
+#define HAS_ARGBUNATTENUATEROW_AVX2
+#define HAS_COMPUTECUMULATIVESUMROW_AVX2
+#define HAS_CONVERT16TO8ROW_AVX2
+#define HAS_COPYROW_AVX
+#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+#define HAS_DETILETOYUY2_AVX2
+#define HAS_DIVIDEROW_16_AVX2
+#define HAS_GAUSSCOL_AVX2
+#define HAS_GAUSSCOL_F32_AVX2
+#define HAS_HALFMERGEUVROW_AVX2
+#define HAS_HALFROW_16_AVX2
+#define HAS_HALFROW_16TO8_AVX2
+#define HAS_HALFROW_AVX2
+#define HAS_HALFWIDTHROW_16TO8_AVX2
+#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2
+#define HAS_I422TOUYVYROW_AVX2
+#define HAS_I422TOYUY2ROW_AVX2
+#define HAS_I444TOARGBROW_AVX2
+#define HAS_I444TORGB24ROW_AVX2
+#define HAS_MERGEAR64ROW_AVX2
+#define HAS_MERGEARGB16TO8ROW_AVX2
+#define HAS_MERGEARGBROW_AVX2
+#define HAS_MERGERGBROW_AVX2
+#define HAS_MERGEUVROW_16_AVX2
+#define HAS_MERGEXR30ROW_AVX2
+#define HAS_MERGEXR64ROW_AVX2
+#define HAS_MERGEXRGB16TO8ROW_AVX2
+#define HAS_MERGEXRGBROW_AVX2
+#define HAS_NV12TORGB24ROW_AVX2
+#define HAS_NV21TORGB24ROW_AVX2
+#define HAS_NV21TOYUV24ROW_AVX2
+#define HAS_RAWTORGBAROW_AVX2
+#define HAS_SCALEMAXSAMPLES_AVX2
+#define HAS_SCALESAMPLES_AVX2
+#define HAS_SCALESUMSAMPLES_AVX2
+#define HAS_SPLITARGBROW_AVX2
+#define HAS_SPLITRGBROW_AVX2
+#define HAS_SPLITUVROW_16_AVX2
+#define HAS_SPLITUVROW_AVX2
+#define HAS_SPLITXRGBROW_AVX2
+#define HAS_UNPACKMT2T_AVX2
+#define HAS_UYVYTOUV422ROW_AVX2
+#define HAS_UYVYTOUVROW_AVX2
+#define HAS_UYVYTOYROW_AVX2
+#define HAS_YUY2TONVUVROW_AVX2
+#define HAS_YUY2TOUV422ROW_AVX2
+#define HAS_YUY2TOUVROW_AVX2
+#define HAS_YUY2TOYROW_AVX2
+#endif
+#if defined(VISUALC_HAS_AVX512)
+#define HAS_RAWTOARGBROW_AVX512BW
+#define HAS_RGB24TOARGBROW_AVX512BW
+#define HAS_RAWTORGB24ROW_AVX512BW
+#define HAS_ARGBSHUFFLEROW_AVX512BW
+#define HAS_I422TORGB24ROW_AVX512VBMI
+#define HAS_RAWTORGB24ROW_AVX512VBMI
+#define HAS_I422TORGB24ROW_AVX512BW
+#define HAS_I422TOAR30ROW_AVX512BW
+#define HAS_RGBTOYMATRIXROW_AVX512BW
+#define HAS_BLENDPLANEROW_AVX512BW
+#define HAS_INTERPOLATEROW_AVX512BW
+#define HAS_ARGBEXTRACTALPHAROW_AVX512BW
+#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+#define HAS_ARGBMIRRORROW_AVX512BW
+#define HAS_RGB24MIRRORROW_AVX512VBMI
+#define HAS_AYUVTOYROW_AVX512VBMI
+#define HAS_AYUVTOUVROW_AVX512VBMI
+#define HAS_AYUVTOVUROW_AVX512VBMI
+#define HAS_AR30TOARGBROW_AVX512BW
+#define HAS_AR30TOABGRROW_AVX512BW
+#define HAS_AR30TOAB30ROW_AVX512BW
+#define HAS_MERGEXR30ROW_10_AVX512BW
+#define HAS_MIRRORROW_16_AVX512BW
+#define HAS_MERGERGBROW_AVX512VBMI
+#define HAS_RAWTORGBAROW_AVX512VBMI
+#define HAS_NV12TOARGBROW_AVX512BW
+#define HAS_NV21TOARGBROW_AVX512BW
+#define HAS_NV12TORGB24ROW_AVX512VBMI
+#define HAS_NV21TORGB24ROW_AVX512VBMI
+#define HAS_NV21TOYUV24ROW_AVX512VBMI
+#define HAS_I422ALPHATOARGBROW_AVX512BW
+#define HAS_SWAPUVROW_AVX512BW
+#define HAS_HALFMERGEUVROW_AVX512BW
+#define HAS_INTERPOLATEROW_16_AVX512BW
+#define HAS_ARGBATTENUATEROW_AVX512BW
+#define HAS_ARGBMULTIPLYROW_AVX512BW
+#define HAS_ARGBTOAR30ROW_AVX512BW
+#define HAS_ABGRTOAR30ROW_AVX512BW
+#define HAS_I400TOARGBROW_AVX512BW
+#define HAS_ARGBGRAYROW_AVX512BW
+#define HAS_ARGBBLENDROW_AVX512BW
+#define HAS_ARGBALPHABLENDROW_AVX512BW
+#define HAS_ARGBORROW_AVX512BW
+#define HAS_ARGBTOYMATRIXROW_AVX512VNNI
+#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+#define HAS_ARGBTOUVMATRIXROW_AVX512VNNI
+#define HAS_ARGBGRAYROW_AVX512VNNI
+// AVX512VNNIVBMI requires both AVX512VNNI and AVX512VBMI.
+#define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+#define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
+#define HAS_I422TOARGBROW_AVX512BW
+#define HAS_ARGB1555TOARGBROW_AVX512BW
+#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI
+#define HAS_ARGBSEPIAROW_AVX512VNNI
+#define HAS_ARGBTORGB565DITHERROW_AVX512BW
+#define HAS_ARGBTORGB565ROW_AVX512BW
+#define HAS_CONVERT8TO8ROW_AVX512BW
+#define HAS_DIVIDEROW_16_AVX512BW
+#define HAS_HALFFLOATROW_AVX512BW
+#define HAS_I210ALPHATOARGBROW_AVX512BW
+#define HAS_I210TOAR30ROW_AVX512BW
+#define HAS_I210TOARGBROW_AVX512BW
+#define HAS_I212TOAR30ROW_AVX512BW
+#define HAS_I212TOARGBROW_AVX512BW
+#define HAS_I410ALPHATOARGBROW_AVX512BW
+#define HAS_I410TOAR30ROW_AVX512BW
+#define HAS_I410TOARGBROW_AVX512BW
+#define HAS_I422TOARGB1555ROW_AVX512BW
+#define HAS_I422TOARGB4444ROW_AVX512BW
+#define HAS_I422TORGB565ROW_AVX512BW
+#define HAS_I422TORGBAROW_AVX512BW
+#define HAS_I444ALPHATOARGBROW_AVX512BW
+#define HAS_I444TOARGBROW_AVX512BW
+#define HAS_I444TORGB24ROW_AVX512VBMI
+#define HAS_MERGEUVROW_16_AVX512BW
+#define HAS_P210TOAR30ROW_AVX512BW
+#define HAS_P210TOARGBROW_AVX512BW
+#define HAS_P410TOAR30ROW_AVX512BW
+#define HAS_P410TOARGBROW_AVX512BW
+#define HAS_UYVYTOARGBROW_AVX512BW
+#define HAS_UYVYTOUVROW_AVX512BW
+#define HAS_UYVYTOYROW_AVX512BW
+#define HAS_YUY2TOARGBROW_AVX512BW
+#define HAS_YUY2TOUVROW_AVX512BW
+#define HAS_YUY2TOYROW_AVX512BW
+#define HAS_ARGBTORAWROW_AVX512BW
+#define HAS_ARGBTORGB24ROW_AVX512BW
+#define HAS_ARGBTOUV444MATRIXROW_AVX512BW
+#define HAS_ARGBTOUVMATRIXROW_AVX512BW
+#define HAS_ARGBTOYMATRIXROW_AVX512BW
+#define HAS_CONVERT16TO8ROW_AVX512BW
+#define HAS_CONVERT8TO16ROW_AVX512BW
+#define HAS_COPYROW_AVX512BW
+#define HAS_HALFROW_16TO8_AVX512BW
+#define HAS_HALFWIDTHROW_16TO8_AVX512BW
+#define HAS_J400TOARGBROW_AVX512BW
+#define HAS_MERGEUVROW_AVX512BW
+#define HAS_MIRRORROW_AVX512BW
+#define HAS_MIRRORUVROW_AVX512BW
+#define HAS_MIRRORSPLITUVROW_AVX512BW
+#define HAS_MULTIPLYROW_16_AVX512BW
+#define HAS_RGBTOUV444MATRIXROW_AVX512BW
+#define HAS_RGBTOUVMATRIXROW_AVX512BW
+#define HAS_SPLITUVROW_AVX512BW
+#endif
#endif
-// The following are available for AVX512 clang x86 platforms:
-// TODO(fbarchard): Port to GCC and Visual C
-// TODO(fbarchard): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI. Issue libyuv:789
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER))) && \
- (defined(CLANG_HAS_AVX512))
-#define HAS_ARGBTORGB24ROW_AVX512VBMI
+// The following are available for AVX512 GCC/clang x64 platforms:
+#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
+ defined(__x86_64__) && \
+ (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512))
+#define HAS_COPYROW_AVX512BW
+#define HAS_RAWTOARGBROW_AVX512BW
+#define HAS_RGB24TOARGBROW_AVX512BW
+#define HAS_RAWTORGB24ROW_AVX512BW
+#define HAS_RAWTORGB24ROW_AVX512VBMI
+#define HAS_ARGBTORGB24ROW_AVX512BW
+#define HAS_ARGBTORAWROW_AVX512BW
+#define HAS_CONVERT16TO8ROW_AVX512BW
+#define HAS_CONVERT8TO16ROW_AVX512BW
+#define HAS_HALFROW_16TO8_AVX512BW
+#define HAS_HALFWIDTHROW_16TO8_AVX512BW
+#define HAS_MULTIPLYROW_16_AVX512BW
+#define HAS_I422TOARGBROW_AVX512BW
+#define HAS_I422TOAR30ROW_AVX512BW
+#define HAS_ARGBSHUFFLEROW_AVX512BW
+#define HAS_ARGBTOUV444MATRIXROW_AVX512BW
+#define HAS_BLENDPLANEROW_AVX512BW
+#define HAS_INTERPOLATEROW_AVX512BW
+#define HAS_RGBTOUV444MATRIXROW_AVX512BW
+#define HAS_ARGBTOYMATRIXROW_AVX512BW
+#define HAS_I422TORGB24ROW_AVX512VBMI
+#define HAS_I422TORGB24ROW_AVX512BW
+#define HAS_ARGBTOUVMATRIXROW_AVX512BW
+#define HAS_J400TOARGBROW_AVX512BW
+#define HAS_MERGEUVROW_AVX512BW
+#define HAS_MIRRORROW_AVX512BW
+#define HAS_MIRRORUVROW_AVX512BW
+#define HAS_MIRRORSPLITUVROW_AVX512BW
+#define HAS_SPLITUVROW_AVX512BW
+#define HAS_RGBTOUVMATRIXROW_AVX512BW
+#define HAS_RGBTOYMATRIXROW_AVX512BW
+#define HAS_ARGBEXTRACTALPHAROW_AVX512BW
+#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+#define HAS_ARGBMIRRORROW_AVX512BW
+#define HAS_RGB24MIRRORROW_AVX512VBMI
+#define HAS_AYUVTOYROW_AVX512VBMI
+#define HAS_AYUVTOUVROW_AVX512VBMI
+#define HAS_AYUVTOVUROW_AVX512VBMI
+#define HAS_AR30TOARGBROW_AVX512BW
+#define HAS_AR30TOABGRROW_AVX512BW
+#define HAS_AR30TOAB30ROW_AVX512BW
+#define HAS_MERGEXR30ROW_10_AVX512BW
+#define HAS_MIRRORROW_16_AVX512BW
+#define HAS_MERGERGBROW_AVX512VBMI
+#define HAS_RAWTORGBAROW_AVX512VBMI
+#define HAS_NV12TOARGBROW_AVX512BW
+#define HAS_NV21TOARGBROW_AVX512BW
+#define HAS_NV12TORGB24ROW_AVX512VBMI
+#define HAS_NV21TORGB24ROW_AVX512VBMI
+#define HAS_NV21TOYUV24ROW_AVX512VBMI
+#define HAS_I422ALPHATOARGBROW_AVX512BW
+#define HAS_SWAPUVROW_AVX512BW
+#define HAS_HALFMERGEUVROW_AVX512BW
+#define HAS_INTERPOLATEROW_16_AVX512BW
+#define HAS_ARGBATTENUATEROW_AVX512BW
+#define HAS_ARGBMULTIPLYROW_AVX512BW
+#define HAS_ARGBTOAR30ROW_AVX512BW
+#define HAS_ABGRTOAR30ROW_AVX512BW
+#define HAS_I400TOARGBROW_AVX512BW
+#define HAS_ARGBGRAYROW_AVX512BW
+#define HAS_ARGBBLENDROW_AVX512BW
+#define HAS_ARGBALPHABLENDROW_AVX512BW
+#define HAS_ARGBORROW_AVX512BW
+#define HAS_ARGBTOYMATRIXROW_AVX512VNNI
+#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+#define HAS_ARGBTOUVMATRIXROW_AVX512VNNI
+#define HAS_ARGBGRAYROW_AVX512VNNI
+// AVX512VNNIVBMI requires both AVX512VNNI and AVX512VBMI.
+#define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+#define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
+#define HAS_ARGB1555TOARGBROW_AVX512BW
+#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI
+#define HAS_ARGBSEPIAROW_AVX512VNNI
+#define HAS_ARGBTORGB565DITHERROW_AVX512BW
+#define HAS_ARGBTORGB565ROW_AVX512BW
+#define HAS_CONVERT8TO8ROW_AVX512BW
+#define HAS_DIVIDEROW_16_AVX512BW
+#define HAS_HALFFLOATROW_AVX512BW
+#define HAS_I210ALPHATOARGBROW_AVX512BW
+#define HAS_I210TOAR30ROW_AVX512BW
+#define HAS_I210TOARGBROW_AVX512BW
+#define HAS_I212TOAR30ROW_AVX512BW
+#define HAS_I212TOARGBROW_AVX512BW
+#define HAS_I410ALPHATOARGBROW_AVX512BW
+#define HAS_I410TOAR30ROW_AVX512BW
+#define HAS_I410TOARGBROW_AVX512BW
+#define HAS_I422TOARGB1555ROW_AVX512BW
+#define HAS_I422TOARGB4444ROW_AVX512BW
+#define HAS_I422TORGB565ROW_AVX512BW
+#define HAS_I422TORGBAROW_AVX512BW
+#define HAS_I444ALPHATOARGBROW_AVX512BW
+#define HAS_I444TOARGBROW_AVX512BW
+#define HAS_I444TORGB24ROW_AVX512VBMI
+#define HAS_MERGEUVROW_16_AVX512BW
+#define HAS_P210TOAR30ROW_AVX512BW
+#define HAS_P210TOARGBROW_AVX512BW
+#define HAS_P410TOAR30ROW_AVX512BW
+#define HAS_P410TOARGBROW_AVX512BW
+#define HAS_UYVYTOARGBROW_AVX512BW
+#define HAS_UYVYTOUVROW_AVX512BW
+#define HAS_UYVYTOYROW_AVX512BW
+#define HAS_YUY2TOARGBROW_AVX512BW
+#define HAS_YUY2TOUVROW_AVX512BW
+#define HAS_YUY2TOYROW_AVX512BW
#endif
// The following are available on Neon platforms:
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
-#define HAS_ABGRTOUVROW_NEON
-#define HAS_ABGRTOYROW_NEON
+#define HAS_AB64TOARGBROW_NEON
+#define HAS_AR30TOAB30ROW_NEON
+#define HAS_AR30TOABGRROW_NEON
+#define HAS_AR30TOARGBROW_NEON
+#define HAS_AR64TOARGBROW_NEON
#define HAS_ARGB1555TOARGBROW_NEON
#define HAS_ARGB1555TOUVROW_NEON
#define HAS_ARGB1555TOYROW_NEON
#define HAS_ARGB4444TOARGBROW_NEON
#define HAS_ARGB4444TOUVROW_NEON
#define HAS_ARGB4444TOYROW_NEON
+#define HAS_ARGBCOPYALPHAROW_NEON
+#define HAS_ARGBCOPYYTOALPHAROW_NEON
#define HAS_ARGBEXTRACTALPHAROW_NEON
#define HAS_ARGBSETROW_NEON
+#define HAS_ARGBTOAB64ROW_NEON
+#define HAS_ARGBTOAR64ROW_NEON
#define HAS_ARGBTOARGB1555ROW_NEON
#define HAS_ARGBTOARGB4444ROW_NEON
#define HAS_ARGBTORAWROW_NEON
#define HAS_ARGBTORGB24ROW_NEON
#define HAS_ARGBTORGB565DITHERROW_NEON
#define HAS_ARGBTORGB565ROW_NEON
-#define HAS_ARGBTOUV444ROW_NEON
-#define HAS_ARGBTOUVJROW_NEON
-#define HAS_ARGBTOUVROW_NEON
-#define HAS_ARGBTOYJROW_NEON
-#define HAS_ARGBTOYROW_NEON
-#define HAS_AYUVTOUVROW_NEON
+#define HAS_ARGBTOUV444MATRIXROW_NEON
+#define HAS_RGBTOUV444MATRIXROW_NEON
+#if !defined(__GNUC__) || defined(__clang__)
+#define HAS_ARGBTOUVMATRIXROW_NEON
+#define HAS_RGBTOUVMATRIXROW_NEON
+#define HAS_RGB565TOUVMATRIXROW_NEON
+#define HAS_ARGB1555TOUVMATRIXROW_NEON
+#define HAS_ARGB4444TOUVMATRIXROW_NEON
+#endif
+#define HAS_ARGBTOYMATRIXROW_NEON
+#define HAS_RGB565TOYMATRIXROW_NEON
+#define HAS_ARGB1555TOYMATRIXROW_NEON
+#define HAS_ARGB4444TOYMATRIXROW_NEON
+#define HAS_RGBTOYMATRIXROW_NEON
#define HAS_AYUVTOVUROW_NEON
#define HAS_AYUVTOYROW_NEON
-#define HAS_BGRATOUVROW_NEON
-#define HAS_BGRATOYROW_NEON
#define HAS_BYTETOFLOATROW_NEON
+#define HAS_CONVERT16TO8ROW_NEON
+#define HAS_CONVERT8TO16ROW_NEON
+#define HAS_CONVERT8TO8ROW_NEON
+#define HAS_COPYROW_16_NEON
#define HAS_COPYROW_NEON
+#define HAS_DETILEROW_16_NEON
+#define HAS_DETILEROW_NEON
+#define HAS_DETILESPLITUVROW_NEON
+#define HAS_DETILETOYUY2_NEON
+#define HAS_DIVIDEROW_16_NEON
#define HAS_HALFFLOATROW_NEON
+#define HAS_HALFMERGEUVROW_NEON
+#define HAS_HALFROW_16TO8_NEON
+#define HAS_HALFWIDTHROW_16TO8_NEON
#define HAS_I400TOARGBROW_NEON
#define HAS_I422ALPHATOARGBROW_NEON
#define HAS_I422TOARGB1555ROW_NEON
@@ -354,11 +755,27 @@ extern "C" {
#define HAS_I422TORGBAROW_NEON
#define HAS_I422TOUYVYROW_NEON
#define HAS_I422TOYUY2ROW_NEON
+#define HAS_I444ALPHATOARGBROW_NEON
#define HAS_I444TOARGBROW_NEON
+#define HAS_I444TORGB24ROW_NEON
+#define HAS_INTERPOLATEROW_16_NEON
+#define HAS_INTERPOLATEROW_NEON
#define HAS_J400TOARGBROW_NEON
+#define HAS_MERGEAR64ROW_NEON
+#define HAS_MERGEARGB16TO8ROW_NEON
+#define HAS_MERGEARGBROW_NEON
+#define HAS_MERGERGBROW_NEON
+#define HAS_MERGEUVROW_16_NEON
#define HAS_MERGEUVROW_NEON
+#define HAS_MERGEXR30ROW_NEON
+#define HAS_MERGEXR64ROW_NEON
+#define HAS_MERGEXRGB16TO8ROW_NEON
+#define HAS_MERGEXRGBROW_NEON
+#define HAS_MIRRORROW_16_NEON
#define HAS_MIRRORROW_NEON
+#define HAS_MIRRORSPLITUVROW_NEON
#define HAS_MIRRORUVROW_NEON
+#define HAS_MULTIPLYROW_16_NEON
#define HAS_NV12TOARGBROW_NEON
#define HAS_NV12TORGB24ROW_NEON
#define HAS_NV12TORGB565ROW_NEON
@@ -367,43 +784,53 @@ extern "C" {
#define HAS_NV21TOYUV24ROW_NEON
#define HAS_RAWTOARGBROW_NEON
#define HAS_RAWTORGB24ROW_NEON
+#define HAS_RAWTORGBAROW_NEON
#define HAS_RAWTOUVROW_NEON
-#define HAS_RAWTOYROW_NEON
#define HAS_RGB24TOARGBROW_NEON
#define HAS_RGB24TOUVROW_NEON
-#define HAS_RGB24TOYROW_NEON
#define HAS_RGB565TOARGBROW_NEON
#define HAS_RGB565TOUVROW_NEON
#define HAS_RGB565TOYROW_NEON
-#define HAS_RGBATOUVROW_NEON
-#define HAS_RGBATOYROW_NEON
#define HAS_SETROW_NEON
+#define HAS_SPLITARGBROW_NEON
#define HAS_SPLITRGBROW_NEON
+#define HAS_SPLITUVROW_16_NEON
#define HAS_SPLITUVROW_NEON
+#define HAS_SPLITXRGBROW_NEON
#define HAS_SWAPUVROW_NEON
+#define HAS_UNPACKMT2T_NEON
#define HAS_UYVYTOARGBROW_NEON
-#define HAS_UYVYTOUV422ROW_NEON
#define HAS_UYVYTOUVROW_NEON
#define HAS_UYVYTOYROW_NEON
#define HAS_YUY2TOARGBROW_NEON
+#define HAS_YUY2TONVUVROW_NEON
#define HAS_YUY2TOUV422ROW_NEON
#define HAS_YUY2TOUVROW_NEON
#define HAS_YUY2TOYROW_NEON
// Effects:
#define HAS_ARGBADDROW_NEON
+#define HAS_ARGBAFFINEROW_NEON
+#define HAS_ARGBALPHABLENDROW_NEON
#define HAS_ARGBATTENUATEROW_NEON
#define HAS_ARGBBLENDROW_NEON
+#define HAS_BLENDPLANEROW_NEON
#define HAS_ARGBCOLORMATRIXROW_NEON
+#define HAS_ARGBCOLORTABLEROW_NEON
#define HAS_ARGBGRAYROW_NEON
+#define HAS_ARGBLUMACOLORTABLEROW_NEON
#define HAS_ARGBMIRRORROW_NEON
#define HAS_ARGBMULTIPLYROW_NEON
+#define HAS_ARGBORROW_NEON
+#define HAS_ARGBPOLYNOMIALROW_NEON
#define HAS_ARGBQUANTIZEROW_NEON
#define HAS_ARGBSEPIAROW_NEON
#define HAS_ARGBSHADEROW_NEON
#define HAS_ARGBSHUFFLEROW_NEON
#define HAS_ARGBSUBTRACTROW_NEON
-#define HAS_INTERPOLATEROW_NEON
+#define HAS_ARGBUNATTENUATEROW_NEON
+#define HAS_RGB24MIRRORROW_NEON
+#define HAS_RGBCOLORTABLEROW_NEON
#define HAS_SOBELROW_NEON
#define HAS_SOBELTOPLANEROW_NEON
#define HAS_SOBELXROW_NEON
@@ -413,170 +840,508 @@ extern "C" {
// The following are available on AArch64 platforms:
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_ABGRTOAR30ROW_NEON
+#define HAS_ARGBTOAR30ROW_NEON
+#define HAS_COMPUTECUMULATIVESUMROW_NEON
+#define HAS_CUMULATIVESUMTOAVERAGEROW_NEON
+#define HAS_I210ALPHATOARGBROW_NEON
+#define HAS_I210TOAR30ROW_NEON
+#define HAS_I210TOARGBROW_NEON
+#define HAS_I212TOAR30ROW_NEON
+#define HAS_I212TOARGBROW_NEON
+#define HAS_I410ALPHATOARGBROW_NEON
+#define HAS_I410TOAR30ROW_NEON
+#define HAS_I410TOARGBROW_NEON
+#define HAS_I422TOAR30ROW_NEON
+#define HAS_P210TOAR30ROW_NEON
+#define HAS_P210TOARGBROW_NEON
+#define HAS_P410TOAR30ROW_NEON
+#define HAS_P410TOARGBROW_NEON
+
+#define HAS_ARGBGRAYROW_NEON_DOTPROD
+#define HAS_ARGBSEPIAROW_NEON_DOTPROD
+#define HAS_ARGBTOYMATRIXROW_NEON_DOTPROD
+#define HAS_RGBTOYMATRIXROW_NEON_DOTPROD
+
+#define HAS_ARGBCOLORMATRIXROW_NEON_I8MM
+#define HAS_ARGBTOUVMATRIXROW_NEON_I8MM
+#define HAS_ARGBTOUV444MATRIXROW_NEON_I8MM
+#endif
+
+// The following are available on AArch64 SVE platforms:
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+#define HAS_ARGB1555TOARGBROW_SVE2
+#define HAS_ARGBALPHABLENDROW_SVE2
+#define HAS_ARGBORROW_SVE2
+#define HAS_ARGBTORAWROW_SVE2
+#define HAS_ARGBTORGB24ROW_SVE2
+#define HAS_ARGBTORGB565DITHERROW_SVE2
+#define HAS_ARGBTORGB565ROW_SVE2
+#define HAS_ARGBTOUVMATRIXROW_SVE2
+#define HAS_ARGBTOYMATRIXROW_SVE2
+#define HAS_AYUVTOUVROW_SVE2
+#define HAS_AYUVTOVUROW_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+// #define HAS_CONVERT16TO8ROW_SVE2
+#define HAS_CONVERT8TO8ROW_SVE2
+#define HAS_COPYROW_SVE2
+#define HAS_DIVIDEROW_16_SVE2
+#define HAS_HALFFLOATROW_SVE2
+#define HAS_HALFMERGEUVROW_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+// #define HAS_HALFROW_16TO8_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+// #define HAS_HALFWIDTHROW_16TO8_SVE2
+#define HAS_I210ALPHATOARGBROW_SVE2
+#define HAS_I210TOAR30ROW_SVE2
+#define HAS_I210TOARGBROW_SVE2
+#define HAS_I212TOAR30ROW_SVE2
+#define HAS_I212TOARGBROW_SVE2
+#define HAS_I400TOARGBROW_SVE2
+#define HAS_I410ALPHATOARGBROW_SVE2
+#define HAS_I410TOAR30ROW_SVE2
+#define HAS_I410TOARGBROW_SVE2
+#define HAS_I422ALPHATOARGBROW_SVE2
+#define HAS_I422TOAR30ROW_SVE2
+#define HAS_I422TOARGB1555ROW_SVE2
+#define HAS_I422TOARGB4444ROW_SVE2
+#define HAS_I422TOARGBROW_SVE2
+#define HAS_I422TORGB24ROW_SVE2
+#define HAS_I422TORGB565ROW_SVE2
+#define HAS_I422TORGBAROW_SVE2
+#define HAS_I444ALPHATOARGBROW_SVE2
+#define HAS_I444TOARGBROW_SVE2
+#define HAS_I444TORGB24ROW_SVE2
+#define HAS_INTERPOLATEROW_SVE2
+#define HAS_MERGEUVROW_SVE2
+#define HAS_NV12TOARGBROW_SVE2
+#define HAS_NV12TORGB24ROW_SVE2
+#define HAS_NV21TOARGBROW_SVE2
+#define HAS_NV21TORGB24ROW_SVE2
+#define HAS_P210TOAR30ROW_SVE2
+#define HAS_P210TOARGBROW_SVE2
+#define HAS_P410TOAR30ROW_SVE2
+#define HAS_P410TOARGBROW_SVE2
+#define HAS_RAWTOARGBROW_SVE2
+#define HAS_RAWTORGB24ROW_SVE2
+#define HAS_RAWTORGBAROW_SVE2
+#define HAS_RGB24TOARGBROW_SVE2
+#define HAS_SPLITUVROW_SVE2
+#define HAS_UYVYTOARGBROW_SVE2
+#define HAS_UYVYTOUVROW_SVE2
+#define HAS_UYVYTOYROW_SVE2
+#define HAS_YUY2TOARGBROW_SVE2
+#define HAS_YUY2TOUVROW_SVE2
+#define HAS_YUY2TOYROW_SVE2
+#endif
+
+// The following are available on AArch64 SME platforms:
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_ARGBALPHABLENDROW_SME
+#define HAS_ARGBMULTIPLYROW_SME
+#define HAS_ARGBORROW_SME
+#define HAS_ARGBTOUVMATRIXROW_SME
+#define HAS_ARGBTOYMATRIXROW_SME
+#define HAS_CONVERT16TO8ROW_SME
+#define HAS_CONVERT8TO16ROW_SME
+#define HAS_CONVERT8TO8ROW_SME
+#define HAS_COPYROW_SME
+#define HAS_HALFROW_16TO8_SME
+#define HAS_HALFWIDTHROW_16TO8_SME
+#define HAS_I210ALPHATOARGBROW_SME
+#define HAS_I210TOAR30ROW_SME
+#define HAS_I210TOARGBROW_SME
+#define HAS_I212TOAR30ROW_SME
+#define HAS_I212TOARGBROW_SME
+#define HAS_I400TOARGBROW_SME
+#define HAS_I410ALPHATOARGBROW_SME
+#define HAS_I410TOAR30ROW_SME
+#define HAS_I410TOARGBROW_SME
+#define HAS_I422ALPHATOARGBROW_SME
+#define HAS_I422TOAR30ROW_SME
+#define HAS_I422TOARGB1555ROW_SME
+#define HAS_I422TOARGB4444ROW_SME
+#define HAS_I422TOARGBROW_SME
+#define HAS_I422TORGB24ROW_SME
+#define HAS_I422TORGB565ROW_SME
+#define HAS_I422TORGBAROW_SME
+#define HAS_I444ALPHATOARGBROW_SME
+#define HAS_I444TOARGBROW_SME
+#define HAS_I444TORGB24ROW_SME
+#define HAS_INTERPOLATEROW_16_SME
+#define HAS_INTERPOLATEROW_SME
+#define HAS_MERGEUVROW_16_SME
+#define HAS_MERGEUVROW_SME
+#define HAS_MULTIPLYROW_16_SME
+#define HAS_NV12TOARGBROW_SME
+#define HAS_NV12TORGB24ROW_SME
+#define HAS_NV21TOARGBROW_SME
+#define HAS_NV21TORGB24ROW_SME
+#define HAS_P210TOAR30ROW_SME
+#define HAS_P210TOARGBROW_SME
+#define HAS_P410TOAR30ROW_SME
+#define HAS_P410TOARGBROW_SME
+#define HAS_UYVYTOARGBROW_SME
+#define HAS_YUY2TOARGBROW_SME
+#endif
+
+// The following are available on AArch64 platforms:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_GAUSSCOL_F32_NEON
+#define HAS_GAUSSROW_F32_NEON
#define HAS_SCALESUMSAMPLES_NEON
#endif
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#define HAS_ABGRTOUVROW_MSA
-#define HAS_ABGRTOYROW_MSA
-#define HAS_ARGB1555TOARGBROW_MSA
-#define HAS_ARGB1555TOUVROW_MSA
-#define HAS_ARGB1555TOYROW_MSA
-#define HAS_ARGB4444TOARGBROW_MSA
-#define HAS_ARGBADDROW_MSA
-#define HAS_ARGBATTENUATEROW_MSA
-#define HAS_ARGBBLENDROW_MSA
-#define HAS_ARGBCOLORMATRIXROW_MSA
-#define HAS_ARGBEXTRACTALPHAROW_MSA
-#define HAS_ARGBGRAYROW_MSA
-#define HAS_ARGBMIRRORROW_MSA
-#define HAS_ARGBMULTIPLYROW_MSA
-#define HAS_ARGBQUANTIZEROW_MSA
-#define HAS_ARGBSEPIAROW_MSA
-#define HAS_ARGBSETROW_MSA
-#define HAS_ARGBSHADEROW_MSA
-#define HAS_ARGBSHUFFLEROW_MSA
-#define HAS_ARGBSUBTRACTROW_MSA
-#define HAS_ARGBTOARGB1555ROW_MSA
-#define HAS_ARGBTOARGB4444ROW_MSA
-#define HAS_ARGBTORAWROW_MSA
-#define HAS_ARGBTORGB24ROW_MSA
-#define HAS_ARGBTORGB565DITHERROW_MSA
-#define HAS_ARGBTORGB565ROW_MSA
-#define HAS_ARGBTOUV444ROW_MSA
-#define HAS_ARGBTOUVJROW_MSA
-#define HAS_ARGBTOUVROW_MSA
-#define HAS_ARGBTOYJROW_MSA
-#define HAS_ARGBTOYROW_MSA
-#define HAS_BGRATOUVROW_MSA
-#define HAS_BGRATOYROW_MSA
-#define HAS_HALFFLOATROW_MSA
-#define HAS_I400TOARGBROW_MSA
-#define HAS_I422ALPHATOARGBROW_MSA
-#define HAS_I422TOARGBROW_MSA
-#define HAS_I422TORGB24ROW_MSA
-#define HAS_I422TORGBAROW_MSA
-#define HAS_I422TOUYVYROW_MSA
-#define HAS_I422TOYUY2ROW_MSA
-#define HAS_I444TOARGBROW_MSA
-#define HAS_INTERPOLATEROW_MSA
-#define HAS_J400TOARGBROW_MSA
-#define HAS_MERGEUVROW_MSA
-#define HAS_MIRRORROW_MSA
-#define HAS_MIRRORUVROW_MSA
-#define HAS_NV12TOARGBROW_MSA
-#define HAS_NV12TORGB565ROW_MSA
-#define HAS_NV21TOARGBROW_MSA
-#define HAS_RAWTOARGBROW_MSA
-#define HAS_RAWTORGB24ROW_MSA
-#define HAS_RAWTOUVROW_MSA
-#define HAS_RAWTOYROW_MSA
-#define HAS_RGB24TOARGBROW_MSA
-#define HAS_RGB24TOUVROW_MSA
-#define HAS_RGB24TOYROW_MSA
-#define HAS_RGB565TOARGBROW_MSA
-#define HAS_RGB565TOUVROW_MSA
-#define HAS_RGB565TOYROW_MSA
-#define HAS_RGBATOUVROW_MSA
-#define HAS_RGBATOYROW_MSA
-#define HAS_SETROW_MSA
-#define HAS_SOBELROW_MSA
-#define HAS_SOBELTOPLANEROW_MSA
-#define HAS_SOBELXROW_MSA
-#define HAS_SOBELXYROW_MSA
-#define HAS_SOBELYROW_MSA
-#define HAS_SPLITUVROW_MSA
-#define HAS_UYVYTOARGBROW_MSA
-#define HAS_UYVYTOUVROW_MSA
-#define HAS_UYVYTOYROW_MSA
-#define HAS_YUY2TOARGBROW_MSA
-#define HAS_YUY2TOUV422ROW_MSA
-#define HAS_YUY2TOUVROW_MSA
-#define HAS_YUY2TOYROW_MSA
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+#define HAS_ARGB1555TOARGBROW_LSX
+#define HAS_ARGB1555TOUVROW_LSX
+#define HAS_ARGB1555TOYROW_LSX
+#define HAS_ARGB4444TOARGBROW_LSX
+#define HAS_ARGBADDROW_LSX
+#define HAS_ARGBALPHABLENDROW_LSX
+#define HAS_ARGBATTENUATEROW_LSX
+#define HAS_ARGBBLENDROW_LSX
+#define HAS_ARGBCOLORMATRIXROW_LSX
+#define HAS_ARGBEXTRACTALPHAROW_LSX
+#define HAS_ARGBGRAYROW_LSX
+#define HAS_ARGBORROW_LSX
+#define HAS_ARGBSEPIAROW_LSX
+#define HAS_ARGBSHADEROW_LSX
+#define HAS_ARGBSHUFFLEROW_LSX
+#define HAS_ARGBSUBTRACTROW_LSX
+#define HAS_ARGBQUANTIZEROW_LSX
+#define HAS_ARGBSETROW_LSX
+#define HAS_ARGBTOARGB1555ROW_LSX
+#define HAS_ARGBTOARGB4444ROW_LSX
+#define HAS_ARGBTORAWROW_LSX
+#define HAS_ARGBTORGB24ROW_LSX
+#define HAS_ARGBTORGB565ROW_LSX
+#define HAS_ARGBTORGB565DITHERROW_LSX
+#define HAS_ARGBMIRRORROW_LSX
+#define HAS_ARGBMULTIPLYROW_LSX
+#define HAS_HALFFLOATROW_LSX
+#define HAS_I400TOARGBROW_LSX
+#define HAS_I444TOARGBROW_LSX
+#define HAS_INTERPOLATEROW_LSX
+#define HAS_I422ALPHATOARGBROW_LSX
+#define HAS_I422TOARGB1555ROW_LSX
+#define HAS_I422TOARGB4444ROW_LSX
+#define HAS_I422TORGB24ROW_LSX
+#define HAS_I422TORGB565ROW_LSX
+#define HAS_I422TORGBAROW_LSX
+#define HAS_I422TOUYVYROW_LSX
+#define HAS_I422TOYUY2ROW_LSX
+#define HAS_J400TOARGBROW_LSX
+#define HAS_MERGEUVROW_LSX
+#define HAS_MIRRORROW_LSX
+#define HAS_MIRRORUVROW_LSX
+#define HAS_MIRRORSPLITUVROW_LSX
+#define HAS_NV12TOARGBROW_LSX
+#define HAS_NV12TORGB565ROW_LSX
+#define HAS_NV21TOARGBROW_LSX
+#define HAS_RAWTOARGBROW_LSX
+#define HAS_RAWTORGB24ROW_LSX
+#define HAS_RAWTOUVROW_LSX
+#define HAS_RGB24TOARGBROW_LSX
+#define HAS_RGB24TOUVROW_LSX
+#define HAS_RGB565TOARGBROW_LSX
+#define HAS_RGB565TOUVROW_LSX
+#define HAS_RGB565TOYROW_LSX
+#define HAS_SETROW_LSX
+#define HAS_SOBELROW_LSX
+#define HAS_SOBELTOPLANEROW_LSX
+#define HAS_SOBELXYROW_LSX
+#define HAS_SPLITUVROW_LSX
+#define HAS_UYVYTOARGBROW_LSX
+#define HAS_UYVYTOUV422ROW_LSX
+#define HAS_UYVYTOUVROW_LSX
+#define HAS_UYVYTOYROW_LSX
+#define HAS_YUY2TOARGBROW_LSX
+#define HAS_YUY2TOUVROW_LSX
+#define HAS_YUY2TOUV422ROW_LSX
+#define HAS_YUY2TOYROW_LSX
+#define HAS_ARGBTOYMATRIXROW_LSX
+#define HAS_COPYROW_LSX
+#define HAS_RGBTOYMATRIXROW_LSX
+#endif
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+#define HAS_I422TOARGBROW_LSX
+#endif
+
+#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
+#define HAS_ARGB1555TOARGBROW_LASX
+#define HAS_ARGB1555TOUVROW_LASX
+#define HAS_ARGB1555TOYROW_LASX
+#define HAS_ARGB4444TOARGBROW_LASX
+#define HAS_ARGBADDROW_LASX
+#define HAS_ARGBALPHABLENDROW_LASX
+#define HAS_ARGBATTENUATEROW_LASX
+#define HAS_ARGBBLENDROW_LASX
+#define HAS_ARGBCOLORMATRIXROW_LASX
+#define HAS_ARGBEXTRACTALPHAROW_LASX
+#define HAS_ARGBGRAYROW_LASX
+#define HAS_ARGBMIRRORROW_LASX
+#define HAS_ARGBMULTIPLYROW_LASX
+#define HAS_ARGBORROW_LASX
+#define HAS_ARGBQUANTIZEROW_LASX
+#define HAS_ARGBSEPIAROW_LASX
+#define HAS_ARGBSETROW_LASX
+#define HAS_ARGBSHADEROW_LASX
+#define HAS_ARGBSHUFFLEROW_LASX
+#define HAS_ARGBSUBTRACTROW_LASX
+#define HAS_ARGBTOARGB1555ROW_LASX
+#define HAS_ARGBTOARGB4444ROW_LASX
+#define HAS_ARGBTORAWROW_LASX
+#define HAS_ARGBTORGB24ROW_LASX
+#define HAS_ARGBTORGB565DITHERROW_LASX
+#define HAS_ARGBTORGB565ROW_LASX
+#define HAS_ARGBTOYMATRIXROW_LASX
+#define HAS_COPYROW_LASX
+#define HAS_HALFFLOATROW_LASX
+#define HAS_I400TOARGBROW_LASX
+#define HAS_I422ALPHATOARGBROW_LASX
+#define HAS_I422TOARGB1555ROW_LASX
+#define HAS_I422TOARGB4444ROW_LASX
+#define HAS_I422TOARGBROW_LASX
+#define HAS_I422TORGB24ROW_LASX
+#define HAS_I422TORGB565ROW_LASX
+#define HAS_I422TORGBAROW_LASX
+#define HAS_I422TOUYVYROW_LASX
+#define HAS_I422TOYUY2ROW_LASX
+#define HAS_I444TOARGBROW_LASX
+#define HAS_INTERPOLATEROW_LASX
+#define HAS_J400TOARGBROW_LASX
+#define HAS_MERGEUVROW_LASX
+#define HAS_MIRRORROW_LASX
+#define HAS_MIRRORSPLITUVROW_LASX
+#define HAS_MIRRORUVROW_LASX
+#define HAS_NV12TOARGBROW_LASX
+#define HAS_NV12TORGB565ROW_LASX
+#define HAS_NV21TOARGBROW_LASX
+#define HAS_RAWTOARGBROW_LASX
+#define HAS_RAWTORGB24ROW_LASX
+#define HAS_RAWTOUVROW_LASX
+#define HAS_RGB24TOARGBROW_LASX
+#define HAS_RGB24TOUVROW_LASX
+#define HAS_RGB565TOARGBROW_LASX
+#define HAS_RGB565TOUVROW_LASX
+#define HAS_RGB565TOYROW_LASX
+#define HAS_RGBTOYMATRIXROW_LASX
+#define HAS_SETROW_LASX
+#define HAS_SOBELROW_LASX
+#define HAS_SOBELTOPLANEROW_LASX
+#define HAS_SOBELXYROW_LASX
+#define HAS_SPLITUVROW_LASX
+#define HAS_UYVYTOARGBROW_LASX
+#define HAS_UYVYTOUV422ROW_LASX
+#define HAS_UYVYTOUVROW_LASX
+#define HAS_UYVYTOYROW_LASX
+#define HAS_YUY2TOARGBROW_LASX
+#define HAS_YUY2TOUV422ROW_LASX
+#define HAS_YUY2TOUVROW_LASX
+#define HAS_YUY2TOYROW_LASX
+#endif
+
+#if !defined(LIBYUV_DISABLE_RVV) && \
+ (defined(__riscv_vector) || defined(LIBYUV_RVV))
+#define HAS_AR64TOARGBROW_RVV
+#define HAS_ARGBCOPYYTOALPHAROW_RVV
+#define HAS_ARGBEXTRACTALPHAROW_RVV
+#define HAS_ARGBMULTIPLYROW_RVV
+#define HAS_ARGBTOAR64ROW_RVV
+#define HAS_ARGBMIRRORROW_RVV
+#define HAS_ARGBSETROW_RVV
+#define HAS_ARGBSHUFFLEROW_RVV
+#define HAS_ARGBTOUV444MATRIXROW_RVV
+#define HAS_ARGBTOUVMATRIXROW_RVV
+#define HAS_ARGBTOYMATRIXROW_RVV
+#define HAS_CONVERT16TO8ROW_RVV
+#define HAS_CONVERT8TO16ROW_RVV
+#define HAS_COPYROW_RVV
+#define HAS_HALFROW_16TO8_RVV
+#define HAS_HALFWIDTHROW_16TO8_RVV
+#define HAS_I422TOAR30ROW_RVV
+#define HAS_INTERPOLATEROW_RVV
+#define HAS_MULTIPLYROW_16_RVV
+#define HAS_RGBTOUV444MATRIXROW_RVV
+#define HAS_RGBTOUVMATRIXROW_RVV
+#define HAS_RGBTOYMATRIXROW_RVV
+#define HAS_SPLITARGBROW_RVV
+#define HAS_SPLITRGBROW_RVV
+#define HAS_SPLITUVROW_RVV
+#define HAS_SPLITXRGBROW_RVV
+#define HAS_SWAPUVROW_RVV
+#define HAS_AB64TOARGBROW_RVV
+#define HAS_AR64TOAB64ROW_RVV
+#define HAS_ARGBALPHABLENDROW_RVV
+#define HAS_ARGBATTENUATEROW_RVV
+#define HAS_ARGBBLENDROW_RVV
+#define HAS_ARGBORROW_RVV
+#define HAS_ARGBTOAB64ROW_RVV
+#define HAS_ARGBTOABGRROW_RVV
+#define HAS_ARGBTOBGRAROW_RVV
+#define HAS_ARGBTORAWROW_RVV
+#define HAS_ARGBTORGB24ROW_RVV
+#define HAS_ARGBTORGBAROW_RVV
+#define HAS_BLENDPLANEROW_RVV
+#define HAS_I400TOARGBROW_RVV
+#define HAS_I422ALPHATOARGBROW_RVV
+#define HAS_I422TOARGBROW_RVV
+#define HAS_I422TORGB24ROW_RVV
+#define HAS_I422TORGBAROW_RVV
+#define HAS_I444ALPHATOARGBROW_RVV
+#define HAS_I444TOARGBROW_RVV
+#define HAS_I444TORGB24ROW_RVV
+#define HAS_J400TOARGBROW_RVV
+#define HAS_MERGEARGBROW_RVV
+#define HAS_MERGERGBROW_RVV
+#define HAS_MERGEUVROW_RVV
+#define HAS_MERGEXRGBROW_RVV
+#define HAS_MIRRORUVROW_RVV
+#define HAS_NV12TOARGBROW_RVV
+#define HAS_NV12TORGB24ROW_RVV
+#define HAS_NV21TOARGBROW_RVV
+#define HAS_NV21TORGB24ROW_RVV
+#define HAS_RAWTOARGBROW_RVV
+#define HAS_RAWTORGB24ROW_RVV
+#define HAS_RAWTORGBAROW_RVV
+#define HAS_RGB24TOARGBROW_RVV
+#define HAS_RGBATOARGBROW_RVV
+#endif
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#define HAS_ARGB1555TOARGBROW_WASMSIMD
+#define HAS_ARGB1555TOUVMATRIXROW_WASMSIMD
+#define HAS_ARGB1555TOYMATRIXROW_WASMSIMD
+#define HAS_ARGB4444TOARGBROW_WASMSIMD
+#define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD
+#define HAS_ARGB4444TOYMATRIXROW_WASMSIMD
+#define HAS_ARGBALPHABLENDROW_WASMSIMD
+#define HAS_ARGBBLENDROW_WASMSIMD
+#define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
+#define HAS_ARGBEXTRACTALPHAROW_WASMSIMD
+#define HAS_ARGBMIRRORROW_WASMSIMD
+#define HAS_ARGBORROW_WASMSIMD
+#define HAS_ARGBSHUFFLEROW_WASMSIMD
+#define HAS_ARGBTOARGB1555ROW_WASMSIMD
+#define HAS_ARGBTOARGB4444ROW_WASMSIMD
+#define HAS_ARGBTORGB565DITHERROW_WASMSIMD
+#define HAS_ARGBTORGB565ROW_WASMSIMD
+#define HAS_ARGBTOUV444MATRIXROW_WASMSIMD
+#define HAS_ARGBTOUVMATRIXROW_WASMSIMD
+#define HAS_ARGBTOYMATRIXROW_WASMSIMD
+#define HAS_BLENDPLANEROW_WASMSIMD
+#define HAS_CONVERT8TO16ROW_WASMSIMD
+#define HAS_I422TOAR30ROW_WASMSIMD
+#define HAS_I422TOARGB1555ROW_WASMSIMD
+#define HAS_I422TOARGB4444ROW_WASMSIMD
+#define HAS_I422TOARGBROW_WASMSIMD
+#define HAS_I422TORGB24ROW_WASMSIMD
+#define HAS_I422TORGB565ROW_WASMSIMD
+#define HAS_INTERPOLATEROW_16_WASMSIMD
+#define HAS_INTERPOLATEROW_WASMSIMD
+#define HAS_J400TOARGBROW_WASMSIMD
+#define HAS_MERGEUVROW_WASMSIMD
+#define HAS_MIRRORROW_WASMSIMD
+#define HAS_MIRRORSPLITUVROW_WASMSIMD
+#define HAS_MIRRORUVROW_WASMSIMD
+#define HAS_MULTIPLYROW_16_WASMSIMD
+#define HAS_NV12TOARGBROW_WASMSIMD
+#define HAS_NV12TORGB565ROW_WASMSIMD
+#define HAS_NV21TOARGBROW_WASMSIMD
+#define HAS_RAWTOARGBROW_WASMSIMD
+#define HAS_RAWTORGB24ROW_WASMSIMD
+#define HAS_RGB24MIRRORROW_WASMSIMD
+#define HAS_RGB24TOARGBROW_WASMSIMD
+#define HAS_RGB565TOARGBROW_WASMSIMD
+#define HAS_RGB565TOUVMATRIXROW_WASMSIMD
+#define HAS_RGB565TOYMATRIXROW_WASMSIMD
+#define HAS_RGBTOUV444MATRIXROW_WASMSIMD
+#define HAS_RGBTOUVMATRIXROW_WASMSIMD
+#define HAS_RGBTOYMATRIXROW_WASMSIMD
+#define HAS_SWAPUVROW_WASMSIMD
#endif
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-#define HAS_ABGRTOUVROW_MMI
-#define HAS_ABGRTOYROW_MMI
-#define HAS_ARGB1555TOARGBROW_MMI
-#define HAS_ARGB1555TOUVROW_MMI
-#define HAS_ARGB1555TOYROW_MMI
-#define HAS_ARGB4444TOARGBROW_MMI
-#define HAS_ARGB4444TOUVROW_MMI
-#define HAS_ARGB4444TOYROW_MMI
-#define HAS_ARGBADDROW_MMI
-#define HAS_ARGBATTENUATEROW_MMI
-#define HAS_ARGBBLENDROW_MMI
-#define HAS_ARGBCOLORMATRIXROW_MMI
-#define HAS_ARGBCOPYALPHAROW_MMI
-#define HAS_ARGBCOPYYTOALPHAROW_MMI
-#define HAS_ARGBEXTRACTALPHAROW_MMI
-#define HAS_ARGBGRAYROW_MMI
-#define HAS_ARGBMIRRORROW_MMI
-#define HAS_ARGBMULTIPLYROW_MMI
-#define HAS_ARGBSEPIAROW_MMI
-#define HAS_ARGBSHADEROW_MMI
-#define HAS_ARGBSHUFFLEROW_MMI
-#define HAS_ARGBSUBTRACTROW_MMI
-#define HAS_ARGBTOARGB1555ROW_MMI
-#define HAS_ARGBTOARGB4444ROW_MMI
-#define HAS_ARGBTORAWROW_MMI
-#define HAS_ARGBTORGB24ROW_MMI
-#define HAS_ARGBTORGB565DITHERROW_MMI
-#define HAS_ARGBTORGB565ROW_MMI
-#define HAS_ARGBTOUV444ROW_MMI
-#define HAS_ARGBTOUVJROW_MMI
-#define HAS_ARGBTOUVROW_MMI
-#define HAS_ARGBTOYJROW_MMI
-#define HAS_ARGBTOYROW_MMI
-#define HAS_BGRATOUVROW_MMI
-#define HAS_BGRATOYROW_MMI
-#define HAS_BLENDPLANEROW_MMI
-#define HAS_COMPUTECUMULATIVESUMROW_MMI
-#define HAS_CUMULATIVESUMTOAVERAGEROW_MMI
-#define HAS_HALFFLOATROW_MMI
-#define HAS_I400TOARGBROW_MMI
-#define HAS_I422TOUYVYROW_MMI
-#define HAS_I422TOYUY2ROW_MMI
-#define HAS_INTERPOLATEROW_MMI
-#define HAS_J400TOARGBROW_MMI
-#define HAS_MERGERGBROW_MMI
-#define HAS_MERGEUVROW_MMI
-#define HAS_MIRRORROW_MMI
-#define HAS_MIRRORUVROW_MMI
-#define HAS_RAWTOARGBROW_MMI
-#define HAS_RAWTORGB24ROW_MMI
-#define HAS_RAWTOUVROW_MMI
-#define HAS_RAWTOYROW_MMI
-#define HAS_RGB24TOARGBROW_MMI
-#define HAS_RGB24TOUVROW_MMI
-#define HAS_RGB24TOYROW_MMI
-#define HAS_RGB565TOARGBROW_MMI
-#define HAS_RGB565TOUVROW_MMI
-#define HAS_RGB565TOYROW_MMI
-#define HAS_RGBATOUVROW_MMI
-#define HAS_RGBATOYROW_MMI
-#define HAS_SOBELROW_MMI
-#define HAS_SOBELTOPLANEROW_MMI
-#define HAS_SOBELXROW_MMI
-#define HAS_SOBELXYROW_MMI
-#define HAS_SOBELYROW_MMI
-#define HAS_SPLITRGBROW_MMI
-#define HAS_SPLITUVROW_MMI
-#define HAS_UYVYTOUVROW_MMI
-#define HAS_UYVYTOYROW_MMI
-#define HAS_YUY2TOUV422ROW_MMI
-#define HAS_YUY2TOUVROW_MMI
-#define HAS_YUY2TOYROW_MMI
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#define HAS_AB64TOARGBROW_HVX
+#define HAS_AR64TOAB64ROW_HVX
+#define HAS_AR64TOARGBROW_HVX
+#define HAS_ARGBADDROW_HVX
+#define HAS_ARGBALPHABLENDROW_HVX
+#define HAS_ARGBATTENUATEROW_HVX
+#define HAS_ARGBBLENDROW_HVX
+#define HAS_ARGBCOPYALPHAROW_HVX
+#define HAS_ARGBCOPYYTOALPHAROW_HVX
+#define HAS_ARGBEXTRACTALPHAROW_HVX
+#define HAS_ARGBMIRRORROW_HVX
+#define HAS_ARGBMULTIPLYROW_HVX
+#define HAS_ARGBORROW_HVX
+#define HAS_ARGBSETROW_HVX
+#define HAS_ARGBSUBTRACTROW_HVX
+#define HAS_ARGBTOAB64ROW_HVX
+#define HAS_ARGBTOABGRROW_HVX
+#define HAS_ARGBTOAR64ROW_HVX
+#define HAS_ARGBTOBGRAROW_HVX
+#define HAS_ARGBTORGBAROW_HVX
+#define HAS_ARGBTOUV444MATRIXROW_HVX
+#define HAS_ARGBTOUVMATRIXROW_HVX
+#define HAS_ARGBTOYMATRIXROW_HVX
+#define HAS_BLENDPLANEROW_HVX
+#define HAS_CONVERT16TO8ROW_HVX
+#define HAS_CONVERT8TO16ROW_HVX
+#define HAS_CONVERT8TO8ROW_HVX
+#define HAS_COPYROW_16_HVX
+#define HAS_COPYROW_HVX
+#define HAS_DIVIDEROW_16_HVX
+#define HAS_HALFROW_16TO8_HVX
+#define HAS_I400TOARGBROW_HVX
+#define HAS_I422ALPHATOARGBROW_HVX
+#define HAS_I422TOARGBROW_HVX
+#define HAS_I422TORGBAROW_HVX
+#define HAS_I444ALPHATOARGBROW_HVX
+#define HAS_I444TOARGBROW_HVX
+#define HAS_INTERPOLATEROW_16_HVX
+#define HAS_INTERPOLATEROW_HVX
+#define HAS_J400TOARGBROW_HVX
+#define HAS_MERGEARGBROW_HVX
+#define HAS_MERGEUVROW_16_HVX
+#define HAS_MERGEUVROW_HVX
+#define HAS_MERGEXRGBROW_HVX
+#define HAS_MIRRORROW_16_HVX
+#define HAS_MIRRORROW_HVX
+#define HAS_MIRRORSPLITUVROW_HVX
+#define HAS_MIRRORUVROW_HVX
+#define HAS_MULTIPLYROW_16_HVX
+#define HAS_NV12TOARGBROW_HVX
+#define HAS_NV21TOARGBROW_HVX
+#define HAS_RGBATOARGBROW_HVX
+#define HAS_SETROW_HVX
+#define HAS_SPLITARGBROW_HVX
+#define HAS_SPLITUVROW_16_HVX
+#define HAS_SPLITUVROW_HVX
+#define HAS_SPLITXRGBROW_HVX
+#define HAS_SWAPUVROW_HVX
+#define HAS_UYVYTOUV422ROW_HVX
+#define HAS_UYVYTOUVROW_HVX
+#define HAS_UYVYTOYROW_HVX
+#define HAS_YUY2TOUV422ROW_HVX
+#define HAS_YUY2TOUVROW_HVX
+#define HAS_YUY2TOYROW_HVX
#endif
+#undef SIMD_ALIGNED
#if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__)
#if defined(VISUALC_HAS_AVX2)
#define SIMD_ALIGNED(var) __declspec(align(32)) var
#else
#define SIMD_ALIGNED(var) __declspec(align(16)) var
#endif
+#define LIBYUV_NOINLINE __declspec(noinline)
typedef __declspec(align(16)) int16_t vec16[8];
typedef __declspec(align(16)) int32_t vec32[4];
+typedef __declspec(align(16)) float vecf32[4];
typedef __declspec(align(16)) int8_t vec8[16];
typedef __declspec(align(16)) uint16_t uvec16[8];
typedef __declspec(align(16)) uint32_t uvec32[4];
@@ -594,8 +1359,10 @@ typedef __declspec(align(32)) uint8_t ulvec8[32];
#else
#define SIMD_ALIGNED(var) var __attribute__((aligned(16)))
#endif
+#define LIBYUV_NOINLINE __attribute__((noinline))
typedef int16_t __attribute__((vector_size(16))) vec16;
typedef int32_t __attribute__((vector_size(16))) vec32;
+typedef float __attribute__((vector_size(16))) vecf32;
typedef int8_t __attribute__((vector_size(16))) vec8;
typedef uint16_t __attribute__((vector_size(16))) uvec16;
typedef uint32_t __attribute__((vector_size(16))) uvec32;
@@ -608,8 +1375,10 @@ typedef uint32_t __attribute__((vector_size(32))) ulvec32;
typedef uint8_t __attribute__((vector_size(32))) ulvec8;
#else
#define SIMD_ALIGNED(var) var
+#define LIBYUV_NOINLINE
typedef int16_t vec16[8];
typedef int32_t vec32[4];
+typedef float vecf32[4];
typedef int8_t vec8[16];
typedef uint16_t uvec16[8];
typedef uint32_t uvec32[4];
@@ -622,65 +1391,66 @@ typedef uint32_t ulvec32[8];
typedef uint8_t ulvec8[32];
#endif
-#if defined(__aarch64__)
-// This struct is for Arm64 color conversion.
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+// This struct is for ARM and RISC-V color conversion.
struct YuvConstants {
- uvec16 kUVToRB;
- uvec16 kUVToRB2;
- uvec16 kUVToG;
- uvec16 kUVToG2;
- vec16 kUVBiasBGR;
- vec32 kYToRgb;
+ uvec8 kUVCoeff;
+ vec16 kRGBCoeffBias;
};
-#elif defined(__arm__)
-// This struct is for ArmV7 color conversion.
-struct YuvConstants {
- uvec8 kUVToRB;
- uvec8 kUVToG;
- vec16 kUVBiasBGR;
- vec32 kYToRgb;
+struct ArgbConstants {
+ uvec8 kRGBToY;
+ vec8 kRGBToU;
+ vec8 kRGBToV;
+ uvec16 kAddY;
+ uvec16 kAddUV;
};
#else
// This struct is for Intel color conversion.
struct YuvConstants {
- int8_t kUVToB[32];
- int8_t kUVToG[32];
- int8_t kUVToR[32];
- int16_t kUVBiasB[16];
- int16_t kUVBiasG[16];
- int16_t kUVBiasR[16];
+ uint8_t kUVToB[32];
+ uint8_t kUVToG[32];
+ uint8_t kUVToR[32];
int16_t kYToRgb[16];
+ int16_t kYBiasToRgb[16];
+};
+struct ArgbConstants {
+ uint8_t kRGBToY[32];
+ int8_t kRGBToU[32];
+ int8_t kRGBToV[32];
+ uint16_t kAddY[16];
+ uint16_t kAddUV[16];
};
// Offsets into YuvConstants structure
#define KUVTOB 0
#define KUVTOG 32
#define KUVTOR 64
-#define KUVBIASB 96
-#define KUVBIASG 128
-#define KUVBIASR 160
-#define KYTORGB 192
-#endif
-
-// Conversion matrix for YUV to RGB
-extern const struct YuvConstants SIMD_ALIGNED(kYuvI601Constants); // BT.601
-extern const struct YuvConstants SIMD_ALIGNED(kYuvJPEGConstants); // JPeg
-extern const struct YuvConstants SIMD_ALIGNED(kYuvH709Constants); // BT.709
+#define KYTORGB 96
+#define KYBIASTORGB 128
-// Conversion matrix for YVU to BGR
-extern const struct YuvConstants SIMD_ALIGNED(kYvuI601Constants); // BT.601
-extern const struct YuvConstants SIMD_ALIGNED(kYvuJPEGConstants); // JPeg
-extern const struct YuvConstants SIMD_ALIGNED(kYvuH709Constants); // BT.709
+#endif
-#define IS_ALIGNED(p, a) (!((uintptr_t)(p) & ((a)-1)))
+#define IS_ALIGNED(p, a) (!((uintptr_t)(p) & ((a) - 1)))
-#define align_buffer_64(var, size) \
- uint8_t* var##_mem = (uint8_t*)(malloc((size) + 63)); /* NOLINT */ \
- uint8_t* var = (uint8_t*)(((intptr_t)(var##_mem) + 63) & ~63) /* NOLINT */
+#ifdef LIBYUV_MSAN
+#define align_buffer_64(var, size) \
+ size_t var##_mem_size = (size); /* NOLINT */ \
+ void* var##_mem = (var##_mem_size > SIZE_MAX - 63) \
+ ? NULL \
+ : calloc(1, var##_mem_size + 63); /* NOLINT */ \
+ uint8_t* var = (uint8_t*)(((intptr_t)var##_mem + 63) & ~63) /* NOLINT */
+#else
+#define align_buffer_64(var, size) \
+ size_t var##_mem_size = (size); /* NOLINT */ \
+ void* var##_mem = (var##_mem_size > SIZE_MAX - 63) \
+ ? NULL \
+ : malloc(var##_mem_size + 63); /* NOLINT */ \
+ uint8_t* var = (uint8_t*)(((intptr_t)var##_mem + 63) & ~63) /* NOLINT */
+#endif
#define free_aligned_buffer_64(var) \
free(var##_mem); \
- var = 0
+ var = NULL
#if defined(__APPLE__) || defined(__x86_64__) || defined(__llvm__)
#define OMITFP
@@ -725,259 +1495,756 @@ extern const struct YuvConstants SIMD_ALIGNED(kYvuH709Constants); // BT.709
#define IACA_UD_BYTES __asm__ __volatile__("\n\t .byte 0x0F, 0x0B");
#else /* Visual C */
-#define IACA_UD_BYTES \
- { __asm _emit 0x0F __asm _emit 0x0B }
+#define IACA_UD_BYTES {__asm _emit 0x0F __asm _emit 0x0B}
#define IACA_SSC_MARK(x) \
- { __asm mov ebx, x __asm _emit 0x64 __asm _emit 0x67 __asm _emit 0x90 }
+ {__asm mov ebx, x __asm _emit 0x64 __asm _emit 0x67 __asm _emit 0x90}
#define IACA_VC64_START __writegsbyte(111, 111);
#define IACA_VC64_END __writegsbyte(222, 222);
#endif
-#define IACA_START \
- { \
- IACA_UD_BYTES \
- IACA_SSC_MARK(111) \
- }
-#define IACA_END \
- { \
- IACA_SSC_MARK(222) \
- IACA_UD_BYTES \
- }
+#define IACA_START {IACA_UD_BYTES IACA_SSC_MARK(111)}
+#define IACA_END {IACA_SSC_MARK(222) IACA_UD_BYTES}
-void I444ToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- const uint8_t* src_a,
- uint8_t* dst_argb,
+void I210AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGBRow_NEON(const uint8_t* src_y,
+void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGBARow_NEON(const uint8_t* src_y,
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
- uint8_t* dst_rgba,
+ uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_NEON(const uint8_t* src_y,
+void I444ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB565Row_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB1555Row_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb1555,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB4444Row_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb4444,
- const struct YuvConstants* yuvconstants,
- int width);
-void NV12ToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_argb,
+void I444ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToRGB565Row_NEON(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width);
-void NV21ToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_argb,
+void I210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToRGB24Row_NEON(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_rgb24,
- const struct YuvConstants* yuvconstants,
- int width);
-void NV21ToRGB24Row_NEON(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_rgb24,
- const struct YuvConstants* yuvconstants,
- int width);
-void NV21ToYUV24Row_NEON(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_yuv24,
- int width);
-void YUY2ToARGBRow_NEON(const uint8_t* src_yuy2,
- uint8_t* dst_argb,
+void I210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-void UYVYToARGBRow_NEON(const uint8_t* src_uyvy,
- uint8_t* dst_argb,
+void I210ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-void I444ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
+void I410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-
-void I422ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
+void I210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGBARow_MSA(const uint8_t* src_y,
+void I422ToAR30Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_SME(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422AlphaToARGBRow_MSA(const uint8_t* src_y,
+void I444AlphaToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_SME(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
const uint8_t* src_a,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_MSA(const uint8_t* src_y,
+void I422ToRGBARow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
- uint8_t* dst_argb,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB565Row_MSA(const uint8_t* src_y,
+void I422ToRGB565Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_SME(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb565,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB4444Row_MSA(const uint8_t* src_y,
+void I422ToARGB1555Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_SME(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
- uint8_t* dst_argb4444,
+ uint8_t* dst_argb1555,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB1555Row_MSA(const uint8_t* src_y,
+void I422ToARGB4444Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_SME(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
- uint8_t* dst_argb1555,
+ uint8_t* dst_argb4444,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToARGBRow_MSA(const uint8_t* src_y,
+void NV12ToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_SME(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToRGB565Row_MSA(const uint8_t* src_y,
+void NV12ToRGB565Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_uv,
- uint8_t* dst_rgb565,
+ uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void NV21ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_vu,
+void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToRGB24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToYUV24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width);
+void YUY2ToARGBRow_NEON(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_SME(const uint8_t* src_yuy2,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void YUY2ToARGBRow_MSA(const uint8_t* src_yuy2,
+void UYVYToARGBRow_NEON(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_SME(const uint8_t* src_uyvy,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void UYVYToARGBRow_MSA(const uint8_t* src_uyvy,
+void I444ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-
-void ARGBToYRow_AVX2(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ABGRToYRow_AVX2(const uint8_t* src_abgr, uint8_t* dst_y, int width);
-void ABGRToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYJRow_AVX2(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYJRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYJRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void BGRAToYRow_SSSE3(const uint8_t* src_bgra, uint8_t* dst_y, int width);
-void ABGRToYRow_SSSE3(const uint8_t* src_abgr, uint8_t* dst_y, int width);
-void RGBAToYRow_SSSE3(const uint8_t* src_rgba, uint8_t* dst_y, int width);
-void RGB24ToYRow_SSSE3(const uint8_t* src_rgb24, uint8_t* dst_y, int width);
-void RAWToYRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_y, int width);
-void ARGBToYRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYJRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width);
-void ARGBToYJRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width);
-void ARGBToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width);
-void ARGBToYJRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width);
-void ARGBToUV444Row_NEON(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUV444Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_MSA(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUV444Row_MMI(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
+void I444AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
int width);
-void ARGBToUVRow_MMI(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
+void I422ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
int width);
-void BGRAToUVRow_NEON(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_NEON(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_NEON(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
+void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void I422ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void NV12ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_LSX(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_LASX(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_Any_NEON(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_Any_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_Any_NEON(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_Any_NEON_I8MM(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_Any_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_SME(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_SME(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
uint8_t* dst_u,
@@ -988,6 +2255,7 @@ void RAWToUVRow_NEON(const uint8_t* src_raw,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+
void RGB565ToUVRow_NEON(const uint8_t* src_rgb565,
int src_stride_rgb565,
uint8_t* dst_u,
@@ -1003,96 +2271,48 @@ void ARGB4444ToUVRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGBToUVJRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToUVRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB24ToUVRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RAWToUVRow_MSA(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB565ToUVRow_MSA(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB1555ToUVRow_MSA(const uint8_t* src_argb1555,
+
+void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555,
int src_stride_argb1555,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGBToUVJRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToUVRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB24ToUVRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
+void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB565ToUVRow_LSX(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB565ToUVRow_LASX(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB24ToUVRow_LSX(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void RAWToUVRow_MMI(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB565ToUVRow_MMI(const uint8_t* src_rgb565,
- int src_stride_rgb565,
+void RGB24ToUVRow_LASX(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGB1555ToUVRow_MMI(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB4444ToUVRow_MMI(const uint8_t* src_argb4444,
- int src_stride_argb4444,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToYRow_NEON(const uint8_t* src_bgra, uint8_t* dst_y, int width);
-void ABGRToYRow_NEON(const uint8_t* src_abgr, uint8_t* dst_y, int width);
-void RGBAToYRow_NEON(const uint8_t* src_rgba, uint8_t* dst_y, int width);
-void RGB24ToYRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_y, int width);
-void RAWToYRow_NEON(const uint8_t* src_raw, uint8_t* dst_y, int width);
+void RAWToUVRow_LSX(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RAWToUVRow_LASX(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
void RGB565ToYRow_NEON(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
void ARGB1555ToYRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_y,
@@ -1100,430 +2320,718 @@ void ARGB1555ToYRow_NEON(const uint8_t* src_argb1555,
void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_y,
int width);
-void BGRAToYRow_MSA(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ABGRToYRow_MSA(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGBAToYRow_MSA(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB24ToYRow_MSA(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RAWToYRow_MSA(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB565ToYRow_MSA(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
-void ARGB1555ToYRow_MSA(const uint8_t* src_argb1555, uint8_t* dst_y, int width);
-void BGRAToYRow_MMI(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ABGRToYRow_MMI(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGBAToYRow_MMI(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB24ToYRow_MMI(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RAWToYRow_MMI(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB565ToYRow_MMI(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
-void ARGB1555ToYRow_MMI(const uint8_t* src_argb1555, uint8_t* dst_y, int width);
-void ARGB4444ToYRow_MMI(const uint8_t* src_argb4444, uint8_t* dst_y, int width);
-
-void ARGBToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ARGBToYJRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void BGRAToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void ABGRToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGBAToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB24ToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RAWToYRow_C(const uint8_t* src_argb, uint8_t* dst_y, int width);
-void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
-void ARGB1555ToYRow_C(const uint8_t* src_argb1555, uint8_t* dst_y, int width);
-void ARGB4444ToYRow_C(const uint8_t* src_argb4444, uint8_t* dst_y, int width);
-void ARGBToYRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYJRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void BGRAToYRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ABGRToYRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGBAToYRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB24ToYRow_Any_SSSE3(const uint8_t* src_rgb24, uint8_t* dst_y, int width);
-void RAWToYRow_Any_SSSE3(const uint8_t* src_raw, uint8_t* dst_y, int width);
-void ARGBToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYJRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void BGRAToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ABGRToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGBAToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB24ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB565ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGB1555ToYRow_Any_NEON(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGB4444ToYRow_Any_NEON(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void BGRAToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ABGRToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGBAToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYJRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB24ToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB565ToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGB1555ToYRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void BGRAToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ABGRToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGBAToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYJRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB24ToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RGB565ToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGB1555ToYRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGB4444ToYRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToUVRow_AVX2(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_AVX2(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_AVX2(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_SSSE3(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_SSSE3(const uint8_t* src_argb,
- int src_stride_argb,
+void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555, uint8_t* dst_y, int width);
+
+void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width);
+void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
+void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
+
+void ARGBToYMatrixRow_C(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_C(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_C(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_C(const uint8_t* src_rgb,
+ int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void BGRAToUVRow_SSSE3(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_SSSE3(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_SSSE3(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_Any_AVX2(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_Any_AVX2(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_Any_AVX2(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_C(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUVRow_Any_SSSE3(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_C(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_C(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_C(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_C(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_C(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_C(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_Any_SSSE3(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToUVRow_Any_SSSE3(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_AVX2(const uint8_t* src_rgb,
+ int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ABGRToUVRow_Any_SSSE3(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_Any_AVX2(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_C(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_AVX2(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_Any_AVX2(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_Any_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_AVX2(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_Any_AVX2(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_Any_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_AVX2(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_Any_AVX2(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_AVX2(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_Any_AVX2(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_NEON(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_NEON(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_NEON(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_NEON(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_NEON(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_NEON(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_Any_NEON(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_Any_NEON(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_Any_NEON(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_Any_NEON(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_Any_NEON(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_Any_NEON(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+
+void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_Any_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_Any_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
+ int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void RGBAToUVRow_Any_SSSE3(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_Any_NEON(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_Any_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_LSX(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_LASX(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
+ int src_stride_argb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUV444Row_Any_NEON(const uint8_t* src_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
+ int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUV444Row_Any_MSA(const uint8_t* src_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void ARGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_Any_LSX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToYMatrixRow_Any_LASX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_LSX(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_LASX(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+
+void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb,
+ int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
- int width);
-void ARGBToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_Any_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+
+// Deprecated.
+void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
+void ARGB1555ToYRow_C(const uint8_t* src_argb1555, uint8_t* dst_y, int width);
+void ARGB4444ToYRow_C(const uint8_t* src_argb4444, uint8_t* dst_y, int width);
+void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
+void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
int width);
-void ARGBToUV444Row_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
+void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
int width);
-void ARGBToUVJRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
+void RGB565ToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGB1555ToYRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGB4444ToYRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void RGB565ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGB1555ToYRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGB4444ToYRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void RGB565ToYRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGB1555ToYRow_Any_LSX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void RGB565ToYRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGB1555ToYRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
void RGB24ToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void RAWToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+
void RGB565ToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void ARGB1555ToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void ARGB4444ToUVRow_Any_NEON(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB1555ToUVRow_Any_LSX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB1555ToUVRow_Any_LASX(const uint8_t* src_ptr,
int src_stride_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGBToUVJRow_Any_MSA(const uint8_t* src_ptr,
+void RGB565ToUVRow_Any_LSX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB565ToUVRow_Any_LASX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB24ToUVRow_Any_LSX(const uint8_t* src_ptr,
int src_stride_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void BGRAToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_Any_MSA(const uint8_t* src_ptr,
+void RGB24ToUVRow_Any_LASX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RAWToUVRow_Any_LSX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RAWToUVRow_Any_LASX(const uint8_t* src_ptr,
int src_stride_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void RGB24ToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RAWToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
+// Deprecated.
+void RGB565ToUVRow_C(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB1555ToUVRow_C(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB4444ToUVRow_C(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void RGB565ToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB1555ToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
+void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void BGRAToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB24ToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RAWToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB565ToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB1555ToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB4444ToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGBToUVJRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void BGRAToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ABGRToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGBAToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB24ToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RAWToUVRow_C(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void RGB565ToUVRow_C(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB1555ToUVRow_C(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void ARGB4444ToUVRow_C(const uint8_t* src_argb4444,
- int src_stride_argb4444,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-
-void ARGBToUV444Row_SSSE3(const uint8_t* src_argb,
+void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGBToUV444Row_Any_SSSE3(const uint8_t* src_ptr,
+void RGB565ToUVRow_Any_AVX2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB1555ToUVRow_Any_AVX2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGB4444ToUVRow_Any_AVX2(const uint8_t* src_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void ARGBToUV444Row_C(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-
+void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width);
-void MirrorRow_MSA(const uint8_t* src, uint8_t* dst, int width);
-void MirrorRow_MMI(const uint8_t* src, uint8_t* dst, int width);
+void MirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width);
+void MirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_C(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void MirrorRow_Any_SSE2(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void MirrorRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void MirrorRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void MirrorUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void MirrorUVRow_SSSE3(const uint8_t* src,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void MirrorUVRow_NEON(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void MirrorUVRow_MSA(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void MirrorUVRow_MMI(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void MirrorUVRow_C(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
+void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_AVX2(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_SSSE3(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_Any_AVX2(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_Any_SSSE3(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_NEON(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_LSX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_LASX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MirrorSplitUVRow_C(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
+void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width);
+void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width);
+void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width);
+void MirrorRow_16_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
+void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBMirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width);
-void ARGBMirrorRow_MSA(const uint8_t* src, uint8_t* dst, int width);
-void ARGBMirrorRow_MMI(const uint8_t* src, uint8_t* dst, int width);
+void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width);
+void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width);
+void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width);
+void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_C(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
@@ -1534,8 +3042,33 @@ void ARGBMirrorRow_Any_SSE2(const uint8_t* src_ptr,
void ARGBMirrorRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBMirrorRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBMirrorRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBMirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width);
+void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width);
+void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width);
+void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width);
+void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width);
+void RGB24MirrorRow_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB24MirrorRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB24MirrorRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void SplitUVRow_C(const uint8_t* src_uv,
uint8_t* dst_u,
@@ -1545,6 +3078,10 @@ void SplitUVRow_SSE2(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void SplitUVRow_AVX512BW(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void SplitUVRow_AVX2(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
@@ -1553,11 +3090,19 @@ void SplitUVRow_NEON(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void SplitUVRow_MSA(const uint8_t* src_uv,
+void SplitUVRow_SVE2(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void SplitUVRow_LSX(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void SplitUVRow_MMI(const uint8_t* src_uv,
+void SplitUVRow_LASX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void SplitUVRow_RVV(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
@@ -1573,15 +3118,112 @@ void SplitUVRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void SplitUVRow_Any_MSA(const uint8_t* src_ptr,
+void SplitUVRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void SplitUVRow_Any_MMI(const uint8_t* src_ptr,
+void SplitUVRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void DetileRow_C(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width);
+void DetileRow_NEON(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width);
+void DetileRow_Any_NEON(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width);
+void DetileRow_AVX2(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width);
+void DetileRow_Any_AVX2(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width);
+void DetileRow_16_C(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width);
+void DetileRow_16_NEON(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width);
+void DetileRow_16_Any_NEON(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width);
+void DetileRow_16_AVX2(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width);
+void DetileRow_16_Any_AVX2(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width);
+void DetileSplitUVRow_C(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-
+void DetileSplitUVRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void DetileSplitUVRow_Any_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void DetileSplitUVRow_NEON(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void DetileSplitUVRow_Any_NEON(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void DetileToYUY2_C(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width);
+void DetileToYUY2_AVX2(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width);
+void DetileToYUY2_Any_AVX2(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width);
+void DetileToYUY2_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width);
+void DetileToYUY2_Any_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width);
+void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size);
+void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size);
+void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size);
void MergeUVRow_C(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
@@ -1594,15 +3236,31 @@ void MergeUVRow_AVX2(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width);
+void MergeUVRow_AVX512BW(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
void MergeUVRow_NEON(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width);
-void MergeUVRow_MSA(const uint8_t* src_u,
+void MergeUVRow_SVE2(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
+void MergeUVRow_SME(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
+void MergeUVRow_LSX(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width);
-void MergeUVRow_MMI(const uint8_t* src_u,
+void MergeUVRow_LASX(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
+void MergeUVRow_RVV(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width);
@@ -1618,14 +3276,67 @@ void MergeUVRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void MergeUVRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void MergeUVRow_Any_MMI(const uint8_t* y_buf,
+void MergeUVRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
+void MergeUVRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+
+void HalfMergeUVRow_C(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+
+void HalfMergeUVRow_NEON(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+
+void HalfMergeUVRow_SVE2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+
+void HalfMergeUVRow_SSSE3(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+
+void HalfMergeUVRow_AVX2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+void HalfMergeUVRow_AVX512BW(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+void HalfMergeUVRow_Any_SSSE3(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
+void HalfMergeUVRow_Any_AVX2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width);
void SplitRGBRow_C(const uint8_t* src_rgb,
uint8_t* dst_r,
@@ -1637,12 +3348,22 @@ void SplitRGBRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_g,
uint8_t* dst_b,
int width);
+void SplitRGBRow_SSE41(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitRGBRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
void SplitRGBRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_r,
uint8_t* dst_g,
uint8_t* dst_b,
int width);
-void SplitRGBRow_MMI(const uint8_t* src_rgb,
+void SplitRGBRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_r,
uint8_t* dst_g,
uint8_t* dst_b,
@@ -1652,93 +3373,599 @@ void SplitRGBRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_g,
uint8_t* dst_b,
int width);
+void SplitRGBRow_Any_SSE41(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitRGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
void SplitRGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_r,
uint8_t* dst_g,
uint8_t* dst_b,
int width);
-void SplitRGBRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_r,
- uint8_t* dst_g,
- uint8_t* dst_b,
- int width);
void MergeRGBRow_C(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
uint8_t* dst_rgb,
int width);
-void MergeRGBRow_SSSE3(const uint8_t* src_r,
- const uint8_t* src_g,
- const uint8_t* src_b,
- uint8_t* dst_rgb,
- int width);
+void MergeRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width);
void MergeRGBRow_NEON(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
uint8_t* dst_rgb,
int width);
-void MergeRGBRow_MMI(const uint8_t* src_r,
+void MergeRGBRow_RVV(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
uint8_t* dst_rgb,
int width);
-void MergeRGBRow_Any_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- int width);
+void MergeRGBRow_AVX512VBMI(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width);
+void MergeRGBRow_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
void MergeRGBRow_Any_NEON(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
uint8_t* dst_rgb,
int width);
-void MergeRGBRow_Any_MMI(const uint8_t* src_r,
- const uint8_t* src_g,
- const uint8_t* src_b,
- uint8_t* dst_rgb,
- int width);
-
-void MergeUVRow_16_C(const uint16_t* src_u,
- const uint16_t* src_v,
+void MergeARGBRow_C(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void MergeARGBRow_SSE2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void MergeARGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void MergeARGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void MergeARGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void MergeARGBRow_Any_SSE2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ int width);
+void MergeARGBRow_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ int width);
+void MergeARGBRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ int width);
+void SplitARGBRow_C(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_RVV(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_Any_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void SplitARGBRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void MergeXRGBRow_C(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void MergeXRGBRow_SSE2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void MergeXRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void MergeXRGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void MergeXRGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void MergeXRGBRow_Any_SSE2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+void MergeXRGBRow_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+void MergeXRGBRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+void SplitXRGBRow_C(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_RVV(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_Any_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitXRGBRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+
+void MergeXR30Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width);
+void MergeAR64Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeARGB16To8Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR64Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeXRGB16To8Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR30Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width);
+void MergeXR30Row_10_AVX512BW(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width);
+void MergeAR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeARGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR30Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width);
+void MergeXR30Row_10_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int /* depth */,
+ int width);
+void MergeAR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeARGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width);
+void MergeXRGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width);
+void MergeXR30Row_Any_AVX2(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeAR64Row_Any_AVX2(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ const uint16_t* a_buf,
+ uint16_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXR64Row_Any_AVX2(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint16_t* dst_ptr,
+ int depth,
+ int width);
+void MergeARGB16To8Row_Any_AVX2(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXRGB16To8Row_Any_AVX2(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXR30Row_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXR30Row_10_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeAR64Row_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ const uint16_t* a_buf,
+ uint16_t* dst_ptr,
+ int depth,
+ int width);
+void MergeARGB16To8Row_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXR64Row_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint16_t* dst_ptr,
+ int depth,
+ int width);
+void MergeXRGB16To8Row_Any_NEON(const uint16_t* r_buf,
+ const uint16_t* g_buf,
+ const uint16_t* b_buf,
+ uint8_t* dst_ptr,
+ int depth,
+ int width);
+
+void MergeUVRow_16_C(const uint16_t* src_u,
+ const uint16_t* src_v,
uint16_t* dst_uv,
- int scale, /* 64 for 10 bit */
+ int depth,
int width);
void MergeUVRow_16_AVX2(const uint16_t* src_u,
const uint16_t* src_v,
uint16_t* dst_uv,
- int scale,
+ int depth,
+ int width);
+void MergeUVRow_16_AVX512BW(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
+void MergeUVRow_16_Any_AVX2(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
+void MergeUVRow_16_NEON(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
int width);
+void MergeUVRow_16_Any_NEON(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
+void MergeUVRow_16_SME(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
+
+void SplitUVRow_16_C(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
+void SplitUVRow_16_AVX2(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
+void SplitUVRow_16_Any_AVX2(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
+void SplitUVRow_16_NEON(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
+void SplitUVRow_16_Any_NEON(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
-void MultiplyRow_16_AVX2(const uint16_t* src_y,
- uint16_t* dst_y,
- int scale,
- int width);
void MultiplyRow_16_C(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
+void MultiplyRow_16_AVX2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void MultiplyRow_16_Any_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void MultiplyRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void MultiplyRow_16_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+void MultiplyRow_16_SME(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void MultiplyRow_16_RVV(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
-void Convert8To16Row_C(const uint8_t* src_y,
+void DivideRow_16_C(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void DivideRow_16_AVX2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void DivideRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void DivideRow_16_Any_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+void DivideRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void DivideRow_16_SVE2(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
+void DivideRow_16_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+
+void Convert8To16Row_C(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
void Convert8To16Row_SSE2(const uint8_t* src_y,
uint16_t* dst_y,
- int scale,
+ int bits,
int width);
void Convert8To16Row_AVX2(const uint8_t* src_y,
uint16_t* dst_y,
- int scale,
+ int bits,
int width);
+void Convert8To16Row_AVX512BW(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
void Convert8To16Row_Any_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
- int scale,
+ int bits,
int width);
void Convert8To16Row_Any_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
- int scale,
+ int bits,
int width);
+void Convert8To16Row_NEON(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
+void Convert8To16Row_Any_NEON(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
+void Convert8To16Row_SME(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
+void Convert8To16Row_RVV(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
void Convert16To8Row_C(const uint16_t* src_y,
uint8_t* dst_y,
@@ -1752,6 +3979,10 @@ void Convert16To8Row_AVX2(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width);
+void Convert16To8Row_AVX512BW(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
void Convert16To8Row_Any_SSSE3(const uint16_t* src_ptr,
uint8_t* dst_ptr,
int scale,
@@ -1760,32 +3991,252 @@ void Convert16To8Row_Any_AVX2(const uint16_t* src_ptr,
uint8_t* dst_ptr,
int scale,
int width);
+void Convert16To8Row_NEON(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
+void Convert16To8Row_Any_NEON(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void Convert16To8Row_SME(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
+void Convert16To8Row_SVE2(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
+void Convert16To8Row_RVV(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
-void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width);
-void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width);
-void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width);
-void CopyRow_MIPS(const uint8_t* src, uint8_t* dst, int count);
+void HalfRow_C(const uint8_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int width);
+void HalfRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int width);
+void HalfRow_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int width);
+
+void HalfRow_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint16_t* dst_uv,
+ int width);
+void HalfRow_16_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint16_t* dst_uv,
+ int width);
+void HalfRow_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int width);
+
+void HalfRow_16To8_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_SSSE3(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_Any_SSSE3(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfRow_16To8_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_Any_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfRow_16To8_SVE2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_SME(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfRow_16To8_RVV(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+
+void HalfWidthRow_16To8_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_Any_SSSE3(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_Any_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_SME(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+
+void Convert8To8Row_C(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_SVE2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_SME(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_AVX2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_AVX512BW(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int bias,
+ int width);
+
+void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int count);
+void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_C(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count);
+void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int count);
+void CopyRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int width);
+void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count);
+void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width);
void ARGBCopyAlphaRow_C(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBCopyAlphaRow_MMI(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyAlphaRow_Any_SSE2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void ARGBCopyAlphaRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBCopyAlphaRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
+void ARGBCopyAlphaRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGBExtractAlphaRow_C(const uint8_t* src_argb, uint8_t* dst_a, int width);
void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
@@ -1794,13 +4245,22 @@ void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_a,
int width);
+void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width);
void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
uint8_t* dst_a,
int width);
-void ARGBExtractAlphaRow_MSA(const uint8_t* src_argb,
+void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb,
uint8_t* dst_a,
int width);
-void ARGBExtractAlphaRow_MMI(const uint8_t* src_argb,
+void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width);
+void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width);
+void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
uint8_t* dst_a,
int width);
void ARGBExtractAlphaRow_Any_SSE2(const uint8_t* src_ptr,
@@ -1812,41 +4272,57 @@ void ARGBExtractAlphaRow_Any_AVX2(const uint8_t* src_ptr,
void ARGBExtractAlphaRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBExtractAlphaRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBExtractAlphaRow_Any_MMI(const uint8_t* src_ptr,
+void ARGBExtractAlphaRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void ARGBExtractAlphaRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBExtractAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGBCopyYToAlphaRow_C(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBCopyYToAlphaRow_MMI(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width);
void ARGBCopyYToAlphaRow_Any_SSE2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void ARGBCopyYToAlphaRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBCopyYToAlphaRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
+void ARGBCopyYToAlphaRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void SetRow_C(uint8_t* dst, uint8_t v8, int width);
-void SetRow_MSA(uint8_t* dst, uint8_t v8, int width);
-void SetRow_X86(uint8_t* dst, uint8_t v8, int width);
void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width);
+void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width);
void SetRow_NEON(uint8_t* dst, uint8_t v8, int width);
-void SetRow_Any_X86(uint8_t* dst_ptr, uint8_t v32, int width);
+void SetRow_LSX(uint8_t* dst, uint8_t v8, int width);
+void SetRow_LASX(uint8_t* dst, uint8_t v8, int width);
+void SetRow_Any_AVX2(uint8_t* dst_ptr, uint8_t v32, int width);
void SetRow_Any_NEON(uint8_t* dst_ptr, uint8_t v32, int width);
+void SetRow_Any_LSX(uint8_t* dst_ptr, uint8_t v32, int width);
+void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width);
void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width);
-void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width);
+void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width);
+void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width);
+void ARGBSetRow_Any_AVX2(uint8_t* dst_ptr, uint32_t v32, int width);
void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width);
-void ARGBSetRow_MSA(uint8_t* dst_argb, uint32_t v32, int width);
-void ARGBSetRow_Any_MSA(uint8_t* dst_ptr, uint32_t v32, int width);
+void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width);
+void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width);
+void ARGBSetRow_Any_LSX(uint8_t* dst_ptr, uint32_t v32, int width);
+void ARGBSetRow_Any_LASX(uint8_t* dst_ptr, uint32_t v32, int width);
// ARGBShufflers for BGRAToARGB etc.
void ARGBShuffleRow_C(const uint8_t* src_argb,
@@ -1861,18 +4337,26 @@ void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
+void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width);
void ARGBShuffleRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
-void ARGBShuffleRow_MSA(const uint8_t* src_argb,
+void ARGBShuffleRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
-void ARGBShuffleRow_MMI(const uint8_t* src_argb,
+void ARGBShuffleRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
+void ARGBShuffleRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width);
void ARGBShuffleRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
@@ -1885,29 +4369,46 @@ void ARGBShuffleRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
int width);
-void ARGBShuffleRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- const uint8_t* param,
- int width);
-void ARGBShuffleRow_Any_MMI(const uint8_t* src_ptr,
+void ARGBShuffleRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
int width);
+void ARGBShuffleRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const uint8_t* param,
+ int width);
void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width);
void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
+void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ int width);
+void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
-void RGB565ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void ARGB1555ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void ARGB4444ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
+void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width);
+void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width);
+
+// Deprecated.
void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width);
void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
+void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width);
void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width);
@@ -1915,43 +4416,66 @@ void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width);
-void RGB24ToARGBRow_MSA(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
-void RGB24ToARGBRow_MMI(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
+void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
+void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
+void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
+void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width);
-void RAWToARGBRow_MSA(const uint8_t* src_raw, uint8_t* dst_argb, int width);
-void RAWToARGBRow_MMI(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
+void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
+void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
-void RAWToRGB24Row_MSA(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
-void RAWToRGB24Row_MMI(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width);
void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width);
-void RGB565ToARGBRow_MSA(const uint8_t* src_rgb565,
- uint8_t* dst_argb,
- int width);
-void RGB565ToARGBRow_MMI(const uint8_t* src_rgb565,
+void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width);
+void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width);
void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
-void ARGB1555ToARGBRow_MSA(const uint8_t* src_argb1555,
- uint8_t* dst_argb,
- int width);
-void ARGB1555ToARGBRow_MMI(const uint8_t* src_argb1555,
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width);
+void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
+void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width);
void ARGB4444ToARGBRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width);
-void ARGB4444ToARGBRow_MSA(const uint8_t* src_argb4444,
- uint8_t* dst_argb,
- int width);
-void ARGB4444ToARGBRow_MMI(const uint8_t* src_argb4444,
+void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width);
+void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width);
void RGB24ToARGBRow_C(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RAWToARGBRow_C(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToRGBARow_C(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
+void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgba,
+ int width);
void RAWToRGB24Row_C(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RGB565ToARGBRow_C(const uint8_t* src_rgb565, uint8_t* dst_argb, int width);
void ARGB1555ToARGBRow_C(const uint8_t* src_argb1555,
@@ -1961,9 +4485,42 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width);
void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width);
+void AR30ToARGBRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_argb, int width);
+void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_argb,
+ int width);
void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width);
+void AR30ToABGRRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_abgr, int width);
+void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_abgr,
+ int width);
void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width);
void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
+void AR30ToAB30Row_AVX2(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
+void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_ab30,
+ int width);
+void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width);
+void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width);
+void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
+void AR30ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AR30ToABGRRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AR30ToAB30Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AR30ToABGRRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AR30ToAB30Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
@@ -1971,19 +4528,23 @@ void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
void RAWToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void RAWToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RGB24ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
+void RAWToRGBARow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void RAWToRGB24Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
-void RGB565ToARGBRow_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGB1555ToARGBRow_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGB4444ToARGBRow_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
void RGB565ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@@ -1997,76 +4558,86 @@ void ARGB4444ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
void RGB24ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void RGB24ToARGBRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void RGB24ToARGBRow_Any_MMI(const uint8_t* src_ptr,
+void RGB24ToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void RGB24ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RAWToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToARGBRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToARGBRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RAWToRGBARow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RAWToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RAWToARGBRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RAWToRGB24Row_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void RAWToRGB24Row_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void RAWToRGB24Row_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RAWToRGB24Row_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RAWToRGB24Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RAWToRGB24Row_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RGB565ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void RGB565ToARGBRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void RGB565ToARGBRow_Any_MMI(const uint8_t* src_ptr,
+void RGB565ToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void RGB565ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGB1555ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGB1555ToARGBRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGB1555ToARGBRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
void ARGB4444ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-
-void ARGB4444ToARGBRow_Any_MSA(const uint8_t* src_ptr,
+void ARGB1555ToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGB4444ToARGBRow_Any_MMI(const uint8_t* src_ptr,
+void ARGB1555ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void ARGB4444ToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void ARGB4444ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width);
void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width);
-void ARGBToRGB565Row_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBToARGB1555Row_SSE2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBToARGB4444Row_SSE2(const uint8_t* src, uint8_t* dst, int width);
+
void ABGRToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width);
void ARGBToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width);
void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width);
-void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width);
+void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
+void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
+// Deprecated.
void ARGBToRGB565DitherRow_C(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width);
-void ARGBToRGB565DitherRow_SSE2(const uint8_t* src,
- uint8_t* dst,
- const uint32_t dither4,
- int width);
+
void ARGBToRGB565DitherRow_AVX2(const uint8_t* src,
uint8_t* dst,
- const uint32_t dither4,
+ uint32_t dither4,
int width);
+void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width);
void ARGBToRGB565Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width);
void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width);
@@ -2074,15 +4645,24 @@ void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width);
void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width);
+void ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width);
+void ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width);
+void ARGBToRGB24Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width);
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width);
+void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_raw, int width);
void ARGBToRGB565Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width);
+void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb565,
+ int width);
void ARGBToARGB1555Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb1555,
int width);
@@ -2091,36 +4671,48 @@ void ARGBToARGB4444Row_NEON(const uint8_t* src_argb,
int width);
void ARGBToRGB565DitherRow_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width);
-void ARGBToRGB24Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToRAWRow_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToRGB565Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToARGB1555Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width);
-void ARGBToARGB4444Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width);
-void ARGBToRGB565DitherRow_MSA(const uint8_t* src_argb,
+void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width);
+void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width);
+void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width);
-void ARGBToRGB24Row_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToRAWRow_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToRGB565Row_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
-void ARGBToARGB1555Row_MMI(const uint8_t* src_argb,
+void ARGBToRGB24Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRGB24Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRGB565Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToARGB1555Row_LSX(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width);
-void ARGBToARGB4444Row_MMI(const uint8_t* src_argb,
+void ARGBToARGB1555Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width);
+void ARGBToARGB4444Row_LSX(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width);
-void ARGBToRGB565DitherRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- const uint32_t dither4,
- int width);
+void ARGBToARGB4444Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width);
+
+void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width);
+void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width);
+void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width);
+void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
+void ARGBToRGB24Row_RVV(const uint8_t* src_argb, uint8_t* dst_rgb24, int width);
+void ARGBToABGRRow_C(const uint8_t* src_argb, uint8_t* dst_abgr, int width);
+void ARGBToBGRARow_C(const uint8_t* src_argb, uint8_t* dst_bgra, int width);
void ARGBToRGBARow_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
void ARGBToRGB24Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
void ARGBToRAWRow_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
@@ -2130,23 +4722,96 @@ void ARGBToARGB4444Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
void ABGRToAR30Row_C(const uint8_t* src_abgr, uint8_t* dst_ar30, int width);
void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width);
-void J400ToARGBRow_SSE2(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void ARGBToAR64Row_C(const uint8_t* src_argb, uint16_t* dst_ar64, int width);
+void ARGBToAB64Row_C(const uint8_t* src_argb, uint16_t* dst_ab64, int width);
+void AR64ToARGBRow_C(const uint16_t* src_ar64, uint8_t* dst_argb, int width);
+void AB64ToARGBRow_C(const uint16_t* src_ab64, uint8_t* dst_argb, int width);
+void AR64ToAB64Row_C(const uint16_t* src_ar64, uint16_t* dst_ab64, int width);
+void RGBAToARGBRow_C(const uint8_t* src_rgba, uint8_t* dst_argb, int width);
+void AR64ShuffleRow_C(const uint8_t* src_ar64,
+ uint8_t* dst_ar64,
+ const uint8_t* shuffler,
+ int width);
+void ARGBToAR64Row_SSSE3(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width);
+void ARGBToAB64Row_SSSE3(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width);
+void AR64ToARGBRow_SSSE3(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width);
+void AB64ToARGBRow_SSSE3(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width);
+void ARGBToAR64Row_AVX2(const uint8_t* src_argb, uint16_t* dst_ar64, int width);
+void ARGBToAB64Row_AVX2(const uint8_t* src_argb, uint16_t* dst_ab64, int width);
+void AR64ToARGBRow_AVX2(const uint16_t* src_ar64, uint8_t* dst_argb, int width);
+void AB64ToARGBRow_AVX2(const uint16_t* src_ab64, uint8_t* dst_argb, int width);
+void ARGBToAR64Row_NEON(const uint8_t* src_argb, uint16_t* dst_ar64, int width);
+void ARGBToAB64Row_NEON(const uint8_t* src_argb, uint16_t* dst_ab64, int width);
+void AR64ToARGBRow_NEON(const uint16_t* src_ar64, uint8_t* dst_argb, int width);
+void AB64ToARGBRow_NEON(const uint16_t* src_ab64, uint8_t* dst_argb, int width);
+void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width);
+void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width);
+void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width);
+void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width);
+void AR64ToAB64Row_RVV(const uint16_t* src_ar64, uint16_t* dst_ab64, int width);
+void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width);
+void ARGBToAR64Row_Any_SSSE3(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void ARGBToAB64Row_Any_SSSE3(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void AR64ToARGBRow_Any_SSSE3(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AB64ToARGBRow_Any_SSSE3(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToAR64Row_Any_AVX2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void ARGBToAB64Row_Any_AVX2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void AR64ToARGBRow_Any_AVX2(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AB64ToARGBRow_Any_AVX2(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToAR64Row_Any_NEON(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void ARGBToAB64Row_Any_NEON(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int width);
+void AR64ToARGBRow_Any_NEON(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void AB64ToARGBRow_Any_NEON(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+
+void J400ToARGBRow_AVX512BW(const uint8_t* src_y, uint8_t* dst_argb, int width);
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width);
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width);
-void J400ToARGBRow_MSA(const uint8_t* src_y, uint8_t* dst_argb, int width);
-void J400ToARGBRow_MMI(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width);
void J400ToARGBRow_C(const uint8_t* src_y, uint8_t* dst_argb, int width);
-void J400ToARGBRow_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
void J400ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void J400ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void J400ToARGBRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void J400ToARGBRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void J400ToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void J400ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void I444ToARGBRow_C(const uint8_t* src_y,
const uint8_t* src_u,
@@ -2154,6 +4819,12 @@ void I444ToARGBRow_C(const uint8_t* src_y,
uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
+void I444ToRGB24Row_C(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_C(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -2178,12 +4849,57 @@ void I210ToARGBRow_C(const uint16_t* src_y,
uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
-void I422AlphaToARGBRow_C(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- const uint8_t* src_a,
- uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants,
+void I212ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_C(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_C(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
int width);
void NV12ToARGBRow_C(const uint8_t* src_y,
const uint8_t* src_uv,
@@ -2222,6 +4938,27 @@ void UYVYToARGBRow_C(const uint8_t* src_uyvy,
uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants,
int width);
+void P210ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
void I422ToRGBARow_C(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -2258,12 +4995,85 @@ void I422ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGBARow_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
- uint8_t* dst_argb,
+ uint8_t* dst_rgba,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+// Deprecated.
+void I422ToRGB565Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444ToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2276,18 +5086,30 @@ void I444ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I444ToARGBRow_SSSE3(const uint8_t* y_buf,
+void I444ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToRGB24Row_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
- uint8_t* dst_argb,
+ uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void I444ToARGBRow_AVX2(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width);
+void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2313,6 +5135,44 @@ void I210ToARGBRow_SSSE3(const uint16_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I212ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToAR30Row_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2325,12 +5185,121 @@ void I210ToARGBRow_AVX2(const uint16_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I210ToAR30Row_AVX2(const uint16_t* y_buf,
const uint16_t* u_buf,
const uint16_t* v_buf,
uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width);
+void I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2345,6 +5314,13 @@ void I422AlphaToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_argb,
@@ -2355,6 +5331,11 @@ void NV12ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToRGB24Row_SSSE3(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb24,
@@ -2365,30 +5346,50 @@ void NV21ToRGB24Row_SSSE3(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToRGB565Row_SSSE3(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width);
+
void NV12ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void NV12ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV21ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
-void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_yuv24,
- int width);
+void NV21ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToRGB565Row_AVX2(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb565,
const struct YuvConstants* yuvconstants,
int width);
+void NV21ToYUV24Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width);
+void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width);
+void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width);
+
void NV21ToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* vu_buf,
uint8_t* dst_argb,
@@ -2399,6 +5400,11 @@ void NV21ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void YUY2ToARGBRow_SSSE3(const uint8_t* yuy2_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
@@ -2411,52 +5417,87 @@ void YUY2ToARGBRow_AVX2(const uint8_t* yuy2_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void UYVYToARGBRow_AVX2(const uint8_t* uyvy_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGBARow_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_rgba,
+void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void P210ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB4444Row_SSSE3(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb4444,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb4444,
+void P410ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB1555Row_SSSE3(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb1555,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB1555Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb1555,
+void P410ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB565Row_SSSE3(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToRGB565Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width);
+void P210ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void I422ToRGBARow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
void I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2469,6 +5510,18 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2487,12 +5540,24 @@ void I444ToARGBRow_Any_SSSE3(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I444ToRGB24Row_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444ToARGBRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I444ToRGB24Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2517,6 +5582,44 @@ void I210ToARGBRow_Any_SSSE3(const uint16_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I212ToAR30Row_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToAR30Row_Any_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2535,6 +5638,58 @@ void I210ToAR30Row_Any_AVX2(const uint16_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I212ToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422AlphaToARGBRow_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2589,20 +5744,15 @@ void NV21ToRGB24Row_Any_AVX2(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void NV21ToYUV24Row_Any_AVX2(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_yuv24,
- int width);
-void NV12ToRGB565Row_Any_SSSE3(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void NV12ToRGB565Row_Any_AVX2(const uint8_t* y_buf,
+void NV21ToYUV24Row_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
int width);
+void NV21ToYUV24Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+
void YUY2ToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
@@ -2619,39 +5769,85 @@ void UYVYToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void P210ToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_Any_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_Any_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGBARow_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB4444Row_Any_SSSE3(const uint8_t* y_buf,
+
+void I422ToRGB24Row_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB4444Row_Any_AVX2(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB1555Row_Any_SSSE3(const uint8_t* y_buf,
+void I422ToARGB4444Row_Any_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB1555Row_Any_AVX2(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToRGB565Row_Any_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
+void NV12ToRGB565Row_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
@@ -2661,55 +5857,119 @@ void I422ToRGB565Row_Any_AVX2(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_Any_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
+void I422ToARGB1555Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_Any_AVX2(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I400ToARGBRow_C(const uint8_t* src_y, uint8_t* rgb_buf, int width);
-void I400ToARGBRow_SSE2(const uint8_t* y_buf, uint8_t* dst_argb, int width);
-void I400ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, int width);
-void I400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width);
-void I400ToARGBRow_MSA(const uint8_t* src_y, uint8_t* dst_argb, int width);
-void I400ToARGBRow_MMI(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void I400ToARGBRow_C(const uint8_t* src_y,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_SSE2(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_AVX2(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_NEON(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_SVE2(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_SME(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_LSX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_LASX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_RVV(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I400ToARGBRow_Any_SSE2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
+ const struct YuvConstants* param,
int width);
void I400ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
+ const struct YuvConstants* param,
int width);
void I400ToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
+ const struct YuvConstants* param,
+ int width);
+void I400ToARGBRow_Any_LSX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
int width);
-void I400ToARGBRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void I400ToARGBRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
// ARGB preattenuated alpha blend.
-void ARGBBlendRow_SSSE3(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width);
-void ARGBBlendRow_NEON(const uint8_t* src_argb0,
+void ARGBBlendRow_AVX2(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBBlendRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBBlendRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBBlendRow_MSA(const uint8_t* src_argb0,
+void ARGBBlendRow_LSX(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBBlendRow_MMI(const uint8_t* src_argb0,
+void ARGBBlendRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBBlendRow_Any_LSX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBBlendRow_Any_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBBlendRow_RVV(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBBlendRow_C(const uint8_t* src_argb0,
+void ARGBBlendRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2735,16 +5995,26 @@ void BlendPlaneRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
int width);
-void BlendPlaneRow_MMI(const uint8_t* src0,
+void BlendPlaneRow_AVX512BW(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width);
+void BlendPlaneRow_NEON(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width);
+void BlendPlaneRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+void BlendPlaneRow_RVV(const uint8_t* src0,
const uint8_t* src1,
const uint8_t* alpha,
uint8_t* dst,
int width);
-void BlendPlaneRow_Any_MMI(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- int width);
void BlendPlaneRow_C(const uint8_t* src0,
const uint8_t* src1,
const uint8_t* alpha,
@@ -2753,11 +6023,11 @@ void BlendPlaneRow_C(const uint8_t* src0,
// ARGB multiply images. Same API as Blend, but these require
// pointer and width alignment for SSE2.
-void ARGBMultiplyRow_C(const uint8_t* src_argb0,
+void ARGBMultiplyRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBMultiplyRow_SSE2(const uint8_t* src_argb0,
+void ARGBMultiplyRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2765,7 +6035,7 @@ void ARGBMultiplyRow_Any_SSE2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBMultiplyRow_AVX2(const uint8_t* src_argb0,
+void ARGBMultiplyRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2773,7 +6043,11 @@ void ARGBMultiplyRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBMultiplyRow_NEON(const uint8_t* src_argb0,
+void ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBMultiplyRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2781,29 +6055,37 @@ void ARGBMultiplyRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBMultiplyRow_MSA(const uint8_t* src_argb0,
+void ARGBMultiplyRow_SME(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBMultiplyRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void ARGBMultiplyRow_MMI(const uint8_t* src_argb0,
+void ARGBMultiplyRow_LSX(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBMultiplyRow_Any_MMI(const uint8_t* y_buf,
+void ARGBMultiplyRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBMultiplyRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
+void ARGBMultiplyRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBMultiplyRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
// ARGB add images.
-void ARGBAddRow_C(const uint8_t* src_argb0,
+void ARGBAddRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBAddRow_SSE2(const uint8_t* src_argb0,
+void ARGBAddRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2811,7 +6093,7 @@ void ARGBAddRow_Any_SSE2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBAddRow_AVX2(const uint8_t* src_argb0,
+void ARGBAddRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2819,7 +6101,7 @@ void ARGBAddRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBAddRow_NEON(const uint8_t* src_argb0,
+void ARGBAddRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2827,30 +6109,30 @@ void ARGBAddRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBAddRow_MSA(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width);
-void ARGBAddRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void ARGBAddRow_MMI(const uint8_t* src_argb0,
+void ARGBAddRow_LSX(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBAddRow_Any_MMI(const uint8_t* y_buf,
+void ARGBAddRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAddRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
+void ARGBAddRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
// ARGB subtract images. Same API as Blend, but these require
// pointer and width alignment for SSE2.
-void ARGBSubtractRow_C(const uint8_t* src_argb0,
+void ARGBSubtractRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBSubtractRow_SSE2(const uint8_t* src_argb0,
+void ARGBSubtractRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2858,7 +6140,7 @@ void ARGBSubtractRow_Any_SSE2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBSubtractRow_AVX2(const uint8_t* src_argb0,
+void ARGBSubtractRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2866,7 +6148,7 @@ void ARGBSubtractRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBSubtractRow_NEON(const uint8_t* src_argb0,
+void ARGBSubtractRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
@@ -2874,22 +6156,170 @@ void ARGBSubtractRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBSubtractRow_MSA(const uint8_t* src_argb0,
+void ARGBSubtractRow_LSX(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBSubtractRow_Any_MSA(const uint8_t* y_buf,
+void ARGBSubtractRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBSubtractRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void ARGBSubtractRow_MMI(const uint8_t* src_argb0,
+void ARGBSubtractRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+
+// ARGB bitwise OR with 32-bit value.
+void ARGBOrRow_C(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_SME(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_LSX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+void ARGBOrRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ uint32_t value,
+ int width);
+void ARGBOrRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width);
+
+// Unattenuated ARGB alpha blend.
+void ARGBAlphaBlendRow_C(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
-void ARGBSubtractRow_Any_MMI(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
+void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
int width);
+void ARGBAlphaBlendRow_Any_SSSE3(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_Any_AVX2(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_SME(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_Any_LSX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAlphaBlendRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
void ARGBToRGB24Row_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
@@ -2897,15 +6327,7 @@ void ARGBToRGB24Row_Any_SSSE3(const uint8_t* src_ptr,
void ARGBToRAWRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToRGB565Row_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToARGB1555Row_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToARGB4444Row_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
+
void ABGRToAR30Row_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@@ -2916,13 +6338,7 @@ void ARGBToRAWRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void ARGBToRGB24Row_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToRGB24Row_Any_AVX512VBMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToRGB565DitherRow_Any_SSE2(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- const uint32_t param,
- int width);
+
void ARGBToRGB565DitherRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint32_t param,
@@ -2961,41 +6377,40 @@ void ARGBToRGB565DitherRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint32_t param,
int width);
-void ARGBToRGB24Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToRAWRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToRGB565Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToARGB1555Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToARGB4444Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBToRGB565DitherRow_Any_MSA(const uint8_t* src_ptr,
+void ARGBToRGB565DitherRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint32_t param,
int width);
-
-void ARGBToRGB24Row_Any_MMI(const uint8_t* src_ptr,
+void ARGBToRGB565DitherRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const uint32_t param,
+ int width);
+void ARGBToRGB24Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToRAWRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void ARGBToRGB565Row_Any_MMI(const uint8_t* src_ptr,
+void ARGBToRGB24Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToRAWRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBToRAWRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBToRGB565Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToARGB1555Row_Any_MMI(const uint8_t* src_ptr,
+void ARGBToRGB565Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToARGB1555Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToARGB4444Row_Any_MMI(const uint8_t* src_ptr,
+void ARGBToARGB1555Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToARGB4444Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBToRGB565DitherRow_Any_MMI(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- const uint32_t param,
- int width);
+void ARGBToARGB4444Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void I444ToARGBRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* u_buf,
@@ -3003,39 +6418,102 @@ void I444ToARGBRow_Any_NEON(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I444ToRGB24Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422AlphaToARGBRow_Any_NEON(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToRGBARow_Any_NEON(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
+void I210ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_Any_NEON(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB4444Row_Any_NEON(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- const struct YuvConstants* yuvconstants,
- int width);
-void I422ToARGB1555Row_Any_NEON(const uint8_t* y_buf,
- const uint8_t* u_buf,
+void I410ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
@@ -3046,6 +6524,12 @@ void I422ToRGB565Row_Any_NEON(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToAR30Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToARGBRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
@@ -3066,9 +6550,9 @@ void NV21ToRGB24Row_Any_NEON(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void NV21ToYUV24Row_Any_NEON(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_yuv24,
+void NV21ToYUV24Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
int width);
void NV12ToRGB565Row_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
@@ -3083,78 +6567,239 @@ void UYVYToARGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I444ToARGBRow_Any_MSA(const uint8_t* y_buf,
+void ARGBToAR30Row_NEON(const uint8_t* src, uint8_t* dst, int width);
+void ABGRToAR30Row_NEON(const uint8_t* src, uint8_t* dst, int width);
+void ABGRToAR30Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToAR30Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void P210ToARGBRow_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGBRow_Any_MSA(const uint8_t* y_buf,
+void I444ToARGBRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGBARow_Any_MSA(const uint8_t* y_buf,
+void I422ToARGBRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422AlphaToARGBRow_Any_MSA(const uint8_t* y_buf,
+void I422ToRGBARow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
const uint8_t* a_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB24Row_Any_MSA(const uint8_t* y_buf,
+void I422AlphaToARGBRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToRGB565Row_Any_MSA(const uint8_t* y_buf,
+void I422ToRGB24Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB4444Row_Any_MSA(const uint8_t* y_buf,
+void I422ToRGB565Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void I422ToARGB1555Row_Any_MSA(const uint8_t* y_buf,
+void I422ToARGB4444Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToARGBRow_Any_MSA(const uint8_t* y_buf,
+void I422ToARGB1555Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+
+void NV12ToARGBRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void NV12ToRGB565Row_Any_MSA(const uint8_t* y_buf,
+void NV12ToARGBRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void NV21ToARGBRow_Any_MSA(const uint8_t* y_buf,
+void NV12ToRGB565Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void YUY2ToARGBRow_Any_MSA(const uint8_t* src_ptr,
+void NV21ToARGBRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
-void UYVYToARGBRow_Any_MSA(const uint8_t* src_ptr,
+void YUY2ToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void UYVYToARGBRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
@@ -3162,6 +6807,10 @@ void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
@@ -3172,100 +6821,148 @@ void YUY2ToUVRow_SSE2(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_SSE2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
+void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
+void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToUVRow_NEON(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
+void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
+void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
+ int stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
+ int stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_NEON(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void YUY2ToYRow_MSA(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
-void YUY2ToYRow_MMI(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
-void YUY2ToUVRow_MSA(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void YUY2ToUVRow_MMI(const uint8_t* src_yuy2,
+void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
+void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
+void YUY2ToUVRow_LSX(const uint8_t* src_yuy2,
int src_stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void YUY2ToUV422Row_MSA(const uint8_t* src_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void YUY2ToUV422Row_MMI(const uint8_t* src_yuy2,
+void YUY2ToUVRow_LASX(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void YUY2ToYRow_C(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToUVRow_C(const uint8_t* src_yuy2,
int src_stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_C(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_C(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void YUY2ToUVRow_Any_AVX2(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_Any_AVX2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToYRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void YUY2ToUVRow_Any_SSE2(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_Any_SSE2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_Any_SSE2(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void YUY2ToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToNVUVRow_Any_NEON(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width);
void YUY2ToUV422Row_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void YUY2ToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void YUY2ToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void YUY2ToUVRow_Any_MSA(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void YUY2ToUVRow_Any_MMI(const uint8_t* src_ptr,
+void YUY2ToYRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void YUY2ToYRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void YUY2ToUVRow_Any_LSX(const uint8_t* src_ptr,
int src_stride_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void YUY2ToUV422Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void YUY2ToUV422Row_Any_MMI(const uint8_t* src_ptr,
+void YUY2ToUVRow_Any_LASX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void YUY2ToUV422Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void YUY2ToUV422Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
void UYVYToUVRow_AVX2(const uint8_t* src_uyvy,
int stride_uyvy,
@@ -3306,26 +7003,26 @@ void UYVYToUV422Row_NEON(const uint8_t* src_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void UYVYToYRow_MSA(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
-void UYVYToYRow_MMI(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
-void UYVYToUVRow_MSA(const uint8_t* src_uyvy,
+void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
+void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
+void UYVYToUVRow_LSX(const uint8_t* src_uyvy,
int src_stride_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void UYVYToUVRow_MMI(const uint8_t* src_uyvy,
- int src_stride_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void UYVYToUV422Row_MSA(const uint8_t* src_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void UYVYToUV422Row_MMI(const uint8_t* src_uyvy,
+void UYVYToUVRow_LASX(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToUV422Row_LSX(const uint8_t* src_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void UYVYToUV422Row_LASX(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void UYVYToYRow_C(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
void UYVYToUVRow_C(const uint8_t* src_uyvy,
@@ -3339,7 +7036,7 @@ void UYVYToUV422Row_C(const uint8_t* src_uyvy,
int width);
void UYVYToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void UYVYToUVRow_Any_AVX2(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
@@ -3349,7 +7046,7 @@ void UYVYToUV422Row_Any_AVX2(const uint8_t* src_ptr,
int width);
void UYVYToYRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void UYVYToUVRow_Any_SSE2(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
@@ -3359,7 +7056,7 @@ void UYVYToUV422Row_Any_SSE2(const uint8_t* src_ptr,
int width);
void UYVYToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void UYVYToUVRow_Any_NEON(const uint8_t* src_ptr,
- int src_stride_ptr,
+ int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
@@ -3367,26 +7064,26 @@ void UYVYToUV422Row_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void UYVYToYRow_Any_MSA(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void UYVYToYRow_Any_MMI(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
-void UYVYToUVRow_Any_MSA(const uint8_t* src_ptr,
+void UYVYToYRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void UYVYToYRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void UYVYToUVRow_Any_LSX(const uint8_t* src_ptr,
int src_stride_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
-void UYVYToUVRow_Any_MMI(const uint8_t* src_ptr,
- int src_stride_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void UYVYToUV422Row_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width);
-void UYVYToUV422Row_Any_MMI(const uint8_t* src_ptr,
+void UYVYToUVRow_Any_LASX(const uint8_t* src_ptr,
+ int src_stride_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToUV422Row_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void UYVYToUV422Row_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void SwapUVRow_C(const uint8_t* src_uv, uint8_t* dst_vu, int width);
void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width);
void SwapUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
@@ -3394,31 +7091,76 @@ void SwapUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_vu, int width);
void SwapUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width);
void SwapUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width);
+void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width);
void AYUVToYRow_C(const uint8_t* src_ayuv, uint8_t* dst_y, int width);
+void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width);
+void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width);
void AYUVToUVRow_C(const uint8_t* src_ayuv,
- int stride_ayuv,
+ int src_stride_ayuv,
uint8_t* dst_uv,
int width);
+void AYUVToUVRow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width);
+void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width);
void AYUVToVURow_C(const uint8_t* src_ayuv,
- int stride_ayuv,
+ int src_stride_ayuv,
uint8_t* dst_vu,
int width);
+void AYUVToVURow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width);
void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width);
void AYUVToUVRow_NEON(const uint8_t* src_ayuv,
- int stride_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width);
+void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
uint8_t* dst_uv,
int width);
void AYUVToVURow_NEON(const uint8_t* src_ayuv,
- int stride_ayuv,
+ int src_stride_ayuv,
uint8_t* dst_vu,
int width);
-void AYUVToYRow_Any_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width);
-void AYUVToUVRow_Any_NEON(const uint8_t* src_ayuv,
- int stride_ayuv,
- uint8_t* dst_uv,
+void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToVURow_Any_AVX2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
int width);
-void AYUVToVURow_Any_NEON(const uint8_t* src_ayuv,
- int stride_ayuv,
+void AYUVToVURow_Any_NEON(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
+void AYUVToVURow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
uint8_t* dst_vu,
int width);
@@ -3492,46 +7234,46 @@ void I422ToUYVYRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
int width);
-void I422ToYUY2Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_yuy2,
- int width);
-void I422ToYUY2Row_MMI(const uint8_t* src_y,
+void I422ToYUY2Row_LSX(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_yuy2,
int width);
-void I422ToUYVYRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uyvy,
- int width);
-void I422ToUYVYRow_MMI(const uint8_t* src_y,
+void I422ToYUY2Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width);
+void I422ToUYVYRow_LSX(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uyvy,
int width);
-void I422ToYUY2Row_Any_MSA(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- int width);
-void I422ToYUY2Row_Any_MMI(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_ptr,
- int width);
-void I422ToUYVYRow_Any_MSA(const uint8_t* y_buf,
+void I422ToUYVYRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width);
+void I422ToYUY2Row_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
int width);
-void I422ToUYVYRow_Any_MMI(const uint8_t* y_buf,
+void I422ToYUY2Row_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+void I422ToUYVYRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
int width);
+void I422ToUYVYRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
// Effects related row functions.
void ARGBAttenuateRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width);
@@ -3541,13 +7283,19 @@ void ARGBAttenuateRow_SSSE3(const uint8_t* src_argb,
void ARGBAttenuateRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width);
+void ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
void ARGBAttenuateRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width);
-void ARGBAttenuateRow_MSA(const uint8_t* src_argb,
+void ARGBAttenuateRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
int width);
-void ARGBAttenuateRow_MMI(const uint8_t* src_argb,
+void ARGBAttenuateRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
int width);
void ARGBAttenuateRow_Any_SSSE3(const uint8_t* src_ptr,
@@ -3559,12 +7307,12 @@ void ARGBAttenuateRow_Any_AVX2(const uint8_t* src_ptr,
void ARGBAttenuateRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
-void ARGBAttenuateRow_Any_MSA(const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- int width);
-void ARGBAttenuateRow_Any_MMI(const uint8_t* src_ptr,
+void ARGBAttenuateRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void ARGBAttenuateRow_Any_LASX(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
// Inverse table for unattenuate, shared by C and SSE2.
extern const uint32_t fixed_invtbl8[256];
@@ -3577,66 +7325,97 @@ void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb,
void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width);
+void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
void ARGBUnattenuateRow_Any_SSE2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void ARGBUnattenuateRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width);
-void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBGrayRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
+void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width);
-void ARGBGrayRow_MSA(const uint8_t* src_argb, uint8_t* dst_argb, int width);
-void ARGBGrayRow_MMI(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
+void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBSepiaRow_C(uint8_t* dst_argb, int width);
-void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width);
void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width);
-void ARGBSepiaRow_MSA(uint8_t* dst_argb, int width);
-void ARGBSepiaRow_MMI(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width);
void ARGBColorMatrixRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
const int8_t* matrix_argb,
int width);
-void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width);
+void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width);
+void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width);
void ARGBColorMatrixRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const int8_t* matrix_argb,
int width);
-void ARGBColorMatrixRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width);
-void ARGBColorMatrixRow_MMI(const uint8_t* src_argb,
+void ARGBColorMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width);
+void ARGBColorMatrixRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
const int8_t* matrix_argb,
int width);
+void ARGBColorMatrixRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width);
void ARGBColorTableRow_C(uint8_t* dst_argb,
const uint8_t* table_argb,
int width);
-void ARGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width);
+void ARGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width);
+void ARGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width);
void RGBColorTableRow_C(uint8_t* dst_argb,
const uint8_t* table_argb,
int width);
-void RGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width);
+void RGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width);
+void RGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width);
void ARGBQuantizeRow_C(uint8_t* dst_argb,
int scale,
int interval_size,
int interval_offset,
int width);
-void ARGBQuantizeRow_SSE2(uint8_t* dst_argb,
+void ARGBQuantizeRow_AVX2(uint8_t* dst_argb,
int scale,
int interval_size,
int interval_offset,
@@ -3646,17 +7425,22 @@ void ARGBQuantizeRow_NEON(uint8_t* dst_argb,
int interval_size,
int interval_offset,
int width);
-void ARGBQuantizeRow_MSA(uint8_t* dst_argb,
+void ARGBQuantizeRow_LSX(uint8_t* dst_argb,
int scale,
int interval_size,
int interval_offset,
int width);
+void ARGBQuantizeRow_LASX(uint8_t* dst_argb,
+ int scale,
+ int interval_size,
+ int interval_offset,
+ int width);
void ARGBShadeRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value);
-void ARGBShadeRow_SSE2(const uint8_t* src_argb,
+void ARGBShadeRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value);
@@ -3664,31 +7448,36 @@ void ARGBShadeRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value);
-void ARGBShadeRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- uint32_t value);
-void ARGBShadeRow_MMI(const uint8_t* src_argb,
+void ARGBShadeRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value);
+void ARGBShadeRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ uint32_t value);
// Used for blur.
-void CumulativeSumToAverageRow_SSE2(const int32_t* topleft,
+void CumulativeSumToAverageRow_AVX2(const int32_t* topleft,
const int32_t* botleft,
int width,
int area,
uint8_t* dst,
int count);
-void ComputeCumulativeSumRow_SSE2(const uint8_t* row,
+void ComputeCumulativeSumRow_AVX2(const uint8_t* row,
+ int32_t* cumsum,
+ const int32_t* previous_cumsum,
+ int width);
+void CumulativeSumToAverageRow_NEON(const int32_t* topleft,
+ const int32_t* botleft,
+ int width,
+ int area,
+ uint8_t* dst,
+ int count);
+void ComputeCumulativeSumRow_NEON(const uint8_t* row,
int32_t* cumsum,
const int32_t* previous_cumsum,
int width);
-
-void ComputeCumulativeSumRow_MMI(const uint8_t* row,
- int32_t* cumsum,
- const int32_t* previous_cumsum,
- int width);
void CumulativeSumToAverageRow_C(const int32_t* tl,
const int32_t* bl,
@@ -3708,7 +7497,13 @@ void ARGBAffineRow_C(const uint8_t* src_argb,
const float* uv_dudv,
int width);
LIBYUV_API
-void ARGBAffineRow_SSE2(const uint8_t* src_argb,
+void ARGBAffineRow_AVX2(const uint8_t* src_argb,
+ int src_argb_stride,
+ uint8_t* dst_argb,
+ const float* src_dudv,
+ int width);
+LIBYUV_API
+void ARGBAffineRow_NEON(const uint8_t* src_argb,
int src_argb_stride,
uint8_t* dst_argb,
const float* src_dudv,
@@ -3720,27 +7515,42 @@ void InterpolateRow_C(uint8_t* dst_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction);
-void InterpolateRow_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction);
void InterpolateRow_AVX2(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction);
+void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction);
void InterpolateRow_NEON(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction);
-void InterpolateRow_MSA(uint8_t* dst_ptr,
+void InterpolateRow_SVE2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction);
+void InterpolateRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction);
+void InterpolateRow_LSX(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction);
-void InterpolateRow_MMI(uint8_t* dst_ptr,
+void InterpolateRow_LASX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_RVV(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int width,
@@ -3750,32 +7560,57 @@ void InterpolateRow_Any_NEON(uint8_t* dst_ptr,
ptrdiff_t src_stride_ptr,
int width,
int source_y_fraction);
-void InterpolateRow_Any_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride_ptr,
- int width,
- int source_y_fraction);
void InterpolateRow_Any_AVX2(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride_ptr,
int width,
int source_y_fraction);
-void InterpolateRow_Any_MSA(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride_ptr,
- int width,
- int source_y_fraction);
-void InterpolateRow_Any_MMI(uint8_t* dst_ptr,
+void InterpolateRow_Any_LSX(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride_ptr,
int width,
int source_y_fraction);
+void InterpolateRow_Any_LASX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride_ptr,
+ int width,
+ int source_y_fraction);
void InterpolateRow_16_C(uint16_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction);
+void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_Any_AVX2(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_AVX512BW(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_NEON(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_Any_NEON(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
// Sobel images.
void SobelXRow_C(const uint8_t* src_y0,
@@ -3783,7 +7618,7 @@ void SobelXRow_C(const uint8_t* src_y0,
const uint8_t* src_y2,
uint8_t* dst_sobelx,
int width);
-void SobelXRow_SSE2(const uint8_t* src_y0,
+void SobelXRow_AVX2(const uint8_t* src_y0,
const uint8_t* src_y1,
const uint8_t* src_y2,
uint8_t* dst_sobelx,
@@ -3793,21 +7628,11 @@ void SobelXRow_NEON(const uint8_t* src_y0,
const uint8_t* src_y2,
uint8_t* dst_sobelx,
int width);
-void SobelXRow_MSA(const uint8_t* src_y0,
- const uint8_t* src_y1,
- const uint8_t* src_y2,
- uint8_t* dst_sobelx,
- int width);
-void SobelXRow_MMI(const uint8_t* src_y0,
- const uint8_t* src_y1,
- const uint8_t* src_y2,
- uint8_t* dst_sobelx,
- int width);
void SobelYRow_C(const uint8_t* src_y0,
const uint8_t* src_y1,
uint8_t* dst_sobely,
int width);
-void SobelYRow_SSE2(const uint8_t* src_y0,
+void SobelYRow_AVX2(const uint8_t* src_y0,
const uint8_t* src_y1,
uint8_t* dst_sobely,
int width);
@@ -3815,19 +7640,11 @@ void SobelYRow_NEON(const uint8_t* src_y0,
const uint8_t* src_y1,
uint8_t* dst_sobely,
int width);
-void SobelYRow_MSA(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
- int width);
-void SobelYRow_MMI(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
- int width);
void SobelRow_C(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
-void SobelRow_SSE2(const uint8_t* src_sobelx,
+void SobelRow_AVX2(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
@@ -3835,19 +7652,19 @@ void SobelRow_NEON(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
-void SobelRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width);
-void SobelRow_MMI(const uint8_t* src_sobelx,
+void SobelRow_LSX(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
+void SobelRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width);
void SobelToPlaneRow_C(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_y,
int width);
-void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx,
+void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_y,
int width);
@@ -3855,19 +7672,19 @@ void SobelToPlaneRow_NEON(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_y,
int width);
-void SobelToPlaneRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_y,
- int width);
-void SobelToPlaneRow_MMI(const uint8_t* src_sobelx,
+void SobelToPlaneRow_LSX(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_y,
int width);
+void SobelToPlaneRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_y,
+ int width);
void SobelXYRow_C(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
-void SobelXYRow_SSE2(const uint8_t* src_sobelx,
+void SobelXYRow_AVX2(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
@@ -3875,15 +7692,15 @@ void SobelXYRow_NEON(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
-void SobelXYRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width);
-void SobelXYRow_MMI(const uint8_t* src_sobelx,
+void SobelXYRow_LSX(const uint8_t* src_sobelx,
const uint8_t* src_sobely,
uint8_t* dst_argb,
int width);
-void SobelRow_Any_SSE2(const uint8_t* y_buf,
+void SobelXYRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width);
+void SobelRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
@@ -3891,15 +7708,15 @@ void SobelRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void SobelRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void SobelRow_Any_MMI(const uint8_t* y_buf,
+void SobelRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void SobelToPlaneRow_Any_SSE2(const uint8_t* y_buf,
+void SobelRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void SobelToPlaneRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
@@ -3907,15 +7724,15 @@ void SobelToPlaneRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void SobelToPlaneRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void SobelToPlaneRow_Any_MMI(const uint8_t* y_buf,
+void SobelToPlaneRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void SobelXYRow_Any_SSE2(const uint8_t* y_buf,
+void SobelToPlaneRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void SobelXYRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
@@ -3923,14 +7740,14 @@ void SobelXYRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
-void SobelXYRow_Any_MSA(const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_ptr,
- int width);
-void SobelXYRow_Any_MMI(const uint8_t* y_buf,
+void SobelXYRow_Any_LSX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
+void SobelXYRow_Any_LASX(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
void ARGBPolynomialRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
@@ -3944,21 +7761,33 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
const float* poly,
int width);
+void ARGBPolynomialRow_Any_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const float* param,
+ int width);
+void ARGBPolynomialRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const float* param,
+ int width);
+void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width);
+void ARGBPolynomialRow_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const float* param,
+ int width);
// Scale and convert to half float.
void HalfFloatRow_C(const uint16_t* src, uint16_t* dst, float scale, int width);
-void HalfFloatRow_SSE2(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width);
-void HalfFloatRow_Any_SSE2(const uint16_t* src_ptr,
- uint16_t* dst_ptr,
- float param,
- int width);
void HalfFloatRow_AVX2(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
+void HalfFloatRow_AVX512BW(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width);
void HalfFloatRow_Any_AVX2(const uint16_t* src_ptr,
uint16_t* dst_ptr,
float param,
@@ -3987,23 +7816,39 @@ void HalfFloatRow_Any_NEON(const uint16_t* src_ptr,
uint16_t* dst_ptr,
float param,
int width);
-void HalfFloat1Row_NEON(const uint16_t* src,
+void HalfFloatRow_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width);
+void HalfFloat1Row_SVE2(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
-void HalfFloat1Row_Any_NEON(const uint16_t* src_ptr,
- uint16_t* dst_ptr,
- float param,
- int width);
-void HalfFloatRow_MSA(const uint16_t* src,
+void HalfFloatRow_LSX(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
-void HalfFloatRow_Any_MSA(const uint16_t* src_ptr,
+void HalfFloatRow_LASX(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width);
+void HalfFloatRow_Any_LSX(const uint16_t* src_ptr,
uint16_t* dst_ptr,
float param,
int width);
+void HalfFloatRow_Any_LASX(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ float param,
+ int width);
void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width);
+void ByteToFloatRow_AVX2(const uint8_t* src,
+ float* dst,
+ float scale,
+ int width);
+void ByteToFloatRow_Any_AVX2(const uint8_t* src_ptr,
+ float* dst_ptr,
+ float param,
+ int width);
void ByteToFloatRow_NEON(const uint8_t* src,
float* dst,
float scale,
@@ -4012,31 +7857,698 @@ void ByteToFloatRow_Any_NEON(const uint8_t* src_ptr,
float* dst_ptr,
float param,
int width);
-
+// Convert FP16 Half Floats to FP32 Floats
+void ConvertFP16ToFP32Row_NEON(const uint16_t* src, // fp16
+ float* dst,
+ int width);
+// Convert a column of FP16 Half Floats to a row of FP32 Floats
+void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
+ int src_stride, // stride in elements
+ float* dst,
+ int width);
+// Convert FP32 Floats to FP16 Half Floats
+void ConvertFP32ToFP16Row_NEON(const float* src,
+ uint16_t* dst, // fp16
+ int width);
void ARGBLumaColorTableRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
const uint8_t* luma,
uint32_t lumacoeff);
-void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- const uint8_t* luma,
- uint32_t lumacoeff);
+void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff);
+void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff);
float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width);
+float ScaleMaxSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width);
float ScaleMaxSamples_NEON(const float* src,
float* dst,
float scale,
int width);
float ScaleSumSamples_C(const float* src, float* dst, float scale, int width);
+float ScaleSumSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width);
float ScaleSumSamples_NEON(const float* src,
float* dst,
float scale,
int width);
void ScaleSamples_C(const float* src, float* dst, float scale, int width);
+void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width);
void ScaleSamples_NEON(const float* src, float* dst, float scale, int width);
+void GaussRow_F32_AVX2(const float* src, float* dst, int width);
+void GaussRow_F32_NEON(const float* src, float* dst, int width);
+void GaussRow_F32_C(const float* src, float* dst, int width);
+
+void GaussCol_F32_AVX2(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width);
+
+void GaussCol_F32_NEON(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width);
+
+void GaussCol_F32_C(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width);
+
+void GaussRow_C(const uint32_t* src, uint16_t* dst, int width);
+void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width);
+void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width);
+void GaussCol_C(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width);
+void GaussCol_AVX2(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width);
+void GaussCol_NEON(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width);
+
+void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width);
+
+void RAWToARGBRow_WASMSIMD(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ int width);
+void RGB24ToARGBRow_WASMSIMD(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
+void ARGBToYMatrixRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_WASMSIMD(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_WASMSIMD(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_WASMSIMD(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void Convert8To16Row_WASMSIMD(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
+void MultiplyRow_16_WASMSIMD(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void MergeUVRow_WASMSIMD(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
+void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
+void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width);
+void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width);
+void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
+void InterpolateRow_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_WASMSIMD(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void RGB565ToARGBRow_WASMSIMD(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width);
+void ARGB1555ToARGBRow_WASMSIMD(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width);
+void ARGB4444ToARGBRow_WASMSIMD(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width);
+void ARGBToRGB565DitherRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width);
+void ARGBToRGB565Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width);
+void ARGBToARGB1555Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb1555,
+ int width);
+void ARGBToARGB4444Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb4444,
+ int width);
+void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width);
+void I422ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void BlendPlaneRow_WASMSIMD(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width);
+
+void RAWToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB24ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUV444MatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGBToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void RGB565ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB1555ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGB4444ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void Convert8To16Row_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+void MultiplyRow_16_Any_WASMSIMD(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int scale,
+ int width);
+void MergeUVRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ int width);
+void MirrorRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void MirrorUVRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void SwapUVRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB24MirrorRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBMirrorRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void InterpolateRow_Any_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_Any_WASMSIMD(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void J400ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void RGB565ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGB1555ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGB4444ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToRGB565DitherRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const uint32_t dither4,
+ int width);
+void ARGBToRGB565Row_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToARGB1555Row_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToARGB4444Row_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBShuffleRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ const uint8_t* shuffler,
+ int width);
+void I422ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB565Row_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void BlendPlaneRow_Any_WASMSIMD(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ int width);
+
+void CopyRow_HVX(const uint8_t* src, uint8_t* dst, int count);
+void CopyRow_16_HVX(const uint16_t* src, uint16_t* dst, int count);
+void SetRow_HVX(uint8_t* dst, uint8_t v8, int width);
+void ARGBSetRow_HVX(uint8_t* dst_argb, uint32_t v32, int width);
+void MirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width);
+void MirrorRow_16_HVX(const uint16_t* src, uint16_t* dst, int width);
+void MirrorUVRow_HVX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
+void MirrorSplitUVRow_HVX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBMirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width);
+void SplitUVRow_HVX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void MergeUVRow_HVX(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
+void SplitUVRow_16_HVX(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width);
+void MergeUVRow_16_HVX(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
+void SplitARGBRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width);
+void MergeARGBRow_HVX(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width);
+void SplitXRGBRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void MergeXRGBRow_HVX(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width);
+void SwapUVRow_HVX(const uint8_t* src_uv, uint8_t* dst_vu, int width);
+void InterpolateRow_HVX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void InterpolateRow_16_HVX(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
+void BlendPlaneRow_HVX(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width);
+void ARGBBlendRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAttenuateRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
+void ARGBMultiplyRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBAddRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBSubtractRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
+void ARGBExtractAlphaRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width);
+void ARGBCopyAlphaRow_HVX(const uint8_t* src, uint8_t* dst, int width);
+void ARGBCopyYToAlphaRow_HVX(const uint8_t* src, uint8_t* dst, int width);
+void Convert16To8Row_HVX(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
+void Convert8To16Row_HVX(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width);
+void Convert8To8Row_HVX(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void MultiplyRow_16_HVX(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void DivideRow_16_HVX(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
+void HalfRow_16To8_HVX(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width);
+void YUY2ToYRow_HVX(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
+void YUY2ToUVRow_HVX(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void YUY2ToUV422Row_HVX(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToYRow_HVX(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
+void UYVYToUVRow_HVX(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void UYVYToUV422Row_HVX(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void I400ToARGBRow_HVX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void J400ToARGBRow_HVX(const uint8_t* src_y, uint8_t* dst_argb, int width);
+void ARGBToABGRRow_HVX(const uint8_t* src_argb, uint8_t* dst_abgr, int width);
+void ARGBToBGRARow_HVX(const uint8_t* src_argb, uint8_t* dst_bgra, int width);
+void ARGBToRGBARow_HVX(const uint8_t* src_argb, uint8_t* dst_rgba, int width);
+void RGBAToARGBRow_HVX(const uint8_t* src_rgba, uint8_t* dst_argb, int width);
+void ARGBToAR64Row_HVX(const uint8_t* src_argb, uint16_t* dst_ar64, int width);
+void ARGBToAB64Row_HVX(const uint8_t* src_argb, uint16_t* dst_ab64, int width);
+void AR64ToARGBRow_HVX(const uint16_t* src_ar64, uint8_t* dst_argb, int width);
+void AB64ToARGBRow_HVX(const uint16_t* src_ab64, uint8_t* dst_argb, int width);
+void AR64ToAB64Row_HVX(const uint16_t* src_ar64, uint16_t* dst_ab64, int width);
+void I422ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void ARGBToYMatrixRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUVMatrixRow_HVX(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+void ARGBToUV444MatrixRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c);
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/include/libyuv/row_sve.h b/libs/libyuv/include/libyuv/row_sve.h
new file mode 100644
index 0000000000..7faf57db84
--- /dev/null
+++ b/libs/libyuv/include/libyuv/row_sve.h
@@ -0,0 +1,2524 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_ROW_SVE_H_
+#define INCLUDE_LIBYUV_ROW_SVE_H_
+
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define STREAMING_COMPATIBLE __arm_streaming_compatible
+#else // defined(LIBYUV_DISABLE_SME) || !defined(CLANG_HAS_SME) ||
+ // !defined(__aarch64__)
+#define STREAMING_COMPATIBLE
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#define YUVTORGB_SVE_SETUP \
+ "ld1rb {z28.b}, p0/z, [%[kUVCoeff], #0] \n" \
+ "ld1rb {z29.b}, p0/z, [%[kUVCoeff], #1] \n" \
+ "ld1rb {z30.b}, p0/z, [%[kUVCoeff], #2] \n" \
+ "ld1rb {z31.b}, p0/z, [%[kUVCoeff], #3] \n" \
+ "ld1rh {z24.h}, p0/z, [%[kRGBCoeffBias], #0] \n" \
+ "ld1rh {z25.h}, p0/z, [%[kRGBCoeffBias], #2] \n" \
+ "ld1rh {z26.h}, p0/z, [%[kRGBCoeffBias], #4] \n" \
+ "ld1rh {z27.h}, p0/z, [%[kRGBCoeffBias], #6] \n"
+
+#define YUVTORGB_SVE_SETUP_AR30 \
+ YUVTORGB_SVE_SETUP \
+ "add z25.h, z25.h, #24 \n" \
+ "sub z26.h, z26.h, #24 \n" \
+ "add z27.h, z27.h, #24 \n"
+
+#define READYUV444_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1b {z1.h}, p1/z, [%[src_u]] \n" \
+ "ld1b {z2.h}, p1/z, [%[src_v]] \n" \
+ "add %[src_y], %[src_y], %[vl] \n" \
+ "add %[src_u], %[src_u], %[vl] \n" \
+ "add %[src_v], %[src_v], %[vl] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 448] \n" \
+ "trn1 z0.b, z0.b, z0.b \n" \
+ "prfm pldl1keep, [%[src_v], 448] \n"
+
+// Read twice as much data from YUV, putting the even elements from the Y data
+// in z0.h and odd elements in z1.h.
+#define READYUV444_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \
+ "ld1b {z2.b}, p1/z, [%[src_u]] \n" \
+ "ld1b {z3.b}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_u] \n" \
+ "incb %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define READYUV400_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "inch %[src_y] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define READYUV422_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1b {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1b {z2.s}, p1/z, [%[src_v]] \n" \
+ "inch %[src_y] \n" \
+ "incw %[src_u] \n" \
+ "incw %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn1 z0.b, z0.b, z0.b \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n"
+
+// Read twice as much data from YUV, putting the even elements from the Y data
+// in z0.h and odd elements in z1.h. U/V data is not duplicated, stored in
+// z2.h/z3.h.
+#define READYUV422_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \
+ "ld1b {z2.h}, p1/z, [%[src_u]] \n" \
+ "ld1b {z3.h}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define READI210_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.s}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "lsl z0.h, z3.h, #6 \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "usra z0.h, z3.h, #4 \n" \
+ "uqshrnb z1.b, z1.h, #2 \n" \
+ "uqshrnb z2.b, z2.h, #2 \n"
+
+#define READI210_SVE_2X \
+ "ld1h {z4.h}, p2/z, [%[src_y]] \n" \
+ "ld1h {z5.h}, p3/z, [%[src_y], #1, mul vl] \n" \
+ "ld1h {z2.h}, p1/z, [%[src_u]] \n" \
+ "ld1h {z3.h}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y], all, mul #2 \n" \
+ "uzp1 z6.h, z4.h, z5.h \n" \
+ "uzp2 z5.h, z4.h, z5.h \n" \
+ "incb %[src_u] \n" \
+ "incb %[src_v] \n" \
+ "lsl z0.h, z6.h, #6 \n" \
+ "lsl z1.h, z5.h, #6 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "usra z0.h, z6.h, #4 \n" \
+ "usra z1.h, z5.h, #4 \n" \
+ "uqshrnb z2.b, z2.h, #2 \n" \
+ "uqshrnb z3.b, z3.h, #2 \n"
+
+#define READP210_SVE \
+ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.h}, p2/z, [%[src_uv]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_uv] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "tbl z1.b, {z1.b}, z22.b \n"
+
+#define READI410_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "lsl z0.h, z3.h, #6 \n" \
+ "usra z0.h, z3.h, #4 \n" \
+ "ld1h {z1.h}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.h}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_u] \n" \
+ "incb %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "uqshrnb z1.b, z1.h, #2 \n" \
+ "uqshrnb z2.b, z2.h, #2 \n"
+
+// We need different predicates for the UV components since we are reading
+// 32-bit (pairs of UV) elements rather than 16-bit Y elements.
+#define READP410_SVE \
+ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1w {z1.s}, p2/z, [%[src_uv]] \n" \
+ "ld1w {z2.s}, p3/z, [%[src_uv], #1, mul vl] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_uv], all, mul #2 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "uzp2 z1.b, z1.b, z2.b \n"
+
+#define READI212_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.s}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "lsl z0.h, z3.h, #4 \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "usra z0.h, z3.h, #8 \n" \
+ "uqshrnb z1.b, z1.h, #4 \n" \
+ "uqshrnb z2.b, z2.h, #4 \n"
+
+#define I400TORGB_SVE \
+ "umulh z18.h, z24.h, z0.h \n" /* Y */ \
+ "movprfx z16, z18 \n" \
+ "usqadd z16.h, p0/m, z16.h, z4.h \n" /* B */ \
+ "movprfx z17, z18 \n" \
+ "usqadd z17.h, p0/m, z17.h, z6.h \n" /* G */ \
+ "usqadd z18.h, p0/m, z18.h, z5.h \n" /* R */
+
+// We need a different predicate for the UV component to handle the tail.
+// If there is a single element remaining then we want to load one Y element
+// but two UV elements.
+#define READNV_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" /* Y0Y0 */ \
+ "ld1b {z2.b}, p2/z, [%[src_uv]] \n" /* U0V0 or V0U0 */ \
+ "incb %[src_y] \n" \
+ "incb %[src_uv] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" /* YYYY */ \
+ "trn1 z0.b, z0.b, z0.b \n" /* YYYY */
+
+// Like NVTORGB_SVE but U/V components are stored in widened .h elements of
+// z1/z2 rather than even/odd .b lanes of z1.
+#define I4XXTORGB_SVE \
+ "umulh z0.h, z24.h, z0.h \n" /* Y */ \
+ "umullb z6.h, z30.b, z1.b \n" \
+ "umullb z4.h, z28.b, z1.b \n" /* DB */ \
+ "umullb z5.h, z29.b, z2.b \n" /* DR */ \
+ "umlalb z6.h, z31.b, z2.b \n" /* DG */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */
+
+#define I444TORGB_SVE_2X \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umullb z6.h, z30.b, z2.b \n" \
+ "umullt z7.h, z30.b, z2.b \n" \
+ "umullb z4.h, z28.b, z2.b \n" /* DB */ \
+ "umullt z2.h, z28.b, z2.b \n" /* DB */ \
+ "umlalb z6.h, z31.b, z3.b \n" /* DG */ \
+ "umlalt z7.h, z31.b, z3.b \n" /* DG */ \
+ "umullb z5.h, z29.b, z3.b \n" /* DR */ \
+ "umullt z3.h, z29.b, z3.b \n" /* DR */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z21.h, z1.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z20.h, z1.h, z2.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "add z22.h, z1.h, z3.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z21.h, z21.h, z7.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R */
+
+// Like I4XXTORGB_SVE but U/V components are stored in even/odd .b lanes of z1
+// rather than widened .h elements of z1/z2.
+#define NVTORGB_SVE \
+ "umulh z0.h, z24.h, z0.h \n" /* Y */ \
+ "umullb z6.h, z30.b, z1.b \n" \
+ "umullb z4.h, z28.b, z1.b \n" /* DB */ \
+ "umullt z5.h, z29.b, z1.b \n" /* DR */ \
+ "umlalt z6.h, z31.b, z1.b \n" /* DG */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */
+
+// The U/V component multiplies do not need to be duplicated in I422, we just
+// need to combine them with Y0/Y1 correctly.
+#define I422TORGB_SVE_2X \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umullb z6.h, z30.b, z2.b \n" \
+ "umullb z4.h, z28.b, z2.b \n" /* DB */ \
+ "umullb z5.h, z29.b, z3.b \n" /* DR */ \
+ "umlalb z6.h, z31.b, z3.b \n" /* DG */ \
+ \
+ "add z17.h, z0.h, z26.h \n" /* G0 */ \
+ "add z21.h, z1.h, z26.h \n" /* G1 */ \
+ "add z16.h, z0.h, z4.h \n" /* B0 */ \
+ "add z20.h, z1.h, z4.h \n" /* B1 */ \
+ "add z18.h, z0.h, z5.h \n" /* R0 */ \
+ "add z22.h, z1.h, z5.h \n" /* R1 */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \
+ "uqsub z21.h, z21.h, z6.h \n" /* G1 */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R1 */
+
+// clang-format off
+#define NVTORGB_SVE_2X(bt_u, bt_v) \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umull" #bt_u " z6.h, z30.b, z2.b \n" \
+ "umull" #bt_u " z4.h, z28.b, z2.b \n" /* DB */ \
+ "umull" #bt_v " z5.h, z29.b, z2.b \n" /* DR */ \
+ "umlal" #bt_v " z6.h, z31.b, z2.b \n" /* DG */ \
+ \
+ "add z17.h, z0.h, z26.h \n" /* G0 */ \
+ "add z21.h, z1.h, z26.h \n" /* G1 */ \
+ "add z16.h, z0.h, z4.h \n" /* B0 */ \
+ "add z20.h, z1.h, z4.h \n" /* B1 */ \
+ "add z18.h, z0.h, z5.h \n" /* R0 */ \
+ "add z22.h, z1.h, z5.h \n" /* R1 */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \
+ "uqsub z21.h, z21.h, z6.h \n" /* G1 */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R1 */
+// clang-format on
+
+#define RGBTOARGB8_SVE_TOP_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
+ "uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
+ "uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
+ "uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
+ "uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
+
+// Convert from 2.14 fixed point RGB to 8 bit ARGB, interleaving as BG and RA
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBTOARGB8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z17.h, #6 \n" /* BG */ \
+ "trn1 z17.b, z18.b, z19.b \n" /* RA */
+
+// Convert from 2.14 fixed point RGBA to 8 bit ARGB, interleaving as BG and RA
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBATOARGB8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.h */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnt z16.b, z17.h, #6 \n" /* BG */ \
+ "uqshrnb z17.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z17.b, z19.h, #2 \n" /* RA */
+
+// Convert from 2.14 fixed point RGB to 8 bit RGBA, interleaving as AB and GR
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBTORGBA8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnt z19.b, z16.h, #6 \n" /* AB */ \
+ "uqshrnb z20.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnt z20.b, z18.h, #6 \n" /* GR */
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+// Store AR30 elements. Inputs are 2.14 fixed point RGB. We expect z23 to be
+// populated with 0x3ff0 (0x3fff would also work) to saturate the R input
+// rather than needing a pair of shifts to saturate and then insert into the
+// correct position in the lane.
+#define STOREAR30_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* ggggggggggxxxxxx */ \
+ "umin z18.h, p0/m, z18.h, z23.h \n" /* 00rrrrrrrrrrxxxx */ \
+ "orr z18.h, z18.h, #0xc000 \n" /* 11rrrrrrrrrrxxxx */ \
+ "sri z18.h, z17.h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "lsl z17.h, z17.h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "sri z17.h, z16.h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "st2h {z17.h, z18.h}, p1, [%[dst_ar30]] \n" \
+ "incb %[dst_ar30], all, mul #2 \n"
+
+#define STOREAR30_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ /* B: z20.h, G: z21.h, R: z22.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl z20.h, p0/m, z20.h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* ggggggggggxxxxxx */ \
+ "uqshl z21.h, p0/m, z21.h, #2 \n" /* ggggggggggxxxxxx */ \
+ "umin z18.h, p0/m, z18.h, z23.h \n" /* 00rrrrrrrrrrxxxx */ \
+ "umin z22.h, p0/m, z22.h, z23.h \n" /* 00rrrrrrrrrrxxxx */ \
+ "orr z18.h, z18.h, #0xc000 \n" /* 11rrrrrrrrrrxxxx */ \
+ "orr z22.h, z22.h, #0xc000 \n" /* 11rrrrrrrrrrxxxx */ \
+ "sri z18.h, z17.h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "sri z22.h, z21.h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "lsl z17.h, z17.h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "lsl z19.h, z21.h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "sri z17.h, z16.h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "sri z19.h, z20.h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "zip2 z16.h, z17.h, z19.h \n" \
+ "zip1 z21.h, z17.h, z19.h \n" \
+ "zip2 z17.h, z18.h, z22.h \n" \
+ "zip1 z22.h, z18.h, z22.h \n" \
+ "st2h {z21.h, z22.h}, p2, [%[dst_ar30]] \n" \
+ "st2h {z16.h, z17.h}, p3, [%[dst_ar30], #2, mul vl] \n" \
+ "incb %[dst_ar30], all, mul #4 \n"
+
+#define YUVTORGB_SVE_REGS \
+ "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", \
+ "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", \
+ "z30", "z31", "p0", "p1", "p2", "p3"
+
+static inline void I444ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+ "incb %[dst_rgb24], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "mov z1.h, #128 \n" // U/V
+ "umullb z6.h, z30.b, z1.b \n"
+ "umullb z4.h, z28.b, z1.b \n" // DB
+ "umullb z5.h, z29.b, z1.b \n" // DR
+ "mla z6.h, p0/m, z31.h, z1.h \n" // DG
+ "sub z4.h, z4.h, z25.h \n"
+ "sub z5.h, z5.h, z27.h \n"
+ "sub z6.h, z26.h, z6.h \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TORGB565_SVE_FROM_TOP_2X \
+ "sri z18.h, z17.h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri z22.h, z21.h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri z18.h, z16.h, #11 \n" /* rrrrrggggggbbbbb */ \
+ "sri z22.h, z20.h, #11 \n" /* rrrrrggggggbbbbb */ \
+ "mov z19.d, z22.d \n"
+
+static inline void I422ToRGB565Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TORGB565_SVE_FROM_TOP_2X
+ "st2h {z18.h, z19.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X
+ RGBTOARGB8_SVE_TOP_2X RGB8TORGB565_SVE_FROM_TOP_2X
+ // Need to permute the data on the final iteration such that the
+ // predicates (.b) line up with the 16-bit element data.
+ "trn1 z20.b, z18.b, z19.b \n"
+ "trn2 z21.b, z18.b, z19.b \n"
+ "st2b {z20.b, z21.b}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_rgb565), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TOARGB1555_SVE_FROM_TOP_2X \
+ "dup z0.h, #0x8000 \n" /* 1000000000000000 */ \
+ "dup z1.h, #0x8000 \n" /* 1000000000000000 */ \
+ "sri z0.h, z18.h, #1 \n" /* 1rrrrrxxxxxxxxxx */ \
+ "sri z1.h, z22.h, #1 \n" /* 1rrrrrxxxxxxxxxx */ \
+ "sri z0.h, z17.h, #6 \n" /* 1rrrrrgggggxxxxx */ \
+ "sri z1.h, z21.h, #6 \n" /* 1rrrrrgggggxxxxx */ \
+ "sri z0.h, z16.h, #11 \n" /* 1rrrrrgggggbbbbb */ \
+ "sri z1.h, z20.h, #11 \n" /* 1rrrrrgggggbbbbb */
+
+static inline void I422ToARGB1555Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TOARGB1555_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X
+ RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X
+ // Need to permute the data on the final iteration such that the
+ // predicates (.b) line up with the 16-bit element data.
+ "trn1 z2.b, z0.b, z1.b \n"
+ "trn2 z3.b, z0.b, z1.b \n"
+ "st2b {z2.b, z3.b}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_argb1555), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TOARGB4444_SVE_FROM_TOP_2X \
+ "dup z0.h, #0xf000 \n" /* 1111000000000000 */ \
+ "dup z1.h, #0xf000 \n" /* 1111000000000000 */ \
+ "sri z0.h, z18.h, #4 \n" /* 1111rrrrxxxxxxxx */ \
+ "sri z1.h, z22.h, #4 \n" /* 1111rrrrxxxxxxxx */ \
+ "sri z0.h, z17.h, #8 \n" /* 1111rrrrggggxxxx */ \
+ "sri z1.h, z21.h, #8 \n" /* 1111rrrrggggxxxx */ \
+ "sri z0.h, z16.h, #12 \n" /* 1111rrrrggggbbbb */ \
+ "sri z1.h, z20.h, #12 \n" /* 1111rrrrggggbbbb */
+
+static inline void I422ToARGB4444Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TOARGB4444_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X
+ RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X
+ // Need to permute the data on the final iteration such that the
+ // predicates (.b) line up with the 16-bit element data.
+ "trn1 z2.b, z0.b, z1.b \n"
+ "trn2 z3.b, z0.b, z1.b \n"
+ "st2b {z2.b, z3.b}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_argb4444), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n"
+ "add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE
+ "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X.
+ const uint16_t limit = 0x3ff0;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP_AR30
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "ptrue p3.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.ge 1b \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.b, wzr, %w[width] \n"
+ "whilelt p2.h, wzr, %w[width] \n"
+ "whilelt p3.h, %w[vl], %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422AlphaToARGBRow_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X
+ "ld1b {z19.b}, p1/z, [%[src_a]] \n"
+ "add %[src_a], %[src_a], %[vl] \n" // Alpha
+ I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X
+ "ld1b {z19.b}, p1/z, [%[src_a]] \n" // Alpha
+ I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I444AlphaToARGBRow_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV444_SVE
+ "ld1b {z19.h}, p1/z, [%[src_a]] \n"
+ "add %[src_a], %[src_a], %[vl] \n" // Alpha
+ I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV444_SVE
+ "ld1b {z19.h}, p1/z, [%[src_a]] \n" // Alpha
+ I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.b, wzr, %w[width] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.b, wzr, %w[width] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_vu]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV12ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+ "incb %[dst_rgb24], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.b, wzr, %w[width] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV21ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+ "incb %[dst_rgb24], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.b, wzr, %w[width] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_vu]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+#define READYUY2_SVE \
+ "ld1w {z0.s}, p2/z, [%[src_yuy2]] \n" /* YUYV */ \
+ "incb %[src_yuy2] \n" \
+ "prfm pldl1keep, [%[src_yuy2], 448] \n" \
+ "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \
+ "trn1 z0.b, z0.b, z0.b \n" /* YYYY */
+
+static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n"
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.h, wzr, %w[width] \n" //
+ READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+#define READUYVY_SVE \
+ "ld1w {z0.s}, p2/z, [%[src_uyvy]] \n" /* UYVY */ \
+ "incb %[src_uyvy] \n" \
+ "prfm pldl1keep, [%[src_uyvy], 448] \n" \
+ "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \
+ "trn2 z0.b, z0.b, z0.b \n" /* YYYY */
+
+static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t nv_uv_start = 0x02000200U;
+ uint32_t nv_uv_step = 0x04040404U;
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n"
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.h, wzr, %w[width] \n" //
+ READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I210AlphaToARGBRow_SVE_SC(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI210_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE
+ "incb %[src_a] \n" //
+ RGBATOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI210_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE RGBATOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "ptrue p3.h \n"
+ "1: \n" //
+ READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p2.h, wzr, %w[width] \n"
+ "whilelt p3.h, %w[vl], %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // (width + 1) / 2
+ "lsr %w[width], %w[width], #1 \n"
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+// P210 has 10 bits in msb of 16 bit NV12 style layout.
+static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.h, wzr, %w[width] \n" //
+ READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READP210_SVE NVTORGB_SVE
+ "subs %w[width], %w[width], %w[vl] \n" //
+ STOREAR30_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "add %w[width], %w[width], #1 \n" // Round up to even
+ "and %w[width], %w[width], #0xfffffffe \n"
+ "whilelt p2.h, wzr, %w[width] \n" //
+ READP210_SVE NVTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410AlphaToARGBRow_SVE_SC(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "cmp %w[width], %w[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE
+ "incb %[src_a] \n" //
+ RGBATOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI410_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE RGBATOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.s \n"
+ "ptrue p3.s \n"
+ "1: \n" //
+ READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "whilelt p2.s, wzr, %w[width] \n"
+ "cntw %x[vl] \n"
+ "whilelt p3.s, %w[vl], %w[width] \n" //
+ READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "+r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.s \n"
+ "ptrue p3.s \n"
+ "1: \n" //
+ READP410_SVE NVTORGB_SVE
+ "subs %w[width], %w[width], %w[vl] \n" //
+ STOREAR30_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n"
+ "whilelt p2.s, wzr, %w[width] \n"
+ "cntw %x[vl] \n"
+ "whilelt p3.s, %w[vl], %w[width] \n" //
+ READP410_SVE NVTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width), // %[width]
+ [vl] "+r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define CONVERT8TO8_SVE \
+ "ld1b {z0.b}, p0/z, [%[src]] \n" \
+ "ld1b {z1.b}, p1/z, [%[src], #1, mul vl] \n" \
+ "incb %[src], all, mul #2 \n" \
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n" \
+ "umulh z0.b, z0.b, z2.b \n" \
+ "umulh z1.b, z1.b, z2.b \n" \
+ "prfm pldl1keep, [%[src], 448] \n" \
+ "add z0.b, z0.b, z3.b \n" \
+ "add z1.b, z1.b, z3.b \n" \
+ "st1b {z0.b}, p0, [%[dst]] \n" \
+ "st1b {z1.b}, p1, [%[dst], #1, mul vl] \n" \
+ "incb %[dst], all, mul #2 \n"
+
+static inline void Convert8To8Row_SVE_SC(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "dup z2.b, %w[scale] \n"
+ "dup z3.b, %w[bias] \n"
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "ptrue p1.b \n"
+ "1: \n" //
+ CONVERT8TO8_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "whilelt p1.b, %w[vl], %w[width] \n" //
+ CONVERT8TO8_SVE
+
+ "99: \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale), // %[scale]
+ [bias] "r"(bias) // %[bias]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0", "p1");
+}
+
+// SVE constants are stored negated such that we can store 128 in int8_t.
+
+// RGB to BT601 coefficients
+// UB 0.875 coefficient = 112
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
+
+static const int8_t kARGBToUVCoefficients[] = {
+ // -UB, -UG, -UR, 0, -VB, -VG, -VR, 0
+ -112, 74, 38, 0, 18, 94, -112, 0,
+};
+
+static const int8_t kABGRToUVCoefficients[] = {
+ // -UR, -UG, -UB, 0, -VR, -VG, -VB, 0
+ 38, 74, -112, 0, -112, 94, 18, 0,
+};
+
+static const int8_t kBGRAToUVCoefficients[] = {
+ // 0, -UR, -UG, -UB, 0, -VR, -VG, -VB
+ 0, 38, 74, -112, 0, -112, 94, 18,
+};
+
+static const int8_t kRGBAToUVCoefficients[] = {
+ // 0, -UB, -UG, -UR, 0, -VB, -VG, -VR
+ 0, -112, 74, 38, 0, 18, 94, -112,
+};
+
+// RGB to JPEG coefficients
+// UB 0.500 coefficient = 128
+// UG -0.33126 coefficient = -85
+// UR -0.16874 coefficient = -43
+// VB -0.08131 coefficient = -21
+// VG -0.41869 coefficient = -107
+// VR 0.500 coefficient = 128
+
+static const int8_t kARGBToUVJCoefficients[] = {
+ // -UB, -UG, -UR, 0, -VB, -VG, -VR, 0
+ -128, 85, 43, 0, 21, 107, -128, 0,
+};
+
+static const int8_t kABGRToUVJCoefficients[] = {
+ // -UR, -UG, -UB, 0, -VR, -VG, -VB, 0
+ 43, 85, -128, 0, -128, 107, 21, 0,
+};
+
+#define ARGBTOUVMATRIX_SVE \
+ "ld1d {z0.d}, p1/z, [%[src0]] \n" /* ABCD(bgra) */ \
+ "ld1d {z1.d}, p2/z, [%[src0], #1, mul vl] \n" /* EFGH(bgra) */ \
+ "ld1d {z2.d}, p3/z, [%[src0], #2, mul vl] \n" /* IJKL(bgra) */ \
+ "ld1d {z3.d}, p4/z, [%[src0], #3, mul vl] \n" /* MNOP(bgra) */ \
+ "ld1d {z4.d}, p1/z, [%[src1]] \n" /* ABCD(bgra) */ \
+ "ld1d {z5.d}, p2/z, [%[src1], #1, mul vl] \n" /* EFGH(bgra) */ \
+ "ld1d {z6.d}, p3/z, [%[src1], #2, mul vl] \n" /* IJKL(bgra) */ \
+ "ld1d {z7.d}, p4/z, [%[src1], #3, mul vl] \n" /* MNOP(bgra) */ \
+ "incb %[src0], all, mul #4 \n" \
+ "incb %[src1], all, mul #4 \n" \
+ \
+ "uaddlb z16.h, z0.b, z4.b \n" /* ABCD(br) */ \
+ "uaddlb z18.h, z1.b, z5.b \n" /* EFGH(br) */ \
+ "uaddlb z20.h, z2.b, z6.b \n" /* IJKL(br) */ \
+ "uaddlb z22.h, z3.b, z7.b \n" /* MNOP(br) */ \
+ "uaddlt z17.h, z0.b, z4.b \n" /* ABCD(ga) */ \
+ "uaddlt z19.h, z1.b, z5.b \n" /* EFGH(ga) */ \
+ "uaddlt z21.h, z2.b, z6.b \n" /* IJKL(ga) */ \
+ "uaddlt z23.h, z3.b, z7.b \n" /* MNOP(ga) */ \
+ \
+ /* Use ADDP on 32-bit elements to add adjacent pairs of 9-bit unsigned */ \
+ "addp z16.s, p0/m, z16.s, z18.s \n" /* ABEFCDGH(br) */ \
+ "addp z17.s, p0/m, z17.s, z19.s \n" /* ABEFCDGH(ga) */ \
+ "addp z20.s, p0/m, z20.s, z22.s \n" /* IJMNKLOP(br) */ \
+ "addp z21.s, p0/m, z21.s, z23.s \n" /* IJMNKLOP(ga) */ \
+ \
+ "rshrnb z0.b, z16.h, #2 \n" /* ABEFCDGH(b0r0) */ \
+ "rshrnb z1.b, z20.h, #2 \n" /* IJMNKLOP(b0r0) */ \
+ "rshrnt z0.b, z17.h, #2 \n" /* ABEFCDGH(bgra) */ \
+ "rshrnt z1.b, z21.h, #2 \n" /* IJMNKLOP(bgra) */ \
+ \
+ "tbl z0.s, {z0.s}, z27.s \n" /* ABCDEFGH */ \
+ "tbl z1.s, {z1.s}, z27.s \n" /* IJKLMNOP */ \
+ \
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n" /* VL per loop */ \
+ \
+ "mov z16.b, #0 \n" \
+ "mov z17.b, #0 \n" \
+ "mov z20.b, #0 \n" \
+ "mov z21.b, #0 \n" \
+ \
+ "usdot z16.s, z0.b, z24.b \n" \
+ "usdot z17.s, z1.b, z24.b \n" \
+ "usdot z20.s, z0.b, z25.b \n" \
+ "usdot z21.s, z1.b, z25.b \n" \
+ \
+ "subhnb z16.b, z26.h, z16.h \n" /* U */ \
+ "subhnb z20.b, z26.h, z20.h \n" /* V */ \
+ "subhnb z17.b, z26.h, z17.h \n" /* U */ \
+ "subhnb z21.b, z26.h, z21.h \n" /* V */ \
+ \
+ "uzp1 z16.h, z16.h, z17.h \n" \
+ "uzp1 z20.h, z20.h, z21.h \n" \
+ \
+ "st1b {z16.h}, p5, [%[dst_u]] \n" /* U */ \
+ "st1b {z20.h}, p5, [%[dst_v]] \n" /* V */ \
+ "inch %[dst_u] \n" \
+ "inch %[dst_v] \n"
+
+static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const int8_t* uvconstants)
+ STREAMING_COMPATIBLE {
+ const uint8_t* src_argb_1 = src_argb + src_stride_argb;
+ uint64_t vl;
+ asm("cntd %x0" : "=r"(vl));
+
+ // Width is a multiple of two here, so halve it.
+ width >>= 1;
+
+ asm volatile(
+ "ptrue p0.b \n"
+ "ld1rw {z24.s}, p0/z, [%[uvconstants]] \n"
+ "ld1rw {z25.s}, p0/z, [%[uvconstants], #4] \n"
+ "mov z26.h, #0x8000 \n" // 128.0 (0x8000)
+
+ // Generate some TBL indices to undo the interleaving from ADDP.
+ "index z0.s, #0, #1 \n"
+ "index z1.s, #1, #1 \n"
+ "uzp1 z27.s, z0.s, z1.s \n"
+
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.lt 2f \n"
+
+ "ptrue p1.d \n"
+ "ptrue p2.d \n"
+ "ptrue p3.d \n"
+ "ptrue p4.d \n"
+ "ptrue p5.h \n"
+ "1: \n" //
+ ARGBTOUVMATRIX_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.eq 99f \n"
+
+ "3: \n"
+ "whilelt p1.d, wzr, %w[width] \n"
+ "whilelt p2.d, %w[vl], %w[width] \n"
+ "whilelt p3.d, %w[vl2], %w[width] \n"
+ "whilelt p4.d, %w[vl3], %w[width] \n"
+ "whilelt p5.h, wzr, %w[width] \n" //
+ ARGBTOUVMATRIX_SVE
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src0] "+r"(src_argb), // %[src0]
+ [src1] "+r"(src_argb_1), // %[src1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width) // %[width]
+ : [uvconstants] "r"(uvconstants), // %[uvconstants]
+ [vl] "r"(vl), // %[vl]
+ [vl2] "r"(vl * 2), // %[vl2]
+ [vl3] "r"(vl * 3) // %[vl3]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
+ "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26",
+ "z27", "p0", "p1", "p2", "p3", "p4", "p5");
+}
+
+#define ARGBTOYMATRIX_SVE(p1, p2, p3, p4, p5) \
+ "ld1w {z0.s}, " p1 \
+ "/z, [%[src]] \n" /* load 4*vl pixels */ \
+ "ld1w {z1.s}, " p2 \
+ "/z, [%[src], #1, mul vl] \n" \
+ "ld1w {z2.s}, " p3 \
+ "/z, [%[src], #2, mul vl] \n" \
+ "ld1w {z3.s}, " p4 \
+ "/z, [%[src], #3, mul vl] \n" \
+ "incb %[src], all, mul #4 \n" \
+ "mov z16.b, #0 \n" \
+ "mov z17.b, #0 \n" \
+ "mov z18.b, #0 \n" \
+ "mov z19.b, #0 \n" \
+ "udot z16.s, z0.b, z24.b \n" \
+ "udot z17.s, z1.b, z24.b \n" \
+ "udot z18.s, z2.b, z24.b \n" \
+ "udot z19.s, z3.b, z24.b \n" \
+ "uzp1 z16.h, z16.h, z17.h \n" \
+ "uzp1 z18.h, z18.h, z19.h \n" \
+ "addhnb z16.b, z16.h, z25.h \n" \
+ "addhnb z18.b, z18.h, z25.h \n" \
+ "uzp1 z20.b, z16.b, z18.b \n" \
+ "st1b {z20.b}, " p5 \
+ ", [%[dst_y]] \n" \
+ "incb %[dst_y] \n" \
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+
+static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c)
+ STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntw %x0" : "=r"(vl));
+
+ asm volatile(
+ "ptrue p0.b \n"
+ "ld1rw {z24.s}, p0/z, [%[c]] \n"
+ "ld1rh {z25.h}, p0/z, [%[c], #48] \n"
+
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.lt 2f \n"
+
+ "ptrue p1.s \n"
+ "1: \n" //
+ ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0")
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.eq 99f \n"
+
+ "3: \n"
+ "whilelt p1.s, wzr, %w[width] \n"
+ "whilelt p2.s, %w[vl], %w[width] \n"
+ "whilelt p3.s, %w[vl2], %w[width] \n"
+ "whilelt p4.s, %w[vl3], %w[width] \n"
+ "whilelt p5.b, wzr, %w[width] \n" //
+ ARGBTOYMATRIX_SVE("p1", "p2", "p3", "p4", "p5")
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width) // %[width]
+ : [c] "r"(c), // %[c]
+ [vl] "r"(vl), // %[vl]
+ [vl2] "r"(vl * 2), // %[vl2]
+ [vl3] "r"(vl * 3) // %[vl3]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z16", "z17", "z18", "z19",
+ "z20", "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5");
+}
+
+#define SSIM4X4_SVE \
+ "add %[tmp_a], %[src_a], %[stride_a] \n" \
+ "add %[tmp_b], %[src_b], %[stride_b] \n" \
+ "ld1w {z0.s}, p0/z, [%[src_a]] \n" \
+ "ld1w {z1.s}, p0/z, [%[tmp_a]] \n" \
+ "add %[tmp_a], %[tmp_a], %[stride_a] \n" \
+ "ld1w {z4.s}, p0/z, [%[src_b]] \n" \
+ "ld1w {z5.s}, p0/z, [%[tmp_b]] \n" \
+ "add %[tmp_b], %[tmp_b], %[stride_b] \n" \
+ "ld1w {z2.s}, p0/z, [%[tmp_a]] \n" \
+ "add %[tmp_a], %[tmp_a], %[stride_a] \n" \
+ "ld1w {z3.s}, p0/z, [%[tmp_a]] \n" \
+ "ld1w {z6.s}, p0/z, [%[tmp_b]] \n" \
+ "add %[tmp_b], %[tmp_b], %[stride_b] \n" \
+ "ld1w {z7.s}, p0/z, [%[tmp_b]] \n" \
+ "incb %[src_a] \n" \
+ "incb %[src_b] \n" \
+ "mov z16.s, #0 \n" \
+ "mov z17.s, #0 \n" \
+ "mov z18.s, #0 \n" \
+ "mov z19.s, #0 \n" \
+ "udot z16.s, z0.b, z20.b \n" \
+ "udot z17.s, z4.b, z20.b \n" \
+ "udot z18.s, z0.b, z0.b \n" \
+ "udot z19.s, z0.b, z4.b \n" \
+ "udot z16.s, z1.b, z20.b \n" \
+ "udot z17.s, z5.b, z20.b \n" \
+ "udot z18.s, z1.b, z1.b \n" \
+ "udot z19.s, z1.b, z5.b \n" \
+ "udot z16.s, z2.b, z20.b \n" \
+ "udot z17.s, z6.b, z20.b \n" \
+ "udot z18.s, z2.b, z2.b \n" \
+ "udot z19.s, z2.b, z6.b \n" \
+ "udot z16.s, z3.b, z20.b \n" \
+ "udot z17.s, z7.b, z20.b \n" \
+ "udot z18.s, z3.b, z3.b \n" \
+ "udot z19.s, z3.b, z7.b \n" \
+ "udot z18.s, z4.b, z4.b \n" \
+ "udot z18.s, z5.b, z5.b \n" \
+ "udot z18.s, z6.b, z6.b \n" \
+ "udot z18.s, z7.b, z7.b \n" \
+ "subs %w[width], %w[width], %w[vl] \n" \
+ "st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]]\n" \
+ "incb %[dst], all, mul #4 \n"
+
+static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ const uint8_t* tmp_a;
+ const uint8_t* tmp_b;
+ width >>= 2;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "mov z20.b, #1 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n" //
+ SSIM4X4_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[width] \n" //
+ SSIM4X4_SVE
+
+ "99: \n"
+ : [src_a] "+r"(src_a), // %[src_a]
+ [src_b] "+r"(src_b), // %[src_b]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp_a] "=&r"(tmp_a), // %[tmp_a]
+ [tmp_b] "=&r"(tmp_b) // %[tmp_b]
+ : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a]
+ [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b]
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
+ "z17", "z18", "z19", "z20", "p0");
+}
+
+static inline float SsimWindowRow_SVE_SC(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p1.s \n"
+ "mov z24.s, #0 \n"
+ "dup z25.s, %w[c1] \n"
+ "dup z26.s, %w[c2] \n"
+
+ "1: \n"
+ "whilelt p0.s, wzr, %w[num_windows] \n"
+ "ld4w {z0.s, z1.s, z2.s, z3.s}, p0/z, [%[row0]] \n"
+ "ld4w {z4.s, z5.s, z6.s, z7.s}, p0/z, [%[row0_next]] \n"
+ "ld4w {z16.s, z17.s, z18.s, z19.s}, p0/z, [%[row1]] \n"
+ "ld4w {z20.s, z21.s, z22.s, z23.s}, p0/z, [%[row1_next]] \n"
+ "incb %[row0], all, mul #4 \n"
+ "incb %[row0_next], all, mul #4 \n"
+ "incb %[row1], all, mul #4 \n"
+ "incb %[row1_next], all, mul #4 \n"
+ "add z0.s, z0.s, z4.s \n"
+ "add z1.s, z1.s, z5.s \n"
+ "add z2.s, z2.s, z6.s \n"
+ "add z3.s, z3.s, z7.s \n"
+ "add z16.s, z16.s, z20.s \n"
+ "add z17.s, z17.s, z21.s \n"
+ "add z18.s, z18.s, z22.s \n"
+ "add z19.s, z19.s, z23.s \n"
+ "add z0.s, z0.s, z16.s \n"
+ "add z1.s, z1.s, z17.s \n"
+ "add z2.s, z2.s, z18.s \n"
+ "add z3.s, z3.s, z19.s \n"
+ "mul z4.s, z0.s, z1.s \n"
+ "lsl z4.s, z4.s, #1 \n"
+ "mul z5.s, z0.s, z0.s \n"
+ "mla z5.s, p1/m, z1.s, z1.s \n"
+ "lsl z3.s, z3.s, #7 \n"
+ "sub z3.s, z3.s, z4.s \n"
+ "add z4.s, z4.s, z25.s \n"
+ "add z3.s, z3.s, z26.s \n"
+ "lsl z2.s, z2.s, #6 \n"
+ "sub z2.s, z2.s, z5.s \n"
+ "add z5.s, z5.s, z25.s \n"
+ "add z2.s, z2.s, z26.s \n"
+ "scvtf z4.s, p1/m, z4.s \n"
+ "scvtf z3.s, p1/m, z3.s \n"
+ "scvtf z5.s, p1/m, z5.s \n"
+ "scvtf z2.s, p1/m, z2.s \n"
+ "fmul z4.s, z4.s, z3.s \n"
+ "fmul z5.s, z5.s, z2.s \n"
+ "fdiv z4.s, p1/m, z4.s, z5.s \n"
+ "fadd z24.s, p0/m, z24.s, z4.s \n"
+ "subs %w[num_windows], %w[num_windows], %w[vl] \n"
+ "b.gt 1b \n"
+
+ "faddv %s[total], p1, z24.s \n"
+ : [row0] "+r"(row0), // %[row0]
+ [row0_next] "+r"(row0_next), // %[row0_next]
+ [row1] "+r"(row1), // %[row1]
+ [row1_next] "+r"(row1_next), // %[row1_next]
+ [num_windows] "+r"(num_windows), // %[num_windows]
+ [vl] "=&r"(vl), // %[vl]
+ [total] "=w"(total) // %[total]
+ : [c1] "r"(26634), // %[c1]
+ [c2] "r"(239708) // %[c2]
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
+ "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26",
+ "p0", "p1");
+ return total;
+}
+
+static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "dup z2.s, %w[value] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "orr z0.d, z0.d, z2.d \n"
+ "orr z1.d, z1.d, z2.d \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.s, %w[vl], %w[width] \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p1/z, [%[src], #1, mul vl] \n"
+ "orr z0.d, z0.d, z2.d \n"
+ "orr z1.d, z1.d, z2.d \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "st1w {z1.s}, p1, [%[dst], #1, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [value] "r"(value) // %[value]
+ : "memory", "cc", "z0", "z1", "z2", "p0", "p1");
+}
+
+#define ARGBALPHABLEND_SVE_2X(pred0, pred1) \
+ "ld1w {z0.s}, " #pred0 "/z, [%[src0]] \n" \
+ "ld1w {z1.s}, " #pred1 "/z, [%[src0], #1, mul vl] \n" \
+ "ld1w {z2.s}, " #pred0 "/z, [%[src1]] \n" \
+ "ld1w {z3.s}, " #pred1 "/z, [%[src1], #1, mul vl] \n" \
+ "incb %[src0], all, mul #2 \n" \
+ "incb %[src1], all, mul #2 \n" \
+ "lsr z4.s, z0.s, #24 \n" \
+ "lsr z5.s, z1.s, #24 \n" \
+ "lsr z6.s, z2.s, #24 \n" \
+ "lsr z7.s, z3.s, #24 \n" \
+ "mov z16.d, z24.d \n" \
+ "mov z17.d, z24.d \n" \
+ "umlalb z16.h, z6.b, z4.b \n" \
+ "umlalb z17.h, z7.b, z5.b \n" \
+ "usra z16.h, z16.h, #8 \n" \
+ "usra z17.h, z17.h, #8 \n" \
+ "lsr z16.h, z16.h, #8 \n" \
+ "lsr z17.h, z17.h, #8 \n" \
+ "sub z16.s, z4.s, z16.s \n" \
+ "sub z17.s, z5.s, z17.s \n" \
+ "add z16.s, z6.s, z16.s \n" \
+ "add z17.s, z7.s, z17.s \n" \
+ "lsl z18.s, z4.s, #8 \n" \
+ "lsl z19.s, z5.s, #8 \n" \
+ "sub z18.s, z18.s, z4.s \n" \
+ "sub z19.s, z19.s, z5.s \n" \
+ "sub z6.s, z6.s, #1 \n" \
+ "sub z7.s, z7.s, #1 \n" \
+ "asr z6.s, z6.s, #31 \n" \
+ "asr z7.s, z7.s, #31 \n" \
+ "udiv z18.s, p0/m, z18.s, z16.s \n" \
+ "udiv z19.s, p0/m, z19.s, z17.s \n" \
+ "mul z18.s, z18.s, z25.s \n" \
+ "mul z19.s, z19.s, z25.s \n" \
+ "orr z18.d, z18.d, z6.d \n" \
+ "orr z19.d, z19.d, z7.d \n" \
+ "not z20.b, p0/m, z18.b \n" \
+ "not z21.b, p0/m, z19.b \n" \
+ "mov z4.d, z24.d \n" \
+ "mov z5.d, z24.d \n" \
+ "mov z6.d, z24.d \n" \
+ "mov z7.d, z24.d \n" \
+ "umlalb z4.h, z2.b, z20.b \n" \
+ "umlalt z5.h, z2.b, z20.b \n" \
+ "umlalb z6.h, z3.b, z21.b \n" \
+ "umlalt z7.h, z3.b, z21.b \n" \
+ "umlalb z4.h, z0.b, z18.b \n" \
+ "umlalt z5.h, z0.b, z18.b \n" \
+ "umlalb z6.h, z1.b, z19.b \n" \
+ "umlalt z7.h, z1.b, z19.b \n" \
+ "usra z4.h, z4.h, #8 \n" \
+ "usra z5.h, z5.h, #8 \n" \
+ "usra z6.h, z6.h, #8 \n" \
+ "usra z7.h, z7.h, #8 \n" \
+ "shrnb z0.b, z4.h, #8 \n" \
+ "shrnt z0.b, z5.h, #8 \n" \
+ "shrnb z1.b, z6.h, #8 \n" \
+ "shrnt z1.b, z7.h, #8 \n" \
+ "and z0.d, z0.d, z26.d \n" \
+ "and z1.d, z1.d, z26.d \n" \
+ "lsl z16.s, z16.s, #24 \n" \
+ "lsl z17.s, z17.s, #24 \n" \
+ "orr z0.d, z0.d, z16.d \n" \
+ "orr z1.d, z1.d, z17.d \n" \
+ "st1w {z0.s}, " #pred0 ", [%[dst]] \n" \
+ "st1w {z1.s}, " #pred1 ", [%[dst], #1, mul vl] \n" \
+ "incb %[dst], all, mul #2 \n"
+
+static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p0.b \n"
+ "mov z24.h, #1 \n"
+ "mov z25.s, #0x01010101 \n"
+ "mov z26.s, #0x00ffffff \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ ARGBALPHABLEND_SVE_2X(p0, p0)
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.s, wzr, %w[width] \n"
+ "whilelt p2.s, %w[vl], %w[width] \n"
+ ARGBALPHABLEND_SVE_2X(p1, p2)
+
+ "99: \n"
+ : [src0] "+r"(src_argb0), // %[src0]
+ [src1] "+r"(src_argb1), // %[src1]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
+ "z17", "z18", "z19", "z20", "z21", "z24", "z25", "z26", "p0", "p1",
+ "p2");
+}
+
+#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_ROW_SVE_H_
diff --git a/libs/libyuv/include/libyuv/scale.h b/libs/libyuv/include/libyuv/scale.h
index 23ba1634f7..ada3f6061a 100644
--- a/libs/libyuv/include/libyuv/scale.h
+++ b/libs/libyuv/include/libyuv/scale.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_SCALE_H_
#define INCLUDE_LIBYUV_SCALE_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -27,27 +29,40 @@ typedef enum FilterMode {
} FilterModeEnum;
// Scale a YUV plane.
+// Returns 0 if successful.
+LIBYUV_API
+int ScalePlane(const uint8_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint8_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int ScalePlane_16(const uint16_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint16_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+// Sample is expected to be in the low 12 bits.
LIBYUV_API
-void ScalePlane(const uint8_t* src,
- int src_stride,
- int src_width,
- int src_height,
- uint8_t* dst,
- int dst_stride,
- int dst_width,
- int dst_height,
- enum FilterMode filtering);
-
-LIBYUV_API
-void ScalePlane_16(const uint16_t* src,
- int src_stride,
- int src_width,
- int src_height,
- uint16_t* dst,
- int dst_stride,
- int dst_width,
- int dst_height,
- enum FilterMode filtering);
+int ScalePlane_12(const uint16_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint16_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
// Scales a YUV 4:2:0 image from the src width and height to the
// dst width and height.
@@ -97,6 +112,25 @@ int I420Scale_16(const uint16_t* src_y,
int dst_height,
enum FilterMode filtering);
+LIBYUV_API
+int I420Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
// Scales a YUV 4:4:4 image from the src width and height to the
// dst width and height.
// If filtering is kFilterNone, a simple nearest-neighbor algorithm is
@@ -145,6 +179,197 @@ int I444Scale_16(const uint16_t* src_y,
int dst_height,
enum FilterMode filtering);
+LIBYUV_API
+int I444Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+// Scales a YUV 4:2:2 image from the src width and height to the
+// dst width and height.
+// If filtering is kFilterNone, a simple nearest-neighbor algorithm is
+// used. This produces basic (blocky) quality at the fastest speed.
+// If filtering is kFilterBilinear, interpolation is used to produce a better
+// quality image, at the expense of speed.
+// If filtering is kFilterBox, averaging is used to produce ever better
+// quality image, at further expense of speed.
+// Returns 0 if successful.
+LIBYUV_API
+int I422Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int I422Scale_16(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int I422Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+// Scales an NV12, NV16 or NV24 image from the src width and height to the
+// dst width and height.
+// If filtering is kFilterNone, a simple nearest-neighbor algorithm is
+// used. This produces basic (blocky) quality at the fastest speed.
+// If filtering is kFilterBilinear, interpolation is used to produce a better
+// quality image, at the expense of speed.
+// kFilterBox is not supported for the UV channel and will be treated as
+// bilinear.
+// Returns 0 if successful.
+
+LIBYUV_API
+int NV12Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int NV16Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int NV24Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+// Scales a P010, P210 or P410 image (10-bit, samples in the MSBs of uint16).
+// Strides are in uint16 elements. Y and UV are converted to 10-bit LSB,
+// scaled with ScalePlane_16 (kFilterBox included), then converted back to
+// 10-bit MSB.
+// Returns 0 if successful.
+
+LIBYUV_API
+int P010Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int P210Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+LIBYUV_API
+int P410Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
#ifdef __cplusplus
// Legacy API. Deprecated.
LIBYUV_API
@@ -164,11 +389,7 @@ int Scale(const uint8_t* src_y,
int dst_stride_v,
int dst_width,
int dst_height,
- LIBYUV_BOOL interpolate);
-
-// For testing, allow disabling of specialized scalers.
-LIBYUV_API
-void SetUseReferenceImpl(LIBYUV_BOOL use);
+ int interpolate);
#endif // __cplusplus
#ifdef __cplusplus
diff --git a/libs/libyuv/include/libyuv/scale_argb.h b/libs/libyuv/include/libyuv/scale_argb.h
index 7641f18e34..c0a55340fd 100644
--- a/libs/libyuv/include/libyuv/scale_argb.h
+++ b/libs/libyuv/include/libyuv/scale_argb.h
@@ -11,6 +11,8 @@
#ifndef INCLUDE_LIBYUV_SCALE_ARGB_H_
#define INCLUDE_LIBYUV_SCALE_ARGB_H_
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#include "libyuv/scale.h" // For FilterMode
diff --git a/libs/libyuv/include/libyuv/scale_rgb.h b/libs/libyuv/include/libyuv/scale_rgb.h
new file mode 100644
index 0000000000..94218fe3df
--- /dev/null
+++ b/libs/libyuv/include/libyuv/scale_rgb.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_SCALE_RGB_H_
+#define INCLUDE_LIBYUV_SCALE_RGB_H_
+
+#include <stdint.h>
+
+#include "libyuv/basic_types.h"
+#include "libyuv/scale.h" // For FilterMode
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// RGB can be RAW, RGB24 or YUV24
+// RGB scales 24 bit images by converting a row at a time to ARGB
+// and using ARGB row functions to scale, then convert to RGB.
+// TODO(fbarchard): Allow input/output formats to be specified.
+LIBYUV_API
+int RGBScale(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ int src_width,
+ int src_height,
+ uint8_t* dst_rgb,
+ int dst_stride_rgb,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_SCALE_UV_H_
diff --git a/libs/libyuv/include/libyuv/scale_row.h b/libs/libyuv/include/libyuv/scale_row.h
index 6e207a9c64..e81d227d91 100644
--- a/libs/libyuv/include/libyuv/scale_row.h
+++ b/libs/libyuv/include/libyuv/scale_row.h
@@ -11,7 +11,10 @@
#ifndef INCLUDE_LIBYUV_SCALE_ROW_H_
#define INCLUDE_LIBYUV_SCALE_ROW_H_
-#include "libyuv/basic_types.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#include "libyuv/cpu_support.h"
#include "libyuv/scale.h"
#ifdef __cplusplus
@@ -19,68 +22,86 @@ namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
-// GCC >= 4.7.0 required for AVX2.
-#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
-#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
-#define GCC_HAS_AVX2 1
-#endif // GNUC >= 4.7
-#endif // __GNUC__
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(_M_IX86) || defined(_M_X64) || defined(__x86_64__) || \
+ defined(__i386__))
#define HAS_FIXEDDIV1_X86
#define HAS_FIXEDDIV_X86
#define HAS_SCALEADDROW_SSE2
-#define HAS_SCALEARGBCOLS_SSE2
-#define HAS_SCALEARGBCOLSUP2_SSE2
-#define HAS_SCALEARGBFILTERCOLS_SSSE3
-#define HAS_SCALEARGBROWDOWN2_SSE2
-#define HAS_SCALEARGBROWDOWNEVEN_SSE2
-#define HAS_SCALECOLSUP2_SSE2
-#define HAS_SCALEFILTERCOLS_SSSE3
+#endif
+
+// The following are available on 32-bit Visual C and gcc/clang x86 platforms:
+#if !defined(LIBYUV_DISABLE_X86) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ ((defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__)) || \
+ defined(__x86_64__) || defined(__i386__))
#define HAS_SCALEROWDOWN2_SSSE3
-#define HAS_SCALEROWDOWN34_SSSE3
-#define HAS_SCALEROWDOWN38_SSSE3
#define HAS_SCALEROWDOWN4_SSSE3
#endif
+// The following are available for gcc/clang x86 platforms:
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_SCALEROWUP2_LINEAR_SSE2
+#define HAS_SCALEROWUP2_LINEAR_SSSE3
+#define HAS_SCALEROWUP2_BILINEAR_SSE2
+#define HAS_SCALEROWUP2_BILINEAR_SSSE3
+#define HAS_SCALEROWUP2_LINEAR_12_SSSE3
+#define HAS_SCALEROWUP2_BILINEAR_12_SSSE3
+#define HAS_SCALEROWUP2_LINEAR_16_SSE2
+#define HAS_SCALEROWUP2_BILINEAR_16_SSE2
+#define HAS_SCALEUVROWUP2_LINEAR_SSSE3
+#define HAS_SCALEUVROWUP2_BILINEAR_SSSE3
+#define HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+#define HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+#endif
+
// The following are available on all x86 platforms, but
// require VS2012, clang 3.4 or gcc 4.7.
-// The code supports NaCL but requires a new compiler and validator.
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
defined(GCC_HAS_AVX2))
#define HAS_SCALEADDROW_AVX2
+#define HAS_SCALEARGBCOLS_AVX2
+#define HAS_SCALEARGBCOLSUP2_AVX2
+#define HAS_SCALEARGBFILTERCOLS_AVX2
+#define HAS_SCALEARGBROWDOWN2_AVX2
+#define HAS_SCALEARGBROWDOWNEVEN_AVX2
+#define HAS_SCALECOLSUP2_AVX2
+#define HAS_SCALEFILTERCOLS_AVX2
+#define HAS_SCALEROWDOWN2_16_AVX2
#define HAS_SCALEROWDOWN2_AVX2
+#define HAS_SCALEROWDOWN34_AVX2
+#define HAS_SCALEROWDOWN38_AVX2
#define HAS_SCALEROWDOWN4_AVX2
+#define HAS_SCALEROWUP2_BILINEAR_12_AVX2
+#define HAS_SCALEROWUP2_BILINEAR_16_AVX2
+#define HAS_SCALEROWUP2_BILINEAR_AVX2
+#define HAS_SCALEROWUP2_LINEAR_12_AVX2
+#define HAS_SCALEROWUP2_LINEAR_16_AVX2
+#define HAS_SCALEROWUP2_LINEAR_AVX2
+#define HAS_SCALEUVADDCOLS3_AVX2
+#define HAS_SCALEUVROWDOWN2_AVX2
+#define HAS_SCALEUVROWDOWN2BOX_AVX2
+#define HAS_SCALEUVROWDOWN2LINEAR_AVX2
+#define HAS_SCALEUVROWDOWNEVEN_AVX2
+#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+#define HAS_SCALEUVROWUP2_BILINEAR_AVX2
+#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+#define HAS_SCALEUVROWUP2_LINEAR_AVX2
+#endif
+
+// The following are available for AVX512 x64 platforms:
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(_M_X64)) && \
+ (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512) || \
+ defined(VISUALC_HAS_AVX512))
+#define HAS_SCALEADDROW_AVX512BW
+#define HAS_SCALEADDCOLS1_AVX512BW
+#define HAS_SCALEADDCOLS2_AVX512BW
+#define HAS_SCALEUVADDCOLS3_AVX512VBMI
+#define HAS_SCALEROWDOWN2_AVX512BW
#endif
// The following are available on Neon platforms:
@@ -96,36 +117,121 @@ extern "C" {
#define HAS_SCALEROWDOWN34_NEON
#define HAS_SCALEROWDOWN38_NEON
#define HAS_SCALEROWDOWN4_NEON
+#define HAS_SCALEUVADDCOLS3_NEON
+#define HAS_SCALEUVROWDOWN2_NEON
+#define HAS_SCALEUVROWDOWN2LINEAR_NEON
+#define HAS_SCALEUVROWDOWN2BOX_NEON
+#define HAS_SCALEUVROWDOWNEVEN_NEON
+#define HAS_SCALEROWUP2_LINEAR_NEON
+#define HAS_SCALEROWUP2_BILINEAR_NEON
+#define HAS_SCALEROWUP2_LINEAR_12_NEON
+#define HAS_SCALEROWUP2_BILINEAR_12_NEON
+#define HAS_SCALEROWUP2_LINEAR_16_NEON
+#define HAS_SCALEROWUP2_BILINEAR_16_NEON
+#define HAS_SCALEUVROWUP2_LINEAR_NEON
+#define HAS_SCALEUVROWUP2_BILINEAR_NEON
+#define HAS_SCALEUVROWUP2_LINEAR_16_NEON
+#define HAS_SCALEUVROWUP2_BILINEAR_16_NEON
+#endif
+
+// The following are available on AArch64 Neon platforms:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_SCALEROWDOWN2_16_NEON
+#endif
+
+// The following are available on AArch64 SME platforms:
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_SCALEARGBROWDOWN2_SME
+#define HAS_SCALEROWDOWN2_16_SME
+#define HAS_SCALEROWDOWN2_SME
+#define HAS_SCALEUVADDCOLS3_SME
+#define HAS_SCALEUVROWDOWN2BOX_SME
+#define HAS_SCALEUVROWDOWN2LINEAR_SME
+#define HAS_SCALEUVROWDOWN2_SME
+#endif
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+#define HAS_SCALEARGBROWDOWN2_LSX
+#define HAS_SCALEARGBROWDOWNEVEN_LSX
+#define HAS_SCALEROWDOWN2_LSX
+#define HAS_SCALEROWDOWN4_LSX
+#define HAS_SCALEROWDOWN38_LSX
+#define HAS_SCALEFILTERCOLS_LSX
+#define HAS_SCALEADDROW_LSX
+#define HAS_SCALEARGBCOLS_LSX
+#define HAS_SCALEARGBFILTERCOLS_LSX
+#define HAS_SCALEROWDOWN34_LSX
+#define HAS_SCALEUVADDCOLS3_LSX
+#endif
+
+#if !defined(LIBYUV_DISABLE_RVV) && \
+ ((defined(__riscv_vector) && defined(__riscv_zve64x)) || \
+ (defined(LIBYUV_RVV) && \
+ (!defined(__riscv_vector) || defined(__riscv_zve64x))))
+#define HAS_SCALEUVROWDOWN4_RVV
+#endif
+
+#if !defined(LIBYUV_DISABLE_RVV) && \
+ (defined(__riscv_vector) || defined(LIBYUV_RVV))
+#define HAS_SCALEARGBROWDOWN2_RVV
+#define HAS_SCALEROWDOWN34_0_BOX_RVV
+#define HAS_SCALEROWDOWN34_1_BOX_RVV
+#define HAS_SCALEROWDOWN38_2_BOX_RVV
+#define HAS_SCALEROWDOWN38_3_BOX_RVV
+#define HAS_SCALEUVROWUP2_BILINEAR_RVV
+#define HAS_SCALEUVROWUP2_LINEAR_RVV
+#define HAS_SCALEROWDOWN34_RVV
+#define HAS_SCALEROWDOWN38_RVV
+#define HAS_SCALEROWUP2_BILINEAR_RVV
+#define HAS_SCALEROWUP2_LINEAR_RVV
+
+#define HAS_SCALEARGBROWDOWN2BOX_RVV
+#define HAS_SCALEARGBROWDOWN2LINEAR_RVV
+#define HAS_SCALEARGBROWDOWNEVENBOX_RVV
+#define HAS_SCALEROWDOWN2BOX_RVV
+#define HAS_SCALEADDROW_RVV
+// TODO: Test ScaleARGBRowDownEven_RVV and enable it
+// #define HAS_SCALEARGBROWDOWNEVEN_RVV
+#define HAS_SCALEUVROWDOWNEVEN_RVV
+#define HAS_SCALEROWDOWN2_RVV
+#define HAS_SCALEROWDOWN2LINEAR_RVV
+#define HAS_SCALEROWDOWN4_RVV
+#define HAS_SCALEROWDOWN4BOX_RVV
+#define HAS_SCALEUVADDCOLS3_RVV
+#define HAS_SCALEUVROWDOWN2_RVV
+#define HAS_SCALEUVROWDOWN2BOX_RVV
+#define HAS_SCALEUVROWDOWN2LINEAR_RVV
+#define HAS_SCALEARGBFILTERCOLS_RVV
#endif
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#define HAS_SCALEADDROW_MSA
-#define HAS_SCALEARGBCOLS_MSA
-#define HAS_SCALEARGBFILTERCOLS_MSA
-#define HAS_SCALEARGBROWDOWN2_MSA
-#define HAS_SCALEARGBROWDOWNEVEN_MSA
-#define HAS_SCALEFILTERCOLS_MSA
-#define HAS_SCALEROWDOWN2_MSA
-#define HAS_SCALEROWDOWN34_MSA
-#define HAS_SCALEROWDOWN38_MSA
-#define HAS_SCALEROWDOWN4_MSA
+// The following are available for WebAssembly SIMD128:
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#define HAS_SCALEADDROW_WASMSIMD
+#define HAS_SCALEARGBCOLS_WASMSIMD
+#define HAS_SCALEARGBCOLSUP2_WASMSIMD
+#define HAS_SCALEARGBFILTERCOLS_WASMSIMD
+#define HAS_SCALEARGBROWDOWN2_WASMSIMD
+#define HAS_SCALEARGBROWDOWNEVEN_WASMSIMD
+#define HAS_SCALEFILTERCOLS_WASMSIMD
+#define HAS_SCALEROWDOWN2_WASMSIMD
+#define HAS_SCALEROWDOWN34_WASMSIMD
+#define HAS_SCALEROWDOWN38_WASMSIMD
+#define HAS_SCALEROWDOWN4_WASMSIMD
+#define HAS_SCALEUVADDCOLS3_WASMSIMD
#endif
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-#define HAS_FIXEDDIV1_MIPS
-#define HAS_FIXEDDIV_MIPS
-#define HAS_SCALEADDROW_16_MMI
-#define HAS_SCALEADDROW_MMI
-#define HAS_SCALEARGBCOLS_MMI
-#define HAS_SCALEARGBCOLSUP2_MMI
-#define HAS_SCALEARGBROWDOWN2_MMI
-#define HAS_SCALEARGBROWDOWNEVEN_MMI
-#define HAS_SCALECOLS_16_MMI
-#define HAS_SCALECOLS_MMI
-#define HAS_SCALEROWDOWN2_16_MMI
-#define HAS_SCALEROWDOWN2_MMI
-#define HAS_SCALEROWDOWN4_16_MMI
-#define HAS_SCALEROWDOWN4_MMI
+// The following are available for Hexagon HVX:
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#define HAS_SCALEADDROW_HVX
+#define HAS_SCALEARGBROWDOWN2_HVX
+#define HAS_SCALEROWDOWN2_16_HVX
+#define HAS_SCALEROWDOWN2_HVX
+#define HAS_SCALEROWDOWN4_HVX
+#define HAS_SCALEUVADDCOLS3_HVX
+#define HAS_SCALEUVROWDOWN2_HVX
+#define HAS_SCALEUVROWDOWN2BOX_HVX
+#define HAS_SCALEUVROWDOWN2LINEAR_HVX
#endif
// Scale ARGB vertically with bilinear interpolation.
@@ -165,17 +271,12 @@ enum FilterMode ScaleFilterReduce(int src_width,
// Divide num by div and return as 16.16 fixed point result.
int FixedDiv_C(int num, int div);
int FixedDiv_X86(int num, int div);
-int FixedDiv_MIPS(int num, int div);
// Divide num - 1 by div - 1 and return as 16.16 fixed point result.
int FixedDiv1_C(int num, int div);
int FixedDiv1_X86(int num, int div);
-int FixedDiv1_MIPS(int num, int div);
#ifdef HAS_FIXEDDIV_X86
#define FixedDiv FixedDiv_X86
#define FixedDiv1 FixedDiv1_X86
-#elif defined HAS_FIXEDDIV_MIPS
-#define FixedDiv FixedDiv_MIPS
-#define FixedDiv1 FixedDiv1_MIPS
#else
#define FixedDiv FixedDiv_C
#define FixedDiv1 FixedDiv1_C
@@ -260,6 +361,40 @@ void ScaleRowDown34_1_Box_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* d,
int dst_width);
+
+void ScaleRowUp2_Linear_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_Any_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_Any_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_Any_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_Any_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+
void ScaleCols_C(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
@@ -375,6 +510,142 @@ void ScaleARGBFilterCols64_C(uint8_t* dst_argb,
int dst_width,
int x32,
int dx);
+void ScaleUVRowDown2_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEven_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEven_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint16_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint16_t* dst_uv,
+ int dst_width);
+
+void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_Any_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_Any_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_Any_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_Any_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+
+void ScaleUVCols_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleUVCols_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleUVCols64_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx);
+void ScaleUVCols64_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx);
+void ScaleUVColsUp2_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int,
+ int);
+void ScaleUVColsUp2_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int,
+ int);
+void ScaleUVFilterCols_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleUVFilterCols_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleUVFilterCols64_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx);
+void ScaleUVFilterCols64_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx);
+void ScaleUVAddCols3_C(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_16_C(int dst_width,
+ int scaleval,
+ const uint32_t* src_uv,
+ uint16_t* dst_uv);
// Specialized scalers for x86.
void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
@@ -401,6 +672,34 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
+void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -418,30 +717,144 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+void ScaleRowDown34_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+
+void ScaleRowUp2_Linear_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_SSE2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_SSSE3(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_12_SSSE3(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_SSE2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
int dst_width);
-void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
+void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
int dst_width);
-void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
int dst_width);
-void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
int dst_width);
+void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_Any_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_Any_SSE2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_Any_SSSE3(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_12_Any_SSSE3(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_Any_SSE2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_Any_SSE2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_Any_SSSE3(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_Any_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_12_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_Any_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+
void ScaleRowDown2_Any_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -474,6 +887,18 @@ void ScaleRowDown2Box_Odd_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
+void ScaleRowDown2_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Linear_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width);
void ScaleRowDown4_Any_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -491,63 +916,78 @@ void ScaleRowDown4Box_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown34_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown34_1_Box_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown34_0_Box_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown38_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown38_3_Box_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
+void ScaleRowDown34_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_1_Box_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_0_Box_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_3_Box_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_2_Box_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
+void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width);
void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
+void ScaleAddCols1_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr);
+void ScaleAddCols2_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr);
-void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx);
-void ScaleColsUp2_SSE2(uint8_t* dst_ptr,
+void ScaleFilterCols_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleColsUp2_AVX2(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
int x,
int dx);
// ARGB Column functions
-void ScaleARGBCols_SSE2(uint8_t* dst_argb,
+void ScaleARGBCols_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx);
-void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
+void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
@@ -572,47 +1012,22 @@ void ScaleARGBCols_Any_NEON(uint8_t* dst_ptr,
int dst_width,
int x,
int dx);
-void ScaleARGBFilterCols_MSA(uint8_t* dst_argb,
+void ScaleARGBFilterCols_RVV(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx);
-void ScaleARGBCols_MSA(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBFilterCols_Any_MSA(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBCols_Any_MSA(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBCols_MMI(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBCols_Any_MMI(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx);
// ARGB Row functions
-void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
+void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
+void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
+void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
@@ -620,47 +1035,59 @@ void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleARGBRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
+void ScaleARGBRowDown2Linear_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width);
void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleARGBRowDown2_MSA(const uint8_t* src_argb,
+void ScaleARGBRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleARGBRowDown2_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Linear_MSA(const uint8_t* src_argb,
+void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Box_MSA(const uint8_t* src_argb,
+void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2_MMI(const uint8_t* src_argb,
+void ScaleARGBRowDown2_LSX(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Linear_MMI(const uint8_t* src_argb,
+void ScaleARGBRowDown2Linear_LSX(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2Box_MMI(const uint8_t* src_argb,
+void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDown2_Any_SSE2(const uint8_t* src_ptr,
+void ScaleARGBRowDown2_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2Linear_Any_SSE2(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Linear_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2Box_Any_SSE2(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Box_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
@@ -676,36 +1103,24 @@ void ScaleARGBRowDown2Box_Any_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBRowDown2_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2Linear_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Linear_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDown2_Any_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleARGBRowDown2Linear_Any_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleARGBRowDown2Box_Any_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb,
+void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb,
+void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
@@ -720,32 +1135,32 @@ void ScaleARGBRowDownEvenBox_NEON(const uint8_t* src_argb,
int src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEven_MSA(const uint8_t* src_argb,
+void ScaleARGBRowDownEven_LSX(const uint8_t* src_argb,
ptrdiff_t src_stride,
int32_t src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEvenBox_MSA(const uint8_t* src_argb,
+void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEven_MMI(const uint8_t* src_argb,
+void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
int32_t src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEvenBox_MMI(const uint8_t* src_argb,
+void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width);
-void ScaleARGBRowDownEven_Any_SSE2(const uint8_t* src_ptr,
+void ScaleARGBRowDownEven_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDownEvenBox_Any_SSE2(const uint8_t* src_ptr,
+void ScaleARGBRowDownEvenBox_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_ptr,
@@ -760,43 +1175,324 @@ void ScaleARGBRowDownEvenBox_Any_NEON(const uint8_t* src_ptr,
int src_stepx,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDownEven_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBRowDownEven_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
int32_t src_stepx,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDownEvenBox_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBRowDownEvenBox_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_ptr,
int dst_width);
-void ScaleARGBRowDownEven_Any_MMI(const uint8_t* src_ptr,
+
+// UV Row functions
+void ScaleUVRowDown2_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleUVRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleUVRowDown2Linear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleUVRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleUVRowDown2_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleUVRowDown2_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDown2Linear_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDown2Box_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDown2_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDown2Linear_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDown2Box_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDownEven_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown4_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int32_t src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEven_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int32_t src_stepx,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDownEven_Any_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowDownEven_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_ptr,
+ int dst_width);
+
+void ScaleUVRowUp2_Linear_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
- int32_t src_stepx,
uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
int dst_width);
-void ScaleARGBRowDownEvenBox_Any_MMI(const uint8_t* src_ptr,
+void ScaleUVRowUp2_Linear_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_Any_SSSE3(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
- int src_stepx,
uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
int dst_width);
+void ScaleUVRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_SSE41(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_SSE41(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_Any_SSE41(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_Any_SSE41(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_Any_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_Any_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVRowUp2_Linear_16_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleUVAddCols3_AVX2(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_Any_AVX2(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_AVX512VBMI(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_NEON(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_Any_NEON(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_SME(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_RVV(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_LSX(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_Any_LSX(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_WASMSIMD(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_Any_WASMSIMD(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
+void ScaleUVAddCols3_HVX(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv);
// ScaleRowDown2Box also used by planar functions
-// NEON downscalers with interpolation.
-
-// Note - not static due to reuse in convert for 444 to 420.
+// NEON/SME downscalers with interpolation.
void ScaleRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2Linear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown4_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -891,6 +1587,55 @@ void ScaleRowDown38_2_Box_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width);
+void ScaleRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_12_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_Any_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_12_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_12_Any_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+void ScaleRowUp2_Linear_16_Any_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width);
+
void ScaleAddRow_NEON(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
void ScaleAddRow_Any_NEON(const uint8_t* src_ptr,
uint16_t* dst_ptr,
@@ -908,197 +1653,425 @@ void ScaleFilterCols_Any_NEON(uint8_t* dst_ptr,
int x,
int dx);
-void ScaleRowDown2_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown2Linear_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2Linear_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown2Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown4_MSA(const uint8_t* src_ptr,
+void ScaleRowDown4_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown4Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown4Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown38_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown38_2_Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_2_Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown38_3_Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_3_Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleAddRow_MSA(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
-void ScaleFilterCols_MSA(uint8_t* dst_ptr,
+void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
+void ScaleFilterCols_LSX(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
int x,
int dx);
-void ScaleRowDown34_MSA(const uint8_t* src_ptr,
+void ScaleARGBFilterCols_LSX(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBCols_LSX(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleRowDown34_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown34_0_Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown34_0_Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* d,
int dst_width);
-void ScaleRowDown34_1_Box_MSA(const uint8_t* src_ptr,
+void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* d,
int dst_width);
-
-void ScaleRowDown2_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown2Linear_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2Linear_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown2Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown2Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown4_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown4_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown4Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown4Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown38_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown38_2_Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_2_Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown38_3_Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown38_3_Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleAddRow_Any_MSA(const uint8_t* src_ptr,
+void ScaleAddRow_Any_LSX(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
-void ScaleFilterCols_Any_MSA(uint8_t* dst_ptr,
+void ScaleFilterCols_Any_LSX(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,
int x,
int dx);
-void ScaleRowDown34_Any_MSA(const uint8_t* src_ptr,
+void ScaleARGBCols_Any_LSX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBFilterCols_Any_LSX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleRowDown34_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown34_0_Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown34_0_Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown34_1_Box_Any_MSA(const uint8_t* src_ptr,
+void ScaleRowDown34_1_Box_Any_LSX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
-void ScaleRowDown2_MMI(const uint8_t* src_ptr,
+void ScaleAddRow_RVV(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
+void ScaleRowDown2_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown2_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-void ScaleRowDown2Linear_MMI(const uint8_t* src_ptr,
+void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown2Linear_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-void ScaleRowDown2Box_MMI(const uint8_t* src_ptr,
+void ScaleRowDown2Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown2Box_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-void ScaleRowDown2Box_Odd_MMI(const uint8_t* src_ptr,
+
+void ScaleRowDown4_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
- uint8_t* dst,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+
+void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
int dst_width);
-void ScaleRowDown4_MMI(const uint8_t* src_ptr,
+
+void ScaleRowDown2_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Linear_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Linear_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown2Box_Odd_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_0_Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown34_1_Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_3_Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown38_2_Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width);
+void ScaleAddRow_Any_WASMSIMD(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width);
+void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleFilterCols_Any_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBRowDown2_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width);
+void ScaleARGBRowDown2Linear_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width);
+void ScaleARGBRowDown2Box_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width);
+void ScaleARGBRowDown2_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBRowDown2Linear_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBRowDown2Box_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBRowDownEven_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width);
+void ScaleARGBRowDownEvenBox_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width);
+void ScaleARGBRowDownEven_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBRowDownEvenBox_Any_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBCols_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBCols_Any_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBFilterCols_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBFilterCols_Any_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx);
+void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
+
+void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
+void ScaleRowDown2_HVX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
-void ScaleRowDown4_16_MMI(const uint16_t* src_ptr,
+void ScaleRowDown2Linear_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_HVX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
- uint16_t* dst,
+ uint8_t* dst,
int dst_width);
-void ScaleRowDown4Box_MMI(const uint8_t* src_ptr,
+void ScaleRowDown2Box_Odd_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2_16_HVX(const uint16_t* src_ptr,
ptrdiff_t src_stride,
- uint8_t* dst,
+ uint16_t* dst,
int dst_width);
-void ScaleRowDown4Box_16_MMI(const uint16_t* src_ptr,
+void ScaleRowDown2Linear_16_HVX(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_16_HVX(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst,
int dst_width);
-void ScaleAddRow_MMI(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
-void ScaleAddRow_16_MMI(const uint16_t* src_ptr,
- uint32_t* dst_ptr,
- int src_width);
-void ScaleColsUp2_MMI(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx);
-void ScaleColsUp2_16_MMI(uint16_t* dst_ptr,
- const uint16_t* src_ptr,
- int dst_width,
- int x,
- int dx);
-void ScaleARGBColsUp2_MMI(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx);
-
-void ScaleRowDown2_Any_MMI(const uint8_t* src_ptr,
+void ScaleRowDown4_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleRowDown4Box_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width);
+void ScaleARGBRowDown2_HVX(const uint8_t* src_argb,
ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+ uint8_t* dst_argb,
int dst_width);
-void ScaleRowDown2Linear_Any_MMI(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Linear_HVX(const uint8_t* src_argb,
ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+ uint8_t* dst_argb,
int dst_width);
-void ScaleRowDown2Box_Any_MMI(const uint8_t* src_ptr,
+void ScaleARGBRowDown2Box_HVX(const uint8_t* src_argb,
ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown4_Any_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width);
-void ScaleRowDown4Box_Any_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
+ uint8_t* dst_argb,
int dst_width);
-void ScaleAddRow_Any_MMI(const uint8_t* src_ptr,
- uint16_t* dst_ptr,
- int src_width);
+void ScaleUVRowDown2_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Linear_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+void ScaleUVRowDown2Box_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/include/libyuv/scale_uv.h b/libs/libyuv/include/libyuv/scale_uv.h
new file mode 100644
index 0000000000..168c5d9495
--- /dev/null
+++ b/libs/libyuv/include/libyuv/scale_uv.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2020 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_SCALE_UV_H_
+#define INCLUDE_LIBYUV_SCALE_UV_H_
+
+#include <stdint.h>
+
+#include "libyuv/basic_types.h"
+#include "libyuv/scale.h" // For FilterMode
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+LIBYUV_API
+int UVScale(const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+// Scale a 16 bit UV image.
+LIBYUV_API
+int UVScale_16(const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_SCALE_UV_H_
diff --git a/libs/libyuv/include/libyuv/version.h b/libs/libyuv/include/libyuv/version.h
index b1883ef8e1..5ba507c6d8 100644
--- a/libs/libyuv/include/libyuv/version.h
+++ b/libs/libyuv/include/libyuv/version.h
@@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
-#define LIBYUV_VERSION 1735
+#define LIBYUV_VERSION 1977
#endif // INCLUDE_LIBYUV_VERSION_H_
diff --git a/libs/libyuv/include/libyuv/video_common.h b/libs/libyuv/include/libyuv/video_common.h
index ffcbdbf1b0..ee989f7869 100644
--- a/libs/libyuv/include/libyuv/video_common.h
+++ b/libs/libyuv/include/libyuv/video_common.h
@@ -13,6 +13,9 @@
#ifndef INCLUDE_LIBYUV_VIDEO_COMMON_H_
#define INCLUDE_LIBYUV_VIDEO_COMMON_H_
+#include <stddef.h>
+#include <stdint.h>
+
#include "libyuv/basic_types.h"
#ifdef __cplusplus
@@ -50,45 +53,72 @@ extern "C" {
// Secondary formats are converted in 2 steps.
// Auxilliary formats call primary converters.
enum FourCC {
- // 9 Primary YUV formats: 5 planar, 2 biplanar, 2 packed.
+ // 15 Primary YUV formats: 6 planar, 7 biplanar, 2 packed.
FOURCC_I420 = FOURCC('I', '4', '2', '0'),
FOURCC_I422 = FOURCC('I', '4', '2', '2'),
FOURCC_I444 = FOURCC('I', '4', '4', '4'),
FOURCC_I400 = FOURCC('I', '4', '0', '0'),
FOURCC_NV21 = FOURCC('N', 'V', '2', '1'),
FOURCC_NV12 = FOURCC('N', 'V', '1', '2'),
+ FOURCC_NV16 = FOURCC('N', 'V', '1', '6'),
+ FOURCC_NV24 = FOURCC('N', 'V', '2', '4'),
+ FOURCC_P010 = FOURCC('P', '0', '1', '0'),
+ FOURCC_P210 = FOURCC('P', '2', '1', '0'),
+ FOURCC_P410 = FOURCC('P', '4', '1', '0'),
FOURCC_YUY2 = FOURCC('Y', 'U', 'Y', '2'),
FOURCC_UYVY = FOURCC('U', 'Y', 'V', 'Y'),
- FOURCC_H010 = FOURCC('H', '0', '1', '0'), // unofficial fourcc. 10 bit lsb
+ FOURCC_I010 = FOURCC('I', '0', '1', '0'), // bt.601 10 bit 420
+ FOURCC_I210 = FOURCC('I', '2', '1', '0'), // bt.601 10 bit 422
- // 1 Secondary YUV format: row biplanar.
+ // 1 Secondary YUV format: row biplanar. deprecated.
FOURCC_M420 = FOURCC('M', '4', '2', '0'),
- // 11 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp, 1 10 bpc
+ // 13 Primary RGB formats: 4 32 bpp, 2 24 bpp, 3 16 bpp, 1 10 bpc 2 64 bpp
FOURCC_ARGB = FOURCC('A', 'R', 'G', 'B'),
FOURCC_BGRA = FOURCC('B', 'G', 'R', 'A'),
FOURCC_ABGR = FOURCC('A', 'B', 'G', 'R'),
FOURCC_AR30 = FOURCC('A', 'R', '3', '0'), // 10 bit per channel. 2101010.
FOURCC_AB30 = FOURCC('A', 'B', '3', '0'), // ABGR version of 10 bit
+ FOURCC_AR64 = FOURCC('A', 'R', '6', '4'), // 16 bit per channel.
+ FOURCC_AB64 = FOURCC('A', 'B', '6', '4'), // ABGR version of 16 bit
FOURCC_24BG = FOURCC('2', '4', 'B', 'G'),
FOURCC_RAW = FOURCC('r', 'a', 'w', ' '),
FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'),
- FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE.
- FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE.
- FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE.
+ FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. deprecated.
+ FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. deprecated.
+ FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. deprecated.
// 1 Primary Compressed YUV format.
FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'),
- // 8 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias.
+ // 23 Auxiliary YUV variations: 3 with U and V planes are swapped, 1 Alias.
FOURCC_YV12 = FOURCC('Y', 'V', '1', '2'),
FOURCC_YV16 = FOURCC('Y', 'V', '1', '6'),
FOURCC_YV24 = FOURCC('Y', 'V', '2', '4'),
FOURCC_YU12 = FOURCC('Y', 'U', '1', '2'), // Linux version of I420.
- FOURCC_J420 = FOURCC('J', '4', '2', '0'),
- FOURCC_J400 = FOURCC('J', '4', '0', '0'), // unofficial fourcc
- FOURCC_H420 = FOURCC('H', '4', '2', '0'), // unofficial fourcc
- FOURCC_H422 = FOURCC('H', '4', '2', '2'), // unofficial fourcc
+ FOURCC_J420 =
+ FOURCC('J', '4', '2', '0'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J422 =
+ FOURCC('J', '4', '2', '2'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J444 =
+ FOURCC('J', '4', '4', '4'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_J400 =
+ FOURCC('J', '4', '0', '0'), // jpeg (bt.601 full), unofficial fourcc
+ FOURCC_F420 = FOURCC('F', '4', '2', '0'), // bt.709 full, unofficial fourcc
+ FOURCC_F422 = FOURCC('F', '4', '2', '2'), // bt.709 full, unofficial fourcc
+ FOURCC_F444 = FOURCC('F', '4', '4', '4'), // bt.709 full, unofficial fourcc
+ FOURCC_H420 = FOURCC('H', '4', '2', '0'), // bt.709, unofficial fourcc
+ FOURCC_H422 = FOURCC('H', '4', '2', '2'), // bt.709, unofficial fourcc
+ FOURCC_H444 = FOURCC('H', '4', '4', '4'), // bt.709, unofficial fourcc
+ FOURCC_U420 = FOURCC('U', '4', '2', '0'), // bt.2020, unofficial fourcc
+ FOURCC_U422 = FOURCC('U', '4', '2', '2'), // bt.2020, unofficial fourcc
+ FOURCC_U444 = FOURCC('U', '4', '4', '4'), // bt.2020, unofficial fourcc
+ FOURCC_F010 = FOURCC('F', '0', '1', '0'), // bt.709 full range 10 bit 420
+ FOURCC_H010 = FOURCC('H', '0', '1', '0'), // bt.709 10 bit 420
+ FOURCC_U010 = FOURCC('U', '0', '1', '0'), // bt.2020 10 bit 420
+ FOURCC_F210 = FOURCC('F', '2', '1', '0'), // bt.709 full range 10 bit 422
+ FOURCC_H210 = FOURCC('H', '2', '1', '0'), // bt.709 10 bit 422
+ FOURCC_U210 = FOURCC('U', '2', '1', '0'), // bt.2020 10 bit 422
// 14 Auxiliary aliases. CanonicalFourCC() maps these to canonical fourcc.
FOURCC_IYUV = FOURCC('I', 'Y', 'U', 'V'), // Alias for I420.
@@ -105,9 +135,9 @@ enum FourCC {
FOURCC_BGR3 = FOURCC('B', 'G', 'R', '3'), // Alias for 24BG.
FOURCC_CM32 = FOURCC(0, 0, 0, 32), // Alias for BGRA kCMPixelFormat_32ARGB
FOURCC_CM24 = FOURCC(0, 0, 0, 24), // Alias for RAW kCMPixelFormat_24RGB
- FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO.
- FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP.
- FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO.
+ FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. deprecated.
+ FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. deprecated.
+ FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. deprecated.
// deprecated formats. Not supported, but defined for backward compatibility.
FOURCC_I411 = FOURCC('I', '4', '1', '1'),
@@ -131,9 +161,11 @@ enum FourCCBpp {
FOURCC_BPP_I400 = 8,
FOURCC_BPP_NV21 = 12,
FOURCC_BPP_NV12 = 12,
+ FOURCC_BPP_NV16 = 16,
+ FOURCC_BPP_NV24 = 24,
FOURCC_BPP_YUY2 = 16,
FOURCC_BPP_UYVY = 16,
- FOURCC_BPP_M420 = 12,
+ FOURCC_BPP_M420 = 12, // deprecated
FOURCC_BPP_Q420 = 12,
FOURCC_BPP_ARGB = 32,
FOURCC_BPP_BGRA = 32,
@@ -141,11 +173,13 @@ enum FourCCBpp {
FOURCC_BPP_RGBA = 32,
FOURCC_BPP_AR30 = 32,
FOURCC_BPP_AB30 = 32,
+ FOURCC_BPP_AR64 = 64,
+ FOURCC_BPP_AB64 = 64,
FOURCC_BPP_24BG = 24,
FOURCC_BPP_RAW = 24,
- FOURCC_BPP_RGBP = 16,
- FOURCC_BPP_RGBO = 16,
- FOURCC_BPP_R444 = 16,
+ FOURCC_BPP_RGBP = 16, // deprecated
+ FOURCC_BPP_RGBO = 16, // deprecated
+ FOURCC_BPP_R444 = 16, // deprecated
FOURCC_BPP_RGGB = 8,
FOURCC_BPP_BGGR = 8,
FOURCC_BPP_GRBG = 8,
@@ -158,7 +192,13 @@ enum FourCCBpp {
FOURCC_BPP_J400 = 8,
FOURCC_BPP_H420 = 12,
FOURCC_BPP_H422 = 16,
- FOURCC_BPP_H010 = 24,
+ FOURCC_BPP_I010 = 15,
+ FOURCC_BPP_I210 = 20,
+ FOURCC_BPP_H010 = 15,
+ FOURCC_BPP_H210 = 20,
+ FOURCC_BPP_P010 = 15,
+ FOURCC_BPP_P210 = 20,
+ FOURCC_BPP_P410 = 30,
FOURCC_BPP_MJPG = 0, // 0 means unknown.
FOURCC_BPP_H264 = 0,
FOURCC_BPP_IYUV = 12,
diff --git a/libs/libyuv/infra/config/OWNERS b/libs/libyuv/infra/config/OWNERS
new file mode 100644
index 0000000000..2c4f90a03c
--- /dev/null
+++ b/libs/libyuv/infra/config/OWNERS
@@ -0,0 +1,3 @@
+fbarchard@chromium.org
+mbonadei@chromium.org
+jansson@google.com
diff --git a/libs/libyuv/infra/config/PRESUBMIT.py b/libs/libyuv/infra/config/PRESUBMIT.py
new file mode 100644
index 0000000000..0aa465e242
--- /dev/null
+++ b/libs/libyuv/infra/config/PRESUBMIT.py
@@ -0,0 +1,17 @@
+# Copyright 2018 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+USE_PYTHON3 = True
+
+
+def CheckChangeOnUpload(input_api, output_api):
+ return input_api.canned_checks.CheckChangedLUCIConfigs(
+ input_api, output_api
+ )
+
+
+def CheckChangeOnCommit(input_api, output_api):
+ return input_api.canned_checks.CheckChangedLUCIConfigs(
+ input_api, output_api
+ )
diff --git a/libs/libyuv/infra/config/README.md b/libs/libyuv/infra/config/README.md
new file mode 100644
index 0000000000..e5e3b5f818
--- /dev/null
+++ b/libs/libyuv/infra/config/README.md
@@ -0,0 +1,2 @@
+This folder contains libyuv project-wide configurations
+for chrome-infra services.
diff --git a/libs/libyuv/infra/config/codereview.settings b/libs/libyuv/infra/config/codereview.settings
new file mode 100644
index 0000000000..6d74227357
--- /dev/null
+++ b/libs/libyuv/infra/config/codereview.settings
@@ -0,0 +1,6 @@
+# This file is used by gcl and git-cl to get repository specific information.
+CODE_REVIEW_SERVER: codereview.chromium.org
+PROJECT: libyuv
+GERRIT_HOST: True
+VIEW_VC: https://chromium.googlesource.com/libyuv/libyuv/+/
+
diff --git a/libs/libyuv/infra/config/commit-queue.cfg b/libs/libyuv/infra/config/commit-queue.cfg
new file mode 100644
index 0000000000..640b530e48
--- /dev/null
+++ b/libs/libyuv/infra/config/commit-queue.cfg
@@ -0,0 +1,144 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see Config message:
+# https://config.luci.app/schemas/projects:commit-queue.cfg
+
+cq_status_host: "chromium-cq-status.appspot.com"
+submit_options {
+ max_burst: 4
+ burst_delay {
+ seconds: 480
+ }
+}
+config_groups {
+ name: "config"
+ gerrit {
+ url: "https://chromium-review.googlesource.com"
+ projects {
+ name: "libyuv/libyuv"
+ ref_regexp: "refs/heads/infra/config"
+ }
+ }
+ verifiers {
+ gerrit_cq_ability {
+ committer_list: "project-libyuv-committers"
+ dry_run_access_list: "project-libyuv-tryjob-access"
+ }
+ tryjob {
+ builders {
+ name: "libyuv/try/presubmit"
+ }
+ retry_config {
+ single_quota: 1
+ global_quota: 2
+ failure_weight: 1
+ transient_failure_weight: 1
+ timeout_weight: 2
+ }
+ }
+ }
+}
+config_groups {
+ name: "master"
+ gerrit {
+ url: "https://chromium-review.googlesource.com"
+ projects {
+ name: "libyuv/libyuv"
+ ref_regexp: "refs/heads/main"
+ ref_regexp: "refs/heads/master"
+ }
+ }
+ verifiers {
+ gerrit_cq_ability {
+ committer_list: "project-libyuv-committers"
+ dry_run_access_list: "project-libyuv-tryjob-access"
+ }
+ tryjob {
+ builders {
+ name: "libyuv/try/android"
+ experiment_percentage: 100
+ }
+ builders {
+ name: "libyuv/try/android_arm64"
+ experiment_percentage: 100
+ }
+ builders {
+ name: "libyuv/try/android_rel"
+ experiment_percentage: 100
+ }
+ builders {
+ name: "libyuv/try/android_x64"
+ }
+ builders {
+ name: "libyuv/try/android_x86"
+ }
+ builders {
+ name: "libyuv/try/ios_arm64"
+ }
+ builders {
+ name: "libyuv/try/ios_arm64_rel"
+ }
+ builders {
+ name: "libyuv/try/linux"
+ }
+ builders {
+ name: "libyuv/try/linux_asan"
+ }
+ builders {
+ name: "libyuv/try/linux_gcc"
+ experiment_percentage: 100
+ }
+ builders {
+ name: "libyuv/try/linux_msan"
+ experiment_percentage: 100
+ }
+ builders {
+ name: "libyuv/try/linux_rel"
+ }
+ builders {
+ name: "libyuv/try/linux_tsan2"
+ }
+ builders {
+ name: "libyuv/try/linux_ubsan"
+ }
+ builders {
+ name: "libyuv/try/linux_ubsan_vptr"
+ }
+ builders {
+ name: "libyuv/try/mac"
+ }
+ builders {
+ name: "libyuv/try/mac_asan"
+ }
+ builders {
+ name: "libyuv/try/mac_rel"
+ }
+ builders {
+ name: "libyuv/try/win"
+ }
+ builders {
+ name: "libyuv/try/win_clang"
+ }
+ builders {
+ name: "libyuv/try/win_clang_rel"
+ }
+ builders {
+ name: "libyuv/try/win_rel"
+ }
+ builders {
+ name: "libyuv/try/win_x64_clang_rel"
+ }
+ builders {
+ name: "libyuv/try/win_x64_rel"
+ }
+ retry_config {
+ single_quota: 1
+ global_quota: 2
+ failure_weight: 1
+ transient_failure_weight: 1
+ timeout_weight: 2
+ }
+ }
+ }
+}
diff --git a/libs/libyuv/infra/config/cr-buildbucket.cfg b/libs/libyuv/infra/config/cr-buildbucket.cfg
new file mode 100644
index 0000000000..a3348afc4f
--- /dev/null
+++ b/libs/libyuv/infra/config/cr-buildbucket.cfg
@@ -0,0 +1,1130 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see BuildbucketCfg message:
+# https://config.luci.app/schemas/projects:buildbucket.cfg
+
+buckets {
+ name: "ci"
+ acls {
+ role: WRITER
+ group: "project-libyuv-admins"
+ }
+ acls {
+ group: "all"
+ }
+ swarming {
+ builders {
+ name: "Android ARM64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android Tester ARM32 Debug (Nexus 5X)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android Tester ARM32 Release (Nexus 5X)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android Tester ARM64 Debug (Nexus 5X)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android32 x86 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Android64 x64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux Asan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux MSan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux Tsan v2"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux UBSan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux UBSan vptr"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux32 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux32 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Linux64 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Mac Asan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Mac64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Mac64 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win32 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win32 Debug (Clang)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win32 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win32 Release (Clang)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win64 Debug (Clang)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win64 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "Win64 Release (Clang)"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "iOS ARM64 Debug"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "iOS ARM64 Release"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.ci"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-trusted\"}"
+ properties_j: "builder_group:\"client.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ }
+ shadow: "ci.shadow"
+ constraints {
+ pools: "luci.flex.ci"
+ }
+}
+buckets {
+ name: "ci.shadow"
+ acls {
+ role: WRITER
+ group: "project-libyuv-admins"
+ }
+ acls {
+ group: "all"
+ }
+ constraints {
+ pools: "luci.flex.ci"
+ service_accounts: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ dynamic_builder_template {}
+}
+buckets {
+ name: "cron"
+ acls {
+ role: WRITER
+ group: "project-libyuv-admins"
+ }
+ acls {
+ group: "all"
+ }
+ swarming {
+ builders {
+ name: "DEPS Autoroller"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Linux"
+ dimensions: "pool:luci.webrtc.cron"
+ recipe {
+ name: "libyuv/roll_deps"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ }
+ execution_timeout_secs: 7200
+ build_numbers: YES
+ service_account: "libyuv-ci-autoroll-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ }
+}
+buckets {
+ name: "try"
+ acls {
+ role: WRITER
+ group: "project-libyuv-admins"
+ }
+ acls {
+ group: "all"
+ }
+ acls {
+ role: SCHEDULER
+ group: "project-libyuv-tryjob-access"
+ }
+ acls {
+ role: SCHEDULER
+ group: "service-account-cq"
+ }
+ swarming {
+ builders {
+ name: "android"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "android_arm64"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "android_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "device_type:walleye"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "android_x64"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "android_x86"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "ios_arm64"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "ios_arm64_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_asan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_gcc"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_msan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_tsan2"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_ubsan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "linux_ubsan_vptr"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "mac"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "mac_asan"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "mac_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cpu:arm64"
+ dimensions: "os:Mac-15"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "presubmit"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Ubuntu-24.04"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "run_presubmit"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ properties_j: "repo_name:\"libyuv\""
+ properties_j: "runhooks:true"
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win_clang"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win_clang_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win_x64_clang_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ builders {
+ name: "win_x64_rel"
+ swarming_host: "chromium-swarm.appspot.com"
+ swarming_tags: "vpython:native-python-wrapper"
+ dimensions: "cores:8"
+ dimensions: "cpu:x86-64"
+ dimensions: "os:Windows-10"
+ dimensions: "pool:luci.flex.try"
+ recipe {
+ name: "libyuv/libyuv"
+ cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+ cipd_version: "refs/heads/main"
+ properties_j: "$build/siso:{\"configs\":[\"builder\"],\"enable_cloud_profiler\":true,\"enable_cloud_trace\":true,\"enable_monitoring\":true,\"project\":\"rbe-webrtc-untrusted\"}"
+ properties_j: "builder_group:\"tryserver.libyuv\""
+ }
+ execution_timeout_secs: 10800
+ build_numbers: YES
+ service_account: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ }
+ shadow: "try.shadow"
+ constraints {
+ pools: "luci.flex.try"
+ service_accounts: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+}
+buckets {
+ name: "try.shadow"
+ acls {
+ role: WRITER
+ group: "project-libyuv-admins"
+ }
+ acls {
+ group: "all"
+ }
+ constraints {
+ pools: "luci.flex.try"
+ service_accounts: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ dynamic_builder_template {}
+}
diff --git a/libs/libyuv/infra/config/luci-logdog.cfg b/libs/libyuv/infra/config/luci-logdog.cfg
new file mode 100644
index 0000000000..01a391261d
--- /dev/null
+++ b/libs/libyuv/infra/config/luci-logdog.cfg
@@ -0,0 +1,9 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see ProjectConfig message:
+# https://config.luci.app/schemas/projects:luci-logdog.cfg
+
+reader_auth_groups: "all"
+writer_auth_groups: "luci-logdog-chromium-writers"
+archive_gs_bucket: "chromium-luci-logdog"
diff --git a/libs/libyuv/infra/config/luci-milo.cfg b/libs/libyuv/infra/config/luci-milo.cfg
new file mode 100644
index 0000000000..cde1848b59
--- /dev/null
+++ b/libs/libyuv/infra/config/luci-milo.cfg
@@ -0,0 +1,246 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see Project message:
+# https://config.luci.app/schemas/projects:luci-milo.cfg
+
+consoles {
+ id: "main"
+ name: "libyuv Main Console"
+ repo_url: "https://chromium.googlesource.com/libyuv/libyuv"
+ refs: "regexp:refs/heads/main"
+ manifest_name: "REVISION"
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android ARM64 Debug"
+ category: "Android|Builder"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android Debug"
+ category: "Android|Builder"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android Release"
+ category: "Android|Builder"
+ short_name: "rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android32 x86 Debug"
+ category: "Android|Builder|x86"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android64 x64 Debug"
+ category: "Android|Builder|x64"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android Tester ARM32 Debug (Nexus 5X)"
+ category: "Android|Tester|ARM 32"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android Tester ARM32 Release (Nexus 5X)"
+ category: "Android|Tester|ARM 32"
+ short_name: "rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Android Tester ARM64 Debug (Nexus 5X)"
+ category: "Android|Tester|ARM 64"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux Asan"
+ category: "Linux"
+ short_name: "asan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux MSan"
+ category: "Linux"
+ short_name: "msan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux Tsan v2"
+ category: "Linux"
+ short_name: "tsan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux UBSan"
+ category: "Linux|UBSan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux UBSan vptr"
+ category: "Linux|UBSan"
+ short_name: "vptr"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux32 Debug"
+ category: "Linux|32"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux32 Release"
+ category: "Linux|32"
+ short_name: "rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux64 Debug"
+ category: "Linux|64"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Linux64 Release"
+ category: "Linux|64"
+ short_name: "rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Mac Asan"
+ category: "Mac"
+ short_name: "asan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Mac64 Debug"
+ category: "Mac"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Mac64 Release"
+ category: "Mac"
+ short_name: "rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win32 Debug"
+ category: "Win|32|Debug"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win32 Debug (Clang)"
+ category: "Win|32|Debug"
+ short_name: "clg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win32 Release"
+ category: "Win|32|Release"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win32 Release (Clang)"
+ category: "Win|32|Release"
+ short_name: "clg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win64 Debug"
+ category: "Win|64|Debug"
+ short_name: "clg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win64 Debug (Clang)"
+ category: "Win|64|Debug"
+ short_name: "clg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win64 Release"
+ category: "Win|64|Release"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/Win64 Release (Clang)"
+ category: "Win|64|Release"
+ short_name: "clg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/iOS ARM64 Debug"
+ category: "iOS|ARM64"
+ short_name: "dbg"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.ci/iOS ARM64 Release"
+ category: "iOS|ARM64"
+ short_name: "rel"
+ }
+ include_experimental_builds: true
+}
+consoles {
+ id: "cron"
+ name: "Cron"
+ builders {
+ name: "buildbucket/luci.libyuv.cron/DEPS Autoroller"
+ }
+ builder_view_only: true
+}
+consoles {
+ id: "try"
+ name: "libyuv Try Builders"
+ builders {
+ name: "buildbucket/luci.libyuv.try/android"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/android_arm64"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/android_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/android_x64"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/android_x86"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/ios_arm64"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/ios_arm64_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_asan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_gcc"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_msan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_tsan2"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_ubsan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/linux_ubsan_vptr"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/mac"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/mac_asan"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/mac_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win_clang"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win_clang_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win_x64_clang_rel"
+ }
+ builders {
+ name: "buildbucket/luci.libyuv.try/win_x64_rel"
+ }
+ builder_view_only: true
+}
+logo_url: "https://storage.googleapis.com/chrome-infra-public/logo/libyuv-logo.png"
diff --git a/libs/libyuv/infra/config/luci-scheduler.cfg b/libs/libyuv/infra/config/luci-scheduler.cfg
new file mode 100644
index 0000000000..33bd6a7d24
--- /dev/null
+++ b/libs/libyuv/infra/config/luci-scheduler.cfg
@@ -0,0 +1,385 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see ProjectConfig message:
+# https://config.luci.app/schemas/projects:luci-scheduler.cfg
+
+job {
+ id: "Android ARM64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android ARM64 Debug"
+ }
+}
+job {
+ id: "Android Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android Debug"
+ }
+}
+job {
+ id: "Android Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android Release"
+ }
+}
+job {
+ id: "Android Tester ARM32 Debug (Nexus 5X)"
+ realm: "ci"
+ acls {
+ role: TRIGGERER
+ granted_to: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android Tester ARM32 Debug (Nexus 5X)"
+ }
+}
+job {
+ id: "Android Tester ARM32 Release (Nexus 5X)"
+ realm: "ci"
+ acls {
+ role: TRIGGERER
+ granted_to: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android Tester ARM32 Release (Nexus 5X)"
+ }
+}
+job {
+ id: "Android Tester ARM64 Debug (Nexus 5X)"
+ realm: "ci"
+ acls {
+ role: TRIGGERER
+ granted_to: "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android Tester ARM64 Debug (Nexus 5X)"
+ }
+}
+job {
+ id: "Android32 x86 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android32 x86 Debug"
+ }
+}
+job {
+ id: "Android64 x64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Android64 x64 Debug"
+ }
+}
+job {
+ id: "DEPS Autoroller"
+ realm: "cron"
+ schedule: "0 14 * * *"
+ acl_sets: "cron"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "cron"
+ builder: "DEPS Autoroller"
+ }
+}
+job {
+ id: "Linux Asan"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux Asan"
+ }
+}
+job {
+ id: "Linux MSan"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux MSan"
+ }
+}
+job {
+ id: "Linux Tsan v2"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux Tsan v2"
+ }
+}
+job {
+ id: "Linux UBSan"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux UBSan"
+ }
+}
+job {
+ id: "Linux UBSan vptr"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux UBSan vptr"
+ }
+}
+job {
+ id: "Linux32 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux32 Debug"
+ }
+}
+job {
+ id: "Linux32 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux32 Release"
+ }
+}
+job {
+ id: "Linux64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux64 Debug"
+ }
+}
+job {
+ id: "Linux64 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Linux64 Release"
+ }
+}
+job {
+ id: "Mac Asan"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Mac Asan"
+ }
+}
+job {
+ id: "Mac64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Mac64 Debug"
+ }
+}
+job {
+ id: "Mac64 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Mac64 Release"
+ }
+}
+job {
+ id: "Win32 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win32 Debug"
+ }
+}
+job {
+ id: "Win32 Debug (Clang)"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win32 Debug (Clang)"
+ }
+}
+job {
+ id: "Win32 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win32 Release"
+ }
+}
+job {
+ id: "Win32 Release (Clang)"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win32 Release (Clang)"
+ }
+}
+job {
+ id: "Win64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win64 Debug"
+ }
+}
+job {
+ id: "Win64 Debug (Clang)"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win64 Debug (Clang)"
+ }
+}
+job {
+ id: "Win64 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win64 Release"
+ }
+}
+job {
+ id: "Win64 Release (Clang)"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "Win64 Release (Clang)"
+ }
+}
+job {
+ id: "iOS ARM64 Debug"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "iOS ARM64 Debug"
+ }
+}
+job {
+ id: "iOS ARM64 Release"
+ realm: "ci"
+ acl_sets: "ci"
+ buildbucket {
+ server: "cr-buildbucket.appspot.com"
+ bucket: "ci"
+ builder: "iOS ARM64 Release"
+ }
+}
+trigger {
+ id: "master-gitiles-trigger"
+ realm: "ci"
+ acl_sets: "ci"
+ triggers: "Android ARM64 Debug"
+ triggers: "Android Debug"
+ triggers: "Android Release"
+ triggers: "Android32 x86 Debug"
+ triggers: "Android64 x64 Debug"
+ triggers: "Linux Asan"
+ triggers: "Linux MSan"
+ triggers: "Linux Tsan v2"
+ triggers: "Linux UBSan"
+ triggers: "Linux UBSan vptr"
+ triggers: "Linux32 Debug"
+ triggers: "Linux32 Release"
+ triggers: "Linux64 Debug"
+ triggers: "Linux64 Release"
+ triggers: "Mac Asan"
+ triggers: "Mac64 Debug"
+ triggers: "Mac64 Release"
+ triggers: "Win32 Debug"
+ triggers: "Win32 Debug (Clang)"
+ triggers: "Win32 Release"
+ triggers: "Win32 Release (Clang)"
+ triggers: "Win64 Debug"
+ triggers: "Win64 Debug (Clang)"
+ triggers: "Win64 Release"
+ triggers: "Win64 Release (Clang)"
+ triggers: "iOS ARM64 Debug"
+ triggers: "iOS ARM64 Release"
+ gitiles {
+ repo: "https://chromium.googlesource.com/libyuv/libyuv"
+ refs: "regexp:refs/heads/main"
+ }
+}
+acl_sets {
+ name: "ci"
+ acls {
+ role: OWNER
+ granted_to: "group:project-libyuv-admins"
+ }
+ acls {
+ granted_to: "group:all"
+ }
+}
+acl_sets {
+ name: "cron"
+ acls {
+ role: OWNER
+ granted_to: "group:project-libyuv-admins"
+ }
+ acls {
+ granted_to: "group:all"
+ }
+}
diff --git a/libs/libyuv/infra/config/main.star b/libs/libyuv/infra/config/main.star
new file mode 100755
index 0000000000..e1634bb938
--- /dev/null
+++ b/libs/libyuv/infra/config/main.star
@@ -0,0 +1,399 @@
+#!/usr/bin/env lucicfg
+# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
+
+"""LUCI project configuration for libyuv CQ and CI."""
+
+lucicfg.check_version("1.30.9")
+
+LIBYUV_GIT = "https://chromium.googlesource.com/libyuv/libyuv"
+LIBYUV_GERRIT = "https://chromium-review.googlesource.com/libyuv/libyuv"
+
+RBE_PROJECT = {
+ "ci": "rbe-webrtc-trusted",
+ "try": "rbe-webrtc-untrusted",
+}
+
+# Use LUCI Scheduler BBv2 names and add Scheduler realms configs.
+lucicfg.enable_experiment("crbug.com/1182002")
+
+lucicfg.config(
+ lint_checks = ["default"],
+ config_dir = ".",
+ tracked_files = [
+ "commit-queue.cfg",
+ "cr-buildbucket.cfg",
+ "luci-logdog.cfg",
+ "luci-milo.cfg",
+ "luci-scheduler.cfg",
+ "project.cfg",
+ "realms.cfg",
+ ],
+)
+
+# Generates project.cfg
+
+luci.project(
+ name = "libyuv",
+ buildbucket = "cr-buildbucket.appspot.com",
+ logdog = "luci-logdog.appspot.com",
+ milo = "luci-milo.appspot.com",
+ notify = "luci-notify.appspot.com",
+ scheduler = "luci-scheduler.appspot.com",
+ swarming = "chromium-swarm.appspot.com",
+ acls = [
+ acl.entry(acl.PROJECT_CONFIGS_READER, groups = ["all"]),
+ acl.entry(acl.LOGDOG_READER, groups = ["all"]),
+ acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
+ acl.entry(acl.SCHEDULER_READER, groups = ["all"]),
+ acl.entry(acl.SCHEDULER_OWNER, groups = ["project-libyuv-admins"]),
+ acl.entry(acl.BUILDBUCKET_READER, groups = ["all"]),
+ acl.entry(acl.BUILDBUCKET_OWNER, groups = ["project-libyuv-admins"]),
+ ],
+ bindings = [
+ luci.binding(
+ roles = "role/swarming.taskTriggerer", # for LED tasks.
+ groups = "project-libyuv-admins",
+ ),
+ luci.binding(
+ roles = "role/configs.validator",
+ users = "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com",
+ ),
+ ],
+)
+
+# Generates luci-logdog.cfg
+
+luci.logdog(
+ gs_bucket = "chromium-luci-logdog",
+)
+
+# Generates luci-scheduler.cfg
+
+luci.gitiles_poller(
+ name = "master-gitiles-trigger",
+ bucket = "ci",
+ repo = LIBYUV_GIT,
+)
+
+# Generates luci-milo.cfg
+
+luci.milo(
+ logo = "https://storage.googleapis.com/chrome-infra-public/logo/libyuv-logo.png",
+)
+
+def libyuv_ci_view(name, category, short_name):
+ return luci.console_view_entry(
+ console_view = "main",
+ builder = name,
+ category = category,
+ short_name = short_name,
+ )
+
+def libyuv_try_view(name):
+ return luci.list_view_entry(
+ list_view = "try",
+ builder = name,
+ )
+
+luci.console_view(
+ name = "main",
+ title = "libyuv Main Console",
+ include_experimental_builds = True,
+ repo = LIBYUV_GIT,
+)
+
+luci.list_view(
+ name = "cron",
+ title = "Cron",
+ entries = ["DEPS Autoroller"],
+)
+
+luci.list_view(
+ name = "try",
+ title = "libyuv Try Builders",
+)
+
+# Generates commit-queue.cfg
+
+def libyuv_try_job_verifier(name, cq_group, experiment_percentage):
+ return luci.cq_tryjob_verifier(
+ builder = name,
+ cq_group = cq_group,
+ experiment_percentage = experiment_percentage,
+ )
+
+luci.cq(
+ status_host = "chromium-cq-status.appspot.com",
+ submit_max_burst = 4,
+ submit_burst_delay = 8 * time.minute,
+)
+
+luci.cq_group(
+ name = "master",
+ watch = [
+ cq.refset(
+ repo = LIBYUV_GERRIT,
+ refs = ["refs/heads/main", "refs/heads/master"],
+ ),
+ ],
+ acls = [
+ acl.entry(acl.CQ_COMMITTER, groups = ["project-libyuv-committers"]),
+ acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-libyuv-tryjob-access"]),
+ ],
+ retry_config = cq.RETRY_ALL_FAILURES,
+ cancel_stale_tryjobs = True,
+)
+
+luci.cq_group(
+ name = "config",
+ watch = [
+ cq.refset(
+ repo = LIBYUV_GERRIT,
+ refs = ["refs/heads/infra/config"],
+ ),
+ ],
+ acls = [
+ acl.entry(acl.CQ_COMMITTER, groups = ["project-libyuv-committers"]),
+ acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-libyuv-tryjob-access"]),
+ ],
+ retry_config = cq.RETRY_ALL_FAILURES,
+ cancel_stale_tryjobs = True,
+)
+
+# Generates cr-buildbucket.cfg
+
+luci.bucket(
+ name = "ci",
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.ci"],
+ ),
+)
+luci.bucket(
+ name = "ci.shadow",
+ shadows = "ci",
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.ci"],
+ service_accounts = [
+ "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
+ ],
+ ),
+ bindings = [
+ # For led permissions.
+ luci.binding(
+ roles = "role/buildbucket.creator",
+ groups = [
+ "chromium-led-users",
+ "mdb/chrome-build-access-sphinx",
+ "mdb/chrome-troopers",
+ "mdb/foundry-x-team",
+ ],
+ ),
+ ],
+ dynamic = True,
+)
+luci.bucket(
+ name = "try",
+ acls = [
+ acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
+ "project-libyuv-tryjob-access",
+ "service-account-cq",
+ ]),
+ ],
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.try"],
+ service_accounts = [
+ "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com",
+ ],
+ ),
+)
+luci.bucket(
+ name = "try.shadow",
+ shadows = "try",
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.try"],
+ service_accounts = [
+ "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com",
+ ],
+ ),
+ bindings = [
+ # For led permissions.
+ luci.binding(
+ roles = "role/buildbucket.creator",
+ groups = [
+ "chromium-led-users",
+ "mdb/chrome-build-access-sphinx",
+ "mdb/chrome-troopers",
+ "mdb/foundry-x-team",
+ ],
+ ),
+ ],
+ dynamic = True,
+)
+luci.bucket(
+ name = "cron",
+)
+
+def get_os_dimensions(os):
+ if os == "android":
+ return {"device_type": "walleye"}
+ if os == "ios" or os == "mac":
+ return {"os": "Mac-15", "cpu": "arm64"}
+ elif os == "win":
+ return {"os": "Windows-10", "cores": "8", "cpu": "x86-64"}
+ elif os == "linux":
+ return {"os": "Ubuntu-24.04", "cores": "8", "cpu": "x86-64"}
+ return {}
+
+def libyuv_ci_builder(name, dimensions, properties, triggered_by):
+ return luci.builder(
+ name = name,
+ dimensions = dimensions,
+ properties = properties,
+ bucket = "ci",
+ service_account = "libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
+ triggered_by = triggered_by,
+ swarming_tags = ["vpython:native-python-wrapper"],
+ execution_timeout = 180 * time.minute,
+ build_numbers = True,
+ executable = luci.recipe(
+ name = "libyuv/libyuv",
+ cipd_package = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
+ ),
+ )
+
+def libyuv_try_builder(name, dimensions, properties, recipe_name = "libyuv/libyuv"):
+ return luci.builder(
+ name = name,
+ dimensions = dimensions,
+ properties = properties,
+ bucket = "try",
+ service_account = "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com",
+ swarming_tags = ["vpython:native-python-wrapper"],
+ execution_timeout = 180 * time.minute,
+ build_numbers = True,
+ executable = luci.recipe(
+ name = recipe_name,
+ cipd_package = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
+ ),
+ )
+
+def get_build_properties(bucket):
+ rbe_project = RBE_PROJECT.get(bucket)
+ return {
+ "$build/siso": {
+ "project": rbe_project,
+ "configs": ["builder"],
+ "enable_cloud_profiler": True,
+ "enable_cloud_trace": True,
+ "enable_monitoring": True,
+ },
+ }
+
+def ci_builder(name, os, category, short_name = None):
+ dimensions = get_os_dimensions(os)
+ properties = get_build_properties("ci")
+
+ dimensions["pool"] = "luci.flex.ci"
+ properties["builder_group"] = "client.libyuv"
+
+ triggered_by = ["master-gitiles-trigger" if os != "android" else "Android Debug"]
+ libyuv_ci_view(name, category, short_name)
+ return libyuv_ci_builder(name, dimensions, properties, triggered_by)
+
+def try_builder(name, os, experiment_percentage = None):
+ dimensions = get_os_dimensions(os)
+ properties = get_build_properties("try")
+
+ dimensions["pool"] = "luci.flex.try"
+ properties["builder_group"] = "tryserver.libyuv"
+
+ if name == "presubmit":
+ recipe_name = "run_presubmit"
+ properties["repo_name"] = "libyuv"
+ properties["runhooks"] = True
+ libyuv_try_job_verifier(name, "config", experiment_percentage)
+ return libyuv_try_builder(name, dimensions, properties, recipe_name)
+
+ libyuv_try_job_verifier(name, "master", experiment_percentage)
+ libyuv_try_view(name)
+ return libyuv_try_builder(name, dimensions, properties)
+
+luci.builder(
+ name = "DEPS Autoroller",
+ bucket = "cron",
+ service_account = "libyuv-ci-autoroll-builder@chops-service-accounts.iam.gserviceaccount.com",
+ dimensions = {
+ "pool": "luci.webrtc.cron",
+ "os": "Linux",
+ "cpu": "x86-64",
+ },
+ swarming_tags = ["vpython:native-python-wrapper"],
+ execution_timeout = 120 * time.minute,
+ build_numbers = True,
+ schedule = "0 14 * * *", # Every 2 hours.
+ executable = luci.recipe(
+ name = "libyuv/roll_deps",
+ cipd_package = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
+ ),
+)
+
+ci_builder("Android ARM64 Debug", "linux", "Android|Builder", "dbg")
+ci_builder("Android Debug", "linux", "Android|Builder", "dbg")
+ci_builder("Android Release", "linux", "Android|Builder", "rel")
+ci_builder("Android32 x86 Debug", "linux", "Android|Builder|x86", "dbg")
+ci_builder("Android64 x64 Debug", "linux", "Android|Builder|x64", "dbg")
+ci_builder("Android Tester ARM32 Debug (Nexus 5X)", "android", "Android|Tester|ARM 32", "dbg")
+ci_builder("Android Tester ARM32 Release (Nexus 5X)", "android", "Android|Tester|ARM 32", "rel")
+ci_builder("Android Tester ARM64 Debug (Nexus 5X)", "android", "Android|Tester|ARM 64", "dbg")
+ci_builder("Linux Asan", "linux", "Linux", "asan")
+ci_builder("Linux MSan", "linux", "Linux", "msan")
+ci_builder("Linux Tsan v2", "linux", "Linux", "tsan")
+ci_builder("Linux UBSan", "linux", "Linux|UBSan")
+ci_builder("Linux UBSan vptr", "linux", "Linux|UBSan", "vptr")
+ci_builder("Linux32 Debug", "linux", "Linux|32", "dbg")
+ci_builder("Linux32 Release", "linux", "Linux|32", "rel")
+ci_builder("Linux64 Debug", "linux", "Linux|64", "dbg")
+ci_builder("Linux64 Release", "linux", "Linux|64", "rel")
+ci_builder("Mac Asan", "mac", "Mac", "asan")
+ci_builder("Mac64 Debug", "mac", "Mac", "dbg")
+ci_builder("Mac64 Release", "mac", "Mac", "rel")
+ci_builder("Win32 Debug", "win", "Win|32|Debug")
+ci_builder("Win32 Debug (Clang)", "win", "Win|32|Debug", "clg")
+ci_builder("Win32 Release", "win", "Win|32|Release")
+ci_builder("Win32 Release (Clang)", "win", "Win|32|Release", "clg")
+ci_builder("Win64 Debug", "win", "Win|64|Debug", "clg")
+ci_builder("Win64 Debug (Clang)", "win", "Win|64|Debug", "clg")
+ci_builder("Win64 Release", "win", "Win|64|Release")
+ci_builder("Win64 Release (Clang)", "win", "Win|64|Release", "clg")
+ci_builder("iOS ARM64 Debug", "ios", "iOS|ARM64", "dbg")
+ci_builder("iOS ARM64 Release", "ios", "iOS|ARM64", "rel")
+
+# TODO(crbug.com/1242847): make this not experimental.
+try_builder("android", "android", experiment_percentage = 100)
+try_builder("android_arm64", "android", experiment_percentage = 100)
+try_builder("android_rel", "android", experiment_percentage = 100)
+
+try_builder("android_x64", "linux")
+try_builder("android_x86", "linux")
+try_builder("ios_arm64", "ios")
+try_builder("ios_arm64_rel", "ios")
+try_builder("linux", "linux")
+try_builder("linux_asan", "linux")
+try_builder("linux_gcc", "linux", experiment_percentage = 100)
+
+# TODO(libyuv:388428508): Make linux_msan not experimental.
+try_builder("linux_msan", "linux", experiment_percentage = 100)
+try_builder("linux_rel", "linux")
+try_builder("linux_tsan2", "linux")
+try_builder("linux_ubsan", "linux")
+try_builder("linux_ubsan_vptr", "linux")
+try_builder("mac", "mac")
+try_builder("mac_asan", "mac")
+try_builder("mac_rel", "mac")
+try_builder("win", "win")
+try_builder("win_clang", "win")
+try_builder("win_clang_rel", "win")
+try_builder("win_rel", "win")
+try_builder("win_x64_clang_rel", "win")
+try_builder("win_x64_rel", "win")
+try_builder("presubmit", "linux")
diff --git a/libs/libyuv/infra/config/project.cfg b/libs/libyuv/infra/config/project.cfg
new file mode 100644
index 0000000000..312b39ef85
--- /dev/null
+++ b/libs/libyuv/infra/config/project.cfg
@@ -0,0 +1,15 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see ProjectCfg message:
+# https://config.luci.app/schemas/projects:project.cfg
+
+name: "libyuv"
+access: "group:all"
+lucicfg {
+ version: "1.46.1"
+ package_dir: "."
+ config_dir: "."
+ entry_point: "main.star"
+ experiments: "crbug.com/1182002"
+}
diff --git a/libs/libyuv/infra/config/realms.cfg b/libs/libyuv/infra/config/realms.cfg
new file mode 100644
index 0000000000..b4bae90ea0
--- /dev/null
+++ b/libs/libyuv/infra/config/realms.cfg
@@ -0,0 +1,111 @@
+# Auto-generated by lucicfg.
+# Do not modify manually.
+#
+# For the schema of this file, see RealmsCfg message:
+# https://config.luci.app/schemas/projects:realms.cfg
+
+realms {
+ name: "@root"
+ bindings {
+ role: "role/buildbucket.owner"
+ principals: "group:project-libyuv-admins"
+ }
+ bindings {
+ role: "role/buildbucket.reader"
+ principals: "group:all"
+ }
+ bindings {
+ role: "role/configs.reader"
+ principals: "group:all"
+ }
+ bindings {
+ role: "role/configs.validator"
+ principals: "user:libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ bindings {
+ role: "role/logdog.reader"
+ principals: "group:all"
+ }
+ bindings {
+ role: "role/logdog.writer"
+ principals: "group:luci-logdog-chromium-writers"
+ }
+ bindings {
+ role: "role/scheduler.owner"
+ principals: "group:project-libyuv-admins"
+ }
+ bindings {
+ role: "role/scheduler.reader"
+ principals: "group:all"
+ }
+ bindings {
+ role: "role/swarming.taskTriggerer"
+ principals: "group:project-libyuv-admins"
+ }
+}
+realms {
+ name: "ci"
+ bindings {
+ role: "role/buildbucket.builderServiceAccount"
+ principals: "user:libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ bindings {
+ role: "role/scheduler.triggerer"
+ principals: "user:libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ conditions {
+ restrict {
+ attribute: "scheduler.job.name"
+ values: "Android Tester ARM32 Debug (Nexus 5X)"
+ values: "Android Tester ARM32 Release (Nexus 5X)"
+ values: "Android Tester ARM64 Debug (Nexus 5X)"
+ }
+ }
+ }
+}
+realms {
+ name: "ci.shadow"
+ bindings {
+ role: "role/buildbucket.builderServiceAccount"
+ principals: "user:libyuv-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ bindings {
+ role: "role/buildbucket.creator"
+ principals: "group:chromium-led-users"
+ principals: "group:mdb/chrome-build-access-sphinx"
+ principals: "group:mdb/chrome-troopers"
+ principals: "group:mdb/foundry-x-team"
+ }
+}
+realms {
+ name: "cron"
+ bindings {
+ role: "role/buildbucket.builderServiceAccount"
+ principals: "user:libyuv-ci-autoroll-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+}
+realms {
+ name: "try"
+ bindings {
+ role: "role/buildbucket.builderServiceAccount"
+ principals: "user:libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ bindings {
+ role: "role/buildbucket.triggerer"
+ principals: "group:project-libyuv-tryjob-access"
+ principals: "group:service-account-cq"
+ }
+}
+realms {
+ name: "try.shadow"
+ bindings {
+ role: "role/buildbucket.builderServiceAccount"
+ principals: "user:libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
+ bindings {
+ role: "role/buildbucket.creator"
+ principals: "group:chromium-led-users"
+ principals: "group:mdb/chrome-build-access-sphinx"
+ principals: "group:mdb/chrome-troopers"
+ principals: "group:mdb/foundry-x-team"
+ }
+}
diff --git a/libs/libyuv/libyuv.bzl b/libs/libyuv/libyuv.bzl
new file mode 100644
index 0000000000..1e910687f8
--- /dev/null
+++ b/libs/libyuv/libyuv.bzl
@@ -0,0 +1,50 @@
+# Copyright 2026 The LibYuv Project Authors. All rights reserved.
+#
+# Shared target definitions for Bazel and Blaze builds.
+
+def libyuv_srcs(prefix = ""):
+ return native.glob(
+ [
+ prefix + "source/*.cc",
+ prefix + "include/libyuv/*.h",
+ ],
+ exclude = [
+ prefix + "source/*neon*.cc",
+ prefix + "source/*rvv*.cc",
+ prefix + "source/*sve*.cc",
+ prefix + "source/*sme*.cc",
+ ],
+ )
+
+def libyuv_hdrs(prefix = ""):
+ return [
+ prefix + "include/libyuv/compare.h",
+ prefix + "include/libyuv/convert.h",
+ prefix + "include/libyuv/convert_from_argb.h",
+ prefix + "include/libyuv/cpu_id.h",
+ prefix + "include/libyuv/row.h", # Bug 553148013: Should be private.
+ ]
+
+def libyuv_neon_srcs(prefix = ""):
+ return native.glob([
+ prefix + "source/*neon*.cc",
+ prefix + "include/libyuv/*.h",
+ ])
+
+def libyuv_rvv_srcs(prefix = ""):
+ return native.glob([
+ prefix + "source/*rvv*.cc",
+ prefix + "include/libyuv/*.h",
+ ])
+
+def libyuv_sve_srcs(prefix = ""):
+ return native.glob([
+ prefix + "source/*sve*.cc",
+ prefix + "include/libyuv/*.h",
+ ])
+
+def libyuv_test_srcs(prefix = ""):
+ return native.glob([
+ prefix + "unit_test/*.cc",
+ prefix + "unit_test/*.h",
+ ])
diff --git a/libs/libyuv/libyuv.gni b/libs/libyuv/libyuv.gni
index 8df40ba2d7..23e38c1b40 100644
--- a/libs/libyuv/libyuv.gni
+++ b/libs/libyuv/libyuv.gni
@@ -6,18 +6,29 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
-import("//build_overrides/build.gni")
import("//build/config/arm.gni")
+import("//build/config/loongarch64.gni")
import("//build/config/mips.gni")
+import("//build_overrides/build.gni")
declare_args() {
libyuv_include_tests = !build_with_chromium
libyuv_disable_jpeg = false
- libyuv_use_neon =
- current_cpu == "arm64" ||
- (current_cpu == "arm" && (arm_use_neon || arm_optionally_use_neon))
+ libyuv_disable_rvv = false
+ libyuv_enable_rowwin = false
+ libyuv_use_neon = current_cpu == "arm64" || current_cpu == "arm"
+ libyuv_use_sve = current_cpu == "arm64"
+
+ # Restrict to (is_linux || is_android) to work around undefined symbol linker
+ # errors on Fuchsia, macOS, and compilation errors on Windows.
+ # TODO: bug 359006069 - Remove the restriction after the linker and
+ # compilation errors are fixed.
+ libyuv_use_sme = current_cpu == "arm64" && (is_linux || is_android)
libyuv_use_msa =
(current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_msa
libyuv_use_mmi =
(current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_mmi
+ libyuv_use_lsx = current_cpu == "loong64" && loongarch64_use_lsx
+ libyuv_use_lasx = current_cpu == "loong64" && loongarch64_use_lasx
+ libyuv_use_rvv = current_cpu == "riscv64" || current_cpu == "riscv32"
}
diff --git a/libs/libyuv/libyuv.gyp b/libs/libyuv/libyuv.gyp
new file mode 100644
index 0000000000..fa4b146a43
--- /dev/null
+++ b/libs/libyuv/libyuv.gyp
@@ -0,0 +1,137 @@
+# Copyright 2011 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+{
+ 'includes': [
+ 'libyuv.gypi',
+ ],
+ # Make sure that if we are being compiled to an xcodeproj, nothing tries to
+ # include a .pch.
+ 'xcode_settings': {
+ 'GCC_PREFIX_HEADER': '',
+ 'GCC_PRECOMPILE_PREFIX_HEADER': 'NO',
+ },
+ 'variables': {
+ 'use_system_libjpeg%': 0,
+ # Can be enabled if your jpeg has GYP support.
+ 'libyuv_disable_jpeg%': 1,
+ # 'chromium_code' treats libyuv as internal and increases warning level.
+ 'chromium_code': 1,
+ # clang compiler default variable usable by other apps that include libyuv.
+ 'clang%': 0,
+ # Link-Time Optimizations.
+ 'use_lto%': 0,
+ 'build_neon': 0,
+ 'conditions': [
+ ['(target_arch == "armv7" or target_arch == "armv7s" or \
+ (target_arch == "arm" and arm_version >= 7) or target_arch == "arm64")\
+ and (arm_neon == 1 or arm_neon_optional == 1)', {
+ 'build_neon': 1,
+ }],
+ ],
+ },
+
+ 'targets': [
+ {
+ 'target_name': 'libyuv',
+ # Change type to 'shared_library' to build .so or .dll files.
+ 'type': 'static_library',
+ 'variables': {
+ 'optimize': 'max', # enable O2 and ltcg.
+ },
+ # Allows libyuv.a redistributable library without external dependencies.
+ 'standalone_static_library': 1,
+ 'conditions': [
+ # Disable -Wunused-parameter
+ ['clang == 1', {
+ 'cflags': [
+ '-Wno-unused-parameter',
+ ],
+ }],
+ ['build_neon != 0', {
+ 'defines': [
+ 'LIBYUV_NEON',
+ ],
+ 'cflags!': [
+ '-mfpu=vfp',
+ '-mfpu=vfpv3',
+ '-mfpu=vfpv3-d16',
+ # '-mthumb', # arm32 not thumb
+ ],
+ 'conditions': [
+ # Disable LTO in libyuv_neon target due to gcc 4.9 compiler bug.
+ ['clang == 0 and use_lto == 1', {
+ 'cflags!': [
+ '-flto',
+ '-ffat-lto-objects',
+ ],
+ }],
+ # arm64 does not need -mfpu=neon option as neon is not optional
+ ['target_arch != "arm64"', {
+ 'cflags': [
+ '-mfpu=neon',
+ # '-marm', # arm32 not thumb
+ ],
+ }],
+ ],
+ }],
+ ['OS != "ios" and libyuv_disable_jpeg != 1', {
+ 'defines': [
+ 'HAVE_JPEG'
+ ],
+ 'conditions': [
+ # Caveat system jpeg support may not support motion jpeg
+ [ 'use_system_libjpeg == 1', {
+ 'dependencies': [
+ '<(DEPTH)/third_party/libjpeg/libjpeg.gyp:libjpeg',
+ ],
+ }, {
+ 'dependencies': [
+ '<(DEPTH)/third_party/libjpeg_turbo/libjpeg.gyp:libjpeg',
+ ],
+ }],
+ [ 'use_system_libjpeg == 1', {
+ 'link_settings': {
+ 'libraries': [
+ '-ljpeg',
+ ],
+ }
+ }],
+ ],
+ }],
+ ], #conditions
+ 'defines': [
+ # Enable the following 3 macros to turn off assembly for specified CPU.
+ # 'LIBYUV_DISABLE_X86',
+ # 'LIBYUV_DISABLE_NEON',
+ # Enable the following macro to build libyuv as a shared library (dll).
+ # 'LIBYUV_USING_SHARED_LIBRARY',
+ # TODO(fbarchard): Make these into gyp defines.
+ ],
+ 'include_dirs': [
+ 'include',
+ '.',
+ ],
+ 'direct_dependent_settings': {
+ 'include_dirs': [
+ 'include',
+ '.',
+ ],
+ },
+ 'sources': [
+ '<@(libyuv_sources)',
+ ],
+ },
+ ], # targets.
+}
+
+# Local Variables:
+# tab-width:2
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=2 shiftwidth=2:
diff --git a/libs/libyuv/libyuv.gypi b/libs/libyuv/libyuv.gypi
new file mode 100644
index 0000000000..91d9ff67fb
--- /dev/null
+++ b/libs/libyuv/libyuv.gypi
@@ -0,0 +1,96 @@
+# Copyright 2014 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+{
+ 'variables': {
+ 'libyuv_sources': [
+ # includes.
+ 'include/libyuv.h',
+ 'include/libyuv/basic_types.h',
+ 'include/libyuv/compare.h',
+ 'include/libyuv/compare_row.h',
+ 'include/libyuv/convert.h',
+ 'include/libyuv/convert_argb.h',
+ 'include/libyuv/convert_from.h',
+ 'include/libyuv/convert_from_argb.h',
+ 'include/libyuv/cpu_id.h',
+ 'include/libyuv/cpu_support.h',
+ 'include/libyuv/loongson_intrinsics.h',
+ 'include/libyuv/mjpeg_decoder.h',
+ 'include/libyuv/planar_functions.h',
+ 'include/libyuv/rotate.h',
+ 'include/libyuv/rotate_argb.h',
+ 'include/libyuv/rotate_row.h',
+ 'include/libyuv/row.h',
+ 'include/libyuv/row_sve.h',
+ 'include/libyuv/scale.h',
+ 'include/libyuv/scale_argb.h',
+ 'include/libyuv/scale_rgb.h',
+ 'include/libyuv/scale_row.h',
+ 'include/libyuv/scale_uv.h',
+ 'include/libyuv/version.h',
+ 'include/libyuv/video_common.h',
+
+ # sources.
+ 'source/compare.cc',
+ 'source/compare_common.cc',
+ 'source/compare_gcc.cc',
+ 'source/compare_hvx.cc',
+ 'source/compare_neon.cc',
+ 'source/compare_neon64.cc',
+ 'source/compare_win.cc',
+ 'source/convert.cc',
+ 'source/convert_argb.cc',
+ 'source/convert_from.cc',
+ 'source/convert_from_argb.cc',
+ 'source/convert_jpeg.cc',
+ 'source/convert_to_argb.cc',
+ 'source/convert_to_i420.cc',
+ 'source/cpu_id.cc',
+ 'source/mjpeg_decoder.cc',
+ 'source/mjpeg_validate.cc',
+ 'source/planar_functions.cc',
+ 'source/rotate.cc',
+ 'source/rotate_any.cc',
+ 'source/rotate_argb.cc',
+ 'source/rotate_common.cc',
+ 'source/rotate_gcc.cc',
+ 'source/rotate_hvx.cc',
+ 'source/rotate_lasx.cc',
+ 'source/rotate_lsx.cc',
+ 'source/rotate_neon.cc',
+ 'source/rotate_neon64.cc',
+ 'source/rotate_rvv.cc',
+ 'source/rotate_win.cc',
+ 'source/row_any.cc',
+ 'source/row_common.cc',
+ 'source/row_gcc.cc',
+ 'source/row_hvx.cc',
+ 'source/row_lasx.cc',
+ 'source/row_lsx.cc',
+ 'source/row_neon.cc',
+ 'source/row_neon64.cc',
+ 'source/row_rvv.cc',
+ 'source/row_win.cc',
+ 'source/scale.cc',
+ 'source/scale_any.cc',
+ 'source/scale_argb.cc',
+ 'source/scale_common.cc',
+ 'source/scale_gcc.cc',
+ 'source/scale_hvx.cc',
+ 'source/scale_lsx.cc',
+ 'source/scale_neon.cc',
+ 'source/scale_neon64.cc',
+ 'source/scale_rgb.cc',
+ 'source/scale_rvv.cc',
+ 'source/scale_uv.cc',
+ 'source/scale_win.cc',
+ 'source/video_common.cc',
+ ],
+ }
+}
diff --git a/libs/libyuv/linux.mk b/libs/libyuv/linux.mk
index e9a26a79b2..f4d9c73b1b 100644
--- a/libs/libyuv/linux.mk
+++ b/libs/libyuv/linux.mk
@@ -13,15 +13,14 @@ LOCAL_OBJ_FILES := \
source/compare.o \
source/compare_common.o \
source/compare_gcc.o \
- source/compare_mmi.o \
- source/compare_msa.o \
- source/compare_neon64.o \
+ source/compare_hvx.o \
source/compare_neon.o \
+ source/compare_neon64.o \
source/compare_win.o \
- source/convert_argb.o \
source/convert.o \
- source/convert_from_argb.o \
+ source/convert_argb.o \
source/convert_from.o \
+ source/convert_from_argb.o \
source/convert_jpeg.o \
source/convert_to_argb.o \
source/convert_to_i420.o \
@@ -29,33 +28,40 @@ LOCAL_OBJ_FILES := \
source/mjpeg_decoder.o \
source/mjpeg_validate.o \
source/planar_functions.o \
+ source/rotate.o \
source/rotate_any.o \
source/rotate_argb.o \
- source/rotate.o \
source/rotate_common.o \
source/rotate_gcc.o \
- source/rotate_mmi.o \
- source/rotate_msa.o \
- source/rotate_neon64.o \
+ source/rotate_hvx.o \
+ source/rotate_lasx.o \
+ source/rotate_lsx.o \
source/rotate_neon.o \
+ source/rotate_neon64.o \
+ source/rotate_rvv.o \
source/rotate_win.o \
source/row_any.o \
source/row_common.o \
source/row_gcc.o \
- source/row_mmi.o \
- source/row_msa.o \
- source/row_neon64.o \
+ source/row_hvx.o \
+ source/row_lasx.o \
+ source/row_lsx.o \
source/row_neon.o \
+ source/row_neon64.o \
+ source/row_rvv.o \
source/row_win.o \
+ source/scale.o \
source/scale_any.o \
source/scale_argb.o \
- source/scale.o \
source/scale_common.o \
source/scale_gcc.o \
- source/scale_mmi.o \
- source/scale_msa.o \
- source/scale_neon64.o \
+ source/scale_hvx.o \
+ source/scale_lsx.o \
source/scale_neon.o \
+ source/scale_neon64.o \
+ source/scale_rgb.o \
+ source/scale_rvv.o \
+ source/scale_uv.o \
source/scale_win.o \
source/video_common.o
@@ -65,19 +71,31 @@ LOCAL_OBJ_FILES := \
.c.o:
$(CC) -c $(CFLAGS) $*.c -o $*.o
-all: libyuv.a yuvconvert cpuid psnr
+all: libyuv.a i444tonv12_eg yuvconvert yuvconstants cpuid psnr compare
libyuv.a: $(LOCAL_OBJ_FILES)
$(AR) $(ARFLAGS) $@ $(LOCAL_OBJ_FILES)
+# A C++ test utility that compares two YUV files.
+compare: util/compare.cc libyuv.a
+ $(CXX) $(CXXFLAGS) -Iutil/ -o $@ util/compare.cc libyuv.a
+
# A C++ test utility that uses libyuv conversion.
yuvconvert: util/yuvconvert.cc libyuv.a
$(CXX) $(CXXFLAGS) -Iutil/ -o $@ util/yuvconvert.cc libyuv.a
+# A C test utility that generates yuvconstants for yuv to rgb.
+yuvconstants: util/yuvconstants.c libyuv.a
+ $(CXX) $(CXXFLAGS) -Iutil/ -lm -o $@ util/yuvconstants.c libyuv.a
+
# A standalone test utility
psnr: util/psnr.cc
$(CXX) $(CXXFLAGS) -Iutil/ -o $@ util/psnr.cc util/psnr_main.cc util/ssim.cc
+# A simple conversion example.
+i444tonv12_eg: util/i444tonv12_eg.cc libyuv.a
+ $(CXX) $(CXXFLAGS) -o $@ util/i444tonv12_eg.cc libyuv.a
+
# A C test utility that uses libyuv conversion from C.
# gcc 4.4 and older require -fno-exceptions to avoid link error on __gxx_personality_v0
# CC=gcc-4.4 CXXFLAGS=-fno-exceptions CXX=g++-4.4 make -f linux.mk
@@ -85,4 +103,4 @@ cpuid: util/cpuid.c libyuv.a
$(CC) $(CFLAGS) -o $@ util/cpuid.c libyuv.a
clean:
- /bin/rm -f source/*.o *.ii *.s libyuv.a yuvconvert cpuid psnr
+ /bin/rm -f source/*.o *.ii *.s libyuv.a i444tonv12_eg yuvconvert yuvconstants cpuid psnr compare
diff --git a/libs/libyuv/pylintrc b/libs/libyuv/pylintrc
index b8bea33489..57be7df051 100644
--- a/libs/libyuv/pylintrc
+++ b/libs/libyuv/pylintrc
@@ -13,5 +13,37 @@ reports=no
[FORMAT]
-# We use two spaces for indents, instead of the usual four spaces or tab.
-indent-string=' '
+# Maximum number of characters on a single line.
+max-line-length=79
+
+# We use four spaces for indents.
+indent-string=' '
+
+[BASIC]
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct method names.
+method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
diff --git a/libs/libyuv/riscv_script/prepare_toolchain_qemu.sh b/libs/libyuv/riscv_script/prepare_toolchain_qemu.sh
new file mode 100755
index 0000000000..2a90173931
--- /dev/null
+++ b/libs/libyuv/riscv_script/prepare_toolchain_qemu.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+set -ev
+
+# Download & build RISC-V Clang toolchain & QEMU emulator.
+# RISC-V Clang is for cross compile with the RISC-V Vector ISA.
+# RISC-V QEMU is used to run the test suite.
+#
+# Requirements: Linux host w/ working C++ compiler, git, cmake, ninja, wget, tar
+
+# NOTE: this script must be run from the top-level directory of the LIBYUV_SRC_DIR.
+
+RISCV_TRIPLE="riscv64-unknown-linux-gnu"
+RISCV_QEMU="qemu-riscv64"
+
+LIBYUV_SRC_DIR=$(pwd)
+BUILD_DIR="$LIBYUV_SRC_DIR"/build-toolchain-qemu
+INSTALL_QEMU="$BUILD_DIR"/riscv-qemu
+INSTALL_CLANG="$BUILD_DIR"/riscv-clang
+
+LLVM_VERSION="16.0.0"
+LLVM_NAME=llvm-project-"$LLVM_VERSION".src
+
+RISCV_GNU_TOOLCHAIN="$BUILD_DIR"/riscv-gnu-toolchain
+RISCV_CLANG_TOOLCHAIN="$BUILD_DIR"/"$LLVM_NAME"
+
+QEMU_NAME="qemu-7.0.0"
+
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+# Download and install RISC-V GNU Toolchain (needed to build Clang)
+if [ ! -d "$RISCV_GNU_TOOLCHAIN" ]
+then
+ git clone git@github.com:riscv/riscv-gnu-toolchain.git
+ pushd "$RISCV_GNU_TOOLCHAIN"
+ git submodule update --init --recursive
+ ./configure --with-cmodel=medany --prefix="$INSTALL_CLANG"
+ ionice nice make linux -j `nproc` install
+ popd
+fi
+
+# Download Clang toolchain & build cross compiler
+if [ ! -d "$RISCV_CLANG_TOOLCHAIN" ]
+then
+ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-"$LLVM_VERSION"/"$LLVM_NAME".tar.xz
+ tar xvJf "$LLVM_NAME".tar.xz
+ pushd "$RISCV_CLANG_TOOLCHAIN"
+ cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_CLANG" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_TARGETS_TO_BUILD="RISCV" \
+ -DLLVM_ENABLE_PROJECTS="clang" \
+ -DLLVM_DEFAULT_TARGET_TRIPLE="$RISCV_TRIPLE" \
+ -DLLVM_INSTALL_TOOLCHAIN_ONLY=On \
+ -DDEFAULT_SYSROOT=../sysroot \
+ -G "Ninja" "$RISCV_CLANG_TOOLCHAIN"/llvm
+ ionice nice ninja -j `nproc`
+ ionice nice ninja -j `nproc` install
+ popd
+ pushd "$INSTALL_CLANG"/bin
+ ln -sf clang "$RISCV_TRIPLE"-clang
+ ln -sf clang++ "$RISCV_TRIPLE"-clang++
+ popd
+fi
+
+# Download QEMU and build the riscv64 Linux usermode emulator
+if [ ! -d "$QEMU_NAME" ]
+then
+ wget https://download.qemu.org/"$QEMU_NAME".tar.xz
+ tar xvJf "$QEMU_NAME".tar.xz
+ pushd "$QEMU_NAME"
+ ./configure --target-list=riscv64-linux-user --prefix="$INSTALL_QEMU"
+ ionice nice make -j `nproc` install
+ popd
+fi
diff --git a/libs/libyuv/riscv_script/riscv-clang.cmake b/libs/libyuv/riscv_script/riscv-clang.cmake
new file mode 100644
index 0000000000..41d23a5c37
--- /dev/null
+++ b/libs/libyuv/riscv_script/riscv-clang.cmake
@@ -0,0 +1,56 @@
+set(CMAKE_CROSSCOMPILING TRUE)
+set(CMAKE_SYSTEM_NAME "Linux")
+set(CMAKE_SYSTEM_PROCESSOR "riscv64")
+
+option(USE_RVV "Enable riscv vector or not." ON)
+option(USE_AUTO_VECTORIZER "Enable riscv auto vectorizer or not." OFF)
+
+# Avoid to use system path for cross-compile
+set(CMAKE_FIND_USE_CMAKE_SYSTEM_PATH FALSE)
+
+set(TOOLCHAIN_PATH "" CACHE STRING "The toolcahin path.")
+if(NOT TOOLCHAIN_PATH)
+ set(TOOLCHAIN_PATH ${CMAKE_SOURCE_DIR}/build-toolchain-qemu/riscv-clang)
+endif()
+
+set(TOOLCHAIN_PREFIX "riscv64-unknown-linux-gnu-" CACHE STRING "The toolcahin prefix.")
+
+# toolchain setting
+set(CMAKE_C_COMPILER "${TOOLCHAIN_PATH}/bin/${TOOLCHAIN_PREFIX}clang")
+set(CMAKE_CXX_COMPILER "${TOOLCHAIN_PATH}/bin/${TOOLCHAIN_PREFIX}clang++")
+
+# CMake will just use the host-side tools for the following tools, so we setup them here.
+set(CMAKE_C_COMPILER_AR "${TOOLCHAIN_PATH}/bin/llvm-ar")
+set(CMAKE_CXX_COMPILER_AR "${TOOLCHAIN_PATH}/bin/llvm-ar")
+set(CMAKE_C_COMPILER_RANLIB "${TOOLCHAIN_PATH}/bin/llvm-ranlib")
+set(CMAKE_CXX_COMPILER_RANLIB "${TOOLCHAIN_PATH}/bin/llvm-ranlib")
+set(CMAKE_OBJDUMP "${TOOLCHAIN_PATH}/bin/llvm-objdump")
+set(CMAKE_OBJCOPY "${TOOLCHAIN_PATH}/bin/llvm-objcopy")
+
+# compile options
+set(RISCV_COMPILER_FLAGS "" CACHE STRING "Compile flags")
+# if user provides RISCV_COMPILER_FLAGS, appeding compile flags is avoided.
+if(RISCV_COMPILER_FLAGS STREQUAL "")
+ message(STATUS "USE_RVV: ${USE_RVV}")
+ message(STATUS "USE_AUTO_VECTORIZER: ${USE_AUTO_VECTORIZER}")
+ if(USE_RVV)
+ list(APPEND RISCV_COMPILER_FLAGS "-march=rv64gcv")
+ if(NOT USE_AUTO_VECTORIZER)
+ # Disable auto-vectorizer
+ add_compile_options(-fno-tree-vectorize -fno-tree-slp-vectorize)
+ endif()
+ else()
+ list(APPEND RISCV_COMPILER_FLAGS "-march=rv64gc")
+ endif()
+endif()
+add_compile_options("-Wuninitialized")
+message(STATUS "RISCV_COMPILER_FLAGS: ${RISCV_COMPILER_FLAGS}")
+
+set(CMAKE_C_FLAGS "${RISCV_COMPILER_FLAGS} ${CMAKE_C_FLAGS}")
+set(CMAKE_CXX_FLAGS "${RISCV_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}")
+
+set(RISCV_LINKER_FLAGS "-lstdc++ -lpthread -lm -ldl")
+set(RISCV_LINKER_FLAGS_EXE)
+set(CMAKE_SHARED_LINKER_FLAGS "${RISCV_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS}")
+set(CMAKE_MODULE_LINKER_FLAGS "${RISCV_LINKER_FLAGS} ${CMAKE_MODULE_LINKER_FLAGS}")
+set(CMAKE_EXE_LINKER_FLAGS "${RISCV_LINKER_FLAGS} ${RISCV_LINKER_FLAGS_EXE} ${CMAKE_EXE_LINKER_FLAGS}")
diff --git a/libs/libyuv/riscv_script/run_qemu.sh b/libs/libyuv/riscv_script/run_qemu.sh
new file mode 100755
index 0000000000..080af3b19d
--- /dev/null
+++ b/libs/libyuv/riscv_script/run_qemu.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -x
+set -e
+
+USE_RVV="${USE_RVV:-OFF}"
+TOOLCHAIN_PATH="${TOOLCHAIN_PATH:-../../build-toolchain-qemu/riscv-clang}"
+QEMU_PREFIX_PATH="${QEMU_PREFIX_PATH:-../../build-toolchain-qemu/riscv-qemu/}"
+
+if [ "${USE_RVV}" = "ON" ];then
+ QEMU_OPTION="-cpu rv64,zba=true,zbb=true,zbc=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0 -L ${TOOLCHAIN_PATH}/sysroot"
+else
+ QEMU_OPTION="-cpu rv64,zba=true,zbb=true,zbc=true,zbs=true -L ${TOOLCHAIN_PATH}/sysroot"
+fi
+
+$QEMU_PREFIX_PATH/bin/qemu-riscv64 $QEMU_OPTION $@
diff --git a/libs/libyuv/source/compare.cc b/libs/libyuv/source/compare.cc
index 7f4828104f..12c5738966 100644
--- a/libs/libyuv/source/compare.cc
+++ b/libs/libyuv/source/compare.cc
@@ -11,12 +11,13 @@
#include "libyuv/compare.h"
#include <float.h>
+#include <limits.h>
#include <math.h>
+#include <string.h>
#ifdef _OPENMP
#include <omp.h>
#endif
-#include "libyuv/basic_types.h"
#include "libyuv/compare_row.h"
#include "libyuv/cpu_id.h"
#include "libyuv/row.h"
@@ -31,6 +32,7 @@ extern "C" {
LIBYUV_API
uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) {
const int kBlockSize = 1 << 15; // 32768;
+ int simd_size = 16; // SIMD for multiple of 16, and C for remainder.
int remainder;
uint32_t (*HashDjb2_SSE)(const uint8_t* src, int count, uint32_t seed) =
HashDjb2_C;
@@ -42,53 +44,76 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) {
#if defined(HAS_HASHDJB2_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
HashDjb2_SSE = HashDjb2_AVX2;
+ simd_size = 32;
+ }
+#endif
+#if defined(HAS_HASHDJB2_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ HashDjb2_SSE = HashDjb2_NEON;
+ }
+#endif
+#if defined(HAS_HASHDJB2_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HashDjb2_SSE = HashDjb2_AVX512BW;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_HASHDJB2_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ HashDjb2_SSE = HashDjb2_RVV;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_HASHDJB2_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ HashDjb2_SSE = HashDjb2_LSX;
+ }
+#endif
+#if defined(HAS_HASHDJB2_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ HashDjb2_SSE = HashDjb2_LASX;
+ }
+#endif
+#if defined(HAS_HASHDJB2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ HashDjb2_SSE = HashDjb2_WASMSIMD;
+ }
+#endif
+#if defined(HAS_HASHDJB2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ HashDjb2_SSE = HashDjb2_HVX;
+ simd_size = 1; // Any count.
}
#endif
- while (count >= (uint64_t)(kBlockSize)) {
+ while (count >= (uint64_t)kBlockSize) {
seed = HashDjb2_SSE(src, kBlockSize, seed);
src += kBlockSize;
count -= kBlockSize;
}
- remainder = (int)count & ~15;
+ remainder = (int)count & ~(simd_size - 1);
if (remainder) {
seed = HashDjb2_SSE(src, remainder, seed);
src += remainder;
count -= remainder;
}
- remainder = (int)count & 15;
+ remainder = (int)count & (simd_size - 1);
if (remainder) {
- seed = HashDjb2_C(src, remainder, seed);
- }
- return seed;
-}
-
-static uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) {
- int x;
- for (x = 0; x < width - 1; x += 2) {
- if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
- return FOURCC_BGRA;
+ if (HashDjb2_SSE == HashDjb2_C) {
+ seed = HashDjb2_C(src, remainder, seed);
+ } else {
+ SIMD_ALIGNED(uint8_t buf[32]);
+ uint32_t pow33 = 1;
+ int i;
+ memset(buf, 0, sizeof(buf));
+ for (i = 0; i < remainder; ++i) {
+ pow33 *= 33;
+ }
+ memcpy(buf + simd_size - remainder, src, remainder);
+ seed = seed * pow33 + HashDjb2_SSE(buf, simd_size, 0);
}
- if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA.
- return FOURCC_ARGB;
- }
- if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255.
- return FOURCC_BGRA;
- }
- if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255.
- return FOURCC_ARGB;
- }
- argb += 8;
}
- if (width & 1) {
- if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
- return FOURCC_BGRA;
- }
- if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA.
- return FOURCC_ARGB;
- }
- }
- return 0;
+ return seed;
}
// Scan an opaque argb image and return fourcc based on alpha offset.
@@ -98,17 +123,56 @@ uint32_t ARGBDetect(const uint8_t* argb,
int stride_argb,
int width,
int height) {
+ uint32_t (*ARGBDetectRow)(const uint8_t* argb, int width) = ARGBDetectRow_C;
uint32_t fourcc = 0;
int h;
+ if (!argb || width <= 0 || height <= 0) {
+ return fourcc;
+ }
// Coalesce rows.
- if (stride_argb == width * 4) {
+ if (stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
stride_argb = 0;
}
+#if defined(HAS_ARGBDETECTROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBDetectRow = ARGBDetectRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBDetectRow = ARGBDetectRow_NEON;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBDetectRow = ARGBDetectRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBDetectRow = ARGBDetectRow_LSX;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBDetectRow = ARGBDetectRow_LASX;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBDetectRow = ARGBDetectRow_WASMSIMD;
+ }
+#endif
+#if defined(HAS_ARGBDETECTROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBDetectRow = ARGBDetectRow_HVX;
+ }
+#endif
for (h = 0; h < height && fourcc == 0; ++h) {
- fourcc = ARGBDetectRow_C(argb, width);
+ fourcc = ARGBDetectRow(argb, width);
argb += stride_argb;
}
return fourcc;
@@ -122,9 +186,8 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
const int kBlockSize = 1 << 15; // 32768;
- const int kSimdSize = 64;
- // SIMD for multiple of 64, and C for remainder
- int remainder = count & (kBlockSize - 1) & ~(kSimdSize - 1);
+ int simd_size = 64; // SIMD for multiple of 64, and C for remainder.
+ int remainder;
uint64_t diff = 0;
int i;
uint32_t (*HammingDistance)(const uint8_t* src_a, const uint8_t* src_b,
@@ -134,6 +197,11 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
HammingDistance = HammingDistance_NEON;
}
#endif
+#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ HammingDistance = HammingDistance_NEON_DotProd;
+ }
+#endif
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
HammingDistance = HammingDistance_SSSE3;
@@ -149,16 +217,46 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
HammingDistance = HammingDistance_AVX2;
}
#endif
-#if defined(HAS_HAMMINGDISTANCE_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- HammingDistance = HammingDistance_MSA;
+#if defined(HAS_HAMMINGDISTANCE_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HammingDistance = HammingDistance_AVX512BW;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ HammingDistance = HammingDistance_AVX512VNNI;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ HammingDistance = HammingDistance_RVV;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ HammingDistance = HammingDistance_LSX;
+ }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ HammingDistance = HammingDistance_LASX;
+ }
+#endif
+#if defined(HAS_HAMMINGDISTANCE_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ HammingDistance = HammingDistance_WASMSIMD;
}
#endif
-#if defined(HAS_HAMMINGDISTANCE_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- HammingDistance = HammingDistance_MMI;
+#if defined(HAS_HAMMINGDISTANCE_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ HammingDistance = HammingDistance_HVX;
+ simd_size = 1; // Any count.
}
#endif
+ remainder = count & (kBlockSize - 1) & ~(simd_size - 1);
#ifdef _OPENMP
#pragma omp parallel for reduction(+ : diff)
@@ -173,9 +271,19 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
src_a += remainder;
src_b += remainder;
}
- remainder = count & (kSimdSize - 1);
+ remainder = count & (simd_size - 1);
if (remainder) {
- diff += HammingDistance_C(src_a, src_b, remainder);
+ if (HammingDistance == HammingDistance_C) {
+ diff += HammingDistance_C(src_a, src_b, remainder);
+ } else {
+ SIMD_ALIGNED(uint8_t buf_a[64]);
+ SIMD_ALIGNED(uint8_t buf_b[64]);
+ memset(buf_a, 0, sizeof(buf_a));
+ memset(buf_b, 0, sizeof(buf_b));
+ memcpy(buf_a, src_a, remainder);
+ memcpy(buf_b, src_b, remainder);
+ diff += HammingDistance(buf_a, buf_b, simd_size);
+ }
}
return diff;
}
@@ -189,7 +297,8 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
// Up to 65536 of those can be summed and remain within a uint32_t.
// After each block of 65536 pixels, accumulate into a uint64_t.
const int kBlockSize = 65536;
- int remainder = count & (kBlockSize - 1) & ~31;
+ int simd_size = 32; // SIMD for multiple of 32, and C for remainder.
+ int remainder;
uint64_t sse = 0;
int i;
uint32_t (*SumSquareError)(const uint8_t* src_a, const uint8_t* src_b,
@@ -199,6 +308,11 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
SumSquareError = SumSquareError_NEON;
}
#endif
+#if defined(HAS_SUMSQUAREERROR_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ SumSquareError = SumSquareError_NEON_DotProd;
+ }
+#endif
#if defined(HAS_SUMSQUAREERROR_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
// Note only used for multiples of 16 so count is not checked.
@@ -211,16 +325,46 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
SumSquareError = SumSquareError_AVX2;
}
#endif
-#if defined(HAS_SUMSQUAREERROR_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SumSquareError = SumSquareError_MSA;
+#if defined(HAS_SUMSQUAREERROR_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SumSquareError = SumSquareError_AVX512BW;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_SUMSQUAREERROR_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ SumSquareError = SumSquareError_AVX512VNNI;
+ simd_size = 1; // Any count.
+ }
+#endif
+#if defined(HAS_SUMSQUAREERROR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SumSquareError = SumSquareError_RVV;
+ simd_size = 1; // Any count.
}
#endif
-#if defined(HAS_SUMSQUAREERROR_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SumSquareError = SumSquareError_MMI;
+#if defined(HAS_SUMSQUAREERROR_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SumSquareError = SumSquareError_LSX;
}
#endif
+#if defined(HAS_SUMSQUAREERROR_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SumSquareError = SumSquareError_LASX;
+ }
+#endif
+#if defined(HAS_SUMSQUAREERROR_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ SumSquareError = SumSquareError_WASMSIMD;
+ }
+#endif
+#if defined(HAS_SUMSQUAREERROR_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SumSquareError = SumSquareError_HVX;
+ simd_size = 1; // Any count.
+ }
+#endif
+ remainder = count & (kBlockSize - 1) & ~(simd_size - 1);
#ifdef _OPENMP
#pragma omp parallel for reduction(+ : sse)
#endif
@@ -234,9 +378,19 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
src_a += remainder;
src_b += remainder;
}
- remainder = count & 31;
+ remainder = count & (simd_size - 1);
if (remainder) {
- sse += SumSquareError_C(src_a, src_b, remainder);
+ if (SumSquareError == SumSquareError_C) {
+ sse += SumSquareError_C(src_a, src_b, remainder);
+ } else {
+ SIMD_ALIGNED(uint8_t buf_a[32]);
+ SIMD_ALIGNED(uint8_t buf_b[32]);
+ memset(buf_a, 0, sizeof(buf_a));
+ memset(buf_b, 0, sizeof(buf_b));
+ memcpy(buf_a, src_a, remainder);
+ memcpy(buf_b, src_b, remainder);
+ sse += SumSquareError(buf_a, buf_b, simd_size);
+ }
}
return sse;
}
@@ -250,8 +404,12 @@ uint64_t ComputeSumSquareErrorPlane(const uint8_t* src_a,
int height) {
uint64_t sse = 0;
int h;
+ if (!src_a || !src_b || width <= 0 || height <= 0) {
+ return sse;
+ }
// Coalesce rows.
- if (stride_a == width && stride_b == width) {
+ if (stride_a == width && stride_b == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
stride_a = stride_b = 0;
@@ -323,18 +481,14 @@ double I420Psnr(const uint8_t* src_y_a,
return SumSquareErrorToPsnr(sse, samples);
}
-static const int64_t cc1 = 26634; // (64^2*(.01*255)^2
-static const int64_t cc2 = 239708; // (64^2*(.03*255)^2
-
-static double Ssim8x8_C(const uint8_t* src_a,
- int stride_a,
- const uint8_t* src_b,
- int stride_b) {
- int64_t sum_a = 0;
- int64_t sum_b = 0;
- int64_t sum_sq_a = 0;
- int64_t sum_sq_b = 0;
- int64_t sum_axb = 0;
+static float Ssim8x8(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b) {
+ int32_t sum_a = 0;
+ int32_t sum_b = 0;
+ int32_t sum_sq_ab = 0;
+ int32_t sum_axb = 0;
int i;
for (i = 0; i < 8; ++i) {
@@ -342,8 +496,7 @@ static double Ssim8x8_C(const uint8_t* src_a,
for (j = 0; j < 8; ++j) {
sum_a += src_a[j];
sum_b += src_b[j];
- sum_sq_a += src_a[j] * src_a[j];
- sum_sq_b += src_b[j] * src_b[j];
+ sum_sq_ab += src_a[j] * src_a[j] + src_b[j] * src_b[j];
sum_axb += src_a[j] * src_b[j];
}
@@ -352,28 +505,88 @@ static double Ssim8x8_C(const uint8_t* src_a,
}
{
- const int64_t count = 64;
- // scale the constants by number of pixels
- const int64_t c1 = (cc1 * count * count) >> 12;
- const int64_t c2 = (cc2 * count * count) >> 12;
-
- const int64_t sum_a_x_sum_b = sum_a * sum_b;
-
- const int64_t ssim_n = (2 * sum_a_x_sum_b + c1) *
- (2 * count * sum_axb - 2 * sum_a_x_sum_b + c2);
-
- const int64_t sum_a_sq = sum_a * sum_a;
- const int64_t sum_b_sq = sum_b * sum_b;
+ const int32_t c1 = 26634; // (64^2*(.01*255)^2)
+ const int32_t c2 = 239708; // (64^2*(.03*255)^2)
+ const int32_t sum_a_x_sum_b_2 = 2 * sum_a * sum_b;
+ const int32_t sum_a_sq_b_sq = sum_a * sum_a + sum_b * sum_b;
+ const int32_t n1 = sum_a_x_sum_b_2 + c1;
+ const int32_t n2 = (sum_axb << 7) - sum_a_x_sum_b_2 + c2;
+ const int32_t d1 = sum_a_sq_b_sq + c1;
+ const int32_t d2 = (sum_sq_ab << 6) - sum_a_sq_b_sq + c2;
+ return ((float)n1 * (float)n2) / ((float)d1 * (float)d2);
+ }
+}
- const int64_t ssim_d =
- (sum_a_sq + sum_b_sq + c1) *
- (count * sum_sq_a - sum_a_sq + count * sum_sq_b - sum_b_sq + c2);
+static void Ssim4x4Row_Rem(void (*Ssim4x4Row)(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width),
+ int simd_size,
+ const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int rem_width) {
+ if (Ssim4x4Row == Ssim4x4Row_C) {
+ Ssim4x4Row_C(src_a, stride_a, src_b, stride_b, dst, rem_width);
+ return;
+ }
+ SIMD_ALIGNED(uint8_t buf_a[32 * 4]);
+ SIMD_ALIGNED(uint8_t buf_b[32 * 4]);
+ SIMD_ALIGNED(uint32_t buf_dst[32]);
+ memset(buf_a, 0, sizeof(buf_a));
+ memset(buf_b, 0, sizeof(buf_b));
+ int r;
+ for (r = 0; r < 4; ++r) {
+ memcpy(buf_a + r * 32, src_a + (ptrdiff_t)r * stride_a, rem_width);
+ memcpy(buf_b + r * 32, src_b + (ptrdiff_t)r * stride_b, rem_width);
+ }
+ Ssim4x4Row(buf_a, 32, buf_b, 32, buf_dst, simd_size);
+ memcpy(dst, buf_dst, (size_t)rem_width * sizeof(uint32_t));
+}
- if (ssim_d == 0.0) {
- return DBL_MAX;
- }
- return ssim_n * 1.0 / ssim_d;
- }
+static float SsimWindowRow_Rem(float (*SsimWindowRow)(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows),
+ int win_simd_size,
+ const uint32_t* row0,
+ const uint32_t* row1,
+ int rem_windows) {
+ if (SsimWindowRow == SsimWindowRow_C) {
+ return SsimWindowRow_C(row0, row1, rem_windows);
+ }
+ SIMD_ALIGNED(uint32_t win0[9 * 4]);
+ SIMD_ALIGNED(uint32_t win1[9 * 4]);
+ memset(win0, 0, sizeof(win0));
+ memset(win1, 0, sizeof(win1));
+ // Set odd tiles (1, 3, 5, 7) in win1 so each unused window k >= 1 has
+ // sum_a = 4439, sum_b = -3, sum_sq = 307887, giving n1 = 2*4439*(-3) + c1 = 0
+ // and SSIM = 0.0f (avoiding float precision loss from adding 1.0f per lane).
+ int k;
+ for (k = 1; k <= win_simd_size; k += 2) {
+ win1[k * 4 + 0] = 4439u;
+ win1[k * 4 + 1] = (uint32_t)-3;
+ win1[k * 4 + 2] = 307887u;
+ }
+ float total = 0.0f;
+ int w;
+ for (w = 0; w < rem_windows; ++w) {
+ const uint32_t* r0 = row0 + w * 4;
+ const uint32_t* r1 = row1 + w * 4;
+ win0[0] = r0[0] + r0[4];
+ win0[1] = r0[1] + r0[5];
+ win0[2] = r0[2] + r0[6];
+ win0[3] = r0[3] + r0[7];
+ win1[0] = r1[0] + r1[4] - 4439u;
+ win1[1] = r1[1] + r1[5] - (uint32_t)-3;
+ win1[2] = r1[2] + r1[6] - 307887u;
+ win1[3] = r1[3] + r1[7];
+ total += SsimWindowRow(win0, win1, win_simd_size);
+ }
+ return total;
}
// We are using a 8x8 moving window with starting location of each 8x8 window
@@ -388,20 +601,231 @@ double CalcFrameSsim(const uint8_t* src_a,
int height) {
int samples = 0;
double ssim_total = 0;
- double (*Ssim8x8)(const uint8_t* src_a, int stride_a, const uint8_t* src_b,
- int stride_b) = Ssim8x8_C;
+ // Return 0 rather than 0.0 / 0 (NaN) when the plane is too small for any
+ // sample windows, so I420Ssim on small images stays finite.
+ if (width <= 8 || height <= 8) {
+ return 0.0;
+ }
+
+ int simd_size = 4;
+ void (*Ssim4x4Row)(const uint8_t* src_a, int stride_a, const uint8_t* src_b,
+ int stride_b, uint32_t* dst, int width) = Ssim4x4Row_C;
+ int win_simd_size = 1;
+ float (*SsimWindowRow)(const uint32_t* row0, const uint32_t* row1,
+ int num_windows) = SsimWindowRow_C;
+#if defined(HAS_SSIM4X4ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Ssim4x4Row = Ssim4x4Row_SSE2;
+ simd_size = 16;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Ssim4x4Row = Ssim4x4Row_AVX2;
+ simd_size = 32;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Ssim4x4Row = Ssim4x4Row_AVX512BW;
+ simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SsimWindowRow = SsimWindowRow_SSE2;
+ win_simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SsimWindowRow = SsimWindowRow_AVX2;
+ win_simd_size = 8;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SsimWindowRow = SsimWindowRow_AVX512BW;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Ssim4x4Row = Ssim4x4Row_NEON;
+ simd_size = 16;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ Ssim4x4Row = Ssim4x4Row_NEON_DotProd;
+ simd_size = 16;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SsimWindowRow = SsimWindowRow_NEON;
+ win_simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Ssim4x4Row = Ssim4x4Row_SVE2;
+ simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SsimWindowRow = SsimWindowRow_SVE2;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Ssim4x4Row = Ssim4x4Row_SME;
+ simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ SsimWindowRow = SsimWindowRow_SME;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Ssim4x4Row = Ssim4x4Row_RVV;
+ simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SsimWindowRow = SsimWindowRow_RVV;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ Ssim4x4Row = Ssim4x4Row_LSX;
+ simd_size = 16;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SsimWindowRow = SsimWindowRow_LSX;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ Ssim4x4Row = Ssim4x4Row_LASX;
+ simd_size = 32;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SsimWindowRow = SsimWindowRow_LASX;
+ win_simd_size = 1;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Ssim4x4Row = Ssim4x4Row_WASMSIMD;
+ simd_size = 16;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ SsimWindowRow = SsimWindowRow_WASMSIMD;
+ win_simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIM4X4ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Ssim4x4Row = Ssim4x4Row_HVX;
+ simd_size = 4;
+ }
+#endif
+#if defined(HAS_SSIMWINDOWROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SsimWindowRow = SsimWindowRow_HVX;
+ win_simd_size = 1;
+ }
+#endif
- // sample point start with each 4x4 location
- int i;
- for (i = 0; i < height - 8; i += 4) {
- int j;
- for (j = 0; j < width - 8; j += 4) {
- ssim_total += Ssim8x8(src_a + j, stride_a, src_b + j, stride_b);
- samples++;
- }
+ const int num_windows = (width - 8 + 3) >> 2;
+ const int num_tiles = num_windows + 1;
+ const int tile_width = num_tiles * 4;
+ const int main_width = tile_width & ~(simd_size - 1);
+ const int rem_width = tile_width & (simd_size - 1);
+ const int main_windows = num_windows & ~(win_simd_size - 1);
+ const int rem_windows = num_windows & (win_simd_size - 1);
- src_a += stride_a * 4;
- src_b += stride_b * 4;
+#ifdef _OPENMP
+#pragma omp parallel reduction(+ : ssim_total, samples)
+#endif
+ {
+ align_buffer_64(row_buf, (size_t)tile_width * 2 * sizeof(uint32_t));
+ if (row_buf) {
+ uint32_t* row0 = (uint32_t*)row_buf;
+ uint32_t* row1 = row0 + tile_width;
+ int prev_i = -8;
+ int i;
+#ifdef _OPENMP
+#pragma omp for schedule(static)
+#endif
+ for (i = 0; i < height - 8; i += 4) {
+ const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
+ const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
+ if (prev_i + 4 != i) {
+ if (main_width > 0) {
+ Ssim4x4Row(a, stride_a, b, stride_b, row0, main_width);
+ }
+ if (rem_width > 0) {
+ Ssim4x4Row_Rem(Ssim4x4Row, simd_size, a + main_width, stride_a,
+ b + main_width, stride_b, row0 + main_width,
+ rem_width);
+ }
+ }
+ a += (ptrdiff_t)stride_a * 4;
+ b += (ptrdiff_t)stride_b * 4;
+ if (main_width > 0) {
+ Ssim4x4Row(a, stride_a, b, stride_b, row1, main_width);
+ }
+ if (rem_width > 0) {
+ Ssim4x4Row_Rem(Ssim4x4Row, simd_size, a + main_width, stride_a,
+ b + main_width, stride_b, row1 + main_width,
+ rem_width);
+ }
+ if (main_windows > 0) {
+ ssim_total += SsimWindowRow(row0, row1, main_windows);
+ }
+ if (rem_windows > 0) {
+ ssim_total += SsimWindowRow_Rem(SsimWindowRow, win_simd_size,
+ row0 + main_windows * 4,
+ row1 + main_windows * 4, rem_windows);
+ }
+ samples += num_windows;
+ uint32_t* tmp = row0;
+ row0 = row1;
+ row1 = tmp;
+ prev_i = i;
+ }
+ free_aligned_buffer_64(row_buf);
+ } else {
+ int i;
+#ifdef _OPENMP
+#pragma omp for schedule(static)
+#endif
+ for (i = 0; i < height - 8; i += 4) {
+ const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
+ const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
+ int j;
+ for (j = 0; j < width - 8; j += 4) {
+ ssim_total += Ssim8x8(a + j, stride_a, b + j, stride_b);
+ samples++;
+ }
+ }
+ }
}
ssim_total /= samples;
diff --git a/libs/libyuv/source/compare_common.cc b/libs/libyuv/source/compare_common.cc
index d4b170ad98..f50bc47082 100644
--- a/libs/libyuv/source/compare_common.cc
+++ b/libs/libyuv/source/compare_common.cc
@@ -8,45 +8,17 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/basic_types.h"
-
#include "libyuv/compare_row.h"
+#include <float.h>
+
+#include "libyuv/video_common.h"
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if ORIGINAL_OPT
-uint32_t HammingDistance_C1(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t diff = 0u;
-
- int i;
- for (i = 0; i < count; ++i) {
- int x = src_a[i] ^ src_b[i];
- if (x & 1)
- ++diff;
- if (x & 2)
- ++diff;
- if (x & 4)
- ++diff;
- if (x & 8)
- ++diff;
- if (x & 16)
- ++diff;
- if (x & 32)
- ++diff;
- if (x & 64)
- ++diff;
- if (x & 128)
- ++diff;
- }
- return diff;
-}
-#endif
-
// Hakmem method for hamming distance.
uint32_t HammingDistance_C(const uint8_t* src_a,
const uint8_t* src_b,
@@ -55,7 +27,11 @@ uint32_t HammingDistance_C(const uint8_t* src_a,
int i;
for (i = 0; i < count - 3; i += 4) {
- uint32_t x = *((const uint32_t*)src_a) ^ *((const uint32_t*)src_b);
+ uint32_t a = (uint32_t)src_a[0] | ((uint32_t)src_a[1] << 8) |
+ ((uint32_t)src_a[2] << 16) | ((uint32_t)src_a[3] << 24);
+ uint32_t b = (uint32_t)src_b[0] | ((uint32_t)src_b[1] << 8) |
+ ((uint32_t)src_b[2] << 16) | ((uint32_t)src_b[3] << 24);
+ uint32_t x = a ^ b;
uint32_t u = x - ((x >> 1) & 0x55555555);
u = ((u >> 2) & 0x33333333) + (u & 0x33333333);
diff += ((((u + (u >> 4)) & 0x0f0f0f0f) * 0x01010101) >> 24);
@@ -98,6 +74,131 @@ uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed) {
return hash;
}
+// Compute 4x4 tile statistics (sum_a, sum_b, sum_sq_a + sum_sq_b, sum_axb)
+// across 4 rows for width pixels (width is a multiple of 4).
+void Ssim4x4Row_C(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ int x;
+ for (x = 0; x < width; x += 4) {
+ uint32_t sum_a = a0[0] + a0[1] + a0[2] + a0[3] + a1[0] + a1[1] + a1[2] +
+ a1[3] + a2[0] + a2[1] + a2[2] + a2[3] + a3[0] + a3[1] +
+ a3[2] + a3[3];
+ uint32_t sum_b = b0[0] + b0[1] + b0[2] + b0[3] + b1[0] + b1[1] + b1[2] +
+ b1[3] + b2[0] + b2[1] + b2[2] + b2[3] + b3[0] + b3[1] +
+ b3[2] + b3[3];
+ uint32_t sum_sq_ab =
+ a0[0] * a0[0] + a0[1] * a0[1] + a0[2] * a0[2] + a0[3] * a0[3] +
+ a1[0] * a1[0] + a1[1] * a1[1] + a1[2] * a1[2] + a1[3] * a1[3] +
+ a2[0] * a2[0] + a2[1] * a2[1] + a2[2] * a2[2] + a2[3] * a2[3] +
+ a3[0] * a3[0] + a3[1] * a3[1] + a3[2] * a3[2] + a3[3] * a3[3] +
+ b0[0] * b0[0] + b0[1] * b0[1] + b0[2] * b0[2] + b0[3] * b0[3] +
+ b1[0] * b1[0] + b1[1] * b1[1] + b1[2] * b1[2] + b1[3] * b1[3] +
+ b2[0] * b2[0] + b2[1] * b2[1] + b2[2] * b2[2] + b2[3] * b2[3] +
+ b3[0] * b3[0] + b3[1] * b3[1] + b3[2] * b3[2] + b3[3] * b3[3];
+ uint32_t sum_axb =
+ a0[0] * b0[0] + a0[1] * b0[1] + a0[2] * b0[2] + a0[3] * b0[3] +
+ a1[0] * b1[0] + a1[1] * b1[1] + a1[2] * b1[2] + a1[3] * b1[3] +
+ a2[0] * b2[0] + a2[1] * b2[1] + a2[2] * b2[2] + a2[3] * b2[3] +
+ a3[0] * b3[0] + a3[1] * b3[1] + a3[2] * b3[2] + a3[3] * b3[3];
+ dst[0] = sum_a;
+ dst[1] = sum_b;
+ dst[2] = sum_sq_ab;
+ dst[3] = sum_axb;
+ a0 += 4;
+ a1 += 4;
+ a2 += 4;
+ a3 += 4;
+ b0 += 4;
+ b1 += 4;
+ b2 += 4;
+ b3 += 4;
+ dst += 4;
+ }
+}
+
+// Combine two consecutive 4x4 tile rows (row0 and row1) into 8x8 windows
+// stepped on a 4-pixel horizontal grid and return the sum of SSIM values.
+float SsimWindowRow_C(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const int32_t c1 = 26634; // (64^2*(.01*255)^2)
+ const int32_t c2 = 239708; // (64^2*(.03*255)^2)
+
+ float ssim_total = 0.0f;
+ int32_t prev_a = (int32_t)(row0[0] + row1[0]);
+ int32_t prev_b = (int32_t)(row0[1] + row1[1]);
+ int32_t prev_sq = (int32_t)(row0[2] + row1[2]);
+ int32_t prev_axb = (int32_t)(row0[3] + row1[3]);
+ int w;
+ for (w = 0; w < num_windows; ++w) {
+ row0 += 4;
+ row1 += 4;
+ const int32_t next_a = (int32_t)(row0[0] + row1[0]);
+ const int32_t next_b = (int32_t)(row0[1] + row1[1]);
+ const int32_t next_sq = (int32_t)(row0[2] + row1[2]);
+ const int32_t next_axb = (int32_t)(row0[3] + row1[3]);
+ const int32_t sum_a = prev_a + next_a;
+ const int32_t sum_b = prev_b + next_b;
+ const int32_t sum_sq_ab = prev_sq + next_sq;
+ const int32_t sum_axb = prev_axb + next_axb;
+ prev_a = next_a;
+ prev_b = next_b;
+ prev_sq = next_sq;
+ prev_axb = next_axb;
+
+ const int32_t sum_a_x_sum_b_2 = 2 * sum_a * sum_b;
+ const int32_t sum_a_sq_b_sq = sum_a * sum_a + sum_b * sum_b;
+ const int32_t n1 = sum_a_x_sum_b_2 + c1;
+ const int32_t n2 = (sum_axb << 7) - sum_a_x_sum_b_2 + c2;
+ const int32_t d1 = sum_a_sq_b_sq + c1;
+ const int32_t d2 = (sum_sq_ab << 6) - sum_a_sq_b_sq + c2;
+ ssim_total += ((float)n1 * (float)n2) / ((float)d1 * (float)d2);
+ }
+ return ssim_total;
+}
+
+uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) {
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
+ return FOURCC_BGRA;
+ }
+ if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA.
+ return FOURCC_ARGB;
+ }
+ if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255.
+ return FOURCC_BGRA;
+ }
+ if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255.
+ return FOURCC_ARGB;
+ }
+ argb += 8;
+ }
+ if (width & 1) {
+ if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
+ return FOURCC_BGRA;
+ }
+ if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA.
+ return FOURCC_ARGB;
+ }
+ }
+ return 0;
+}
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/compare_gcc.cc b/libs/libyuv/source/compare_gcc.cc
index 676527c1b1..5828ceaa7b 100644
--- a/libs/libyuv/source/compare_gcc.cc
+++ b/libs/libyuv/source/compare_gcc.cc
@@ -8,222 +8,200 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/basic_types.h"
-
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
+#include "libyuv/video_common.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+// This module is for GCC and clang x86 and x64.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (!defined(_MSC_VER) || defined(__clang__))
+
+// "memory" clobber prevents the reads from being removed
-#if defined(__x86_64__)
+#ifdef HAS_HAMMINGDISTANCE_SSE42
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint64_t diff = 0u;
+ uint64_t diff;
asm volatile(
- "xor %3,%3 \n"
- "xor %%r8,%%r8 \n"
- "xor %%r9,%%r9 \n"
- "xor %%r10,%%r10 \n"
+ "xor %3,%3 \n"
+ "xor %%r8,%%r8 \n"
+ "xor %%r9,%%r9 \n"
+ "xor %%r10,%%r10 \n"
// Process 32 bytes per loop.
LABELALIGN
- "1: \n"
- "mov (%0),%%rcx \n"
- "mov 0x8(%0),%%rdx \n"
- "xor (%1),%%rcx \n"
- "xor 0x8(%1),%%rdx \n"
- "popcnt %%rcx,%%rcx \n"
- "popcnt %%rdx,%%rdx \n"
- "mov 0x10(%0),%%rsi \n"
- "mov 0x18(%0),%%rdi \n"
- "xor 0x10(%1),%%rsi \n"
- "xor 0x18(%1),%%rdi \n"
- "popcnt %%rsi,%%rsi \n"
- "popcnt %%rdi,%%rdi \n"
- "add $0x20,%0 \n"
- "add $0x20,%1 \n"
- "add %%rcx,%3 \n"
- "add %%rdx,%%r8 \n"
- "add %%rsi,%%r9 \n"
- "add %%rdi,%%r10 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
-
- "add %%r8, %3 \n"
- "add %%r9, %3 \n"
- "add %%r10, %3 \n"
- : "+r"(src_a), // %0
- "+r"(src_b), // %1
- "+r"(count), // %2
- "=r"(diff) // %3
- :
- : "memory", "cc", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10");
+ "1: \n"
+ "mov (%0),%%rcx \n"
+ "mov 0x8(%0),%%rdx \n"
+ "xor (%1),%%rcx \n"
+ "xor 0x8(%1),%%rdx \n"
+ "popcnt %%rcx,%%rcx \n"
+ "popcnt %%rdx,%%rdx \n"
+ "mov 0x10(%0),%%rsi \n"
+ "mov 0x18(%0),%%rdi \n"
+ "xor 0x10(%1),%%rsi \n"
+ "xor 0x18(%1),%%rdi \n"
+ "popcnt %%rsi,%%rsi \n"
+ "popcnt %%rdi,%%rdi \n"
+ "add $0x20,%0 \n"
+ "add $0x20,%1 \n"
+ "add %%rcx,%3 \n"
+ "add %%rdx,%%r8 \n"
+ "add %%rsi,%%r9 \n"
+ "add %%rdi,%%r10 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
- return static_cast<uint32_t>(diff);
-}
-#else
-uint32_t HammingDistance_SSE42(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t diff = 0u;
-
- asm volatile(
- // Process 16 bytes per loop.
- LABELALIGN
- "1: \n"
- "mov (%0),%%ecx \n"
- "mov 0x4(%0),%%edx \n"
- "xor (%1),%%ecx \n"
- "xor 0x4(%1),%%edx \n"
- "popcnt %%ecx,%%ecx \n"
- "add %%ecx,%3 \n"
- "popcnt %%edx,%%edx \n"
- "add %%edx,%3 \n"
- "mov 0x8(%0),%%ecx \n"
- "mov 0xc(%0),%%edx \n"
- "xor 0x8(%1),%%ecx \n"
- "xor 0xc(%1),%%edx \n"
- "popcnt %%ecx,%%ecx \n"
- "add %%ecx,%3 \n"
- "popcnt %%edx,%%edx \n"
- "add %%edx,%3 \n"
- "add $0x10,%0 \n"
- "add $0x10,%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "add %%r8, %3 \n"
+ "add %%r9, %3 \n"
+ "add %%r10, %3 \n"
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
- "+r"(diff) // %3
+ "=&r"(diff) // %3
:
- : "memory", "cc", "ecx", "edx");
+ : "cc", "memory", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10");
- return diff;
+ return (uint32_t)(diff);
}
-#endif
+#endif // HAS_HAMMINGDISTANCE_SSE42
+#if defined(HAS_HAMMINGDISTANCE_SSSE3) || defined(HAS_HAMMINGDISTANCE_AVX2) || \
+ defined(HAS_HAMMINGDISTANCE_AVX512BW)
static const vec8 kNibbleMask = {15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15};
static const vec8 kBitCount = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
+#endif
+#ifdef HAS_HAMMINGDISTANCE_SSSE3
uint32_t HammingDistance_SSSE3(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint32_t diff = 0u;
+ uint32_t diff;
asm volatile(
- "movdqa %4,%%xmm2 \n"
- "movdqa %5,%%xmm3 \n"
- "pxor %%xmm0,%%xmm0 \n"
- "pxor %%xmm1,%%xmm1 \n"
- "sub %0,%1 \n"
+ "movdqa %4,%%xmm2 \n"
+ "movdqa %5,%%xmm3 \n"
+ "pxor %%xmm0,%%xmm0 \n"
+ "pxor %%xmm1,%%xmm1 \n"
+ "sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "movdqa (%0),%%xmm4 \n"
- "movdqa 0x10(%0), %%xmm5 \n"
- "pxor (%0,%1), %%xmm4 \n"
- "movdqa %%xmm4,%%xmm6 \n"
- "pand %%xmm2,%%xmm6 \n"
- "psrlw $0x4,%%xmm4 \n"
- "movdqa %%xmm3,%%xmm7 \n"
- "pshufb %%xmm6,%%xmm7 \n"
- "pand %%xmm2,%%xmm4 \n"
- "movdqa %%xmm3,%%xmm6 \n"
- "pshufb %%xmm4,%%xmm6 \n"
- "paddb %%xmm7,%%xmm6 \n"
- "pxor 0x10(%0,%1),%%xmm5 \n"
- "add $0x20,%0 \n"
- "movdqa %%xmm5,%%xmm4 \n"
- "pand %%xmm2,%%xmm5 \n"
- "psrlw $0x4,%%xmm4 \n"
- "movdqa %%xmm3,%%xmm7 \n"
- "pshufb %%xmm5,%%xmm7 \n"
- "pand %%xmm2,%%xmm4 \n"
- "movdqa %%xmm3,%%xmm5 \n"
- "pshufb %%xmm4,%%xmm5 \n"
- "paddb %%xmm7,%%xmm5 \n"
- "paddb %%xmm5,%%xmm6 \n"
- "psadbw %%xmm1,%%xmm6 \n"
- "paddd %%xmm6,%%xmm0 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
-
- "pshufd $0xaa,%%xmm0,%%xmm1 \n"
- "paddd %%xmm1,%%xmm0 \n"
- "movd %%xmm0, %3 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm4 \n"
+ "movdqu 0x10(%0), %%xmm5 \n"
+ "movdqu (%0,%1), %%xmm6 \n"
+ "pxor %%xmm6, %%xmm4 \n"
+ "movdqa %%xmm4,%%xmm6 \n"
+ "pand %%xmm2,%%xmm6 \n"
+ "psrlw $0x4,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm7 \n"
+ "pshufb %%xmm6,%%xmm7 \n"
+ "pand %%xmm2,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm6 \n"
+ "pshufb %%xmm4,%%xmm6 \n"
+ "paddb %%xmm7,%%xmm6 \n"
+ "movdqu 0x10(%0,%1),%%xmm7 \n"
+ "pxor %%xmm7,%%xmm5 \n"
+ "add $0x20,%0 \n"
+ "movdqa %%xmm5,%%xmm4 \n"
+ "pand %%xmm2,%%xmm5 \n"
+ "psrlw $0x4,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm7 \n"
+ "pshufb %%xmm5,%%xmm7 \n"
+ "pand %%xmm2,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm5 \n"
+ "pshufb %%xmm4,%%xmm5 \n"
+ "paddb %%xmm7,%%xmm5 \n"
+ "paddb %%xmm5,%%xmm6 \n"
+ "psadbw %%xmm1,%%xmm6 \n"
+ "paddd %%xmm6,%%xmm0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+
+ "pshufd $0xaa,%%xmm0,%%xmm1 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "movd %%xmm0, %3 \n"
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
"=r"(diff) // %3
: "m"(kNibbleMask), // %4
"m"(kBitCount) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
return diff;
}
+#endif // HAS_HAMMINGDISTANCE_SSSE3
#ifdef HAS_HAMMINGDISTANCE_AVX2
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint32_t diff = 0u;
+ uint32_t diff;
asm volatile(
"vbroadcastf128 %4,%%ymm2 \n"
"vbroadcastf128 %5,%%ymm3 \n"
- "vpxor %%ymm0,%%ymm0,%%ymm0 \n"
- "vpxor %%ymm1,%%ymm1,%%ymm1 \n"
- "sub %0,%1 \n"
+ "vpxor %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpxor %%ymm1,%%ymm1,%%ymm1 \n"
+ "sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "vmovdqa (%0),%%ymm4 \n"
- "vmovdqa 0x20(%0), %%ymm5 \n"
- "vpxor (%0,%1), %%ymm4, %%ymm4 \n"
- "vpand %%ymm2,%%ymm4,%%ymm6 \n"
- "vpsrlw $0x4,%%ymm4,%%ymm4 \n"
- "vpshufb %%ymm6,%%ymm3,%%ymm6 \n"
- "vpand %%ymm2,%%ymm4,%%ymm4 \n"
- "vpshufb %%ymm4,%%ymm3,%%ymm4 \n"
- "vpaddb %%ymm4,%%ymm6,%%ymm6 \n"
- "vpxor 0x20(%0,%1),%%ymm5,%%ymm4 \n"
- "add $0x40,%0 \n"
- "vpand %%ymm2,%%ymm4,%%ymm5 \n"
- "vpsrlw $0x4,%%ymm4,%%ymm4 \n"
- "vpshufb %%ymm5,%%ymm3,%%ymm5 \n"
- "vpand %%ymm2,%%ymm4,%%ymm4 \n"
- "vpshufb %%ymm4,%%ymm3,%%ymm4 \n"
- "vpaddb %%ymm5,%%ymm4,%%ymm4 \n"
- "vpaddb %%ymm6,%%ymm4,%%ymm4 \n"
- "vpsadbw %%ymm1,%%ymm4,%%ymm4 \n"
- "vpaddd %%ymm0,%%ymm4,%%ymm0 \n"
- "sub $0x40,%2 \n"
- "jg 1b \n"
-
- "vpermq $0xb1,%%ymm0,%%ymm1 \n"
- "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xaa,%%ymm0,%%ymm1 \n"
- "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovd %%xmm0, %3 \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm4 \n"
+ "vmovdqu 0x20(%0), %%ymm5 \n"
+ "vpxor (%0,%1), %%ymm4, %%ymm4 \n"
+ "vpand %%ymm2,%%ymm4,%%ymm6 \n"
+ "vpsrlw $0x4,%%ymm4,%%ymm4 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm6 \n"
+ "vpand %%ymm2,%%ymm4,%%ymm4 \n"
+ "vpshufb %%ymm4,%%ymm3,%%ymm4 \n"
+ "vpaddb %%ymm4,%%ymm6,%%ymm6 \n"
+ "vpxor 0x20(%0,%1),%%ymm5,%%ymm4 \n"
+ "add $0x40,%0 \n"
+ "vpand %%ymm2,%%ymm4,%%ymm5 \n"
+ "vpsrlw $0x4,%%ymm4,%%ymm4 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm5 \n"
+ "vpand %%ymm2,%%ymm4,%%ymm4 \n"
+ "vpshufb %%ymm4,%%ymm3,%%ymm4 \n"
+ "vpaddb %%ymm5,%%ymm4,%%ymm4 \n"
+ "vpaddb %%ymm6,%%ymm4,%%ymm4 \n"
+ "vpsadbw %%ymm1,%%ymm4,%%ymm4 \n"
+ "vpaddd %%ymm0,%%ymm4,%%ymm0 \n"
+ "sub $0x40,%2 \n"
+ "jg 1b \n"
+
+ "vpermq $0xb1,%%ymm0,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xaa,%%ymm0,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovd %%xmm0,%3 \n"
+ "vzeroupper \n"
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
"=r"(diff) // %3
: "m"(kNibbleMask), // %4
"m"(kBitCount) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
return diff;
}
@@ -234,44 +212,89 @@ uint32_t SumSquareError_SSE2(const uint8_t* src_a,
int count) {
uint32_t sse;
asm volatile(
- "pxor %%xmm0,%%xmm0 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ "pxor %%xmm0,%%xmm0 \n"
+ "pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "movdqu (%1),%%xmm2 \n"
- "lea 0x10(%1),%1 \n"
- "movdqa %%xmm1,%%xmm3 \n"
- "psubusb %%xmm2,%%xmm1 \n"
- "psubusb %%xmm3,%%xmm2 \n"
- "por %%xmm2,%%xmm1 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "punpckhbw %%xmm5,%%xmm2 \n"
- "pmaddwd %%xmm1,%%xmm1 \n"
- "pmaddwd %%xmm2,%%xmm2 \n"
- "paddd %%xmm1,%%xmm0 \n"
- "paddd %%xmm2,%%xmm0 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
-
- "pshufd $0xee,%%xmm0,%%xmm1 \n"
- "paddd %%xmm1,%%xmm0 \n"
- "pshufd $0x1,%%xmm0,%%xmm1 \n"
- "paddd %%xmm1,%%xmm0 \n"
- "movd %%xmm0,%3 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqu (%1),%%xmm2 \n"
+ "lea 0x10(%1),%1 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "psubusb %%xmm2,%%xmm1 \n"
+ "psubusb %%xmm3,%%xmm2 \n"
+ "por %%xmm2,%%xmm1 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "punpcklbw %%xmm5,%%xmm1 \n"
+ "punpckhbw %%xmm5,%%xmm2 \n"
+ "pmaddwd %%xmm1,%%xmm1 \n"
+ "pmaddwd %%xmm2,%%xmm2 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "paddd %%xmm2,%%xmm0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "pshufd $0xee,%%xmm0,%%xmm1 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "pshufd $0x1,%%xmm0,%%xmm1 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "movd %%xmm0,%3 \n"
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
- "=g"(sse) // %3
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "=r"(sse) // %3
+ :
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
return sse;
}
+#ifdef HAS_SUMSQUAREERROR_AVX2
+uint32_t SumSquareError_AVX2(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t sse;
+ asm volatile(
+ "vpxor %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %0,%1 \n"
+
+ // Process 32 bytes per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm1 \n"
+ "vmovdqu (%0,%1),%%ymm2 \n"
+ "add $0x20,%0 \n"
+ "vpsubusb %%ymm2,%%ymm1,%%ymm3 \n"
+ "vpsubusb %%ymm1,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm2,%%ymm3,%%ymm1 \n"
+ "vpunpcklbw %%ymm5,%%ymm1,%%ymm2 \n"
+ "vpunpckhbw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpmaddwd %%ymm2,%%ymm2,%%ymm2 \n"
+ "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0x1,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,%3 \n"
+ "vzeroupper \n"
+ : "+r"(src_a), // %0
+ "+r"(src_b), // %1
+ "+r"(count), // %2
+ "=r"(sse) // %3
+ :
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_AVX2
+
static const uvec32 kHash16x33 = {0x92d9e201, 0, 0, 0}; // 33 ^ 16
static const uvec32 kHashMul0 = {
0x0c3525e1, // 33 ^ 15
@@ -301,58 +324,1251 @@ static const uvec32 kHashMul3 = {
uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
uint32_t hash;
asm volatile(
- "movd %2,%%xmm0 \n"
- "pxor %%xmm7,%%xmm7 \n"
- "movdqa %4,%%xmm6 \n"
+ "movd %2,%%xmm0 \n"
+ "pxor %%xmm7,%%xmm7 \n"
+ "movdqa %4,%%xmm6 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "pmulld %%xmm6,%%xmm0 \n"
- "movdqa %5,%%xmm5 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "punpcklbw %%xmm7,%%xmm2 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "punpcklwd %%xmm7,%%xmm3 \n"
- "pmulld %%xmm5,%%xmm3 \n"
- "movdqa %6,%%xmm5 \n"
- "movdqa %%xmm2,%%xmm4 \n"
- "punpckhwd %%xmm7,%%xmm4 \n"
- "pmulld %%xmm5,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "punpckhbw %%xmm7,%%xmm1 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "punpcklwd %%xmm7,%%xmm2 \n"
- "pmulld %%xmm5,%%xmm2 \n"
- "movdqa %8,%%xmm5 \n"
- "punpckhwd %%xmm7,%%xmm1 \n"
- "pmulld %%xmm5,%%xmm1 \n"
- "paddd %%xmm4,%%xmm3 \n"
- "paddd %%xmm2,%%xmm1 \n"
- "paddd %%xmm3,%%xmm1 \n"
- "pshufd $0xe,%%xmm1,%%xmm2 \n"
- "paddd %%xmm2,%%xmm1 \n"
- "pshufd $0x1,%%xmm1,%%xmm2 \n"
- "paddd %%xmm2,%%xmm1 \n"
- "paddd %%xmm1,%%xmm0 \n"
- "sub $0x10,%1 \n"
- "jg 1b \n"
- "movd %%xmm0,%3 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "pmulld %%xmm6,%%xmm0 \n"
+ "movdqa %5,%%xmm5 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "punpcklbw %%xmm7,%%xmm2 \n"
+ "movdqa %%xmm2,%%xmm3 \n"
+ "punpcklwd %%xmm7,%%xmm3 \n"
+ "pmulld %%xmm5,%%xmm3 \n"
+ "movdqa %6,%%xmm5 \n"
+ "movdqa %%xmm2,%%xmm4 \n"
+ "punpckhwd %%xmm7,%%xmm4 \n"
+ "pmulld %%xmm5,%%xmm4 \n"
+ "movdqa %7,%%xmm5 \n"
+ "punpckhbw %%xmm7,%%xmm1 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "punpcklwd %%xmm7,%%xmm2 \n"
+ "pmulld %%xmm5,%%xmm2 \n"
+ "movdqa %8,%%xmm5 \n"
+ "punpckhwd %%xmm7,%%xmm1 \n"
+ "pmulld %%xmm5,%%xmm1 \n"
+ "paddd %%xmm4,%%xmm3 \n"
+ "paddd %%xmm2,%%xmm1 \n"
+ "paddd %%xmm3,%%xmm1 \n"
+ "pshufd $0xe,%%xmm1,%%xmm2 \n"
+ "paddd %%xmm2,%%xmm1 \n"
+ "pshufd $0x1,%%xmm1,%%xmm2 \n"
+ "paddd %%xmm2,%%xmm1 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "sub $0x10,%1 \n"
+ "jg 1b \n"
+ "movd %%xmm0,%3 \n"
: "+r"(src), // %0
"+r"(count), // %1
"+rm"(seed), // %2
- "=g"(hash) // %3
+ "=r"(hash) // %3
: "m"(kHash16x33), // %4
"m"(kHashMul0), // %5
"m"(kHashMul1), // %6
"m"(kHashMul2), // %7
"m"(kHashMul3) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
return hash;
}
-#endif // defined(__x86_64__) || (defined(__i386__) && !defined(__pic__)))
+#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
+ defined(HAS_HAMMINGDISTANCE_AVX512VNNI) || \
+ defined(HAS_SUMSQUAREERROR_AVX512BW) || \
+ defined(HAS_SUMSQUAREERROR_AVX512VNNI) || \
+ defined(HAS_HASHDJB2_AVX512BW) || defined(HAS_SSIM4X4ROW_AVX512BW)
+// Mask of the low n bytes, for n = 0 to 63.
+static uint64_t LowBytesMask64(int n) {
+ return n ? ~0ULL >> (64 - n) : 0ULL;
+}
+#endif
+
+#ifdef HAS_HAMMINGDISTANCE_AVX512BW
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512BW
+uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ const uint64_t mask = LowBytesMask64(count & 63);
+ asm volatile(
+ "vbroadcasti32x4 %[nibble],%%zmm2 \n"
+ "vbroadcasti32x4 %[bitcount],%%zmm3 \n"
+ "vpxord %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxord %%zmm1,%%zmm1,%%zmm1 \n"
+ "kmovq %[mask],%%k1 \n"
+ "sub %[src_a],%[src_b] \n"
+ "sub $0x40,%[count] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_a]),%%zmm4 \n"
+ "vpxorq (%[src_a],%[src_b]),%%zmm4,%%zmm4 \n"
+ "add $0x40,%[src_a] \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm5 \n"
+ "vpsrlw $0x4,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
+ "vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpsadbw %%zmm1,%%zmm4,%%zmm4 \n"
+ "vpaddq %%zmm4,%%zmm0,%%zmm0 \n"
+ "sub $0x40,%[count] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 63 bytes, masked loads.
+ "2: \n"
+ "add $0x40,%[count] \n"
+ "je 3f \n"
+ "vmovdqu8 (%[src_a]),%%zmm4%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm5%{%%k1%}%{z%} \n"
+ "vpxorq %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm5 \n"
+ "vpsrlw $0x4,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
+ "vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpsadbw %%zmm1,%%zmm4,%%zmm4 \n"
+ "vpaddq %%zmm4,%%zmm0,%%zmm0 \n"
+
+ // Add 8 qwords.
+ "3: \n"
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n"
+ "vpaddq %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpaddq %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n"
+ "vpaddq %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,%[diff] \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [diff] "=r"(diff)
+ : [nibble] "m"(kNibbleMask), [bitcount] "m"(kBitCount), [mask] "m"(mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return diff;
+}
+#endif // HAS_HAMMINGDISTANCE_AVX512BW
+
+#ifdef HAS_HAMMINGDISTANCE_AVX512VNNI
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, accumulating byte popcounts with vpdpbusd.
+uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ const uint64_t mask = LowBytesMask64(count & 63);
+ asm volatile(
+ "vbroadcasti32x4 %[nibble],%%zmm2 \n"
+ "vbroadcasti32x4 %[bitcount],%%zmm3 \n"
+ "vpxord %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpternlogd $0xff,%%zmm1,%%zmm1,%%zmm1 \n"
+ "vpsrlw $0xf,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm0,%%zmm1,%%zmm1 \n"
+ "kmovq %[mask],%%k1 \n"
+ "sub %[src_a],%[src_b] \n"
+ "sub $0x40,%[count] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_a]),%%zmm4 \n"
+ "vpxorq (%[src_a],%[src_b]),%%zmm4,%%zmm4 \n"
+ "add $0x40,%[src_a] \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm5 \n"
+ "vpsrlw $0x4,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
+ "vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpdpbusd %%zmm4,%%zmm1,%%zmm0 \n"
+ "sub $0x40,%[count] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 63 bytes, masked loads.
+ "2: \n"
+ "add $0x40,%[count] \n"
+ "je 3f \n"
+ "vmovdqu8 (%[src_a]),%%zmm4%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm5%{%%k1%}%{z%} \n"
+ "vpxorq %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm5 \n"
+ "vpsrlw $0x4,%%zmm4,%%zmm4 \n"
+ "vpandq %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
+ "vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpdpbusd %%zmm4,%%zmm1,%%zmm0 \n"
+
+ // Add 16 dwords.
+ "3: \n"
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0x1,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,%[diff] \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [diff] "=r"(diff)
+ : [nibble] "m"(kNibbleMask), [bitcount] "m"(kBitCount), [mask] "m"(mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return diff;
+}
+#endif // HAS_HAMMINGDISTANCE_AVX512VNNI
+
+#ifdef HAS_SUMSQUAREERROR_AVX512BW
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512BW
+uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t sse;
+ const uint64_t mask = LowBytesMask64(count & 63);
+ asm volatile(
+ "vpxord %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxord %%zmm5,%%zmm5,%%zmm5 \n"
+ "kmovq %[mask],%%k1 \n"
+ "sub %[src_a],%[src_b] \n"
+ "sub $0x40,%[count] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_a]),%%zmm1 \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm2 \n"
+ "add $0x40,%[src_a] \n"
+ "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" // abs difference trick
+ "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
+ "vporq %%zmm3,%%zmm2,%%zmm1 \n"
+ "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" // u16
+ "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" // square + hadd to u32
+ "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+ "sub $0x40,%[count] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 63 bytes, masked loads.
+ "2: \n"
+ "add $0x40,%[count] \n"
+ "je 3f \n"
+ "vmovdqu8 (%[src_a]),%%zmm1%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm2%{%%k1%}%{z%} \n"
+ "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n"
+ "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
+ "vporq %%zmm3,%%zmm2,%%zmm1 \n"
+ "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n"
+ "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+
+ // Add 16 dwords.
+ "3: \n"
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0x1,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,%[sse] \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [sse] "=r"(sse)
+ : [mask] "m"(mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_AVX512BW
+
+#ifdef HAS_SUMSQUAREERROR_AVX512VNNI
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512VNNI
+uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t sse;
+ const uint64_t mask = LowBytesMask64(count & 63);
+ asm volatile(
+ "vpxord %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxord %%zmm4,%%zmm4,%%zmm4 \n"
+ "vpxord %%zmm5,%%zmm5,%%zmm5 \n"
+ "kmovq %[mask],%%k1 \n"
+ "sub %[src_a],%[src_b] \n"
+ "sub $0x40,%[count] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_a]),%%zmm1 \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm2 \n"
+ "add $0x40,%[src_a] \n"
+ "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" // abs difference trick
+ "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
+ "vporq %%zmm3,%%zmm2,%%zmm1 \n"
+ "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" // u16
+ "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" // square + hadd + acc
+ "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n"
+ "sub $0x40,%[count] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 63 bytes, masked loads.
+ "2: \n"
+ "add $0x40,%[count] \n"
+ "je 3f \n"
+ "vmovdqu8 (%[src_a]),%%zmm1%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src_a],%[src_b]),%%zmm2%{%%k1%}%{z%} \n"
+ "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n"
+ "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
+ "vporq %%zmm3,%%zmm2,%%zmm1 \n"
+ "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n"
+ "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n"
+ "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n"
+
+ // Add 16 dwords.
+ "3: \n"
+ "vpaddd %%zmm4,%%zmm0,%%zmm0 \n"
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0x1,%%xmm0,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,%[sse] \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [sse] "=r"(sse)
+ : [mask] "m"(mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_AVX512VNNI
+
+#if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW)
+// HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
+// The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that
+// vpmaddwd can be used instead of vpmulld, which is 2 uops and 10 cycles.
+// m = hi * 65536 + lo, with lo treated as signed and hi adjusted by the carry,
+// so src * m = src * lo + ((src * hi) << 16) modulo 2 ^ 32.
+// The first 64 are lo and the next 64 are hi.
+static const uint16_t kHashMulLoHi[128] = {
+ // lo for 33 ^ 63 .. 33 ^ 0
+ 0x0be1, 0x93c1, 0x9fa1, 0xaf81, 0x4361, 0xdb41, 0xf721, 0x1701, //
+ 0xbae1, 0x62c1, 0x8ea1, 0xbe81, 0x7261, 0x2a41, 0x6621, 0xa601, //
+ 0x69e1, 0x31c1, 0x7da1, 0xcd81, 0xa161, 0x7941, 0xd521, 0x3501, //
+ 0x18e1, 0x00c1, 0x6ca1, 0xdc81, 0xd061, 0xc841, 0x4421, 0xc401, //
+ 0xc7e1, 0xcfc1, 0x5ba1, 0xeb81, 0xff61, 0x1741, 0xb321, 0x5301, //
+ 0x76e1, 0x9ec1, 0x4aa1, 0xfa81, 0x2e61, 0x6641, 0x2221, 0xe201, //
+ 0x25e1, 0x6dc1, 0x39a1, 0x0981, 0x5d61, 0xb541, 0x9121, 0x7101, //
+ 0xd4e1, 0x3cc1, 0x28a1, 0x1881, 0x8c61, 0x0441, 0x0021, 0x0001, //
+ // hi for 33 ^ 63 .. 33 ^ 0
+ 0xd8be, 0xef4c, 0x6458, 0x6022, 0x40f9, 0x193e, 0x6d5f, 0x31dc, //
+ 0x0945, 0xc238, 0x1567, 0xa38f, 0x3380, 0x301b, 0x664e, 0x031a, //
+ 0xd18c, 0xb8c6, 0x79f6, 0xae5d, 0x0549, 0xc219, 0x7a3f, 0x1338, //
+ 0x0095, 0xc1f5, 0xa107, 0xaf8c, 0x0552, 0x3e39, 0x3830, 0x1138, //
+ 0x655f, 0xccc5, 0x9999, 0xd61c, 0x829c, 0x1379, 0x2f23, 0xac18, //
+ 0xcee9, 0xc836, 0x72ac, 0x510d, 0xcc27, 0xb0da, 0xee16, 0x92da, //
+ 0x0c35, 0xa347, 0x3b40, 0x4f5f, 0x30f3, 0x855d, 0x040b, 0x747c, //
+ 0xec42, 0x4cfa, 0x0255, 0x0012, 0x0001, 0x0000, 0x0000, 0x0000, //
+};
+#endif
+
+#ifdef HAS_HASHDJB2_AVX2
+// Sum of 32 words in ymm0 to ymm1 times 32 multipliers at offset 'o' words,
+// added to xmm4 (lo) and xmm5 (hi).
+#define HASH32_AVX2(o) \
+ "vpmaddwd " #o \
+ "*2(%[tbl]),%%ymm0,%%ymm2 \n" \
+ "vpmaddwd " #o \
+ "*2+0x20(%[tbl]),%%ymm1,%%ymm3 \n" \
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n" \
+ "vpaddd %%ymm3,%%ymm4,%%ymm4 \n" \
+ "vpmaddwd " #o \
+ "*2+0x80(%[tbl]),%%ymm0,%%ymm2 \n" \
+ "vpmaddwd " #o \
+ "*2+0xa0(%[tbl]),%%ymm1,%%ymm3 \n" \
+ "vpaddd %%ymm2,%%ymm5,%%ymm5 \n" \
+ "vpaddd %%ymm3,%%ymm5,%%ymm5 \n"
+
+// lo + (hi << 16), add 8 dwords, and hash = hash * mul + sum.
+#define HASHSUM_AVX2(mul) \
+ "vpslld $0x10,%%ymm5,%%ymm5 \n" \
+ "vpaddd %%ymm5,%%ymm4,%%ymm4 \n" \
+ "vextracti128 $1,%%ymm4,%%xmm5 \n" \
+ "vpaddd %%xmm5,%%xmm4,%%xmm4 \n" \
+ "vpshufd $0xee,%%xmm4,%%xmm5 \n" \
+ "vpaddd %%xmm5,%%xmm4,%%xmm4 \n" \
+ "vpshufd $0x1,%%xmm4,%%xmm5 \n" \
+ "vpaddd %%xmm5,%%xmm4,%%xmm4 \n" \
+ "vmovd %%xmm4,%[sum] \n" \
+ "imul $" #mul \
+ ",%[seed],%[seed] \n" \
+ "add %[sum],%[seed] \n"
+
+// Count is a multiple of 32. Process 64 bytes per loop with one horizontal
+// add, then 32 bytes if needed.
+uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t sum;
+ asm volatile(
+ "sub $0x40,%[count] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%[src]),%%ymm0 \n"
+ "vpmovzxbw 0x10(%[src]),%%ymm1 \n"
+ "vpmaddwd (%[tbl]),%%ymm0,%%ymm4 \n"
+ "vpmaddwd 0x20(%[tbl]),%%ymm1,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n"
+ "vpmaddwd 0x80(%[tbl]),%%ymm0,%%ymm5 \n"
+ "vpmaddwd 0xa0(%[tbl]),%%ymm1,%%ymm3 \n"
+ "vpaddd %%ymm3,%%ymm5,%%ymm5 \n"
+ "vpmovzxbw 0x20(%[src]),%%ymm0 \n"
+ "vpmovzxbw 0x30(%[src]),%%ymm1 \n"
+ "add $0x40,%[src] \n" //
+ HASH32_AVX2(32) //
+ HASHSUM_AVX2(0xf07f8801) // 33 ^ 64
+ "sub $0x40,%[count] \n"
+ "jge 1b \n"
+
+ // Remainder of 32 bytes.
+ "2: \n"
+ "add $0x40,%[count] \n"
+ "je 3f \n"
+ "vpmovzxbw (%[src]),%%ymm0 \n"
+ "vpmovzxbw 0x10(%[src]),%%ymm1 \n"
+ "vpxor %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n" //
+ HASH32_AVX2(32) //
+ HASHSUM_AVX2(0x1137c401) // 33 ^ 32
+
+ "3: \n"
+ "vzeroupper \n"
+ :
+ [src] "+r"(src), [count] "+r"(count), [seed] "+r"(seed), [sum] "=&r"(sum)
+ : [tbl] "r"(kHashMulLoHi)
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return seed;
+}
+#undef HASH32_AVX2
+#undef HASHSUM_AVX2
+#endif // HAS_HASHDJB2_AVX2
+
+#ifdef HAS_HASHDJB2_AVX512BW
+static uint32_t Pow33(int n) {
+ uint32_t r = 1u;
+ uint32_t b = 33u;
+ while (n) {
+ if (n & 1) {
+ r *= b;
+ }
+ b *= b;
+ n >>= 1;
+ }
+ return r;
+}
+
+// Sum of 64 words in zmm0 and zmm1 times 33 ^ (63 - i), using lo multipliers
+// in zmm4/zmm5 and hi multipliers in zmm6/zmm7, then hash = hash * mul + sum.
+#define HASH64_AVX512BW(mul) \
+ "vpmaddwd %%zmm6,%%zmm0,%%zmm2 \n" \
+ "vpmaddwd %%zmm7,%%zmm1,%%zmm3 \n" \
+ "vpmaddwd %%zmm4,%%zmm0,%%zmm0 \n" \
+ "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n" \
+ "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" \
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpslld $0x10,%%zmm2,%%zmm2 \n" \
+ "vpaddd %%zmm2,%%zmm0,%%zmm0 \n" \
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n" \
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" \
+ "vextracti128 $1,%%ymm0,%%xmm1 \n" \
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" \
+ "vpshufd $0xee,%%xmm0,%%xmm1 \n" \
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" \
+ "vpshufd $0x1,%%xmm0,%%xmm1 \n" \
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" \
+ "vmovd %%xmm0,%[sum] \n" \
+ "imul " mul \
+ " \n" \
+ "add %[sum],%[seed] \n"
+
+// Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded
+// into the high n bytes with a masked load ending at src + count, so the
+// same multipliers apply, and hash is multiplied by 33 ^ n.
+LIBYUV_TARGET_AVX512BW
+uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
+ const int n = count & 63;
+ const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL;
+ const uint32_t pow_n = Pow33(n);
+ intptr_t width = count;
+ uint32_t sum;
+ asm volatile(
+ "vmovdqu64 (%[tbl]),%%zmm4 \n"
+ "vmovdqu64 0x40(%[tbl]),%%zmm5 \n"
+ "vmovdqu64 0x80(%[tbl]),%%zmm6 \n"
+ "vmovdqu64 0xc0(%[tbl]),%%zmm7 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%[src]),%%zmm0 \n"
+ "vpmovzxbw 0x20(%[src]),%%zmm1 \n"
+ "add $0x40,%[src] \n" //
+ HASH64_AVX512BW("$0xf07f8801,%[seed],%[seed]") // 33 ^ 64
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 63 bytes, masked load into the high bytes.
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "je 3f \n"
+ "kmovq %[mask],%%k1 \n"
+ "vmovdqu8 -0x40(%[src],%[width]),%%zmm0%{%%k1%}%{z%} \n"
+ "vextracti64x4 $1,%%zmm0,%%ymm1 \n"
+ "vpmovzxbw %%ymm0,%%zmm0 \n"
+ "vpmovzxbw %%ymm1,%%zmm1 \n" //
+ HASH64_AVX512BW("%[pow_n],%[seed]")
+
+ "3: \n"
+ "vzeroupper \n"
+ :
+ [src] "+r"(src), [width] "+r"(width), [seed] "+r"(seed), [sum] "=&r"(sum)
+ : [tbl] "r"(kHashMulLoHi), [pow_n] "m"(pow_n), [mask] "m"(mask)
+ : "cc", "memory", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ return seed;
+}
+#undef HASH64_AVX512BW
+#endif // HAS_HASHDJB2_AVX512BW
+
+#ifdef HAS_SSIM4X4ROW_SSE2
+#define SSIM_ROW0_SSE2(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "movdqa %%xmm0,%%xmm2 \n" \
+ "psrlw $0x8,%%xmm0 \n" \
+ "psllw $0x8,%%xmm2 \n" \
+ "psrlw $0x8,%%xmm2 \n" \
+ "movdqa %%xmm0,%%xmm4 \n" \
+ "paddw %%xmm2,%%xmm4 \n" \
+ "movdqa %%xmm0,%%xmm6 \n" \
+ "pmaddwd %%xmm6,%%xmm6 \n" \
+ "movdqa %%xmm2,%%xmm3 \n" \
+ "pmaddwd %%xmm3,%%xmm3 \n" \
+ "paddd %%xmm3,%%xmm6 \n" \
+ "movdqa %%xmm1,%%xmm3 \n" \
+ "psrlw $0x8,%%xmm1 \n" \
+ "psllw $0x8,%%xmm3 \n" \
+ "psrlw $0x8,%%xmm3 \n" \
+ "movdqa %%xmm1,%%xmm5 \n" \
+ "paddw %%xmm3,%%xmm5 \n" \
+ "pmaddwd %%xmm1,%%xmm0 \n" \
+ "pmaddwd %%xmm3,%%xmm2 \n" \
+ "movdqa %%xmm0,%%xmm7 \n" \
+ "paddd %%xmm2,%%xmm7 \n" \
+ "pmaddwd %%xmm1,%%xmm1 \n" \
+ "pmaddwd %%xmm3,%%xmm3 \n" \
+ "paddd %%xmm1,%%xmm6 \n" \
+ "paddd %%xmm3,%%xmm6 \n"
+
+#define SSIM_ROW_SSE2(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "movdqa %%xmm0,%%xmm2 \n" \
+ "psrlw $0x8,%%xmm0 \n" \
+ "psllw $0x8,%%xmm2 \n" \
+ "psrlw $0x8,%%xmm2 \n" \
+ "paddw %%xmm0,%%xmm4 \n" \
+ "paddw %%xmm2,%%xmm4 \n" \
+ "movdqa %%xmm0,%%xmm3 \n" \
+ "pmaddwd %%xmm3,%%xmm3 \n" \
+ "paddd %%xmm3,%%xmm6 \n" \
+ "movdqa %%xmm2,%%xmm3 \n" \
+ "pmaddwd %%xmm3,%%xmm3 \n" \
+ "paddd %%xmm3,%%xmm6 \n" \
+ "movdqa %%xmm1,%%xmm3 \n" \
+ "psrlw $0x8,%%xmm1 \n" \
+ "psllw $0x8,%%xmm3 \n" \
+ "psrlw $0x8,%%xmm3 \n" \
+ "paddw %%xmm1,%%xmm5 \n" \
+ "paddw %%xmm3,%%xmm5 \n" \
+ "pmaddwd %%xmm1,%%xmm0 \n" \
+ "pmaddwd %%xmm3,%%xmm2 \n" \
+ "paddd %%xmm0,%%xmm7 \n" \
+ "paddd %%xmm2,%%xmm7 \n" \
+ "pmaddwd %%xmm1,%%xmm1 \n" \
+ "pmaddwd %%xmm3,%%xmm3 \n" \
+ "paddd %%xmm1,%%xmm6 \n" \
+ "paddd %%xmm3,%%xmm6 \n"
+
+void Ssim4x4Row_SSE2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ asm volatile(
+ LABELALIGN "1: \n" //
+ SSIM_ROW0_SSE2("movdqu (%[src_a]),%%xmm0\n",
+ "movdqu (%[src_b]),%%xmm1\n") //
+ "add %[stride_a],%[src_a] \n"
+ "add %[stride_b],%[src_b] \n" //
+ SSIM_ROW_SSE2("movdqu (%[src_a]),%%xmm0\n",
+ "movdqu (%[src_b]),%%xmm1\n") //
+ SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a]),%%xmm0\n",
+ "movdqu (%[src_b],%[stride_b]),%%xmm1\n") //
+ SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a],2),%%xmm0\n",
+ "movdqu (%[src_b],%[stride_b],2),%%xmm1\n") //
+ "sub %[stride_a],%[src_a] \n"
+ "sub %[stride_b],%[src_b] \n"
+ "add $0x10,%[src_a] \n"
+ "add $0x10,%[src_b] \n"
+ "pcmpeqd %%xmm0,%%xmm0 \n"
+ "psrlw $0xf,%%xmm0 \n"
+ "pmaddwd %%xmm0,%%xmm4 \n"
+ "pmaddwd %%xmm0,%%xmm5 \n"
+ "movdqa %%xmm4,%%xmm0 \n"
+ "punpckldq %%xmm5,%%xmm0 \n"
+ "punpckhdq %%xmm5,%%xmm4 \n"
+ "movdqa %%xmm6,%%xmm1 \n"
+ "punpckldq %%xmm7,%%xmm1 \n"
+ "punpckhdq %%xmm7,%%xmm6 \n"
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklqdq %%xmm1,%%xmm2 \n"
+ "punpckhqdq %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm4,%%xmm3 \n"
+ "punpcklqdq %%xmm6,%%xmm3 \n"
+ "punpckhqdq %%xmm6,%%xmm4 \n"
+ "movdqu %%xmm2,0x00(%[dst]) \n"
+ "movdqu %%xmm0,0x10(%[dst]) \n"
+ "movdqu %%xmm3,0x20(%[dst]) \n"
+ "movdqu %%xmm4,0x30(%[dst]) \n"
+ "add $0x40,%[dst] \n"
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
+ [width] "+rm"(width)
+ : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#undef SSIM_ROW0_SSE2
+#undef SSIM_ROW_SSE2
+#endif // HAS_SSIM4X4ROW_SSE2
+
+#ifdef HAS_SSIM4X4ROW_AVX2
+#define SSIM_ROW0_AVX2(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \
+ "vpsllw $0x8,%%ymm0,%%ymm0 \n" \
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
+ "vpaddw %%ymm0,%%ymm2,%%ymm4 \n" \
+ "vpmaddwd %%ymm2,%%ymm2,%%ymm6 \n" \
+ "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \
+ "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
+ "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n" \
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \
+ "vpaddw %%ymm1,%%ymm3,%%ymm5 \n" \
+ "vpmaddwd %%ymm3,%%ymm2,%%ymm7 \n" \
+ "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \
+ "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \
+ "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
+ "vpaddd %%ymm1,%%ymm6,%%ymm6 \n"
+
+#define SSIM_ROW_AVX2(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \
+ "vpsllw $0x8,%%ymm0,%%ymm0 \n" \
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
+ "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \
+ "vpaddw %%ymm0,%%ymm4,%%ymm4 \n" \
+ "vpmaddwd %%ymm2,%%ymm2,%%ymm3 \n" \
+ "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
+ "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \
+ "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
+ "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n" \
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \
+ "vpaddw %%ymm3,%%ymm5,%%ymm5 \n" \
+ "vpaddw %%ymm1,%%ymm5,%%ymm5 \n" \
+ "vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n" \
+ "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \
+ "vpaddd %%ymm2,%%ymm7,%%ymm7 \n" \
+ "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \
+ "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
+ "vpaddd %%ymm1,%%ymm6,%%ymm6 \n"
+
+void Ssim4x4Row_AVX2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ asm volatile(
+ LABELALIGN "1: \n" //
+ SSIM_ROW0_AVX2("vmovdqu (%[src_a]),%%ymm0\n",
+ "vmovdqu (%[src_b]),%%ymm1\n") //
+ "add %[stride_a],%[src_a] \n"
+ "add %[stride_b],%[src_b] \n" //
+ SSIM_ROW_AVX2("vmovdqu (%[src_a]),%%ymm0\n",
+ "vmovdqu (%[src_b]),%%ymm1\n") //
+ SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a]),%%ymm0\n",
+ "vmovdqu (%[src_b],%[stride_b]),%%ymm1\n") //
+ SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a],2),%%ymm0\n",
+ "vmovdqu (%[src_b],%[stride_b],2),%%ymm1\n") //
+ "sub %[stride_a],%[src_a] \n"
+ "sub %[stride_b],%[src_b] \n"
+ "add $0x20,%[src_a] \n"
+ "add $0x20,%[src_b] \n"
+ "vpcmpeqd %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0xf,%%ymm0,%%ymm0 \n"
+ "vpmaddwd %%ymm0,%%ymm4,%%ymm4 \n"
+ "vpmaddwd %%ymm0,%%ymm5,%%ymm5 \n"
+ "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n"
+ "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n"
+ "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n"
+ "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n"
+ "vpunpcklqdq %%ymm2,%%ymm0,%%ymm4 \n"
+ "vpunpckhqdq %%ymm2,%%ymm0,%%ymm5 \n"
+ "vpunpcklqdq %%ymm3,%%ymm1,%%ymm6 \n"
+ "vpunpckhqdq %%ymm3,%%ymm1,%%ymm7 \n"
+ "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n"
+ "vperm2i128 $0x20,%%ymm7,%%ymm6,%%ymm1 \n"
+ "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n"
+ "vperm2i128 $0x31,%%ymm7,%%ymm6,%%ymm3 \n"
+ "vmovdqu %%ymm0,0x00(%[dst]) \n"
+ "vmovdqu %%ymm1,0x20(%[dst]) \n"
+ "vmovdqu %%ymm2,0x40(%[dst]) \n"
+ "vmovdqu %%ymm3,0x60(%[dst]) \n"
+ "add $0x80,%[dst] \n"
+ "subl $0x20,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
+ [width] "+rm"(width)
+ : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#undef SSIM_ROW0_AVX2
+#undef SSIM_ROW_AVX2
+#endif // HAS_SSIM4X4ROW_AVX2
+
+#ifdef HAS_SSIM4X4ROW_AVX512BW
+#define SSIM_ROW0_AVX512BW(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \
+ "vpsllw $0x8,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \
+ "vpaddw %%zmm0,%%zmm2,%%zmm4 \n" \
+ "vpmaddwd %%zmm2,%%zmm2,%%zmm6 \n" \
+ "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \
+ "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
+ "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n" \
+ "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \
+ "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" \
+ "vpmaddwd %%zmm3,%%zmm2,%%zmm7 \n" \
+ "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \
+ "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \
+ "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \
+ "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
+ "vpaddd %%zmm1,%%zmm6,%%zmm6 \n"
+
+#define SSIM_ROW_AVX512BW(LOAD_A, LOAD_B) \
+ LOAD_A \
+ LOAD_B \
+ "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \
+ "vpsllw $0x8,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \
+ "vpaddw %%zmm2,%%zmm4,%%zmm4 \n" \
+ "vpaddw %%zmm0,%%zmm4,%%zmm4 \n" \
+ "vpmaddwd %%zmm2,%%zmm2,%%zmm3 \n" \
+ "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
+ "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \
+ "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
+ "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n" \
+ "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \
+ "vpaddw %%zmm3,%%zmm5,%%zmm5 \n" \
+ "vpaddw %%zmm1,%%zmm5,%%zmm5 \n" \
+ "vpmaddwd %%zmm3,%%zmm2,%%zmm2 \n" \
+ "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpaddd %%zmm2,%%zmm7,%%zmm7 \n" \
+ "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \
+ "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \
+ "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \
+ "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
+ "vpaddd %%zmm1,%%zmm6,%%zmm6 \n"
+
+#define SSIM_TRANSPOSE_AVX512BW \
+ "vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $0xf,%%zmm0,%%zmm0 \n" \
+ "vpmaddwd %%zmm0,%%zmm4,%%zmm4 \n" \
+ "vpmaddwd %%zmm0,%%zmm5,%%zmm5 \n" \
+ "vpunpckldq %%zmm5,%%zmm4,%%zmm0 \n" \
+ "vpunpckhdq %%zmm5,%%zmm4,%%zmm1 \n" \
+ "vpunpckldq %%zmm7,%%zmm6,%%zmm2 \n" \
+ "vpunpckhdq %%zmm7,%%zmm6,%%zmm3 \n" \
+ "vpunpcklqdq %%zmm2,%%zmm0,%%zmm4 \n" \
+ "vpunpckhqdq %%zmm2,%%zmm0,%%zmm5 \n" \
+ "vpunpcklqdq %%zmm3,%%zmm1,%%zmm6 \n" \
+ "vpunpckhqdq %%zmm3,%%zmm1,%%zmm7 \n" \
+ "vshufi32x4 $0x44,%%zmm5,%%zmm4,%%zmm0 \n" \
+ "vshufi32x4 $0x44,%%zmm7,%%zmm6,%%zmm1 \n" \
+ "vshufi32x4 $0xee,%%zmm5,%%zmm4,%%zmm2 \n" \
+ "vshufi32x4 $0xee,%%zmm7,%%zmm6,%%zmm3 \n" \
+ "vshufi32x4 $0x88,%%zmm1,%%zmm0,%%zmm4 \n" \
+ "vshufi32x4 $0xdd,%%zmm1,%%zmm0,%%zmm5 \n" \
+ "vshufi32x4 $0x88,%%zmm3,%%zmm2,%%zmm6 \n" \
+ "vshufi32x4 $0xdd,%%zmm3,%%zmm2,%%zmm7 \n"
+
+// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60
+// pixels (1 to 15 tiles) with masked loads and stores.
+LIBYUV_TARGET_AVX512BW
+void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const int rem = width & 60;
+ const uint16_t load_mask = rem ? (uint16_t)((1U << (rem >> 2)) - 1U) : 0U;
+ const uint64_t store_mask = LowBytesMask64(rem);
+ asm volatile(
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN "1: \n" //
+ SSIM_ROW0_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n",
+ "vmovdqu8 (%[src_b]),%%zmm1\n") //
+ "add %[stride_a],%[src_a] \n"
+ "add %[stride_b],%[src_b] \n" //
+ SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n",
+ "vmovdqu8 (%[src_b]),%%zmm1\n") //
+ SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a]),%%zmm0\n",
+ "vmovdqu8 (%[src_b],%[stride_b]),%%zmm1\n") //
+ SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a],2),%%zmm0\n",
+ "vmovdqu8 (%[src_b],%[stride_b],2),%%zmm1\n") //
+ "sub %[stride_a],%[src_a] \n"
+ "sub %[stride_b],%[src_b] \n"
+ "add $0x40,%[src_a] \n"
+ "add $0x40,%[src_b] \n" //
+ SSIM_TRANSPOSE_AVX512BW
+ "vmovdqu32 %%zmm4,0x00(%[dst]) \n"
+ "vmovdqu32 %%zmm5,0x40(%[dst]) \n"
+ "vmovdqu32 %%zmm6,0x80(%[dst]) \n"
+ "vmovdqu32 %%zmm7,0xc0(%[dst]) \n"
+ "add $0x100,%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "je 3f \n"
+ "kmovw %[load_mask],%%k1 \n" //
+ SSIM_ROW0_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n",
+ "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") //
+ "add %[stride_a],%[src_a] \n"
+ "add %[stride_b],%[src_b] \n" //
+ SSIM_ROW_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n",
+ "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") //
+ SSIM_ROW_AVX512BW(
+ "vmovdqu32 (%[src_a],%[stride_a]),%%zmm0%{%%k1%}%{z%}\n",
+ "vmovdqu32 (%[src_b],%[stride_b]),%%zmm1%{%%k1%}%{z%}\n") //
+ SSIM_ROW_AVX512BW(
+ "vmovdqu32 (%[src_a],%[stride_a],2),%%zmm0%{%%k1%}%{z%}\n",
+ "vmovdqu32 (%[src_b],%[stride_b],2),%%zmm1%{%%k1%}%{z%}\n") //
+ SSIM_TRANSPOSE_AVX512BW
+ "kmovq %[store_mask],%%k1 \n"
+ "vmovdqu32 %%zmm4,0x00(%[dst])%{%%k1%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 %%zmm5,0x40(%[dst])%{%%k1%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 %%zmm6,0x80(%[dst])%{%%k1%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 %%zmm7,0xc0(%[dst])%{%%k1%} \n"
+
+ "3: \n"
+ "vzeroupper \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
+ [width] "+r"(width)
+ : [stride_a] "r"((ptrdiff_t)stride_a),
+ [stride_b] "r"((ptrdiff_t)stride_b), [load_mask] "m"(load_mask),
+ [store_mask] "m"(store_mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#undef SSIM_ROW0_AVX512BW
+#undef SSIM_ROW_AVX512BW
+#undef SSIM_TRANSPOSE_AVX512BW
+#endif // HAS_SSIM4X4ROW_AVX512BW
+
+#if defined(HAS_SSIMWINDOWROW_SSE2) || defined(HAS_SSIMWINDOWROW_AVX2) || \
+ defined(HAS_SSIMWINDOWROW_AVX512BW)
+static const uvec32 kSsimC1 = {26634, 26634, 26634, 26634};
+static const uvec32 kSsimC2 = {239708, 239708, 239708, 239708};
+#endif
+
+#ifdef HAS_SSIMWINDOWROW_SSE2
+// num_windows is a multiple of 4.
+float SsimWindowRow_SSE2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ float total;
+ asm volatile(
+ "sub %[row0],%[row1] \n"
+ "xorps %%xmm0,%%xmm0 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu 0x00(%[row0]),%%xmm1 \n"
+ "movdqu 0x00(%[row0],%[row1]),%%xmm5 \n"
+ "paddd %%xmm5,%%xmm1 \n"
+ "movdqu 0x10(%[row0]),%%xmm2 \n"
+ "movdqu 0x10(%[row0],%[row1]),%%xmm5 \n"
+ "paddd %%xmm5,%%xmm2 \n"
+ "movdqu 0x20(%[row0]),%%xmm3 \n"
+ "movdqu 0x20(%[row0],%[row1]),%%xmm5 \n"
+ "paddd %%xmm5,%%xmm3 \n"
+ "movdqu 0x30(%[row0]),%%xmm4 \n"
+ "movdqu 0x30(%[row0],%[row1]),%%xmm5 \n"
+ "paddd %%xmm5,%%xmm4 \n"
+ "movdqu 0x40(%[row0]),%%xmm5 \n"
+ "add $0x40,%[row0] \n"
+ "paddd %%xmm2,%%xmm1 \n"
+ "paddd %%xmm3,%%xmm2 \n"
+ "paddd %%xmm4,%%xmm3 \n"
+ "paddd %%xmm5,%%xmm4 \n"
+ "movdqu 0x00(%[row0],%[row1]),%%xmm5 \n"
+ "paddd %%xmm5,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm5 \n"
+ "punpckldq %%xmm2,%%xmm1 \n"
+ "punpckhdq %%xmm2,%%xmm5 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "punpckldq %%xmm4,%%xmm3 \n"
+ "punpckhdq %%xmm4,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "punpcklqdq %%xmm3,%%xmm1 \n"
+ "punpckhqdq %%xmm3,%%xmm4 \n"
+ "movdqa %%xmm5,%%xmm3 \n"
+ "punpcklqdq %%xmm2,%%xmm3 \n"
+ "punpckhqdq %%xmm2,%%xmm5 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "pmaddwd %%xmm4,%%xmm2 \n"
+ "pmaddwd %%xmm1,%%xmm1 \n"
+ "pmaddwd %%xmm4,%%xmm4 \n"
+ "paddd %%xmm4,%%xmm1 \n"
+ "pslld $0x1,%%xmm2 \n"
+ "pslld $0x7,%%xmm5 \n"
+ "psubd %%xmm2,%%xmm5 \n"
+ "paddd %[c1],%%xmm2 \n"
+ "paddd %[c2],%%xmm5 \n"
+ "pslld $0x6,%%xmm3 \n"
+ "psubd %%xmm1,%%xmm3 \n"
+ "paddd %[c1],%%xmm1 \n"
+ "paddd %[c2],%%xmm3 \n"
+ "cvtdq2ps %%xmm2,%%xmm2 \n"
+ "cvtdq2ps %%xmm5,%%xmm5 \n"
+ "cvtdq2ps %%xmm1,%%xmm1 \n"
+ "cvtdq2ps %%xmm3,%%xmm3 \n"
+ "mulps %%xmm5,%%xmm2 \n"
+ "mulps %%xmm3,%%xmm1 \n"
+ "divps %%xmm1,%%xmm2 \n"
+ "addps %%xmm2,%%xmm0 \n"
+ "sub $0x4,%[num_windows] \n"
+ "jg 1b \n"
+
+ "movhlps %%xmm0,%%xmm1 \n"
+ "addps %%xmm1,%%xmm0 \n"
+ "movaps %%xmm0,%%xmm1 \n"
+ "shufps $0x1,%%xmm0,%%xmm1 \n"
+ "addss %%xmm1,%%xmm0 \n"
+ "movss %%xmm0,%[total] \n"
+ : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
+ [total] "=m"(total)
+ : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return total;
+}
+#endif // HAS_SSIMWINDOWROW_SSE2
+
+#ifdef HAS_SSIMWINDOWROW_AVX2
+// num_windows is a multiple of 8.
+float SsimWindowRow_AVX2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ float total;
+ asm volatile(
+ "sub %[row0],%[row1] \n"
+ "vxorps %%ymm0,%%ymm0,%%ymm0 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu 0x00(%[row0]),%%ymm1 \n"
+ "vpaddd 0x00(%[row0],%[row1]),%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x10(%[row0]),%%ymm5 \n"
+ "vpaddd 0x10(%[row0],%[row1]),%%ymm5,%%ymm5 \n"
+ "vpaddd %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x20(%[row0]),%%ymm2 \n"
+ "vpaddd 0x20(%[row0],%[row1]),%%ymm2,%%ymm2 \n"
+ "vmovdqu 0x30(%[row0]),%%ymm5 \n"
+ "vpaddd 0x30(%[row0],%[row1]),%%ymm5,%%ymm5 \n"
+ "vpaddd %%ymm5,%%ymm2,%%ymm2 \n"
+ "vmovdqu 0x40(%[row0]),%%ymm3 \n"
+ "vpaddd 0x40(%[row0],%[row1]),%%ymm3,%%ymm3 \n"
+ "vmovdqu 0x50(%[row0]),%%ymm5 \n"
+ "vpaddd 0x50(%[row0],%[row1]),%%ymm5,%%ymm5 \n"
+ "vpaddd %%ymm5,%%ymm3,%%ymm3 \n"
+ "vmovdqu 0x60(%[row0]),%%ymm4 \n"
+ "vpaddd 0x60(%[row0],%[row1]),%%ymm4,%%ymm4 \n"
+ "vmovdqu 0x70(%[row0]),%%ymm5 \n"
+ "vpaddd 0x70(%[row0],%[row1]),%%ymm5,%%ymm5 \n"
+ "vpaddd %%ymm5,%%ymm4,%%ymm4 \n"
+ "add $0x80,%[row0] \n"
+ "vpunpckldq %%ymm2,%%ymm1,%%ymm5 \n"
+ "vpunpckhdq %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpunpckldq %%ymm4,%%ymm3,%%ymm2 \n"
+ "vpunpckhdq %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpunpcklqdq %%ymm2,%%ymm5,%%ymm4 \n"
+ "vpunpckhqdq %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpunpcklqdq %%ymm3,%%ymm1,%%ymm5 \n"
+ "vpunpckhqdq %%ymm3,%%ymm1,%%ymm3 \n"
+ "vpmaddwd %%ymm2,%%ymm4,%%ymm1 \n"
+ "vpmaddwd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpmaddwd %%ymm2,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n"
+ "vpslld $0x1,%%ymm1,%%ymm1 \n"
+ "vpslld $0x7,%%ymm3,%%ymm3 \n"
+ "vpsubd %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpslld $0x6,%%ymm5,%%ymm5 \n"
+ "vpsubd %%ymm4,%%ymm5,%%ymm5 \n"
+ "vpbroadcastd %[c1],%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n"
+ "vpbroadcastd %[c2],%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddd %%ymm2,%%ymm5,%%ymm5 \n"
+ "vcvtdq2ps %%ymm1,%%ymm1 \n"
+ "vcvtdq2ps %%ymm3,%%ymm3 \n"
+ "vcvtdq2ps %%ymm4,%%ymm4 \n"
+ "vcvtdq2ps %%ymm5,%%ymm5 \n"
+ "vmulps %%ymm3,%%ymm1,%%ymm1 \n"
+ "vmulps %%ymm5,%%ymm4,%%ymm4 \n"
+ "vdivps %%ymm4,%%ymm1,%%ymm1 \n"
+ "vaddps %%ymm1,%%ymm0,%%ymm0 \n"
+ "sub $0x8,%[num_windows] \n"
+ "jg 1b \n"
+
+ "vextractf128 $0x1,%%ymm0,%%xmm1 \n"
+ "vaddps %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n"
+ "vaddps %%xmm1,%%xmm0,%%xmm0 \n"
+ "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n"
+ "vaddss %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovss %%xmm0,%[total] \n"
+ "vzeroupper \n"
+ : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
+ [total] "=m"(total)
+ : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return total;
+}
+#endif // HAS_SSIMWINDOWROW_AVX2
+
+#ifdef HAS_SSIMWINDOWROW_AVX512BW
+#define SSIM_WINDOW_AVX512BW \
+ "vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" \
+ "vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" \
+ "vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" \
+ "vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" \
+ "vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" \
+ "vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" \
+ "vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" \
+ "vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" \
+ "vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" \
+ "vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" \
+ "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" \
+ "vpaddd %%zmm2,%%zmm4,%%zmm4 \n" \
+ "vpslld $0x1,%%zmm1,%%zmm1 \n" \
+ "vpslld $0x7,%%zmm3,%%zmm3 \n" \
+ "vpsubd %%zmm1,%%zmm3,%%zmm3 \n" \
+ "vpslld $0x6,%%zmm5,%%zmm5 \n" \
+ "vpsubd %%zmm4,%%zmm5,%%zmm5 \n" \
+ "vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" \
+ "vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" \
+ "vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" \
+ "vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" \
+ "vcvtdq2ps %%zmm1,%%zmm1 \n" \
+ "vcvtdq2ps %%zmm3,%%zmm3 \n" \
+ "vcvtdq2ps %%zmm4,%%zmm4 \n" \
+ "vcvtdq2ps %%zmm5,%%zmm5 \n" \
+ "vmulps %%zmm3,%%zmm1,%%zmm1 \n" \
+ "vmulps %%zmm5,%%zmm4,%%zmm4 \n" \
+ "vdivps %%zmm4,%%zmm1,%%zmm1 \n"
+
+// Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with
+// masked loads and masked accumulation.
+LIBYUV_TARGET_AVX512BW
+float SsimWindowRow_AVX512BW(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const int rem = num_windows & 15;
+ const uint64_t load_mask = rem ? (~0ULL >> (60 - rem * 4)) : 0ULL;
+ const uint16_t win_mask =
+ (uint16_t)(((0x01110U >> (16 - (rem & 3) * 4)) << (rem >> 2)) +
+ (0x1111U << (rem >> 2)) - 0x1111U);
+ float total;
+ asm volatile(
+ "sub %[row0],%[row1] \n"
+ "vxorps %%xmm0,%%xmm0,%%xmm0 \n"
+ "sub $0x10,%[num_windows] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 0x00(%[row0]),%%zmm1 \n"
+ "vpaddd 0x00(%[row0],%[row1]),%%zmm1,%%zmm1 \n"
+ "vmovdqu32 0x40(%[row0]),%%zmm2 \n"
+ "vpaddd 0x40(%[row0],%[row1]),%%zmm2,%%zmm2 \n"
+ "vmovdqu32 0x80(%[row0]),%%zmm3 \n"
+ "vpaddd 0x80(%[row0],%[row1]),%%zmm3,%%zmm3 \n"
+ "vmovdqu32 0xc0(%[row0]),%%zmm4 \n"
+ "vpaddd 0xc0(%[row0],%[row1]),%%zmm4,%%zmm4 \n"
+ "valignd $0x4,%%zmm1,%%zmm2,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm1,%%zmm1 \n"
+ "valignd $0x4,%%zmm2,%%zmm3,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm2,%%zmm2 \n"
+ "valignd $0x4,%%zmm3,%%zmm4,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm3,%%zmm3 \n"
+ "vmovdqu32 0xd0(%[row0]),%%zmm5 \n"
+ "vpaddd 0xd0(%[row0],%[row1]),%%zmm5,%%zmm5 \n"
+ "add $0x100,%[row0] \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" //
+ SSIM_WINDOW_AVX512BW
+ "vaddps %%zmm1,%%zmm0,%%zmm0 \n"
+ "sub $0x10,%[num_windows] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[num_windows] \n"
+ "je 3f \n"
+ "kmovq %[load_mask],%%k1 \n"
+ "vmovdqu32 0x00(%[row0]),%%zmm1%{%%k1%}%{z%} \n"
+ "vpaddd 0x00(%[row0],%[row1]),%%zmm1,%%zmm1%{%%k1%}%{z%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 0x40(%[row0]),%%zmm2%{%%k1%}%{z%} \n"
+ "vpaddd 0x40(%[row0],%[row1]),%%zmm2,%%zmm2%{%%k1%}%{z%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 0x80(%[row0]),%%zmm3%{%%k1%}%{z%} \n"
+ "vpaddd 0x80(%[row0],%[row1]),%%zmm3,%%zmm3%{%%k1%}%{z%} \n"
+ "kshiftrq $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 0xc0(%[row0]),%%zmm4%{%%k1%}%{z%} \n"
+ "vpaddd 0xc0(%[row0],%[row1]),%%zmm4,%%zmm4%{%%k1%}%{z%} \n"
+ "valignd $0x4,%%zmm1,%%zmm2,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm1,%%zmm1 \n"
+ "valignd $0x4,%%zmm2,%%zmm3,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm2,%%zmm2 \n"
+ "valignd $0x4,%%zmm3,%%zmm4,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm3,%%zmm3 \n"
+ "valignd $0x4,%%zmm4,%%zmm4,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" //
+ SSIM_WINDOW_AVX512BW
+ "kmovw %[win_mask],%%k1 \n"
+ "vaddps %%zmm1,%%zmm0,%%zmm0%{%%k1%} \n"
+
+ "3: \n"
+ "vextractf32x8 $0x1,%%zmm0,%%ymm1 \n"
+ "vaddps %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextractf128 $0x1,%%ymm0,%%xmm1 \n"
+ "vaddps %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n"
+ "vaddps %%xmm1,%%xmm0,%%xmm0 \n"
+ "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n"
+ "vaddss %%xmm1,%%xmm0,%[total] \n"
+ "vzeroupper \n"
+ : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
+ [total] "=x"(total)
+ : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2), [load_mask] "m"(load_mask),
+ [win_mask] "m"(win_mask)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ return total;
+}
+#undef SSIM_WINDOW_AVX512BW
+#endif // HAS_SSIMWINDOWROW_AVX512BW
+
+#ifdef HAS_ARGBDETECTROW_AVX2
+uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
+ uint32_t diff = 0;
+ asm volatile(
+ "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n" LABELALIGN
+ "1: \n"
+ "vpcmpeqb (%[argb]),%%ymm1,%%ymm0 \n"
+ "vpmovmskb %%ymm0,%[diff] \n"
+ "not %[diff] \n"
+ "and $0x99999999,%[diff] \n"
+ "jnz 99f \n"
+ "add $0x20,%[argb] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n" LABELALIGN
+ "3: \n"
+ "vmovd (%[argb]),%%xmm0 \n"
+ "add $0x4,%[argb] \n"
+ "vpcmpeqb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpmovmskb %%xmm0,%[diff] \n"
+ "not %[diff] \n"
+ "and $0x9,%[diff] \n"
+ "jnz 99f \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+ "99: \n"
+ "vzeroupper \n"
+ : [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff)
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+ if (diff) {
+ return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_AVX2
+
+#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
+ // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN)
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/source/compare_hvx.cc b/libs/libyuv/source/compare_hvx.cc
new file mode 100644
index 0000000000..fac674adb5
--- /dev/null
+++ b/libs/libyuv/source/compare_hvx.cc
@@ -0,0 +1,381 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/video_common.h"
+
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+
+#include <hexagon_protos.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <string.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+static inline HVX_Vector vloada(const void* ptr) {
+ HVX_Vector v;
+ memcpy(&v, (const HVX_Vector*)ptr, sizeof(v));
+ return v;
+}
+
+static inline HVX_UVector vloadu(const void* ptr) {
+ HVX_UVector v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+}
+
+static inline void vstoreu(void* ptr, HVX_UVector v) {
+ memcpy(ptr, &v, sizeof(v));
+}
+
+// Partial load of n bytes (1 <= n <= 128), zeroing lanes past n.
+static inline HVX_Vector vloadu_partial_zero(const void* ptr, int n) {
+ if (n >= 128) {
+ return vloadu(ptr);
+ }
+ const uintptr_t addr = (uintptr_t)ptr;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector v0 = vloada(ptr);
+ HVX_Vector v1 =
+ (offset + (uintptr_t)n > 128) ? vloada((const uint8_t*)ptr + 128) : v0;
+ HVX_Vector aligned = Q6_V_valign_VVR(v1, v0, addr);
+ HVX_VectorPred mask = Q6_Q_vsetq_R(n);
+ return Q6_V_vmux_QVV(mask, aligned, Q6_V_vzero());
+}
+
+// Partial store of n bytes (1 <= n <= 128) without overwriting past n.
+static inline void vstoreu_partial(void* ptr, HVX_Vector v, int n) {
+ if (n >= 128) {
+ vstoreu(ptr, v);
+ return;
+ }
+ const uintptr_t addr = (uintptr_t)ptr;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector value = Q6_V_vlalign_VVR(v, v, addr);
+ HVX_VectorPred ql_not = Q6_Q_vsetq_R(addr);
+ HVX_VectorPred qr = Q6_Q_vsetq2_R(offset + (uintptr_t)n);
+ if (offset + (uintptr_t)n > 128) {
+ Q6_vmem_QRIV(qr, (uint8_t*)ptr + 128, value);
+ qr = Q6_Q_vcmp_eq_VbVb(value, value);
+ }
+ ql_not = Q6_Q_or_QQn(ql_not, qr);
+ Q6_vmem_QnRIV(ql_not, ptr, value);
+}
+
+static inline uint32_t horizontal_sum_u32(HVX_Vector x) {
+ x = Q6_Vw_vadd_VwVw(x, Q6_V_vror_VR(x, 64));
+ x = Q6_Vw_vadd_VwVw(x, Q6_V_vror_VR(x, 32));
+ x = Q6_Vw_vadd_VwVw(x, Q6_V_vror_VR(x, 16));
+ x = Q6_Vw_vadd_VwVw(x, Q6_V_vror_VR(x, 8));
+ x = Q6_Vw_vadd_VwVw(x, Q6_V_vror_VR(x, 4));
+ uint32_t result;
+ memcpy(&result, &x, sizeof(result));
+ return result;
+}
+
+#if defined(HAS_HAMMINGDISTANCE_HVX)
+uint32_t HammingDistance_HVX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ HVX_Vector sum = Q6_V_vzero();
+ while (count >= 128) {
+ HVX_Vector x = Q6_V_vxor_VV(vloadu(src_a), vloadu(src_b));
+ HVX_Vector c = Q6_Vh_vpopcount_Vh(x);
+ sum = Q6_Vuw_vrmpyacc_VuwVubRub(sum, c, 0x01010101);
+ src_a += 128;
+ src_b += 128;
+ count -= 128;
+ }
+ if (count > 0) {
+ HVX_Vector x = Q6_V_vxor_VV(vloadu_partial_zero(src_a, count),
+ vloadu_partial_zero(src_b, count));
+ HVX_Vector c = Q6_Vh_vpopcount_Vh(x);
+ sum = Q6_Vuw_vrmpyacc_VuwVubRub(sum, c, 0x01010101);
+ }
+ return horizontal_sum_u32(sum);
+}
+#endif // defined(HAS_HAMMINGDISTANCE_HVX)
+
+#if defined(HAS_SUMSQUAREERROR_HVX)
+uint32_t SumSquareError_HVX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ HVX_Vector sum = Q6_V_vzero();
+ while (count >= 128) {
+ HVX_Vector d = Q6_Vub_vabsdiff_VubVub(vloadu(src_a), vloadu(src_b));
+ sum = Q6_Vuw_vrmpyacc_VuwVubVub(sum, d, d);
+ src_a += 128;
+ src_b += 128;
+ count -= 128;
+ }
+ if (count > 0) {
+ HVX_Vector d = Q6_Vub_vabsdiff_VubVub(vloadu_partial_zero(src_a, count),
+ vloadu_partial_zero(src_b, count));
+ sum = Q6_Vuw_vrmpyacc_VuwVubVub(sum, d, d);
+ }
+ return horizontal_sum_u32(sum);
+}
+#endif // defined(HAS_SUMSQUAREERROR_HVX)
+
+#if defined(HAS_HASHDJB2_HVX)
+static const uint32_t kHashMul32[32] __attribute__((aligned(128))) = {
+ 0x655ec7e1, 0xccc4cfc1, 0x99995ba1, 0xd61beb81, // 33 ^ 31 .. 28
+ 0x829bff61, 0x13791741, 0x2f22b321, 0xac185301, // 33 ^ 27 .. 24
+ 0xcee976e1, 0xc8359ec1, 0x72ac4aa1, 0x510cfa81, // 33 ^ 23 .. 20
+ 0xcc272e61, 0xb0da6641, 0xee162221, 0x92d9e201, // 33 ^ 19 .. 16
+ 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, // 33 ^ 15 .. 12
+ 0x30f35d61, 0x855cb541, 0x040a9121, 0x747c7101, // 33 ^ 11 .. 8
+ 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, // 33 ^ 7 .. 4
+ 0x00008c61, 0x00000441, 0x00000021, 0x00000001, // 33 ^ 3 .. 0
+};
+
+static inline HVX_Vector vmpy_u32(HVX_Vector a, HVX_Vector b) {
+ HVX_Vector hi = Q6_Vw_vmpyieo_VhVh(a, b);
+ return Q6_Vw_vmpyieacc_VwVwVuh(hi, a, b);
+}
+
+static uint32_t Pow33(int n) {
+ uint32_t r = 1u;
+ uint32_t b = 33u;
+ while (n) {
+ if (n & 1) {
+ r *= b;
+ }
+ b *= b;
+ n >>= 1;
+ }
+ return r;
+}
+
+static inline uint32_t HashSum128_HVX(HVX_Vector s,
+ HVX_Vector mul0,
+ HVX_Vector mul1,
+ HVX_Vector mul2,
+ HVX_Vector mul3) {
+ HVX_VectorPair h01 = Q6_Wuh_vunpack_Vub(s);
+ HVX_VectorPair w01 = Q6_Wuw_vunpack_Vuh(Q6_V_lo_W(h01));
+ HVX_VectorPair w23 = Q6_Wuw_vunpack_Vuh(Q6_V_hi_W(h01));
+ HVX_Vector sum = Q6_Vw_vmpyie_VwVuh(mul0, Q6_V_lo_W(w01));
+ sum = Q6_Vw_vmpyieacc_VwVwVuh(sum, mul1, Q6_V_hi_W(w01));
+ sum = Q6_Vw_vmpyieacc_VwVwVuh(sum, mul2, Q6_V_lo_W(w23));
+ sum = Q6_Vw_vmpyieacc_VwVwVuh(sum, mul3, Q6_V_hi_W(w23));
+ return horizontal_sum_u32(sum);
+}
+
+uint32_t HashDjb2_HVX(const uint8_t* src, int count, uint32_t seed) {
+ const HVX_Vector c32 = Q6_V_vsplat_R(0x1137c401u); // 33 ^ 32
+ const HVX_Vector mul3 = vloada(kHashMul32);
+ const HVX_Vector mul2 = vmpy_u32(mul3, c32);
+ const HVX_Vector mul1 = vmpy_u32(mul2, c32);
+ const HVX_Vector mul0 = vmpy_u32(mul1, c32);
+ while (count >= 128) {
+ seed = seed * 0x193f1001u + // 33 ^ 128
+ HashSum128_HVX(vloadu(src), mul0, mul1, mul2, mul3);
+ src += 128;
+ count -= 128;
+ }
+ if (count > 0) {
+ HVX_Vector v = vloadu_partial_zero(src, count);
+ HVX_Vector s = Q6_V_valign_VVR(v, Q6_V_vzero(), count);
+ seed = seed * Pow33(count) + HashSum128_HVX(s, mul0, mul1, mul2, mul3);
+ }
+ return seed;
+}
+#endif // defined(HAS_HASHDJB2_HVX)
+
+#if defined(HAS_SSIM4X4ROW_HVX)
+#define SSIM_4X4_HVX(va0, vb0, va1, vb1, va2, vb2, va3, vb3) \
+ HVX_Vector sum_a = Q6_Vuw_vrmpy_VubRub(va0, 0x01010101); \
+ sum_a = Q6_Vuw_vrmpyacc_VuwVubRub(sum_a, va1, 0x01010101); \
+ sum_a = Q6_Vuw_vrmpyacc_VuwVubRub(sum_a, va2, 0x01010101); \
+ sum_a = Q6_Vuw_vrmpyacc_VuwVubRub(sum_a, va3, 0x01010101); \
+ HVX_Vector sum_b = Q6_Vuw_vrmpy_VubRub(vb0, 0x01010101); \
+ sum_b = Q6_Vuw_vrmpyacc_VuwVubRub(sum_b, vb1, 0x01010101); \
+ sum_b = Q6_Vuw_vrmpyacc_VuwVubRub(sum_b, vb2, 0x01010101); \
+ sum_b = Q6_Vuw_vrmpyacc_VuwVubRub(sum_b, vb3, 0x01010101); \
+ HVX_Vector sum_sq = Q6_Vuw_vrmpy_VubVub(va0, va0); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, vb0, vb0); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, va1, va1); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, vb1, vb1); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, va2, va2); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, vb2, vb2); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, va3, va3); \
+ sum_sq = Q6_Vuw_vrmpyacc_VuwVubVub(sum_sq, vb3, vb3); \
+ HVX_Vector sum_axb = Q6_Vuw_vrmpy_VubVub(va0, vb0); \
+ sum_axb = Q6_Vuw_vrmpyacc_VuwVubVub(sum_axb, va1, vb1); \
+ sum_axb = Q6_Vuw_vrmpyacc_VuwVubVub(sum_axb, va2, vb2); \
+ sum_axb = Q6_Vuw_vrmpyacc_VuwVubVub(sum_axb, va3, vb3); \
+ HVX_VectorPair ab = Q6_W_vshuff_VVR(sum_b, sum_a, -4); \
+ HVX_VectorPair sqx = Q6_W_vshuff_VVR(sum_axb, sum_sq, -4); \
+ HVX_VectorPair out01 = Q6_W_vshuff_VVR(Q6_V_lo_W(sqx), Q6_V_lo_W(ab), -8); \
+ HVX_VectorPair out23 = Q6_W_vshuff_VVR(Q6_V_hi_W(sqx), Q6_V_hi_W(ab), -8)
+
+// Width is a multiple of 4.
+void Ssim4x4Row_HVX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+
+ while (width >= 128) {
+ SSIM_4X4_HVX(vloadu(a0), vloadu(b0), vloadu(a1), vloadu(b1), vloadu(a2),
+ vloadu(b2), vloadu(a3), vloadu(b3));
+ vstoreu(dst + 0, Q6_V_lo_W(out01));
+ vstoreu(dst + 32, Q6_V_hi_W(out01));
+ vstoreu(dst + 64, Q6_V_lo_W(out23));
+ vstoreu(dst + 96, Q6_V_hi_W(out23));
+ a0 += 128;
+ a1 += 128;
+ a2 += 128;
+ a3 += 128;
+ b0 += 128;
+ b1 += 128;
+ b2 += 128;
+ b3 += 128;
+ dst += 128;
+ width -= 128;
+ }
+ if (width > 0) {
+ SSIM_4X4_HVX(vloadu_partial_zero(a0, width), vloadu_partial_zero(b0, width),
+ vloadu_partial_zero(a1, width), vloadu_partial_zero(b1, width),
+ vloadu_partial_zero(a2, width), vloadu_partial_zero(b2, width),
+ vloadu_partial_zero(a3, width),
+ vloadu_partial_zero(b3, width));
+ int bytes = width * 4;
+ if (bytes > 0) {
+ vstoreu_partial(dst + 0, Q6_V_lo_W(out01), bytes < 128 ? bytes : 128);
+ }
+ if (bytes > 128) {
+ vstoreu_partial(dst + 32, Q6_V_hi_W(out01),
+ bytes - 128 < 128 ? bytes - 128 : 128);
+ }
+ if (bytes > 256) {
+ vstoreu_partial(dst + 64, Q6_V_lo_W(out23),
+ bytes - 256 < 128 ? bytes - 256 : 128);
+ }
+ if (bytes > 384) {
+ vstoreu_partial(dst + 96, Q6_V_hi_W(out23), bytes - 384);
+ }
+ }
+}
+#undef SSIM_4X4_HVX
+#endif // defined(HAS_SSIM4X4ROW_HVX)
+
+#if defined(HAS_SSIMWINDOWROW_HVX)
+float SsimWindowRow_HVX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const HVX_Vector vc1 = Q6_V_vsplat_R(26634);
+ const HVX_Vector vc2 = Q6_V_vsplat_R(239708);
+ float total = 0.0f;
+ while (num_windows > 0) {
+ int count = num_windows < 8 ? num_windows : 8;
+ HVX_Vector t0, t1;
+ if (num_windows >= 8) {
+ t0 = Q6_Vw_vadd_VwVw(vloadu(row0), vloadu(row1));
+ t1 = Q6_Vw_vadd_VwVw(vloadu(row0 + 4), vloadu(row1 + 4));
+ } else {
+ int bytes0 = count * 16;
+ int bytes1 = (count + 1) * 16;
+ t0 = Q6_Vw_vadd_VwVw(vloadu_partial_zero(row0, bytes0),
+ vloadu_partial_zero(row1, bytes0));
+ t1 = Q6_Vw_vadd_VwVw(vloadu_partial_zero(row0 + 4, bytes1 - 16),
+ vloadu_partial_zero(row1 + 4, bytes1 - 16));
+ }
+ HVX_Vector w = Q6_Vw_vadd_VwVw(t0, t1);
+ HVX_Vector w_b = Q6_V_vror_VR(w, 4);
+ HVX_Vector w_sq = Q6_V_vror_VR(w, 8);
+ HVX_Vector w_axb = Q6_V_vror_VR(w, 12);
+ HVX_Vector ab2 = Q6_Vw_vasl_VwR(Q6_Vw_vmpyie_VwVuh(w, w_b), 1);
+ HVX_Vector a2b2 =
+ Q6_Vw_vmpyieacc_VwVwVuh(Q6_Vw_vmpyie_VwVuh(w, w), w_b, w_b);
+ HVX_Vector vn1 = Q6_Vw_vadd_VwVw(ab2, vc1);
+ HVX_Vector vn2 =
+ Q6_Vw_vadd_VwVw(Q6_Vw_vsub_VwVw(Q6_Vw_vasl_VwR(w_axb, 7), ab2), vc2);
+ HVX_Vector vd1 = Q6_Vw_vadd_VwVw(a2b2, vc1);
+ HVX_Vector vd2 =
+ Q6_Vw_vadd_VwVw(Q6_Vw_vsub_VwVw(Q6_Vw_vasl_VwR(w_sq, 6), a2b2), vc2);
+ int32_t n1[32] __attribute__((aligned(128)));
+ int32_t n2[32] __attribute__((aligned(128)));
+ int32_t d1[32] __attribute__((aligned(128)));
+ int32_t d2[32] __attribute__((aligned(128)));
+ vstoreu(n1, vn1);
+ vstoreu(n2, vn2);
+ vstoreu(d1, vd1);
+ vstoreu(d2, vd2);
+ int i;
+ for (i = 0; i < count; ++i) {
+ float num = (float)n1[i * 4] * (float)n2[i * 4];
+ float den = (float)d1[i * 4] * (float)d2[i * 4];
+ total += num / den;
+ }
+ row0 += 32;
+ row1 += 32;
+ num_windows -= 8;
+ }
+ return total;
+}
+#endif // defined(HAS_SSIMWINDOWROW_HVX)
+
+#if defined(HAS_ARGBDETECTROW_HVX)
+uint32_t ARGBDetectRow_HVX(const uint8_t* argb, int width) {
+ const HVX_Vector mask_rgb = Q6_V_vsplat_R(0x00ffff00);
+ const HVX_Vector ones = Q6_Vb_vsplat_R(-1);
+ const HVX_Vector zero = Q6_V_vzero();
+ while (width > 0) {
+ HVX_Vector v;
+ if (width >= 32) {
+ v = vloadu(argb);
+ } else {
+ int bytes = width * 4;
+ v = Q6_V_vmux_QVV(Q6_Q_vsetq_R(bytes), vloadu_partial_zero(argb, bytes),
+ ones);
+ }
+ v = Q6_V_vor_VV(v, mask_rgb);
+ HVX_VectorPred eq = Q6_Q_vcmp_eq_VbVb(v, ones);
+ HVX_Vector diff_v = Q6_V_vmux_QVV(eq, zero, ones);
+ if (horizontal_sum_u32(Q6_Vuw_vrmpy_VubRub(diff_v, 0x01010101)) != 0) {
+ uint32_t diff[32] __attribute__((aligned(128)));
+ vstoreu(diff, diff_v);
+ int i;
+ for (i = 0; i < 32; ++i) {
+ if (diff[i] != 0) {
+ return (diff[i] & 0xffu) ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ }
+ }
+ argb += 128;
+ width -= 32;
+ }
+ return 0;
+}
+#endif // defined(HAS_ARGBDETECTROW_HVX)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
diff --git a/libs/libyuv/source/compare_mmi.cc b/libs/libyuv/source/compare_mmi.cc
deleted file mode 100644
index 7640d9468c..0000000000
--- a/libs/libyuv/source/compare_mmi.cc
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2012 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "libyuv/basic_types.h"
-
-#include "libyuv/compare_row.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-// This module is for Mips MMI.
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-// Hakmem method for hamming distance.
-uint32_t HammingDistance_MMI(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t diff = 0u;
-
- uint64_t temp = 0, temp1 = 0, ta = 0, tb = 0;
- uint64_t c1 = 0x5555555555555555;
- uint64_t c2 = 0x3333333333333333;
- uint64_t c3 = 0x0f0f0f0f0f0f0f0f;
- uint32_t c4 = 0x01010101;
- uint64_t s1 = 1, s2 = 2, s3 = 4;
- __asm__ volatile(
- "1: \n\t"
- "ldc1 %[ta], 0(%[src_a]) \n\t"
- "ldc1 %[tb], 0(%[src_b]) \n\t"
- "xor %[temp], %[ta], %[tb] \n\t"
- "psrlw %[temp1], %[temp], %[s1] \n\t" // temp1=x>>1
- "and %[temp1], %[temp1], %[c1] \n\t" // temp1&=c1
- "psubw %[temp1], %[temp], %[temp1] \n\t" // x-temp1
- "and %[temp], %[temp1], %[c2] \n\t" // t = (u&c2)
- "psrlw %[temp1], %[temp1], %[s2] \n\t" // u>>2
- "and %[temp1], %[temp1], %[c2] \n\t" // u>>2 & c2
- "paddw %[temp1], %[temp1], %[temp] \n\t" // t1 = t1+t
- "psrlw %[temp], %[temp1], %[s3] \n\t" // u>>4
- "paddw %[temp1], %[temp1], %[temp] \n\t" // u+(u>>4)
- "and %[temp1], %[temp1], %[c3] \n\t" //&c3
- "dmfc1 $t0, %[temp1] \n\t"
- "dsrl32 $t0, $t0, 0 \n\t "
- "mul $t0, $t0, %[c4] \n\t"
- "dsrl $t0, $t0, 24 \n\t"
- "dadd %[diff], %[diff], $t0 \n\t"
- "dmfc1 $t0, %[temp1] \n\t"
- "mul $t0, $t0, %[c4] \n\t"
- "dsrl $t0, $t0, 24 \n\t"
- "dadd %[diff], %[diff], $t0 \n\t"
- "daddiu %[src_a], %[src_a], 8 \n\t"
- "daddiu %[src_b], %[src_b], 8 \n\t"
- "addiu %[count], %[count], -8 \n\t"
- "bgtz %[count], 1b \n\t"
- "nop \n\t"
- : [diff] "+r"(diff), [src_a] "+r"(src_a), [src_b] "+r"(src_b),
- [count] "+r"(count), [ta] "+f"(ta), [tb] "+f"(tb), [temp] "+f"(temp),
- [temp1] "+f"(temp1)
- : [c1] "f"(c1), [c2] "f"(c2), [c3] "f"(c3), [c4] "r"(c4), [s1] "f"(s1),
- [s2] "f"(s2), [s3] "f"(s3)
- : "memory");
- return diff;
-}
-
-uint32_t SumSquareError_MMI(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t sse = 0u;
- uint32_t sse_hi = 0u, sse_lo = 0u;
-
- uint64_t src1, src2;
- uint64_t diff, diff_hi, diff_lo;
- uint64_t sse_sum, sse_tmp;
-
- const uint64_t mask = 0x0ULL;
-
- __asm__ volatile(
- "xor %[sse_sum], %[sse_sum], %[sse_sum] \n\t"
-
- "1: \n\t"
- "ldc1 %[src1], 0x00(%[src_a]) \n\t"
- "ldc1 %[src2], 0x00(%[src_b]) \n\t"
- "pasubub %[diff], %[src1], %[src2] \n\t"
- "punpcklbh %[diff_lo], %[diff], %[mask] \n\t"
- "punpckhbh %[diff_hi], %[diff], %[mask] \n\t"
- "pmaddhw %[sse_tmp], %[diff_lo], %[diff_lo] \n\t"
- "paddw %[sse_sum], %[sse_sum], %[sse_tmp] \n\t"
- "pmaddhw %[sse_tmp], %[diff_hi], %[diff_hi] \n\t"
- "paddw %[sse_sum], %[sse_sum], %[sse_tmp] \n\t"
-
- "daddiu %[src_a], %[src_a], 0x08 \n\t"
- "daddiu %[src_b], %[src_b], 0x08 \n\t"
- "daddiu %[count], %[count], -0x08 \n\t"
- "bnez %[count], 1b \n\t"
-
- "mfc1 %[sse_lo], %[sse_sum] \n\t"
- "mfhc1 %[sse_hi], %[sse_sum] \n\t"
- "daddu %[sse], %[sse_hi], %[sse_lo] \n\t"
- : [sse] "+&r"(sse), [diff] "=&f"(diff), [src1] "=&f"(src1),
- [src2] "=&f"(src2), [diff_lo] "=&f"(diff_lo), [diff_hi] "=&f"(diff_hi),
- [sse_sum] "=&f"(sse_sum), [sse_tmp] "=&f"(sse_tmp),
- [sse_hi] "+&r"(sse_hi), [sse_lo] "+&r"(sse_lo)
- : [src_a] "r"(src_a), [src_b] "r"(src_b), [count] "r"(count),
- [mask] "f"(mask)
- : "memory");
-
- return sse;
-}
-
-#endif // !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
diff --git a/libs/libyuv/source/compare_msa.cc b/libs/libyuv/source/compare_msa.cc
deleted file mode 100644
index 0b807d37be..0000000000
--- a/libs/libyuv/source/compare_msa.cc
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2017 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "libyuv/basic_types.h"
-
-#include "libyuv/compare_row.h"
-#include "libyuv/row.h"
-
-// This module is for GCC MSA
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#include "libyuv/macros_msa.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-uint32_t HammingDistance_MSA(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t diff = 0u;
- int i;
- v16u8 src0, src1, src2, src3;
- v2i64 vec0 = {0}, vec1 = {0};
-
- for (i = 0; i < count; i += 32) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_a, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_a, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_b, 0);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_b, 16);
- src0 ^= src2;
- src1 ^= src3;
- vec0 += __msa_pcnt_d((v2i64)src0);
- vec1 += __msa_pcnt_d((v2i64)src1);
- src_a += 32;
- src_b += 32;
- }
-
- vec0 += vec1;
- diff = (uint32_t)__msa_copy_u_w((v4i32)vec0, 0);
- diff += (uint32_t)__msa_copy_u_w((v4i32)vec0, 2);
- return diff;
-}
-
-uint32_t SumSquareError_MSA(const uint8_t* src_a,
- const uint8_t* src_b,
- int count) {
- uint32_t sse = 0u;
- int i;
- v16u8 src0, src1, src2, src3;
- v8i16 vec0, vec1, vec2, vec3;
- v4i32 reg0 = {0}, reg1 = {0}, reg2 = {0}, reg3 = {0};
- v2i64 tmp0;
-
- for (i = 0; i < count; i += 32) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_a, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_a, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_b, 0);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_b, 16);
- vec0 = (v8i16)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- vec1 = (v8i16)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- vec2 = (v8i16)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- vec3 = (v8i16)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- vec0 = __msa_hsub_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = __msa_hsub_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = __msa_hsub_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = __msa_hsub_u_h((v16u8)vec3, (v16u8)vec3);
- reg0 = __msa_dpadd_s_w(reg0, vec0, vec0);
- reg1 = __msa_dpadd_s_w(reg1, vec1, vec1);
- reg2 = __msa_dpadd_s_w(reg2, vec2, vec2);
- reg3 = __msa_dpadd_s_w(reg3, vec3, vec3);
- src_a += 32;
- src_b += 32;
- }
-
- reg0 += reg1;
- reg2 += reg3;
- reg0 += reg2;
- tmp0 = __msa_hadd_s_d(reg0, reg0);
- sse = (uint32_t)__msa_copy_u_w((v4i32)tmp0, 0);
- sse += (uint32_t)__msa_copy_u_w((v4i32)tmp0, 2);
- return sse;
-}
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
-
-#endif // !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
diff --git a/libs/libyuv/source/compare_neon.cc b/libs/libyuv/source/compare_neon.cc
index 2a2181e0cb..529571822e 100644
--- a/libs/libyuv/source/compare_neon.cc
+++ b/libs/libyuv/source/compare_neon.cc
@@ -8,16 +8,22 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/basic_types.h"
-
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
+#include "libyuv/video_common.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
@@ -29,24 +35,24 @@ uint32_t HammingDistance_NEON(const uint8_t* src_a,
uint32_t diff;
asm volatile(
- "vmov.u16 q4, #0 \n" // accumulator
+ "vmov.u16 q4, #0 \n" // accumulator
- "1: \n"
- "vld1.8 {q0, q1}, [%0]! \n"
- "vld1.8 {q2, q3}, [%1]! \n"
- "veor.32 q0, q0, q2 \n"
- "veor.32 q1, q1, q3 \n"
- "vcnt.i8 q0, q0 \n"
- "vcnt.i8 q1, q1 \n"
- "subs %2, %2, #32 \n"
- "vadd.u8 q0, q0, q1 \n" // 16 byte counts
- "vpadal.u8 q4, q0 \n" // 8 shorts
- "bgt 1b \n"
-
- "vpaddl.u16 q0, q4 \n" // 4 ints
- "vpadd.u32 d0, d0, d1 \n"
- "vpadd.u32 d0, d0, d0 \n"
- "vmov.32 %3, d0[0] \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n"
+ "vld1.8 {q2, q3}, [%1]! \n"
+ "veor.32 q0, q0, q2 \n"
+ "veor.32 q1, q1, q3 \n"
+ "vcnt.i8 q0, q0 \n"
+ "vcnt.i8 q1, q1 \n"
+ "subs %2, %2, #32 \n"
+ "vadd.u8 q0, q0, q1 \n" // 16 byte counts
+ "vpadal.u8 q4, q0 \n" // 8 shorts
+ "bgt 1b \n"
+
+ "vpaddl.u16 q0, q4 \n" // 4 ints
+ "vpadd.u32 d0, d0, d1 \n"
+ "vpadd.u32 d0, d0, d0 \n"
+ "vmov.32 %3, d0[0] \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
:
@@ -59,35 +65,267 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a,
int count) {
uint32_t sse;
asm volatile(
- "vmov.u8 q8, #0 \n"
- "vmov.u8 q10, #0 \n"
- "vmov.u8 q9, #0 \n"
- "vmov.u8 q11, #0 \n"
+ "vmov.u8 q8, #0 \n"
+ "vmov.u8 q10, #0 \n"
+ "vmov.u8 q9, #0 \n"
+ "vmov.u8 q11, #0 \n"
- "1: \n"
- "vld1.8 {q0}, [%0]! \n"
- "vld1.8 {q1}, [%1]! \n"
- "subs %2, %2, #16 \n"
- "vsubl.u8 q2, d0, d2 \n"
- "vsubl.u8 q3, d1, d3 \n"
- "vmlal.s16 q8, d4, d4 \n"
- "vmlal.s16 q9, d6, d6 \n"
- "vmlal.s16 q10, d5, d5 \n"
- "vmlal.s16 q11, d7, d7 \n"
- "bgt 1b \n"
-
- "vadd.u32 q8, q8, q9 \n"
- "vadd.u32 q10, q10, q11 \n"
- "vadd.u32 q11, q8, q10 \n"
- "vpaddl.u32 q1, q11 \n"
- "vadd.u64 d0, d2, d3 \n"
- "vmov.32 %3, d0[0] \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n"
+ "vld1.8 {q1}, [%1]! \n"
+ "subs %2, %2, #16 \n"
+ "vsubl.u8 q2, d0, d2 \n"
+ "vsubl.u8 q3, d1, d3 \n"
+ "vmlal.s16 q8, d4, d4 \n"
+ "vmlal.s16 q9, d6, d6 \n"
+ "vmlal.s16 q10, d5, d5 \n"
+ "vmlal.s16 q11, d7, d7 \n"
+ "bgt 1b \n"
+
+ "vadd.u32 q8, q8, q9 \n"
+ "vadd.u32 q10, q10, q11 \n"
+ "vadd.u32 q11, q8, q10 \n"
+ "vpaddl.u32 q1, q11 \n"
+ "vadd.u64 d0, d2, d3 \n"
+ "vmov.32 %3, d0[0] \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
:
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
return sse;
}
+#define SSIM_ROW_NEON32(LOAD_A, LOAD_B, PADD_U8, PADD_U16) \
+ LOAD_A \
+ LOAD_B \
+ PADD_U8 " q8, q0 \n" PADD_U8 \
+ " q9, q1 \n" \
+ "vmull.u8 q2, d0, d0 \n" \
+ "vmull.u8 q3, d1, d1 \n" PADD_U16 \
+ " q10, q2 \n" PADD_U16 \
+ " q11, q3 \n" \
+ "vmull.u8 q2, d2, d2 \n" \
+ "vmull.u8 q3, d3, d3 \n" \
+ "vpadal.u16 q10, q2 \n" \
+ "vpadal.u16 q11, q3 \n" \
+ "vmull.u8 q2, d0, d2 \n" \
+ "vmull.u8 q3, d1, d3 \n" PADD_U16 \
+ " q12, q2 \n" PADD_U16 \
+ " q13, q3 \n"
+
+void Ssim4x4Row_NEON(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ asm volatile(
+ "1: \n" //
+ SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
+ "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpaddl.u8",
+ "vpaddl.u16") //
+ SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
+ "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8",
+ "vpadal.u16") //
+ SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
+ "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8",
+ "vpadal.u16") //
+ SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]]\n",
+ "vld1.8 {d2, d3}, [%[src_b]]\n", "vpadal.u8",
+ "vpadal.u16") //
+ "sub %[src_a], %[src_a], %[stride_a], lsl #1 \n"
+ "sub %[src_a], %[src_a], %[stride_a] \n"
+ "add %[src_a], %[src_a], #16 \n"
+ "sub %[src_b], %[src_b], %[stride_b], lsl #1 \n"
+ "sub %[src_b], %[src_b], %[stride_b] \n"
+ "add %[src_b], %[src_b], #16 \n"
+ "vpaddl.u16 q8, q8 \n"
+ "vpaddl.u16 q9, q9 \n"
+ "vpadd.u32 d20, d20, d21 \n"
+ "vpadd.u32 d21, d22, d23 \n"
+ "vpadd.u32 d22, d24, d25 \n"
+ "vpadd.u32 d23, d26, d27 \n"
+ "subs %[width], %[width], #16 \n"
+ "vst4.32 {d16, d18, d20, d22}, [%[dst]]! \n"
+ "vst4.32 {d17, d19, d21, d23}, [%[dst]]! \n"
+ "bgt 1b \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
+ [width] "+r"(width)
+ : [stride_a] "r"(stride_a), [stride_b] "r"(stride_b)
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
+ "q13");
+}
+#undef SSIM_ROW_NEON32
+
+static const uvec32 kDjb2Multiplicands[] = {
+ {0x0c3525e1, // 33^15
+ 0xa3476dc1, // 33^14
+ 0x3b4039a1, // 33^13
+ 0x4f5f0981}, // 33^12
+ {0x30f35d61, // 33^11
+ 0x855cb541, // 33^10
+ 0x040a9121, // 33^9
+ 0x747c7101}, // 33^8
+ {0xec41d4e1, // 33^7
+ 0x4cfa3cc1, // 33^6
+ 0x025528a1, // 33^5
+ 0x00121881}, // 33^4
+ {0x00008c61, // 33^3
+ 0x00000441, // 33^2
+ 0x00000021, // 33^1
+ 0x00000001}, // 33^0
+};
+
+uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ const uint32_t c16 = 0x92d9e201; // 33^16
+ const void* muls0 = &kDjb2Multiplicands[0];
+ const void* muls1 = &kDjb2Multiplicands[2];
+ uint32_t tmp;
+ asm volatile(
+ "vld1.32 {d16, d17, d18, d19}, [%[muls0]] \n"
+ "vld1.32 {d20, d21, d22, d23}, [%[muls1]] \n"
+ "1: \n"
+ "vld1.8 {d0, d1}, [%[src]]! \n"
+ "subs %[count], %[count], #16 \n"
+ "vmovl.u8 q1, d0 \n"
+ "vmovl.u8 q2, d1 \n"
+ "vmovl.u16 q0, d2 \n"
+ "vmovl.u16 q1, d3 \n"
+ "vmovl.u16 q3, d5 \n"
+ "vmovl.u16 q2, d4 \n"
+ "vmul.i32 q0, q0, q8 \n"
+ "vmla.i32 q0, q1, q9 \n"
+ "vmla.i32 q0, q2, q10 \n"
+ "vmla.i32 q0, q3, q11 \n"
+ "vpadd.i32 d0, d0, d1 \n"
+ "vpadd.i32 d0, d0, d0 \n"
+ "vmov.32 %[tmp], d0[0] \n"
+ "mla %[hash], %[hash], %[c16], %[tmp] \n"
+ "bgt 1b \n"
+ :
+ [src] "+r"(src), [hash] "+r"(hash), [count] "+r"(count), [tmp] "=&r"(tmp)
+ : [muls0] "r"(muls0), [muls1] "r"(muls1), [c16] "r"(c16)
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
+ return hash;
+}
+
+float SsimWindowRow_NEON(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ asm volatile(
+ "vmov.i32 q12, #0 \n"
+ "vdup.32 q13, %[c1] \n"
+ "vdup.32 q14, %[c2] \n"
+ "1: \n"
+ "vld4.32 {d0, d2, d4, d6}, [%[row0]]! \n"
+ "vld4.32 {d1, d3, d5, d7}, [%[row0]]! \n"
+ "vld4.32 {d16, d18, d20, d22}, [%[row0_next]]! \n"
+ "vld4.32 {d17, d19, d21, d23}, [%[row0_next]]! \n"
+ "vadd.i32 q0, q0, q8 \n"
+ "vadd.i32 q1, q1, q9 \n"
+ "vadd.i32 q2, q2, q10 \n"
+ "vadd.i32 q3, q3, q11 \n"
+ "vld4.32 {d16, d18, d20, d22}, [%[row1]]! \n"
+ "vld4.32 {d17, d19, d21, d23}, [%[row1]]! \n"
+ "vadd.i32 q0, q0, q8 \n"
+ "vadd.i32 q1, q1, q9 \n"
+ "vadd.i32 q2, q2, q10 \n"
+ "vadd.i32 q3, q3, q11 \n"
+ "vld4.32 {d16, d18, d20, d22}, [%[row1_next]]! \n"
+ "vld4.32 {d17, d19, d21, d23}, [%[row1_next]]! \n"
+ "vadd.i32 q0, q0, q8 \n"
+ "vadd.i32 q1, q1, q9 \n"
+ "vadd.i32 q2, q2, q10 \n"
+ "vadd.i32 q3, q3, q11 \n"
+ "vmul.i32 q8, q0, q1 \n"
+ "vshl.i32 q8, q8, #1 \n"
+ "vmul.i32 q9, q0, q0 \n"
+ "vmla.i32 q9, q1, q1 \n"
+ "vshl.i32 q3, q3, #7 \n"
+ "vsub.i32 q3, q3, q8 \n"
+ "vadd.i32 q8, q8, q13 \n"
+ "vadd.i32 q3, q3, q14 \n"
+ "vshl.i32 q2, q2, #6 \n"
+ "vsub.i32 q2, q2, q9 \n"
+ "vadd.i32 q9, q9, q13 \n"
+ "vadd.i32 q2, q2, q14 \n"
+ "vcvt.f32.s32 q8, q8 \n"
+ "vcvt.f32.s32 q3, q3 \n"
+ "vcvt.f32.s32 q9, q9 \n"
+ "vcvt.f32.s32 q2, q2 \n"
+ "vmul.f32 q3, q8, q3 \n"
+ "vmul.f32 q2, q9, q2 \n"
+ "vdiv.f32 s12, s12, s8 \n"
+ "vdiv.f32 s13, s13, s9 \n"
+ "vdiv.f32 s14, s14, s10 \n"
+ "vdiv.f32 s15, s15, s11 \n"
+ "subs %[num_windows], %[num_windows], #4 \n"
+ "vadd.f32 q12, q12, q3 \n"
+ "bgt 1b \n"
+ "vpadd.f32 d0, d24, d25 \n"
+ "vpadd.f32 d0, d0, d0 \n"
+ "vmov.f32 %[total], s0 \n"
+ : [row0] "+r"(row0), [row0_next] "+r"(row0_next), [row1] "+r"(row1),
+ [row1_next] "+r"(row1_next), [num_windows] "+r"(num_windows),
+ [total] "=r"(total)
+ : [c1] "r"(26634), [c2] "r"(239708)
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
+ "q13", "q14");
+ return total;
+}
+
+uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) {
+ uint32_t diff_lo = 0;
+ uint32_t diff_hi = 0;
+ uint32_t tmp;
+ asm volatile(
+ "vmov.i32 q2, #0x00ff0000 \n"
+ "vorr.i32 q2, #0x0000ff00 \n"
+ "vmov.i8 q3, #0xff \n"
+ "subs %[width], %[width], #4 \n"
+ "blt 2f \n"
+ "1: \n"
+ "vld1.8 {q0}, [%[argb]]! \n"
+ "vorr q0, q0, q2 \n"
+ "vceq.i8 q0, q0, q3 \n"
+ "vmvn q0, q0 \n"
+ "vshrn.i16 d0, q0, #4 \n"
+ "vmov %[diff_lo], %[diff_hi], d0 \n"
+ "orrs %[tmp], %[diff_lo], %[diff_hi] \n"
+ "bne 99f \n"
+ "subs %[width], %[width], #4 \n"
+ "bge 1b \n"
+ "2: \n"
+ "adds %[width], %[width], #4 \n"
+ "ble 99f \n"
+ "3: \n"
+ "vld1.32 {d0[0]}, [%[argb]]! \n"
+ "vorr d0, d0, d4 \n"
+ "vceq.i8 d0, d0, d6 \n"
+ "vmvn d0, d0 \n"
+ "vshrn.i16 d0, q0, #4 \n"
+ "vmov.u16 %[diff_lo], d0[0] \n"
+ "cmp %[diff_lo], #0 \n"
+ "bne 99f \n"
+ "subs %[width], %[width], #1 \n"
+ "bgt 3b \n"
+ "99: \n"
+ : [argb] "+r"(argb), [width] "+r"(width), [diff_lo] "+r"(diff_lo),
+ [diff_hi] "+r"(diff_hi), [tmp] "=&r"(tmp)
+ :
+ : "memory", "cc", "q0", "q2", "q3");
+ if (diff_lo) {
+ return (__builtin_ctz(diff_lo) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ if (diff_hi) {
+ return (__builtin_ctz(diff_hi) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/compare_neon64.cc b/libs/libyuv/source/compare_neon64.cc
index 6e8f672ab7..c1b4741589 100644
--- a/libs/libyuv/source/compare_neon64.cc
+++ b/libs/libyuv/source/compare_neon64.cc
@@ -8,16 +8,22 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/basic_types.h"
-
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
+#include "libyuv/video_common.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
// 256 bits at a time
@@ -27,25 +33,27 @@ uint32_t HammingDistance_NEON(const uint8_t* src_a,
int count) {
uint32_t diff;
asm volatile(
- "movi v4.8h, #0 \n"
+ "movi v4.8h, #0 \n"
- "1: \n"
- "ld1 {v0.16b, v1.16b}, [%0], #32 \n"
- "ld1 {v2.16b, v3.16b}, [%1], #32 \n"
- "eor v0.16b, v0.16b, v2.16b \n"
- "eor v1.16b, v1.16b, v3.16b \n"
- "cnt v0.16b, v0.16b \n"
- "cnt v1.16b, v1.16b \n"
- "subs %w2, %w2, #32 \n"
- "add v0.16b, v0.16b, v1.16b \n"
- "uadalp v4.8h, v0.16b \n"
- "b.gt 1b \n"
-
- "uaddlv s4, v4.8h \n"
- "fmov %w3, s4 \n"
+ "1: \n"
+ "ld1 {v0.16b, v1.16b}, [%0], #32 \n"
+ "ld1 {v2.16b, v3.16b}, [%1], #32 \n"
+ "eor v0.16b, v0.16b, v2.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "eor v1.16b, v1.16b, v3.16b \n"
+ "cnt v0.16b, v0.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "cnt v1.16b, v1.16b \n"
+ "subs %w2, %w2, #32 \n"
+ "add v0.16b, v0.16b, v1.16b \n"
+ "uadalp v4.8h, v0.16b \n"
+ "b.gt 1b \n"
+
+ "uaddlv s4, v4.8h \n"
+ "fmov %w3, s4 \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
:
- : "cc", "v0", "v1", "v2", "v3", "v4");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4");
return diff;
}
@@ -54,34 +62,395 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a,
int count) {
uint32_t sse;
asm volatile(
- "eor v16.16b, v16.16b, v16.16b \n"
- "eor v18.16b, v18.16b, v18.16b \n"
- "eor v17.16b, v17.16b, v17.16b \n"
- "eor v19.16b, v19.16b, v19.16b \n"
+ "movi v16.16b, #0 \n"
+ "movi v17.16b, #0 \n"
+ "movi v18.16b, #0 \n"
+ "movi v19.16b, #0 \n"
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n"
- "ld1 {v1.16b}, [%1], #16 \n"
- "subs %w2, %w2, #16 \n"
- "usubl v2.8h, v0.8b, v1.8b \n"
- "usubl2 v3.8h, v0.16b, v1.16b \n"
- "smlal v16.4s, v2.4h, v2.4h \n"
- "smlal v17.4s, v3.4h, v3.4h \n"
- "smlal2 v18.4s, v2.8h, v2.8h \n"
- "smlal2 v19.4s, v3.8h, v3.8h \n"
- "b.gt 1b \n"
-
- "add v16.4s, v16.4s, v17.4s \n"
- "add v18.4s, v18.4s, v19.4s \n"
- "add v19.4s, v16.4s, v18.4s \n"
- "addv s0, v19.4s \n"
- "fmov %w3, s0 \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n"
+ "ld1 {v1.16b}, [%1], #16 \n"
+ "subs %w2, %w2, #16 \n"
+ "usubl v2.8h, v0.8b, v1.8b \n"
+ "usubl2 v3.8h, v0.16b, v1.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "smlal v16.4s, v2.4h, v2.4h \n"
+ "smlal v17.4s, v3.4h, v3.4h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "smlal2 v18.4s, v2.8h, v2.8h \n"
+ "smlal2 v19.4s, v3.8h, v3.8h \n"
+ "b.gt 1b \n"
+
+ "add v16.4s, v16.4s, v17.4s \n"
+ "add v18.4s, v18.4s, v19.4s \n"
+ "add v19.4s, v16.4s, v18.4s \n"
+ "addv s0, v19.4s \n"
+ "fmov %w3, s0 \n"
+ : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ return sse;
+}
+
+static const uvec32 kDjb2Multiplicands[] = {
+ {0x0c3525e1, // 33^15
+ 0xa3476dc1, // 33^14
+ 0x3b4039a1, // 33^13
+ 0x4f5f0981}, // 33^12
+ {0x30f35d61, // 33^11
+ 0x855cb541, // 33^10
+ 0x040a9121, // 33^9
+ 0x747c7101}, // 33^8
+ {0xec41d4e1, // 33^7
+ 0x4cfa3cc1, // 33^6
+ 0x025528a1, // 33^5
+ 0x00121881}, // 33^4
+ {0x00008c61, // 33^3
+ 0x00000441, // 33^2
+ 0x00000021, // 33^1
+ 0x00000001}, // 33^0
+};
+
+static const uvec32 kDjb2WidenIndices[] = {
+ {0xffffff00U, 0xffffff01U, 0xffffff02U, 0xffffff03U},
+ {0xffffff04U, 0xffffff05U, 0xffffff06U, 0xffffff07U},
+ {0xffffff08U, 0xffffff09U, 0xffffff0aU, 0xffffff0bU},
+ {0xffffff0cU, 0xffffff0dU, 0xffffff0eU, 0xffffff0fU},
+};
+
+uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ const uint32_t c16 = 0x92d9e201; // 33^16
+ uint32_t tmp, tmp2;
+ asm("ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[kIdx]] \n"
+ "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[kMuls]] \n"
+
+ // count is always a multiple of 16.
+ // maintain two accumulators, reduce and then final sum in scalar since
+ // this has better performance on little cores.
+ "1: \n"
+ "ldr q0, [%[src]], #16 \n"
+ "subs %w[count], %w[count], #16 \n"
+ "tbl v3.16b, {v0.16b}, v19.16b \n"
+ "tbl v2.16b, {v0.16b}, v18.16b \n"
+ "tbl v1.16b, {v0.16b}, v17.16b \n"
+ "tbl v0.16b, {v0.16b}, v16.16b \n"
+ "mul v3.4s, v3.4s, v7.4s \n"
+ "mul v2.4s, v2.4s, v6.4s \n"
+ "mla v3.4s, v1.4s, v5.4s \n"
+ "mla v2.4s, v0.4s, v4.4s \n"
+ "addv s1, v3.4s \n"
+ "addv s0, v2.4s \n"
+ "fmov %w[tmp2], s1 \n"
+ "fmov %w[tmp], s0 \n"
+ "add %w[tmp], %w[tmp], %w[tmp2] \n"
+ "madd %w[hash], %w[hash], %w[c16], %w[tmp] \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [hash] "+r"(hash), // %[hash]
+ [count] "+r"(count), // %[count]
+ [tmp] "=&r"(tmp), // %[tmp]
+ [tmp2] "=&r"(tmp2) // %[tmp2]
+ : [kMuls] "r"(kDjb2Multiplicands), // %[kMuls]
+ [kIdx] "r"(kDjb2WidenIndices), // %[kIdx]
+ [c16] "r"(c16) // %[c16]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
+ return hash;
+}
+
+uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ asm volatile(
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "movi v6.16b, #1 \n"
+
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "ldp q2, q3, [%1], #32 \n"
+ "eor v0.16b, v0.16b, v2.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "eor v1.16b, v1.16b, v3.16b \n"
+ "cnt v0.16b, v0.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "cnt v1.16b, v1.16b \n"
+ "subs %w2, %w2, #32 \n"
+ "udot v4.4s, v0.16b, v6.16b \n"
+ "udot v5.4s, v1.16b, v6.16b \n"
+ "b.gt 1b \n"
+
+ "add v0.4s, v4.4s, v5.4s \n"
+ "addv s0, v0.4s \n"
+ "fmov %w3, s0 \n"
+ : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
+ return diff;
+}
+
+uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ // count is guaranteed to be a multiple of 32.
+ uint32_t sse;
+ asm volatile(
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+
+ "1: \n"
+ "ldp q0, q2, [%0], #32 \n"
+ "ldp q1, q3, [%1], #32 \n"
+ "subs %w2, %w2, #32 \n"
+ "uabd v0.16b, v0.16b, v1.16b \n"
+ "uabd v1.16b, v2.16b, v3.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "udot v4.4s, v0.16b, v0.16b \n"
+ "udot v5.4s, v1.16b, v1.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "b.gt 1b \n"
+
+ "add v0.4s, v4.4s, v5.4s \n"
+ "addv s0, v0.4s \n"
+ "fmov %w3, s0 \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
:
- : "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5");
return sse;
}
+#define SSIM_ROW_NEON64(VA, VB, PADD_U8, PADD_U16) \
+ PADD_U8 " v4.8h, " #VA ".16b \n" PADD_U8 \
+ " v5.8h, " #VB \
+ ".16b \n" \
+ "umull v6.8h, " #VA ".8b, " #VA \
+ ".8b \n" \
+ "umull2 v7.8h, " #VA ".16b, " #VA ".16b \n" PADD_U16 \
+ " v20.4s, v6.8h \n" PADD_U16 \
+ " v21.4s, v7.8h \n" \
+ "umull v6.8h, " #VB ".8b, " #VB \
+ ".8b \n" \
+ "umull2 v7.8h, " #VB ".16b, " #VB \
+ ".16b \n" \
+ "uadalp v20.4s, v6.8h \n" \
+ "uadalp v21.4s, v7.8h \n" \
+ "umull v6.8h, " #VA ".8b, " #VB \
+ ".8b \n" \
+ "umull2 v7.8h, " #VA ".16b, " #VB ".16b \n" PADD_U16 \
+ " v22.4s, v6.8h \n" PADD_U16 \
+ " v23.4s, v7.8h \n"
+
+void Ssim4x4Row_NEON(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ asm volatile(
+ "1: \n"
+ "ldr q0, [%[a0]], #16 \n"
+ "ldr q1, [%[b0]], #16 \n"
+ "ldr q2, [%[a1]], #16 \n"
+ "ldr q3, [%[b1]], #16 \n" //
+ SSIM_ROW_NEON64(v0, v1, "uaddlp", "uaddlp") //
+ SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") //
+ "ldr q0, [%[a2]], #16 \n"
+ "ldr q1, [%[b2]], #16 \n"
+ "ldr q2, [%[a3]], #16 \n"
+ "ldr q3, [%[b3]], #16 \n" //
+ SSIM_ROW_NEON64(v0, v1, "uadalp", "uadalp") //
+ SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") //
+ "uaddlp v16.4s, v4.8h \n"
+ "uaddlp v17.4s, v5.8h \n"
+ "addp v18.4s, v20.4s, v21.4s \n"
+ "addp v19.4s, v22.4s, v23.4s \n"
+ "subs %w[width], %w[width], #16 \n"
+ "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n"
+ "b.gt 1b \n"
+ : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3),
+ [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3),
+ [dst] "+r"(dst), [width] "+r"(width)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#undef SSIM_ROW_NEON64
+
+void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ asm volatile(
+ "movi v6.16b, #1 \n"
+ "1: \n"
+ "movi v16.4s, #0 \n"
+ "movi v17.4s, #0 \n"
+ "movi v18.4s, #0 \n"
+ "movi v19.4s, #0 \n"
+ "ldr q0, [%[a0]], #16 \n"
+ "ldr q1, [%[b0]], #16 \n"
+ "ldr q2, [%[a1]], #16 \n"
+ "ldr q3, [%[b1]], #16 \n"
+ "udot v16.4s, v0.16b, v6.16b \n"
+ "udot v17.4s, v1.16b, v6.16b \n"
+ "udot v18.4s, v0.16b, v0.16b \n"
+ "udot v18.4s, v1.16b, v1.16b \n"
+ "udot v19.4s, v0.16b, v1.16b \n"
+ "udot v16.4s, v2.16b, v6.16b \n"
+ "udot v17.4s, v3.16b, v6.16b \n"
+ "udot v18.4s, v2.16b, v2.16b \n"
+ "udot v18.4s, v3.16b, v3.16b \n"
+ "udot v19.4s, v2.16b, v3.16b \n"
+ "ldr q0, [%[a2]], #16 \n"
+ "ldr q1, [%[b2]], #16 \n"
+ "ldr q2, [%[a3]], #16 \n"
+ "ldr q3, [%[b3]], #16 \n"
+ "udot v16.4s, v0.16b, v6.16b \n"
+ "udot v17.4s, v1.16b, v6.16b \n"
+ "udot v18.4s, v0.16b, v0.16b \n"
+ "udot v18.4s, v1.16b, v1.16b \n"
+ "udot v19.4s, v0.16b, v1.16b \n"
+ "udot v16.4s, v2.16b, v6.16b \n"
+ "udot v17.4s, v3.16b, v6.16b \n"
+ "udot v18.4s, v2.16b, v2.16b \n"
+ "udot v18.4s, v3.16b, v3.16b \n"
+ "udot v19.4s, v2.16b, v3.16b \n"
+ "subs %w[width], %w[width], #16 \n"
+ "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n"
+ "b.gt 1b \n"
+ : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3),
+ [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3),
+ [dst] "+r"(dst), [width] "+r"(width)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v6", "v16", "v17", "v18",
+ "v19");
+}
+
+float SsimWindowRow_NEON(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ asm volatile(
+ "movi v24.4s, #0 \n"
+ "dup v25.4s, %w[c1] \n"
+ "dup v26.4s, %w[c2] \n"
+ "1: \n"
+ "ld4 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[row0]], #64 \n"
+ "ld4 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[row0_next]], #64 \n"
+ "ld4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[row1]], #64 \n"
+ "ld4 {v20.4s, v21.4s, v22.4s, v23.4s}, [%[row1_next]], #64 \n"
+ "add v0.4s, v0.4s, v4.4s \n"
+ "add v1.4s, v1.4s, v5.4s \n"
+ "add v2.4s, v2.4s, v6.4s \n"
+ "add v3.4s, v3.4s, v7.4s \n"
+ "add v16.4s, v16.4s, v20.4s \n"
+ "add v17.4s, v17.4s, v21.4s \n"
+ "add v18.4s, v18.4s, v22.4s \n"
+ "add v19.4s, v19.4s, v23.4s \n"
+ "add v0.4s, v0.4s, v16.4s \n"
+ "add v1.4s, v1.4s, v17.4s \n"
+ "add v2.4s, v2.4s, v18.4s \n"
+ "add v3.4s, v3.4s, v19.4s \n"
+ "mul v4.4s, v0.4s, v1.4s \n"
+ "shl v4.4s, v4.4s, #1 \n"
+ "mul v5.4s, v0.4s, v0.4s \n"
+ "mla v5.4s, v1.4s, v1.4s \n"
+ "shl v3.4s, v3.4s, #7 \n"
+ "sub v3.4s, v3.4s, v4.4s \n"
+ "add v4.4s, v4.4s, v25.4s \n"
+ "add v3.4s, v3.4s, v26.4s \n"
+ "shl v2.4s, v2.4s, #6 \n"
+ "sub v2.4s, v2.4s, v5.4s \n"
+ "add v5.4s, v5.4s, v25.4s \n"
+ "add v2.4s, v2.4s, v26.4s \n"
+ "scvtf v4.4s, v4.4s \n"
+ "scvtf v3.4s, v3.4s \n"
+ "scvtf v5.4s, v5.4s \n"
+ "scvtf v2.4s, v2.4s \n"
+ "fmul v4.4s, v4.4s, v3.4s \n"
+ "fmul v5.4s, v5.4s, v2.4s \n"
+ "fdiv v4.4s, v4.4s, v5.4s \n"
+ "subs %w[num_windows], %w[num_windows], #4 \n"
+ "fadd v24.4s, v24.4s, v4.4s \n"
+ "b.gt 1b \n"
+ "faddp v24.4s, v24.4s, v24.4s \n"
+ "faddp %s[total], v24.2s \n"
+ : [row0] "+r"(row0), [row0_next] "+r"(row0_next), [row1] "+r"(row1),
+ [row1_next] "+r"(row1_next), [num_windows] "+r"(num_windows),
+ [total] "=w"(total)
+ : [c1] "r"(26634), [c2] "r"(239708)
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26");
+ return total;
+}
+
+uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) {
+ uint64_t diff = 0;
+ asm volatile(
+ "movi v2.2d, #0x00ffff0000ffff00 \n"
+ "movi v3.16b, #0xff \n"
+ "subs %w[width], %w[width], #4 \n"
+ "b.lt 2f \n"
+ "1: \n"
+ "ldr q0, [%[argb]], #16 \n"
+ "orr v0.16b, v0.16b, v2.16b \n"
+ "cmeq v0.16b, v0.16b, v3.16b \n"
+ "mvn v0.16b, v0.16b \n"
+ "shrn v0.8b, v0.8h, #4 \n"
+ "fmov %x[diff], d0 \n"
+ "cbnz %x[diff], 99f \n"
+ "subs %w[width], %w[width], #4 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w[width], %w[width], #4 \n"
+ "b.le 99f \n"
+ "3: \n"
+ "ldr s0, [%[argb]], #4 \n"
+ "orr v0.8b, v0.8b, v2.8b \n"
+ "cmeq v0.8b, v0.8b, v3.8b \n"
+ "mvn v0.8b, v0.8b \n"
+ "shrn v0.8b, v0.8h, #4 \n"
+ "fmov %x[diff], d0 \n"
+ "and %x[diff], %x[diff], #0xffff \n"
+ "cbnz %x[diff], 99f \n"
+ "subs %w[width], %w[width], #1 \n"
+ "b.gt 3b \n"
+ "99: \n"
+ : [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff)
+ :
+ : "memory", "cc", "v0", "v2", "v3");
+ if (diff) {
+ return (__builtin_ctzll(diff) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/compare_wasm.cc b/libs/libyuv/source/compare_wasm.cc
new file mode 100644
index 0000000000..5447cf84be
--- /dev/null
+++ b/libs/libyuv/source/compare_wasm.cc
@@ -0,0 +1,345 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+#include "libyuv/video_common.h"
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#include <wasm_simd128.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Add the 4 32 bit lanes.
+static inline uint32_t SumLanes_WASMSIMD(v128_t sum) {
+ sum = wasm_i32x4_add(sum, wasm_i64x2_shuffle(sum, sum, 1, 0));
+ sum = wasm_i32x4_add(sum, wasm_i32x4_shuffle(sum, sum, 1, 0, 3, 2));
+ return wasm_u32x4_extract_lane(sum, 0);
+}
+
+#if defined(HAS_HAMMINGDISTANCE_WASMSIMD)
+// Bit counts of 64 bytes are added as bytes, then accumulated into 32 bit
+// lanes using relaxed i8x16 dot product when available or pairwise widening
+// adds. Count is a multiple of 64.
+uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ v128_t sum = wasm_i32x4_splat(0);
+#ifdef __wasm_relaxed_simd__
+ const v128_t ones = wasm_i8x16_splat(1);
+#endif
+ while (count > 0) {
+ v128_t c0 = wasm_i8x16_popcnt(
+ wasm_v128_xor(wasm_v128_load(src_a), wasm_v128_load(src_b)));
+ v128_t c1 = wasm_i8x16_popcnt(
+ wasm_v128_xor(wasm_v128_load(src_a + 16), wasm_v128_load(src_b + 16)));
+ v128_t c2 = wasm_i8x16_popcnt(
+ wasm_v128_xor(wasm_v128_load(src_a + 32), wasm_v128_load(src_b + 32)));
+ v128_t c3 = wasm_i8x16_popcnt(
+ wasm_v128_xor(wasm_v128_load(src_a + 48), wasm_v128_load(src_b + 48)));
+ v128_t c = wasm_i8x16_add(wasm_i8x16_add(c0, c1), wasm_i8x16_add(c2, c3));
+#ifdef __wasm_relaxed_simd__
+ sum = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(c, ones, sum);
+#else
+ sum = wasm_i32x4_add(sum, wasm_u32x4_extadd_pairwise_u16x8(
+ wasm_u16x8_extadd_pairwise_u8x16(c)));
+#endif
+ src_a += 64;
+ src_b += 64;
+ count -= 64;
+ }
+ return SumLanes_WASMSIMD(sum);
+}
+#endif // HAS_HAMMINGDISTANCE_WASMSIMD
+
+#if defined(HAS_SUMSQUAREERROR_WASMSIMD)
+// Differences are widened to signed 16 bits and squared with i32x4.dot.
+// Count is a multiple of 16.
+uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ v128_t sum = wasm_i32x4_splat(0);
+ while (count > 0) {
+ v128_t a = wasm_v128_load(src_a);
+ v128_t b = wasm_v128_load(src_b);
+ v128_t lo = wasm_i16x8_sub(wasm_u16x8_extend_low_u8x16(a),
+ wasm_u16x8_extend_low_u8x16(b));
+ v128_t hi = wasm_i16x8_sub(wasm_u16x8_extend_high_u8x16(a),
+ wasm_u16x8_extend_high_u8x16(b));
+ sum = wasm_i32x4_add(sum, wasm_i32x4_add(wasm_i32x4_dot_i16x8(lo, lo),
+ wasm_i32x4_dot_i16x8(hi, hi)));
+ src_a += 16;
+ src_b += 16;
+ count -= 16;
+ }
+ return SumLanes_WASMSIMD(sum);
+}
+#endif // HAS_SUMSQUAREERROR_WASMSIMD
+
+#if defined(HAS_HASHDJB2_WASMSIMD)
+static const uint32_t kHashMul_WASMSIMD[24] = {
+ // 33 ^ (30 - 2 * i) for byte pair i of 32 bytes.
+ 0xccc4cfc1, 0xd61beb81, 0x13791741, 0xac185301, // 33 ^ 30 .. 24
+ 0xc8359ec1, 0x510cfa81, 0xb0da6641, 0x92d9e201, // 33 ^ 22 .. 16
+ 0xa3476dc1, 0x4f5f0981, 0x855cb541, 0x747c7101, // 33 ^ 14 .. 8
+ 0x4cfa3cc1, 0x00121881, 0x00000441, 0x00000001, // 33 ^ 6 .. 0
+ 0x1137c401, 0x1137c401, 0x1137c401, 0x1137c401, // 33 ^ 32
+ 0x00010021, 0x00010021, 0x00010021, 0x00010021, // 16 bit 33, 1
+};
+
+// Sum of 16 bytes times 33 ^ n, as 4 lanes. Byte pairs are combined as
+// src[2 * i] * 33 + src[2 * i + 1] with i32x4.dot, then multiplied by
+// 33 ^ n for the pair.
+static inline v128_t HashMul16_WASMSIMD(const uint8_t* src,
+ const v128_t* m,
+ v128_t k33) {
+ v128_t s = wasm_v128_load(src);
+ v128_t lo = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_low_u8x16(s), k33);
+ v128_t hi = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_high_u8x16(s), k33);
+ return wasm_i32x4_add(wasm_i32x4_mul(lo, m[0]), wasm_i32x4_mul(hi, m[1]));
+}
+
+// HashDjb2 of n bytes is hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
+// Products are summed per lane, multiplying the sums by 33 ^ 32 for each 32
+// bytes, so one horizontal add is done at the end.
+// Count is a multiple of 16.
+uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) {
+ const v128_t* m = (const v128_t*)kHashMul_WASMSIMD;
+ v128_t sum = wasm_i32x4_make(0, 0, 0, (int32_t)seed);
+ // Load constants into Wasm locals outside the loop, instead of v128.const
+ // in the loop.
+ asm("" : "+r"(m));
+ const v128_t k33_32 = m[4];
+ const v128_t k33 = m[5];
+ if (count & 16) {
+ // Lane 3 of m[1] is 33 ^ 16, and lanes 0..2 of sum are 0.
+ sum = wasm_i32x4_add(wasm_i32x4_mul(sum, m[1]),
+ HashMul16_WASMSIMD(src, m + 2, k33));
+ src += 16;
+ count -= 16;
+ }
+ while (count > 0) {
+ sum = wasm_i32x4_add(
+ wasm_i32x4_mul(sum, k33_32),
+ wasm_i32x4_add(HashMul16_WASMSIMD(src, m, k33),
+ HashMul16_WASMSIMD(src + 16, m + 2, k33)));
+ src += 32;
+ count -= 32;
+ }
+ return SumLanes_WASMSIMD(sum);
+}
+#endif // HAS_HASHDJB2_WASMSIMD
+
+#if defined(HAS_SSIM4X4ROW_WASMSIMD)
+void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const uint8_t* src_a1 = src_a + stride_a;
+ const uint8_t* src_a2 = src_a1 + stride_a;
+ const uint8_t* src_a3 = src_a2 + stride_a;
+ const uint8_t* src_b1 = src_b + stride_b;
+ const uint8_t* src_b2 = src_b1 + stride_b;
+ const uint8_t* src_b3 = src_b2 + stride_b;
+ const v128_t mask = wasm_i16x8_splat(0x00ff);
+ for (; width > 0; width -= 16) {
+ v128_t a0 = wasm_v128_load(src_a);
+ v128_t a1 = wasm_v128_load(src_a1);
+ v128_t a2 = wasm_v128_load(src_a2);
+ v128_t a3 = wasm_v128_load(src_a3);
+ v128_t b0 = wasm_v128_load(src_b);
+ v128_t b1 = wasm_v128_load(src_b1);
+ v128_t b2 = wasm_v128_load(src_b2);
+ v128_t b3 = wasm_v128_load(src_b3);
+
+ v128_t a0_ev = wasm_v128_and(a0, mask);
+ v128_t a0_od = wasm_u16x8_shr(a0, 8);
+ v128_t a1_ev = wasm_v128_and(a1, mask);
+ v128_t a1_od = wasm_u16x8_shr(a1, 8);
+ v128_t a2_ev = wasm_v128_and(a2, mask);
+ v128_t a2_od = wasm_u16x8_shr(a2, 8);
+ v128_t a3_ev = wasm_v128_and(a3, mask);
+ v128_t a3_od = wasm_u16x8_shr(a3, 8);
+
+ v128_t b0_ev = wasm_v128_and(b0, mask);
+ v128_t b0_od = wasm_u16x8_shr(b0, 8);
+ v128_t b1_ev = wasm_v128_and(b1, mask);
+ v128_t b1_od = wasm_u16x8_shr(b1, 8);
+ v128_t b2_ev = wasm_v128_and(b2, mask);
+ v128_t b2_od = wasm_u16x8_shr(b2, 8);
+ v128_t b3_ev = wasm_v128_and(b3, mask);
+ v128_t b3_od = wasm_u16x8_shr(b3, 8);
+
+ v128_t sum_a_16 =
+ wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(a0_ev, a0_od),
+ wasm_i16x8_add(a1_ev, a1_od)),
+ wasm_i16x8_add(wasm_i16x8_add(a2_ev, a2_od),
+ wasm_i16x8_add(a3_ev, a3_od)));
+ v128_t sum_b_16 =
+ wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(b0_ev, b0_od),
+ wasm_i16x8_add(b1_ev, b1_od)),
+ wasm_i16x8_add(wasm_i16x8_add(b2_ev, b2_od),
+ wasm_i16x8_add(b3_ev, b3_od)));
+
+ v128_t sum_a = wasm_u32x4_extadd_pairwise_u16x8(sum_a_16);
+ v128_t sum_b = wasm_u32x4_extadd_pairwise_u16x8(sum_b_16);
+
+ v128_t sum_axb = wasm_i32x4_add(
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, b0_ev),
+ wasm_i32x4_dot_i16x8(a0_od, b0_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, b1_ev),
+ wasm_i32x4_dot_i16x8(a1_od, b1_od))),
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, b2_ev),
+ wasm_i32x4_dot_i16x8(a2_od, b2_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, b3_ev),
+ wasm_i32x4_dot_i16x8(a3_od, b3_od))));
+
+ v128_t sq0 =
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, a0_ev),
+ wasm_i32x4_dot_i16x8(a0_od, a0_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(b0_ev, b0_ev),
+ wasm_i32x4_dot_i16x8(b0_od, b0_od)));
+ v128_t sq1 =
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, a1_ev),
+ wasm_i32x4_dot_i16x8(a1_od, a1_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(b1_ev, b1_ev),
+ wasm_i32x4_dot_i16x8(b1_od, b1_od)));
+ v128_t sq2 =
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, a2_ev),
+ wasm_i32x4_dot_i16x8(a2_od, a2_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(b2_ev, b2_ev),
+ wasm_i32x4_dot_i16x8(b2_od, b2_od)));
+ v128_t sq3 =
+ wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, a3_ev),
+ wasm_i32x4_dot_i16x8(a3_od, a3_od)),
+ wasm_i32x4_add(wasm_i32x4_dot_i16x8(b3_ev, b3_ev),
+ wasm_i32x4_dot_i16x8(b3_od, b3_od)));
+ v128_t sum_sq_ab =
+ wasm_i32x4_add(wasm_i32x4_add(sq0, sq1), wasm_i32x4_add(sq2, sq3));
+
+ v128_t t0 = wasm_i32x4_shuffle(sum_a, sum_b, 0, 4, 1, 5);
+ v128_t t1 = wasm_i32x4_shuffle(sum_a, sum_b, 2, 6, 3, 7);
+ v128_t t2 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 0, 4, 1, 5);
+ v128_t t3 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 2, 6, 3, 7);
+
+ wasm_v128_store(dst, wasm_i64x2_shuffle(t0, t2, 0, 2));
+ wasm_v128_store(dst + 4, wasm_i64x2_shuffle(t0, t2, 1, 3));
+ wasm_v128_store(dst + 8, wasm_i64x2_shuffle(t1, t3, 0, 2));
+ wasm_v128_store(dst + 12, wasm_i64x2_shuffle(t1, t3, 1, 3));
+
+ src_a += 16;
+ src_a1 += 16;
+ src_a2 += 16;
+ src_a3 += 16;
+ src_b += 16;
+ src_b1 += 16;
+ src_b2 += 16;
+ src_b3 += 16;
+ dst += 16;
+ }
+}
+#endif // HAS_SSIM4X4ROW_WASMSIMD
+
+#if defined(HAS_SSIMWINDOWROW_WASMSIMD)
+float SsimWindowRow_WASMSIMD(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const v128_t vc1 = wasm_i32x4_splat(26634);
+ const v128_t vc2 = wasm_i32x4_splat(239708);
+ v128_t total = wasm_f32x4_splat(0.0f);
+ v128_t t0 = wasm_i32x4_add(wasm_v128_load(row0), wasm_v128_load(row1));
+ int w;
+ for (w = 0; w < num_windows; w += 4) {
+ v128_t t1 =
+ wasm_i32x4_add(wasm_v128_load(row0 + 4), wasm_v128_load(row1 + 4));
+ v128_t t2 =
+ wasm_i32x4_add(wasm_v128_load(row0 + 8), wasm_v128_load(row1 + 8));
+ v128_t t3 =
+ wasm_i32x4_add(wasm_v128_load(row0 + 12), wasm_v128_load(row1 + 12));
+ v128_t t4 =
+ wasm_i32x4_add(wasm_v128_load(row0 + 16), wasm_v128_load(row1 + 16));
+ row0 += 16;
+ row1 += 16;
+ v128_t w0 = wasm_i32x4_add(t0, t1);
+ v128_t w1 = wasm_i32x4_add(t1, t2);
+ v128_t w2 = wasm_i32x4_add(t2, t3);
+ v128_t w3 = wasm_i32x4_add(t3, t4);
+ t0 = t4;
+
+ v128_t u0 = wasm_i32x4_shuffle(w0, w1, 0, 4, 1, 5);
+ v128_t u1 = wasm_i32x4_shuffle(w0, w1, 2, 6, 3, 7);
+ v128_t u2 = wasm_i32x4_shuffle(w2, w3, 0, 4, 1, 5);
+ v128_t u3 = wasm_i32x4_shuffle(w2, w3, 2, 6, 3, 7);
+ v128_t sum_a = wasm_i64x2_shuffle(u0, u2, 0, 2);
+ v128_t sum_b = wasm_i64x2_shuffle(u0, u2, 1, 3);
+ v128_t sum_sq = wasm_i64x2_shuffle(u1, u3, 0, 2);
+ v128_t sum_axb = wasm_i64x2_shuffle(u1, u3, 1, 3);
+
+ v128_t ab2 = wasm_i32x4_shl(wasm_i32x4_dot_i16x8(sum_a, sum_b), 1);
+ v128_t a2b2 = wasm_i32x4_add(wasm_i32x4_dot_i16x8(sum_a, sum_a),
+ wasm_i32x4_dot_i16x8(sum_b, sum_b));
+ v128_t n1 = wasm_i32x4_add(ab2, vc1);
+ v128_t n2 =
+ wasm_i32x4_add(wasm_i32x4_sub(wasm_i32x4_shl(sum_axb, 7), ab2), vc2);
+ v128_t d1 = wasm_i32x4_add(a2b2, vc1);
+ v128_t d2 =
+ wasm_i32x4_add(wasm_i32x4_sub(wasm_i32x4_shl(sum_sq, 6), a2b2), vc2);
+
+ v128_t num = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(n1),
+ wasm_f32x4_convert_i32x4(n2));
+ v128_t den = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(d1),
+ wasm_f32x4_convert_i32x4(d2));
+ total = wasm_f32x4_add(total, wasm_f32x4_div(num, den));
+ }
+ total = wasm_f32x4_add(total, wasm_i64x2_shuffle(total, total, 1, 0));
+ total = wasm_f32x4_add(total, wasm_i32x4_shuffle(total, total, 1, 0, 3, 2));
+ return wasm_f32x4_extract_lane(total, 0);
+}
+#endif // HAS_SSIMWINDOWROW_WASMSIMD
+
+#if defined(HAS_ARGBDETECTROW_WASMSIMD)
+uint32_t ARGBDetectRow_WASMSIMD(const uint8_t* argb, int width) {
+ const v128_t mask_rgb = wasm_i32x4_splat(0x00ffff00);
+ const v128_t ones = wasm_i8x16_splat(-1);
+ while (width >= 4) {
+ v128_t v = wasm_v128_or(wasm_v128_load(argb), mask_rgb);
+ uint32_t diff =
+ (~(uint32_t)wasm_i8x16_bitmask(wasm_i8x16_eq(v, ones))) & 0xffffu;
+ if (diff) {
+ return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ argb += 16;
+ width -= 4;
+ }
+ while (width > 0) {
+ v128_t v = wasm_v128_or(wasm_v128_load32_zero(argb), mask_rgb);
+ uint32_t diff =
+ (~(uint32_t)wasm_i8x16_bitmask(wasm_i8x16_eq(v, ones))) & 0x9u;
+ if (diff) {
+ return (diff & 1u) ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ argb += 4;
+ width -= 1;
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_WASMSIMD
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
diff --git a/libs/libyuv/source/compare_win.cc b/libs/libyuv/source/compare_win.cc
index d57d3d9d1c..a25cf6efae 100644
--- a/libs/libyuv/source/compare_win.cc
+++ b/libs/libyuv/source/compare_win.cc
@@ -8,13 +8,22 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/basic_types.h"
-
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
+#include "libyuv/video_common.h"
+
+// This module is for Visual C 32/64 bit, and for clang with
+// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
-#if defined(_MSC_VER)
-#include <intrin.h> // For __popcnt
+#include <immintrin.h> // For SSE2 to AVX512 intrinsics
+#include <string.h> // For memcpy
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
#endif
#ifdef __cplusplus
@@ -22,220 +31,998 @@ namespace libyuv {
extern "C" {
#endif
-// This module is for 32 bit Visual C x86 and clangcl
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
+// Target attributes allow gcc and clang to build each function for its
+// instruction set regardless of the baseline.
+#if defined(__clang__) || defined(__GNUC__)
+#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
+#define LIBYUV_TARGET_SSE41 __attribute__((target("sse4.1")))
+#define LIBYUV_TARGET_SSE42 __attribute__((target("sse4.2,popcnt")))
+#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
+#define LIBYUV_TARGET_AVX512BW \
+ __attribute__((target("avx512bw,avx512vl,avx512f")))
+#define LIBYUV_TARGET_AVX512VNNI \
+ __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f")))
+#else
+#define LIBYUV_TARGET_SSE2
+#define LIBYUV_TARGET_SSE41
+#define LIBYUV_TARGET_SSE42
+#define LIBYUV_TARGET_AVX2
+#define LIBYUV_TARGET_AVX512BW
+#define LIBYUV_TARGET_AVX512VNNI
+#endif // defined(__clang__) || defined(__GNUC__)
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+// Count is a multiple of 8.
+LIBYUV_TARGET_SSE42
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
uint32_t diff = 0u;
-
int i;
- for (i = 0; i < count - 3; i += 4) {
- uint32_t x = *((uint32_t*)src_a) ^ *((uint32_t*)src_b); // NOLINT
- src_a += 4;
- src_b += 4;
- diff += __popcnt(x);
+ for (i = 0; i < count; i += 8) {
+ uint64_t a;
+ uint64_t b;
+ memcpy(&a, src_a + i, 8);
+ memcpy(&b, src_b + i, 8);
+#if defined(__x86_64__) || defined(_M_X64)
+ diff += (uint32_t)_mm_popcnt_u64(a ^ b);
+#else
+ diff += _mm_popcnt_u32((uint32_t)(a ^ b)) +
+ _mm_popcnt_u32((uint32_t)((a ^ b) >> 32));
+#endif
}
return diff;
}
+#endif // HAS_HAMMINGDISTANCE_SSE42
-__declspec(naked) uint32_t
- SumSquareError_SSE2(const uint8_t* src_a, const uint8_t* src_b, int count) {
- __asm {
- mov eax, [esp + 4] // src_a
- mov edx, [esp + 8] // src_b
- mov ecx, [esp + 12] // count
- pxor xmm0, xmm0
- pxor xmm5, xmm5
-
- wloop:
- movdqu xmm1, [eax]
- lea eax, [eax + 16]
- movdqu xmm2, [edx]
- lea edx, [edx + 16]
- movdqa xmm3, xmm1 // abs trick
- psubusb xmm1, xmm2
- psubusb xmm2, xmm3
- por xmm1, xmm2
- movdqa xmm2, xmm1
- punpcklbw xmm1, xmm5
- punpckhbw xmm2, xmm5
- pmaddwd xmm1, xmm1
- pmaddwd xmm2, xmm2
- paddd xmm0, xmm1
- paddd xmm0, xmm2
- sub ecx, 16
- jg wloop
-
- pshufd xmm1, xmm0, 0xee
- paddd xmm0, xmm1
- pshufd xmm1, xmm0, 0x01
- paddd xmm0, xmm1
- movd eax, xmm0
- ret
- }
-}
-
-// Visual C 2012 required for AVX2.
-#if _MSC_VER >= 1700
-// C4752: found Intel(R) Advanced Vector Extensions; consider using /arch:AVX.
-#pragma warning(disable : 4752)
-__declspec(naked) uint32_t
- SumSquareError_AVX2(const uint8_t* src_a, const uint8_t* src_b, int count) {
- __asm {
- mov eax, [esp + 4] // src_a
- mov edx, [esp + 8] // src_b
- mov ecx, [esp + 12] // count
- vpxor ymm0, ymm0, ymm0 // sum
- vpxor ymm5, ymm5, ymm5 // constant 0 for unpck
- sub edx, eax
-
- wloop:
- vmovdqu ymm1, [eax]
- vmovdqu ymm2, [eax + edx]
- lea eax, [eax + 32]
- vpsubusb ymm3, ymm1, ymm2 // abs difference trick
- vpsubusb ymm2, ymm2, ymm1
- vpor ymm1, ymm2, ymm3
- vpunpcklbw ymm2, ymm1, ymm5 // u16. mutates order.
- vpunpckhbw ymm1, ymm1, ymm5
- vpmaddwd ymm2, ymm2, ymm2 // square + hadd to u32.
- vpmaddwd ymm1, ymm1, ymm1
- vpaddd ymm0, ymm0, ymm1
- vpaddd ymm0, ymm0, ymm2
- sub ecx, 32
- jg wloop
-
- vpshufd ymm1, ymm0, 0xee // 3, 2 + 1, 0 both lanes.
- vpaddd ymm0, ymm0, ymm1
- vpshufd ymm1, ymm0, 0x01 // 1 + 0 both lanes.
- vpaddd ymm0, ymm0, ymm1
- vpermq ymm1, ymm0, 0x02 // high + low lane.
- vpaddd ymm0, ymm0, ymm1
- vmovd eax, xmm0
- vzeroupper
- ret
- }
-}
-#endif // _MSC_VER >= 1700
-
-uvec32 kHash16x33 = {0x92d9e201, 0, 0, 0}; // 33 ^ 16
-uvec32 kHashMul0 = {
- 0x0c3525e1, // 33 ^ 15
- 0xa3476dc1, // 33 ^ 14
- 0x3b4039a1, // 33 ^ 13
- 0x4f5f0981, // 33 ^ 12
-};
-uvec32 kHashMul1 = {
- 0x30f35d61, // 33 ^ 11
- 0x855cb541, // 33 ^ 10
- 0x040a9121, // 33 ^ 9
- 0x747c7101, // 33 ^ 8
-};
-uvec32 kHashMul2 = {
- 0xec41d4e1, // 33 ^ 7
- 0x4cfa3cc1, // 33 ^ 6
- 0x025528a1, // 33 ^ 5
- 0x00121881, // 33 ^ 4
+#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
+ defined(HAS_HAMMINGDISTANCE_AVX512VNNI) || \
+ defined(HAS_SUMSQUAREERROR_AVX512BW) || \
+ defined(HAS_SUMSQUAREERROR_AVX512VNNI) || \
+ defined(HAS_HASHDJB2_AVX512BW) || defined(HAS_SSIM4X4ROW_AVX512BW)
+// Mask of the low n bytes, for n = 0 to 64.
+static __mmask64 LowBytesMask64(int n) {
+ return (__mmask64)(n >= 64 ? ~0ULL : (1ULL << n) - 1);
+}
+#endif
+
+#if defined(HAS_HAMMINGDISTANCE_AVX2)
+// Count is a multiple of 64.
+LIBYUV_TARGET_AVX2
+uint32_t HammingDistance_AVX2(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ const __m256i kNibbleMask = _mm256_set1_epi8(15);
+ const __m256i kBitCount = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
+ const __m256i zero = _mm256_setzero_si256();
+ __m256i sum = _mm256_setzero_si256();
+ __m128i sum128;
+ int i;
+ for (i = 0; i < count; i += 64) {
+ __m256i x0 =
+ _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i)),
+ _mm256_loadu_si256((const __m256i*)(src_b + i)));
+ __m256i x1 =
+ _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_b + i + 32)));
+ __m256i c0 = _mm256_add_epi8(
+ _mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x0, kNibbleMask)),
+ _mm256_shuffle_epi8(
+ kBitCount,
+ _mm256_and_si256(_mm256_srli_epi16(x0, 4), kNibbleMask)));
+ __m256i c1 = _mm256_add_epi8(
+ _mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x1, kNibbleMask)),
+ _mm256_shuffle_epi8(
+ kBitCount,
+ _mm256_and_si256(_mm256_srli_epi16(x1, 4), kNibbleMask)));
+ sum = _mm256_add_epi64(sum, _mm256_sad_epu8(_mm256_add_epi8(c0, c1), zero));
+ }
+ sum128 = _mm_add_epi64(_mm256_castsi256_si128(sum),
+ _mm256_extracti128_si256(sum, 1));
+ sum128 = _mm_add_epi64(sum128, _mm_unpackhi_epi64(sum128, sum128));
+ _mm256_zeroupper();
+ return (uint32_t)_mm_cvtsi128_si32(sum128);
+}
+#endif // HAS_HAMMINGDISTANCE_AVX2
+
+#if defined(HAS_HAMMINGDISTANCE_AVX512BW)
+// Bit count of 64 bytes, summed to 8 qwords.
+LIBYUV_TARGET_AVX512BW
+static __m512i BitCount64_AVX512BW(__m512i x) {
+ const __m512i kNibbleMask = _mm512_set1_epi8(15);
+ const __m512i kBitCount = _mm512_broadcast_i32x4(
+ _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
+ __m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask));
+ __m512i hi = _mm512_shuffle_epi8(
+ kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask));
+ return _mm512_sad_epu8(_mm512_add_epi8(lo, hi), _mm512_setzero_si512());
+}
+
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512BW
+uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ __m512i sum = _mm512_setzero_si512();
+ while (count >= 64) {
+ sum = _mm512_add_epi64(
+ sum, BitCount64_AVX512BW(_mm512_xor_si512(_mm512_loadu_si512(src_a),
+ _mm512_loadu_si512(src_b))));
+ src_a += 64;
+ src_b += 64;
+ count -= 64;
+ }
+ if (count > 0) {
+ const __mmask64 k = LowBytesMask64(count);
+ sum = _mm512_add_epi64(sum, BitCount64_AVX512BW(_mm512_xor_si512(
+ _mm512_maskz_loadu_epi8(k, src_a),
+ _mm512_maskz_loadu_epi8(k, src_b))));
+ }
+ return (uint32_t)_mm512_reduce_add_epi64(sum);
+}
+#endif // HAS_HAMMINGDISTANCE_AVX512BW
+
+#if defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
+// Bit count of 64 bytes accumulated into 16 dwords with vpdpbusd.
+LIBYUV_TARGET_AVX512VNNI
+static __m512i BitCount64_AVX512VNNI(__m512i x, __m512i sum) {
+ const __m512i kNibbleMask = _mm512_set1_epi8(15);
+ const __m512i kOnes = _mm512_set1_epi8(1);
+ const __m512i kBitCount = _mm512_broadcast_i32x4(
+ _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
+ __m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask));
+ __m512i hi = _mm512_shuffle_epi8(
+ kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask));
+ return _mm512_dpbusd_epi32(sum, kOnes, _mm512_add_epi8(lo, hi));
+}
+
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512VNNI
+uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ __m512i sum = _mm512_setzero_si512();
+ while (count >= 64) {
+ sum = BitCount64_AVX512VNNI(
+ _mm512_xor_si512(_mm512_loadu_si512(src_a), _mm512_loadu_si512(src_b)),
+ sum);
+ src_a += 64;
+ src_b += 64;
+ count -= 64;
+ }
+ if (count > 0) {
+ const __mmask64 k = LowBytesMask64(count);
+ sum = BitCount64_AVX512VNNI(
+ _mm512_xor_si512(_mm512_maskz_loadu_epi8(k, src_a),
+ _mm512_maskz_loadu_epi8(k, src_b)),
+ sum);
+ }
+ return (uint32_t)_mm512_reduce_add_epi32(sum);
+}
+#endif // HAS_HAMMINGDISTANCE_AVX512VNNI
+
+#if defined(HAS_SUMSQUAREERROR_SSE2)
+// Count is a multiple of 16.
+LIBYUV_TARGET_SSE2
+uint32_t SumSquareError_SSE2(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ const __m128i zero = _mm_setzero_si128();
+ __m128i sum = _mm_setzero_si128();
+ int i;
+ for (i = 0; i < count; i += 16) {
+ __m128i a = _mm_loadu_si128((const __m128i*)(src_a + i));
+ __m128i b = _mm_loadu_si128((const __m128i*)(src_b + i));
+ __m128i d = _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
+ __m128i lo = _mm_unpacklo_epi8(d, zero);
+ __m128i hi = _mm_unpackhi_epi8(d, zero);
+ sum = _mm_add_epi32(sum, _mm_madd_epi16(lo, lo));
+ sum = _mm_add_epi32(sum, _mm_madd_epi16(hi, hi));
+ }
+ sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xee));
+ sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
+ return (uint32_t)_mm_cvtsi128_si32(sum);
+}
+#endif // HAS_SUMSQUAREERROR_SSE2
+
+#if defined(HAS_SUMSQUAREERROR_AVX2)
+// Count is a multiple of 32.
+LIBYUV_TARGET_AVX2
+uint32_t SumSquareError_AVX2(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ const __m256i zero = _mm256_setzero_si256();
+ __m256i sum = _mm256_setzero_si256();
+ __m128i sum128;
+ int i;
+ for (i = 0; i < count; i += 32) {
+ __m256i a = _mm256_loadu_si256((const __m256i*)(src_a + i));
+ __m256i b = _mm256_loadu_si256((const __m256i*)(src_b + i));
+ __m256i d = _mm256_or_si256(_mm256_subs_epu8(a, b), _mm256_subs_epu8(b, a));
+ __m256i lo = _mm256_unpacklo_epi8(d, zero);
+ __m256i hi = _mm256_unpackhi_epi8(d, zero);
+ sum = _mm256_add_epi32(sum, _mm256_madd_epi16(lo, lo));
+ sum = _mm256_add_epi32(sum, _mm256_madd_epi16(hi, hi));
+ }
+ sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum),
+ _mm256_extracti128_si256(sum, 1));
+ sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0xee));
+ sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0x01));
+ _mm256_zeroupper();
+ return (uint32_t)_mm_cvtsi128_si32(sum128);
+}
+#endif // HAS_SUMSQUAREERROR_AVX2
+
+#if defined(HAS_SUMSQUAREERROR_AVX512BW)
+// Sum of squared differences of 64 bytes, as 16 dwords.
+LIBYUV_TARGET_AVX512BW
+static __m512i SumSquare64_AVX512BW(__m512i a, __m512i b) {
+ const __m512i zero = _mm512_setzero_si512();
+ __m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
+ __m512i lo = _mm512_unpacklo_epi8(d, zero);
+ __m512i hi = _mm512_unpackhi_epi8(d, zero);
+ return _mm512_add_epi32(_mm512_madd_epi16(lo, lo), _mm512_madd_epi16(hi, hi));
+}
+
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512BW
+uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ __m512i sum = _mm512_setzero_si512();
+ while (count >= 64) {
+ sum =
+ _mm512_add_epi32(sum, SumSquare64_AVX512BW(_mm512_loadu_si512(src_a),
+ _mm512_loadu_si512(src_b)));
+ src_a += 64;
+ src_b += 64;
+ count -= 64;
+ }
+ if (count > 0) {
+ const __mmask64 k = LowBytesMask64(count);
+ sum = _mm512_add_epi32(
+ sum, SumSquare64_AVX512BW(_mm512_maskz_loadu_epi8(k, src_a),
+ _mm512_maskz_loadu_epi8(k, src_b)));
+ }
+ return (uint32_t)_mm512_reduce_add_epi32(sum);
+}
+#endif // HAS_SUMSQUAREERROR_AVX512BW
+
+#if defined(HAS_SUMSQUAREERROR_AVX512VNNI)
+// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
+// loads that read zeros, so any count is supported.
+LIBYUV_TARGET_AVX512VNNI
+uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ const __m512i zero = _mm512_setzero_si512();
+ __m512i sum0 = _mm512_setzero_si512();
+ __m512i sum1 = _mm512_setzero_si512();
+ while (count >= 64) {
+ __m512i a = _mm512_loadu_si512(src_a);
+ __m512i b = _mm512_loadu_si512(src_b);
+ __m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
+ __m512i lo = _mm512_unpacklo_epi8(d, zero);
+ __m512i hi = _mm512_unpackhi_epi8(d, zero);
+ sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
+ sum1 = _mm512_dpwssd_epi32(sum1, hi, hi);
+ src_a += 64;
+ src_b += 64;
+ count -= 64;
+ }
+ if (count > 0) {
+ const __mmask64 k = LowBytesMask64(count);
+ __m512i a = _mm512_maskz_loadu_epi8(k, src_a);
+ __m512i b = _mm512_maskz_loadu_epi8(k, src_b);
+ __m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
+ __m512i lo = _mm512_unpacklo_epi8(d, zero);
+ __m512i hi = _mm512_unpackhi_epi8(d, zero);
+ sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
+ sum1 = _mm512_dpwssd_epi32(sum1, hi, hi);
+ }
+ return (uint32_t)_mm512_reduce_add_epi32(_mm512_add_epi32(sum0, sum1));
+}
+#endif // HAS_SUMSQUAREERROR_AVX512VNNI
+
+#if defined(HAS_HASHDJB2_SSE41)
+// 33 ^ (15 - i) for byte i of 16.
+static const uint32_t kHashMul16[16] = {
+ 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, // 33 ^ 15 .. 12
+ 0x30f35d61, 0x855cb541, 0x040a9121, 0x747c7101, // 33 ^ 11 .. 8
+ 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, // 33 ^ 7 .. 4
+ 0x00008c61, 0x00000441, 0x00000021, 0x00000001, // 33 ^ 3 .. 0
};
-uvec32 kHashMul3 = {
- 0x00008c61, // 33 ^ 3
- 0x00000441, // 33 ^ 2
- 0x00000021, // 33 ^ 1
- 0x00000001, // 33 ^ 0
+static const uint32_t kHash16x33 = 0x92d9e201; // 33 ^ 16
+
+// Count is a multiple of 16.
+LIBYUV_TARGET_SSE41
+uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
+ const __m128i zero = _mm_setzero_si128();
+ const __m128i mul0 = _mm_loadu_si128((const __m128i*)(kHashMul16));
+ const __m128i mul1 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 4));
+ const __m128i mul2 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 8));
+ const __m128i mul3 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 12));
+ int i;
+ for (i = 0; i < count; i += 16) {
+ __m128i s = _mm_loadu_si128((const __m128i*)(src + i));
+ __m128i lo = _mm_unpacklo_epi8(s, zero);
+ __m128i hi = _mm_unpackhi_epi8(s, zero);
+ __m128i sum = _mm_add_epi32(
+ _mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(lo, zero), mul0),
+ _mm_mullo_epi32(_mm_unpackhi_epi16(lo, zero), mul1)),
+ _mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(hi, zero), mul2),
+ _mm_mullo_epi32(_mm_unpackhi_epi16(hi, zero), mul3)));
+ sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x0e));
+ sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
+ seed = seed * kHash16x33 + (uint32_t)_mm_cvtsi128_si32(sum);
+ }
+ return seed;
+}
+#endif // HAS_HASHDJB2_SSE41
+
+#if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW)
+// HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
+// The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that
+// vpmaddwd can be used instead of vpmulld, which is 2 uops and 10 cycles.
+// m = hi * 65536 + lo, with lo treated as signed and hi adjusted by the carry,
+// so src * m = src * lo + ((src * hi) << 16) modulo 2 ^ 32.
+// The first 64 are lo and the next 64 are hi.
+static const uint16_t kHashMulLoHi[128] = {
+ // lo for 33 ^ 63 .. 33 ^ 0
+ 0x0be1, 0x93c1, 0x9fa1, 0xaf81, 0x4361, 0xdb41, 0xf721, 0x1701, //
+ 0xbae1, 0x62c1, 0x8ea1, 0xbe81, 0x7261, 0x2a41, 0x6621, 0xa601, //
+ 0x69e1, 0x31c1, 0x7da1, 0xcd81, 0xa161, 0x7941, 0xd521, 0x3501, //
+ 0x18e1, 0x00c1, 0x6ca1, 0xdc81, 0xd061, 0xc841, 0x4421, 0xc401, //
+ 0xc7e1, 0xcfc1, 0x5ba1, 0xeb81, 0xff61, 0x1741, 0xb321, 0x5301, //
+ 0x76e1, 0x9ec1, 0x4aa1, 0xfa81, 0x2e61, 0x6641, 0x2221, 0xe201, //
+ 0x25e1, 0x6dc1, 0x39a1, 0x0981, 0x5d61, 0xb541, 0x9121, 0x7101, //
+ 0xd4e1, 0x3cc1, 0x28a1, 0x1881, 0x8c61, 0x0441, 0x0021, 0x0001, //
+ // hi for 33 ^ 63 .. 33 ^ 0
+ 0xd8be, 0xef4c, 0x6458, 0x6022, 0x40f9, 0x193e, 0x6d5f, 0x31dc, //
+ 0x0945, 0xc238, 0x1567, 0xa38f, 0x3380, 0x301b, 0x664e, 0x031a, //
+ 0xd18c, 0xb8c6, 0x79f6, 0xae5d, 0x0549, 0xc219, 0x7a3f, 0x1338, //
+ 0x0095, 0xc1f5, 0xa107, 0xaf8c, 0x0552, 0x3e39, 0x3830, 0x1138, //
+ 0x655f, 0xccc5, 0x9999, 0xd61c, 0x829c, 0x1379, 0x2f23, 0xac18, //
+ 0xcee9, 0xc836, 0x72ac, 0x510d, 0xcc27, 0xb0da, 0xee16, 0x92da, //
+ 0x0c35, 0xa347, 0x3b40, 0x4f5f, 0x30f3, 0x855d, 0x040b, 0x747c, //
+ 0xec42, 0x4cfa, 0x0255, 0x0012, 0x0001, 0x0000, 0x0000, 0x0000, //
};
+#endif
-__declspec(naked) uint32_t
- HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
- __asm {
- mov eax, [esp + 4] // src
- mov ecx, [esp + 8] // count
- movd xmm0, [esp + 12] // seed
-
- pxor xmm7, xmm7 // constant 0 for unpck
- movdqa xmm6, xmmword ptr kHash16x33
-
- wloop:
- movdqu xmm1, [eax] // src[0-15]
- lea eax, [eax + 16]
- pmulld xmm0, xmm6 // hash *= 33 ^ 16
- movdqa xmm5, xmmword ptr kHashMul0
- movdqa xmm2, xmm1
- punpcklbw xmm2, xmm7 // src[0-7]
- movdqa xmm3, xmm2
- punpcklwd xmm3, xmm7 // src[0-3]
- pmulld xmm3, xmm5
- movdqa xmm5, xmmword ptr kHashMul1
- movdqa xmm4, xmm2
- punpckhwd xmm4, xmm7 // src[4-7]
- pmulld xmm4, xmm5
- movdqa xmm5, xmmword ptr kHashMul2
- punpckhbw xmm1, xmm7 // src[8-15]
- movdqa xmm2, xmm1
- punpcklwd xmm2, xmm7 // src[8-11]
- pmulld xmm2, xmm5
- movdqa xmm5, xmmword ptr kHashMul3
- punpckhwd xmm1, xmm7 // src[12-15]
- pmulld xmm1, xmm5
- paddd xmm3, xmm4 // add 16 results
- paddd xmm1, xmm2
- paddd xmm1, xmm3
-
- pshufd xmm2, xmm1, 0x0e // upper 2 dwords
- paddd xmm1, xmm2
- pshufd xmm2, xmm1, 0x01
- paddd xmm1, xmm2
- paddd xmm0, xmm1
- sub ecx, 16
- jg wloop
-
- movd eax, xmm0 // return hash
- ret
- }
-}
-
-// Visual C 2012 required for AVX2.
-#if _MSC_VER >= 1700
-__declspec(naked) uint32_t
- HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) {
- __asm {
- mov eax, [esp + 4] // src
- mov ecx, [esp + 8] // count
- vmovd xmm0, [esp + 12] // seed
-
- wloop:
- vpmovzxbd xmm3, [eax] // src[0-3]
- vpmulld xmm0, xmm0, xmmword ptr kHash16x33 // hash *= 33 ^ 16
- vpmovzxbd xmm4, [eax + 4] // src[4-7]
- vpmulld xmm3, xmm3, xmmword ptr kHashMul0
- vpmovzxbd xmm2, [eax + 8] // src[8-11]
- vpmulld xmm4, xmm4, xmmword ptr kHashMul1
- vpmovzxbd xmm1, [eax + 12] // src[12-15]
- vpmulld xmm2, xmm2, xmmword ptr kHashMul2
- lea eax, [eax + 16]
- vpmulld xmm1, xmm1, xmmword ptr kHashMul3
- vpaddd xmm3, xmm3, xmm4 // add 16 results
- vpaddd xmm1, xmm1, xmm2
- vpaddd xmm1, xmm1, xmm3
- vpshufd xmm2, xmm1, 0x0e // upper 2 dwords
- vpaddd xmm1, xmm1,xmm2
- vpshufd xmm2, xmm1, 0x01
- vpaddd xmm1, xmm1, xmm2
- vpaddd xmm0, xmm0, xmm1
- sub ecx, 16
- jg wloop
-
- vmovd eax, xmm0 // return hash
- vzeroupper
- ret
- }
-}
-#endif // _MSC_VER >= 1700
-
-#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
+#if defined(HAS_HASHDJB2_AVX2)
+// lo + (hi << 16), then add 8 dwords.
+LIBYUV_TARGET_AVX2
+static uint32_t HashSum_AVX2(__m256i lo, __m256i hi) {
+ __m256i s = _mm256_add_epi32(lo, _mm256_slli_epi32(hi, 16));
+ __m128i s128 =
+ _mm_add_epi32(_mm256_castsi256_si128(s), _mm256_extracti128_si256(s, 1));
+ s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x0e));
+ s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x01));
+ return (uint32_t)_mm_cvtsi128_si32(s128);
+}
+
+// Count is a multiple of 32. Process 64 bytes per loop with one horizontal
+// add, then 32 bytes if needed.
+LIBYUV_TARGET_AVX2
+uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) {
+ const __m256i* m = (const __m256i*)kHashMulLoHi;
+ while (count >= 64) {
+ __m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
+ __m256i w1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
+ __m256i w2 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32)));
+ __m256i w3 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48)));
+ __m256i lo = _mm256_add_epi32(
+ _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 0)),
+ _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 1))),
+ _mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 2)),
+ _mm256_madd_epi16(w3, _mm256_loadu_si256(m + 3))));
+ __m256i hi = _mm256_add_epi32(
+ _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 4)),
+ _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 5))),
+ _mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 6)),
+ _mm256_madd_epi16(w3, _mm256_loadu_si256(m + 7))));
+ seed = seed * 0xf07f8801u + HashSum_AVX2(lo, hi); // 33 ^ 64
+ src += 64;
+ count -= 64;
+ }
+ if (count > 0) {
+ __m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
+ __m256i w1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
+ __m256i lo =
+ _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 2)),
+ _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 3)));
+ __m256i hi =
+ _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 6)),
+ _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 7)));
+ seed = seed * 0x1137c401u + HashSum_AVX2(lo, hi); // 33 ^ 32
+ }
+ _mm256_zeroupper();
+ return seed;
+}
+#endif // HAS_HASHDJB2_AVX2
+
+#if defined(HAS_HASHDJB2_AVX512BW)
+static uint32_t Pow33(int n) {
+ uint32_t r = 1u;
+ uint32_t b = 33u;
+ while (n) {
+ if (n & 1) {
+ r *= b;
+ }
+ b *= b;
+ n >>= 1;
+ }
+ return r;
+}
+
+// Sum of 64 bytes, zero extended to words in w0 and w1, times 33 ^ (63 - i).
+LIBYUV_TARGET_AVX512BW
+static uint32_t HashSum64_AVX512BW(__m512i w0, __m512i w1) {
+ const __m512i* m = (const __m512i*)kHashMulLoHi;
+ __m512i lo =
+ _mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m)),
+ _mm512_madd_epi16(w1, _mm512_loadu_si512(m + 1)));
+ __m512i hi =
+ _mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m + 2)),
+ _mm512_madd_epi16(w1, _mm512_loadu_si512(m + 3)));
+ return (uint32_t)_mm512_reduce_add_epi32(
+ _mm512_add_epi32(lo, _mm512_slli_epi32(hi, 16)));
+}
+
+// hash = hash * 33 ^ 64 + sum(src[i] * 33 ^ (63 - i)) for each 64 bytes.
+// The remainder of n = 1 to 63 bytes is loaded into the high n bytes with a
+// masked load ending at src + count, so the same multipliers apply, and
+// hash is multiplied by 33 ^ n.
+LIBYUV_TARGET_AVX512BW
+uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
+ int n;
+ while (count >= 64) {
+ seed = seed * 0xf07f8801u + // 33 ^ 64
+ HashSum64_AVX512BW(
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src)),
+ _mm512_cvtepu8_epi16(
+ _mm256_loadu_si256((const __m256i*)(src + 32))));
+ src += 64;
+ count -= 64;
+ }
+ n = count;
+ if (n) {
+ const __mmask64 k = (__mmask64)~LowBytesMask64(64 - n);
+ __m512i s = _mm512_maskz_loadu_epi8(k, src + n - 64);
+ seed = seed * Pow33(n) +
+ HashSum64_AVX512BW(
+ _mm512_cvtepu8_epi16(_mm512_castsi512_si256(s)),
+ _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(s, 1)));
+ }
+ return seed;
+}
+#endif // HAS_HASHDJB2_AVX512BW
+
+#if defined(HAS_SSIM4X4ROW_SSE2)
+// Width is a multiple of 16.
+LIBYUV_TARGET_SSE2
+void Ssim4x4Row_SSE2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ const __m128i kMaskFF = _mm_set1_epi16(0x00ff);
+ const __m128i kOnes16 = _mm_set1_epi16(1);
+ int x;
+ for (x = 0; x < width; x += 16) {
+ __m128i acc_a = _mm_setzero_si128();
+ __m128i acc_b = _mm_setzero_si128();
+ __m128i acc_sq = _mm_setzero_si128();
+ __m128i acc_axb = _mm_setzero_si128();
+#define SSIM_ROW_SSE2_WIN(ar, br) \
+ do { \
+ __m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \
+ __m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \
+ __m128i a_ev = _mm_and_si128(va, kMaskFF); \
+ __m128i a_od = _mm_srli_epi16(va, 8); \
+ __m128i b_ev = _mm_and_si128(vb, kMaskFF); \
+ __m128i b_od = _mm_srli_epi16(vb, 8); \
+ acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \
+ acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \
+ acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \
+ _mm_madd_epi16(a_od, a_od))); \
+ acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \
+ _mm_madd_epi16(b_od, b_od))); \
+ acc_axb = \
+ _mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \
+ _mm_madd_epi16(a_od, b_od))); \
+ } while (0)
+ SSIM_ROW_SSE2_WIN(a0, b0);
+ SSIM_ROW_SSE2_WIN(a1, b1);
+ SSIM_ROW_SSE2_WIN(a2, b2);
+ SSIM_ROW_SSE2_WIN(a3, b3);
+#undef SSIM_ROW_SSE2_WIN
+ acc_a = _mm_madd_epi16(acc_a, kOnes16);
+ acc_b = _mm_madd_epi16(acc_b, kOnes16);
+ {
+ __m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b);
+ __m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b);
+ __m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb);
+ __m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb);
+ _mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2));
+ _mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2));
+ _mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3));
+ _mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3));
+ }
+ dst += 16;
+ }
+}
+#endif // HAS_SSIM4X4ROW_SSE2
+
+#if defined(HAS_SSIM4X4ROW_AVX2)
+// Width is a multiple of 32.
+LIBYUV_TARGET_AVX2
+void Ssim4x4Row_AVX2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ const __m256i kMaskFF = _mm256_set1_epi16(0x00ff);
+ const __m256i kOnes16 = _mm256_set1_epi16(1);
+ int x;
+ for (x = 0; x < width; x += 32) {
+ __m256i acc_a = _mm256_setzero_si256();
+ __m256i acc_b = _mm256_setzero_si256();
+ __m256i acc_sq = _mm256_setzero_si256();
+ __m256i acc_axb = _mm256_setzero_si256();
+#define SSIM_ROW_AVX2_WIN(ar, br) \
+ do { \
+ __m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \
+ __m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \
+ __m256i a_ev = _mm256_and_si256(va, kMaskFF); \
+ __m256i a_od = _mm256_srli_epi16(va, 8); \
+ __m256i b_ev = _mm256_and_si256(vb, kMaskFF); \
+ __m256i b_od = _mm256_srli_epi16(vb, 8); \
+ acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \
+ acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \
+ acc_sq = _mm256_add_epi32( \
+ acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \
+ _mm256_madd_epi16(a_od, a_od))); \
+ acc_sq = _mm256_add_epi32( \
+ acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \
+ _mm256_madd_epi16(b_od, b_od))); \
+ acc_axb = _mm256_add_epi32( \
+ acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \
+ _mm256_madd_epi16(a_od, b_od))); \
+ } while (0)
+ SSIM_ROW_AVX2_WIN(a0, b0);
+ SSIM_ROW_AVX2_WIN(a1, b1);
+ SSIM_ROW_AVX2_WIN(a2, b2);
+ SSIM_ROW_AVX2_WIN(a3, b3);
+#undef SSIM_ROW_AVX2_WIN
+ acc_a = _mm256_madd_epi16(acc_a, kOnes16);
+ acc_b = _mm256_madd_epi16(acc_b, kOnes16);
+ {
+ __m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b);
+ __m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b);
+ __m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb);
+ __m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb);
+ __m256i u0 = _mm256_unpacklo_epi64(t0, t2);
+ __m256i u1 = _mm256_unpackhi_epi64(t0, t2);
+ __m256i u2 = _mm256_unpacklo_epi64(t1, t3);
+ __m256i u3 = _mm256_unpackhi_epi64(t1, t3);
+ _mm256_storeu_si256((__m256i*)(dst + 0),
+ _mm256_permute2x128_si256(u0, u1, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst + 8),
+ _mm256_permute2x128_si256(u2, u3, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst + 16),
+ _mm256_permute2x128_si256(u0, u1, 0x31));
+ _mm256_storeu_si256((__m256i*)(dst + 24),
+ _mm256_permute2x128_si256(u2, u3, 0x31));
+ }
+ dst += 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SSIM4X4ROW_AVX2
+
+#if defined(HAS_SSIM4X4ROW_AVX512BW)
+// Width is a multiple of 4.
+LIBYUV_TARGET_AVX512BW
+void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const ptrdiff_t sa = stride_a;
+ const ptrdiff_t sb = stride_b;
+ const uint8_t* a0 = src_a;
+ const uint8_t* a1 = a0 + sa;
+ const uint8_t* a2 = a1 + sa;
+ const uint8_t* a3 = a2 + sa;
+ const uint8_t* b0 = src_b;
+ const uint8_t* b1 = b0 + sb;
+ const uint8_t* b2 = b1 + sb;
+ const uint8_t* b3 = b2 + sb;
+ const __m512i kMaskFF = _mm512_set1_epi16(0x00ff);
+ const __m512i kOnes16 = _mm512_set1_epi16(1);
+#define SSIM_ROW_AVX512_WIN(va, vb) \
+ do { \
+ __m512i a_ev = _mm512_and_si512(va, kMaskFF); \
+ __m512i a_od = _mm512_srli_epi16(va, 8); \
+ __m512i b_ev = _mm512_and_si512(vb, kMaskFF); \
+ __m512i b_od = _mm512_srli_epi16(vb, 8); \
+ acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \
+ acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \
+ acc_sq = _mm512_add_epi32( \
+ acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \
+ _mm512_madd_epi16(a_od, a_od))); \
+ acc_sq = _mm512_add_epi32( \
+ acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \
+ _mm512_madd_epi16(b_od, b_od))); \
+ acc_axb = _mm512_add_epi32( \
+ acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \
+ _mm512_madd_epi16(a_od, b_od))); \
+ } while (0)
+#define SSIM_TRANSPOSE_AVX512_WIN() \
+ acc_a = _mm512_madd_epi16(acc_a, kOnes16); \
+ acc_b = _mm512_madd_epi16(acc_b, kOnes16); \
+ __m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \
+ __m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \
+ __m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \
+ __m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \
+ __m512i u0 = _mm512_unpacklo_epi64(t0, t2); \
+ __m512i u1 = _mm512_unpackhi_epi64(t0, t2); \
+ __m512i u2 = _mm512_unpacklo_epi64(t1, t3); \
+ __m512i u3 = _mm512_unpackhi_epi64(t1, t3); \
+ __m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \
+ __m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \
+ __m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \
+ __m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \
+ __m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \
+ __m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \
+ __m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \
+ __m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd)
+
+ while (width >= 64) {
+ __m512i acc_a = _mm512_setzero_si512();
+ __m512i acc_b = _mm512_setzero_si512();
+ __m512i acc_sq = _mm512_setzero_si512();
+ __m512i acc_axb = _mm512_setzero_si512();
+ SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0));
+ SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1));
+ SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2));
+ SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3));
+ SSIM_TRANSPOSE_AVX512_WIN();
+ _mm512_storeu_si512(dst + 0, out0);
+ _mm512_storeu_si512(dst + 16, out1);
+ _mm512_storeu_si512(dst + 32, out2);
+ _mm512_storeu_si512(dst + 48, out3);
+ a0 += 64;
+ a1 += 64;
+ a2 += 64;
+ a3 += 64;
+ b0 += 64;
+ b1 += 64;
+ b2 += 64;
+ b3 += 64;
+ dst += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U);
+ const uint64_t store_mask = LowBytesMask64(width);
+ __m512i acc_a = _mm512_setzero_si512();
+ __m512i acc_b = _mm512_setzero_si512();
+ __m512i acc_sq = _mm512_setzero_si512();
+ __m512i acc_axb = _mm512_setzero_si512();
+ SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0),
+ _mm512_maskz_loadu_epi32(load_mask, b0));
+ SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1),
+ _mm512_maskz_loadu_epi32(load_mask, b1));
+ SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2),
+ _mm512_maskz_loadu_epi32(load_mask, b2));
+ SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3),
+ _mm512_maskz_loadu_epi32(load_mask, b3));
+ SSIM_TRANSPOSE_AVX512_WIN();
+ _mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0);
+ _mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1);
+ _mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2);
+ _mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3);
+ }
+#undef SSIM_ROW_AVX512_WIN
+#undef SSIM_TRANSPOSE_AVX512_WIN
+}
+#endif // HAS_SSIM4X4ROW_AVX512BW
+
+#if defined(HAS_SSIMWINDOWROW_SSE2)
+// num_windows is a multiple of 4.
+LIBYUV_TARGET_SSE2
+float SsimWindowRow_SSE2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const __m128i vc1 = _mm_set1_epi32(26634);
+ const __m128i vc2 = _mm_set1_epi32(239708);
+ __m128 total = _mm_setzero_ps();
+ __m128i t0 = _mm_add_epi32(_mm_loadu_si128((const __m128i*)row0),
+ _mm_loadu_si128((const __m128i*)row1));
+ int w;
+ for (w = 0; w < num_windows; w += 4) {
+ __m128i t1 = _mm_add_epi32(_mm_loadu_si128((const __m128i*)(row0 + 4)),
+ _mm_loadu_si128((const __m128i*)(row1 + 4)));
+ __m128i t2 = _mm_add_epi32(_mm_loadu_si128((const __m128i*)(row0 + 8)),
+ _mm_loadu_si128((const __m128i*)(row1 + 8)));
+ __m128i t3 = _mm_add_epi32(_mm_loadu_si128((const __m128i*)(row0 + 12)),
+ _mm_loadu_si128((const __m128i*)(row1 + 12)));
+ __m128i t4 = _mm_add_epi32(_mm_loadu_si128((const __m128i*)(row0 + 16)),
+ _mm_loadu_si128((const __m128i*)(row1 + 16)));
+ row0 += 16;
+ row1 += 16;
+ __m128i w0 = _mm_add_epi32(t0, t1);
+ __m128i w1 = _mm_add_epi32(t1, t2);
+ __m128i w2 = _mm_add_epi32(t2, t3);
+ __m128i w3 = _mm_add_epi32(t3, t4);
+ t0 = t4;
+
+ __m128i u0 = _mm_unpacklo_epi32(w0, w1);
+ __m128i u1 = _mm_unpackhi_epi32(w0, w1);
+ __m128i u2 = _mm_unpacklo_epi32(w2, w3);
+ __m128i u3 = _mm_unpackhi_epi32(w2, w3);
+ __m128i sum_a = _mm_unpacklo_epi64(u0, u2);
+ __m128i sum_b = _mm_unpackhi_epi64(u0, u2);
+ __m128i sum_sq = _mm_unpacklo_epi64(u1, u3);
+ __m128i sum_axb = _mm_unpackhi_epi64(u1, u3);
+
+ __m128i ab2 = _mm_slli_epi32(_mm_madd_epi16(sum_a, sum_b), 1);
+ __m128i a2b2 = _mm_add_epi32(_mm_madd_epi16(sum_a, sum_a),
+ _mm_madd_epi16(sum_b, sum_b));
+ __m128i n1 = _mm_add_epi32(ab2, vc1);
+ __m128i n2 =
+ _mm_add_epi32(_mm_sub_epi32(_mm_slli_epi32(sum_axb, 7), ab2), vc2);
+ __m128i d1 = _mm_add_epi32(a2b2, vc1);
+ __m128i d2 =
+ _mm_add_epi32(_mm_sub_epi32(_mm_slli_epi32(sum_sq, 6), a2b2), vc2);
+
+ __m128 num = _mm_mul_ps(_mm_cvtepi32_ps(n1), _mm_cvtepi32_ps(n2));
+ __m128 den = _mm_mul_ps(_mm_cvtepi32_ps(d1), _mm_cvtepi32_ps(d2));
+ total = _mm_add_ps(total, _mm_div_ps(num, den));
+ }
+ total = _mm_add_ps(total, _mm_movehl_ps(total, total));
+ total = _mm_add_ss(total, _mm_shuffle_ps(total, total, 1));
+ return _mm_cvtss_f32(total);
+}
+#endif // HAS_SSIMWINDOWROW_SSE2
+
+#if defined(HAS_SSIMWINDOWROW_AVX2)
+// num_windows is a multiple of 8.
+LIBYUV_TARGET_AVX2
+float SsimWindowRow_AVX2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const __m256i vc1 = _mm256_set1_epi32(26634);
+ const __m256i vc2 = _mm256_set1_epi32(239708);
+ __m256 total = _mm256_setzero_ps();
+ __m128 total128;
+ int w;
+ for (w = 0; w < num_windows; w += 8) {
+ __m256i e01 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 0)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 0)));
+ __m256i o12 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 4)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 4)));
+ __m256i e23 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 8)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 8)));
+ __m256i o34 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 12)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 12)));
+ __m256i e45 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 16)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 16)));
+ __m256i o56 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 20)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 20)));
+ __m256i e67 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 24)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 24)));
+ __m256i o78 =
+ _mm256_add_epi32(_mm256_loadu_si256((const __m256i*)(row0 + 28)),
+ _mm256_loadu_si256((const __m256i*)(row1 + 28)));
+ row0 += 32;
+ row1 += 32;
+ __m256i w0 = _mm256_add_epi32(e01, o12);
+ __m256i w1 = _mm256_add_epi32(e23, o34);
+ __m256i w2 = _mm256_add_epi32(e45, o56);
+ __m256i w3 = _mm256_add_epi32(e67, o78);
+
+ __m256i u0 = _mm256_unpacklo_epi32(w0, w1);
+ __m256i u1 = _mm256_unpackhi_epi32(w0, w1);
+ __m256i u2 = _mm256_unpacklo_epi32(w2, w3);
+ __m256i u3 = _mm256_unpackhi_epi32(w2, w3);
+ __m256i sum_a = _mm256_unpacklo_epi64(u0, u2);
+ __m256i sum_b = _mm256_unpackhi_epi64(u0, u2);
+ __m256i sum_sq = _mm256_unpacklo_epi64(u1, u3);
+ __m256i sum_axb = _mm256_unpackhi_epi64(u1, u3);
+
+ __m256i ab2 = _mm256_slli_epi32(_mm256_madd_epi16(sum_a, sum_b), 1);
+ __m256i a2b2 = _mm256_add_epi32(_mm256_madd_epi16(sum_a, sum_a),
+ _mm256_madd_epi16(sum_b, sum_b));
+ __m256i n1 = _mm256_add_epi32(ab2, vc1);
+ __m256i n2 = _mm256_add_epi32(
+ _mm256_sub_epi32(_mm256_slli_epi32(sum_axb, 7), ab2), vc2);
+ __m256i d1 = _mm256_add_epi32(a2b2, vc1);
+ __m256i d2 = _mm256_add_epi32(
+ _mm256_sub_epi32(_mm256_slli_epi32(sum_sq, 6), a2b2), vc2);
+
+ __m256 num = _mm256_mul_ps(_mm256_cvtepi32_ps(n1), _mm256_cvtepi32_ps(n2));
+ __m256 den = _mm256_mul_ps(_mm256_cvtepi32_ps(d1), _mm256_cvtepi32_ps(d2));
+ total = _mm256_add_ps(total, _mm256_div_ps(num, den));
+ }
+ total128 = _mm_add_ps(_mm256_castps256_ps128(total),
+ _mm256_extractf128_ps(total, 1));
+ total128 = _mm_add_ps(total128, _mm_movehl_ps(total128, total128));
+ total128 = _mm_add_ss(total128, _mm_shuffle_ps(total128, total128, 1));
+ _mm256_zeroupper();
+ return _mm_cvtss_f32(total128);
+}
+#endif // HAS_SSIMWINDOWROW_AVX2
+
+#if defined(HAS_SSIMWINDOWROW_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+float SsimWindowRow_AVX512BW(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const __m512i vc1 = _mm512_set1_epi32(26634);
+ const __m512i vc2 = _mm512_set1_epi32(239708);
+ __m512 total = _mm512_setzero_ps();
+#define SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3) \
+ __m512i u0 = _mm512_unpacklo_epi32(w0, w1); \
+ __m512i u1 = _mm512_unpackhi_epi32(w0, w1); \
+ __m512i u2 = _mm512_unpacklo_epi32(w2, w3); \
+ __m512i u3 = _mm512_unpackhi_epi32(w2, w3); \
+ __m512i sum_a = _mm512_unpacklo_epi64(u0, u2); \
+ __m512i sum_b = _mm512_unpackhi_epi64(u0, u2); \
+ __m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); \
+ __m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); \
+ __m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); \
+ __m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), \
+ _mm512_madd_epi16(sum_b, sum_b)); \
+ __m512i n1 = _mm512_add_epi32(ab2, vc1); \
+ __m512i n2 = _mm512_add_epi32( \
+ _mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); \
+ __m512i d1 = _mm512_add_epi32(a2b2, vc1); \
+ __m512i d2 = _mm512_add_epi32( \
+ _mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); \
+ __m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); \
+ __m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); \
+ __m512 ssim = _mm512_div_ps(num, den)
+
+ while (num_windows >= 16) {
+ __m512i t0 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 0),
+ _mm512_loadu_si512(row1 + 0));
+ __m512i t1 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 16),
+ _mm512_loadu_si512(row1 + 16));
+ __m512i t2 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 32),
+ _mm512_loadu_si512(row1 + 32));
+ __m512i t3 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 48),
+ _mm512_loadu_si512(row1 + 48));
+ __m512i t4 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 52),
+ _mm512_loadu_si512(row1 + 52));
+ row0 += 64;
+ row1 += 64;
+ num_windows -= 16;
+ __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4));
+ __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4));
+ __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4));
+ __m512i w3 = _mm512_add_epi32(t3, t4);
+ SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3);
+ total = _mm512_add_ps(total, ssim);
+ }
+ if (num_windows > 0) {
+ const uint64_t load_mask = ~0ULL >> (60 - num_windows * 4);
+ const __mmask16 win_mask =
+ (__mmask16)(((0x01110U >> (16 - (num_windows & 3) * 4))
+ << (num_windows >> 2)) +
+ (0x1111U << (num_windows >> 2)) - 0x1111U);
+ const __mmask16 m0 = (__mmask16)load_mask;
+ const __mmask16 m1 = (__mmask16)(load_mask >> 16);
+ const __mmask16 m2 = (__mmask16)(load_mask >> 32);
+ const __mmask16 m3 = (__mmask16)(load_mask >> 48);
+ __m512i t0 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m0, row0 + 0),
+ _mm512_maskz_loadu_epi32(m0, row1 + 0));
+ __m512i t1 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m1, row0 + 16),
+ _mm512_maskz_loadu_epi32(m1, row1 + 16));
+ __m512i t2 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m2, row0 + 32),
+ _mm512_maskz_loadu_epi32(m2, row1 + 32));
+ __m512i t3 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m3, row0 + 48),
+ _mm512_maskz_loadu_epi32(m3, row1 + 48));
+ __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4));
+ __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4));
+ __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4));
+ __m512i w3 = _mm512_add_epi32(t3, _mm512_alignr_epi32(t3, t3, 4));
+ SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3);
+ total = _mm512_mask_add_ps(total, win_mask, total, ssim);
+ }
+#undef SSIM_WINDOW_AVX512_WIN
+ return _mm512_reduce_add_ps(total);
+}
+#endif // HAS_SSIMWINDOWROW_AVX512BW
+
+#if defined(HAS_ARGBDETECTROW_AVX2)
+LIBYUV_TARGET_AVX2
+uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
+ if (width >= 8) {
+ const __m256i mask_rgb = _mm256_set1_epi32(0x00ffff00);
+ const __m256i ones = _mm256_set1_epi8(-1);
+ do {
+ __m256i v =
+ _mm256_or_si256(_mm256_loadu_si256((const __m256i*)argb), mask_rgb);
+ uint32_t diff =
+ ~(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, ones));
+ if (diff) {
+#if defined(_MSC_VER) && !defined(__clang__)
+ unsigned long bit; // NOLINT(runtime/int)
+ _BitScanForward(&bit, diff);
+#else
+ int bit = __builtin_ctz(diff);
+#endif
+ _mm256_zeroupper();
+ return (bit & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ argb += 32;
+ width -= 8;
+ } while (width >= 8);
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ const __m128i mask_rgb = _mm_set1_epi32(0x00ffff00);
+ const __m128i ones = _mm_set1_epi8(-1);
+ do {
+ __m128i v = _mm_or_si128(_mm_loadu_si32(argb), mask_rgb);
+ uint32_t diff =
+ (~(uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, ones))) & 0x9u;
+ if (diff) {
+ return (diff & 1u) ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ argb += 4;
+ width -= 1;
+ } while (width > 0);
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_AVX2
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
+
+#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
+ // (Visual C or LIBYUV_ENABLE_ROWWIN)
diff --git a/libs/libyuv/source/convert.cc b/libs/libyuv/source/convert.cc
index 614fa48241..e76ba7c341 100644
--- a/libs/libyuv/source/convert.cc
+++ b/libs/libyuv/source/convert.cc
@@ -10,24 +10,35 @@
#include "libyuv/convert.h"
-#include "libyuv/basic_types.h"
+#include <limits.h>
+
+#include "libyuv/convert_from_argb.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/row.h"
-#include "libyuv/scale.h" // For ScalePlane()
+#include "libyuv/scale.h" // For ScalePlane()
+#include "libyuv/scale_row.h" // For FixedDiv
+#include "libyuv/scale_uv.h" // For UVScale()
#ifdef __cplusplus
namespace libyuv {
+
+extern const struct ArgbConstants kArgbI601Constants;
+extern const struct ArgbConstants kArgbJPEGConstants;
extern "C" {
#endif
+// Subsample amount uses a shift.
+// v is value
+// a is amount to add to round up
+// s is shift to subsample down
#define SUBSAMPLE(v, a, s) (v < 0) ? (-((-v + a) >> s)) : ((v + a) >> s)
static __inline int Abs(int v) {
return v >= 0 ? v : -v;
}
-// Any I4xx To I420 format with mirroring.
+// Any I4xx To I420 format
static int I4xxToI420(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -44,27 +55,31 @@ static int I4xxToI420(const uint8_t* src_y,
int src_y_height,
int src_uv_width,
int src_uv_height) {
- const int dst_y_width = Abs(src_y_width);
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v ||
+ src_y_width <= 0 || src_y_height == 0 || src_y_height == INT_MIN ||
+ src_uv_width <= 0 || src_uv_height == 0) {
+ return -1;
+ }
+ const int dst_y_width = src_y_width;
const int dst_y_height = Abs(src_y_height);
const int dst_uv_width = SUBSAMPLE(dst_y_width, 1, 1);
const int dst_uv_height = SUBSAMPLE(dst_y_height, 1, 1);
- if (src_uv_width == 0 || src_uv_height == 0) {
- return -1;
- }
if (dst_y) {
- ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, dst_y,
- dst_stride_y, dst_y_width, dst_y_height, kFilterBilinear);
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, src_y_width,
+ src_y_height);
}
- ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
- dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
- ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, dst_v,
- dst_stride_v, dst_uv_width, dst_uv_height, kFilterBilinear);
- return 0;
+ r = ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
+ dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, dst_v,
+ dst_stride_v, dst_uv_width, dst_uv_height, kFilterBilinear);
+ return r;
}
-// Copy I420 with optional flipping.
-// TODO(fbarchard): Use Scale plane which supports mirroring, but ensure
-// is does row coalescing.
+// Copy I420 with optional vertical flipping using negative height.
LIBYUV_API
int I420Copy(const uint8_t* src_y,
int src_stride_y,
@@ -82,16 +97,17 @@ int I420Copy(const uint8_t* src_y,
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -124,16 +140,17 @@ int I010Copy(const uint16_t* src_y,
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -148,270 +165,417 @@ int I010Copy(const uint16_t* src_y,
return 0;
}
-// Convert 10 bit YUV to 8 bit.
-LIBYUV_API
-int I010ToI420(const uint16_t* src_y,
- int src_stride_y,
- const uint16_t* src_u,
- int src_stride_u,
- const uint16_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+static int Planar16bitTo8bit(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int depth) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
+ int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ int scale = 1 << (24 - depth);
// Negative height means invert the image.
if (height < 0) {
height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ uv_height = -uv_height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(uv_height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(uv_height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
// Convert Y plane.
- Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, 16384, width,
- height);
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
+ }
// Convert UV planes.
- Convert16To8Plane(src_u, src_stride_u, dst_u, dst_stride_u, 16384, halfwidth,
- halfheight);
- Convert16To8Plane(src_v, src_stride_v, dst_v, dst_stride_v, 16384, halfwidth,
- halfheight);
+ Convert16To8Plane(src_u, src_stride_u, dst_u, dst_stride_u, scale, uv_width,
+ uv_height);
+ Convert16To8Plane(src_v, src_stride_v, dst_v, dst_stride_v, scale, uv_width,
+ uv_height);
return 0;
}
-// 422 chroma is 1/2 width, 1x height
-// 420 chroma is 1/2 width, 1/2 height
-LIBYUV_API
-int I422ToI420(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- const int src_uv_width = SUBSAMPLE(width, 1, 1);
- return I4xxToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
- dst_v, dst_stride_v, width, height, src_uv_width, height);
-}
+static int I41xToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ const int scale = 1 << (24 - depth);
+ const int uv_width = SUBSAMPLE(width, 1, 1);
+ void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
+ int width) = Convert16To8Row_C;
+ void (*HalfWidthRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv, int scale, int width) =
+ (width & 1) ? HalfWidthRow_16To8_Odd_C : HalfWidthRow_16To8_C;
-// TODO(fbarchard): Implement row conversion.
-LIBYUV_API
-int I422ToNV21(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (height - 1) * src_stride_u;
- src_v = src_v + (height - 1) * src_stride_v;
- src_stride_y = -src_stride_y;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
- // Allocate u and v buffers
- align_buffer_64(plane_u, halfwidth * halfheight * 2);
- uint8_t* plane_v = plane_u + halfwidth * halfheight;
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Convert16To8Row = Convert16To8Row_SVE2;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Convert16To8Row = Convert16To8Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert16To8Row = Convert16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert16To8Row = Convert16To8Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert16To8Row = Convert16To8Row_HVX;
+ }
+#endif
- I422ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
- dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth, width,
- height);
- MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu,
- halfwidth, halfheight);
- free_aligned_buffer_64(plane_u);
+ if (IS_ALIGNED(width, 2)) {
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_NEON;
+ if (IS_ALIGNED(uv_width, 8)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_NEON;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_SVE2;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_SME;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_SSSE3;
+ if (IS_ALIGNED(uv_width, 8)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX2;
+ if (IS_ALIGNED(uv_width, 16)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV;
+ }
+#endif
+ } else {
+#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_AVX2;
+ }
+#endif
+ }
+
+ for (y = 0; y < height - 1; y += 2) {
+ HalfWidthRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width);
+ HalfWidthRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width);
+ if (dst_y) {
+ Convert16To8Row(src_y, dst_y, scale, width);
+ Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width);
+ src_y += src_stride_y * 2;
+ dst_y += dst_stride_y * 2;
+ }
+ src_u += src_stride_u * 2;
+ src_v += src_stride_v * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ HalfWidthRow_16To8(src_u, 0, dst_u, scale, uv_width);
+ HalfWidthRow_16To8(src_v, 0, dst_v, scale, uv_width);
+ if (dst_y) {
+ Convert16To8Row(src_y, dst_y, scale, width);
+ }
+ }
return 0;
}
-#ifdef I422TONV21_ROW_VERSION
-// Unittest fails for this version.
-// 422 chroma is 1/2 width, 1x height
-// 420 chroma is 1/2 width, 1/2 height
-// Swap src_u and src_v to implement I422ToNV12
-LIBYUV_API
-int I422ToNV21(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
+static int I21xToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int depth) {
int y;
- void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
- uint8_t* dst_uv, int width) = MergeUVRow_C;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
- ptrdiff_t src_stride, int dst_width,
- int source_y_fraction) = InterpolateRow_C;
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_vu || width <= 0 || height == 0) {
+ const int scale = 1 << (24 - depth);
+ const int uv_width = SUBSAMPLE(width, 1, 1);
+ void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
+ int width) = Convert16To8Row_C;
+ void (*HalfRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv, int scale, int width) =
+ HalfRow_16To8_C;
+
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
- src_stride_y = -src_stride_y;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
-#if defined(HAS_MERGEUVROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- MergeUVRow = MergeUVRow_Any_SSE2;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow = MergeUVRow_SSE2;
+
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_AVX2)
+#if defined(HAS_CONVERT16TO8ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Convert16To8Row = Convert16To8Row_SVE2;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Convert16To8Row = Convert16To8Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- MergeUVRow = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(halfwidth, 32)) {
- MergeUVRow = MergeUVRow_AVX2;
+ Convert16To8Row = Convert16To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert16To8Row = Convert16To8Row_AVX2;
}
}
#endif
-#if defined(HAS_MERGEUVROW_NEON)
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert16To8Row = Convert16To8Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert16To8Row = Convert16To8Row_HVX;
+ }
+#endif
+
+#if defined(HAS_HALFROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeUVRow = MergeUVRow_Any_NEON;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow = MergeUVRow_NEON;
+ HalfRow_16To8 = HalfRow_16To8_Any_NEON;
+ if (IS_ALIGNED(uv_width, 16)) {
+ HalfRow_16To8 = HalfRow_16To8_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow = MergeUVRow_Any_MSA;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow = MergeUVRow_MSA;
- }
+#if defined(HAS_HALFROW_16TO8_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ HalfRow_16To8 = HalfRow_16To8_SVE2;
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- MergeUVRow = MergeUVRow_MMI;
- }
+#if defined(HAS_HALFROW_16TO8_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ HalfRow_16To8 = HalfRow_16To8_SME;
}
#endif
-#if defined(HAS_INTERPOLATEROW_SSSE3)
+#if defined(HAS_HALFROW_16TO8_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
+ HalfRow_16To8 = HalfRow_16To8_Any_SSSE3;
+ if (IS_ALIGNED(uv_width, 16)) {
+ HalfRow_16To8 = HalfRow_16To8_SSSE3;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_AVX2)
+#if defined(HAS_HALFROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_AVX2;
+ HalfRow_16To8 = HalfRow_16To8_Any_AVX2;
+ if (IS_ALIGNED(uv_width, 32)) {
+ HalfRow_16To8 = HalfRow_16To8_AVX2;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_NEON;
- }
+#if defined(HAS_HALFROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HalfRow_16To8 = HalfRow_16To8_AVX512BW;
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
- }
+#if defined(HAS_HALFROW_16TO8_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ HalfRow_16To8 = HalfRow_16To8_RVV;
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- InterpolateRow = InterpolateRow_MMI;
- }
+#if defined(HAS_HALFROW_16TO8_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ HalfRow_16To8 = HalfRow_16To8_HVX;
}
#endif
- if (dst_y) {
- CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, halfwidth, height);
+ for (y = 0; y < height - 1; y += 2) {
+ HalfRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width);
+ HalfRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width);
+ if (dst_y) {
+ Convert16To8Row(src_y, dst_y, scale, width);
+ Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width);
+ src_y += src_stride_y * 2;
+ dst_y += dst_stride_y * 2;
+ }
+ src_u += src_stride_u * 2;
+ src_v += src_stride_v * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
}
- {
- // Allocate 2 rows of vu.
- int awidth = halfwidth * 2;
- align_buffer_64(row_vu_0, awidth * 2);
- uint8_t* row_vu_1 = row_vu_0 + awidth;
+ if (height & 1) {
+ HalfRow_16To8(src_u, 0, dst_u, scale, uv_width);
+ HalfRow_16To8(src_v, 0, dst_v, scale, uv_width);
+ if (dst_y) {
+ Convert16To8Row(src_y, dst_y, scale, width);
+ }
+ }
+ return 0;
+}
- for (y = 0; y < height - 1; y += 2) {
- MergeUVRow(src_v, src_u, row_vu_0, halfwidth);
- MergeUVRow(src_v + src_stride_v, src_u + src_stride_u, row_vu_1,
- halfwidth);
- InterpolateRow(dst_vu, row_vu_0, awidth, awidth, 128);
- src_u += src_stride_u * 2;
- src_v += src_stride_v * 2;
- dst_vu += dst_stride_vu;
- }
- if (height & 1) {
- MergeUVRow(src_v, src_u, dst_vu, halfwidth);
- }
- free_aligned_buffer_64(row_vu_0);
- }
- return 0;
+// Convert 10 bit YUV to 8 bit.
+LIBYUV_API
+int I010ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 1, 10);
}
-#endif // I422TONV21_ROW_VERSION
-// 444 chroma is 1x width, 1x height
-// 420 chroma is 1/2 width, 1/2 height
LIBYUV_API
-int I444ToI420(const uint8_t* src_y,
+int I210ToI420(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
+ const uint16_t* src_u,
int src_stride_u,
- const uint8_t* src_v,
+ const uint16_t* src_v,
int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
@@ -421,55 +585,39 @@ int I444ToI420(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- return I4xxToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ return I21xToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
- dst_v, dst_stride_v, width, height, width, height);
+ dst_v, dst_stride_v, width, height, 10);
}
-// TODO(fbarchard): Implement row conversion.
LIBYUV_API
-int I444ToNV21(const uint8_t* src_y,
+int I210ToI422(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_u,
+ const uint16_t* src_u,
int src_stride_u,
- const uint8_t* src_v,
+ const uint16_t* src_v,
int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
int width,
int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (height - 1) * src_stride_u;
- src_v = src_v + (height - 1) * src_stride_v;
- src_stride_y = -src_stride_y;
- src_stride_u = -src_stride_u;
- src_stride_v = -src_stride_v;
- }
- // Allocate u and v buffers
- align_buffer_64(plane_u, halfwidth * halfheight * 2);
- uint8_t* plane_v = plane_u + halfwidth * halfheight;
-
- I444ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
- dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth, width,
- height);
- MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu,
- halfwidth, halfheight);
- free_aligned_buffer_64(plane_u);
- return 0;
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 0, 10);
}
-// I400 is greyscale typically used in MJPG
LIBYUV_API
-int I400ToI420(const uint8_t* src_y,
+int I410ToI420(const uint16_t* src_y,
int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -478,174 +626,60 @@ int I400ToI420(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
- }
- if (dst_y) {
- CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
- }
- SetPlane(dst_u, dst_stride_u, halfwidth, halfheight, 128);
- SetPlane(dst_v, dst_stride_v, halfwidth, halfheight, 128);
- return 0;
+ return I41xToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, 10);
}
-// I400 is greyscale typically used in MJPG
LIBYUV_API
-int I400ToNV21(const uint8_t* src_y,
+int I410ToI444(const uint16_t* src_y,
int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
int width,
int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!dst_vu || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
- }
- if (dst_y) {
- CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
- }
- SetPlane(dst_vu, dst_stride_vu, halfwidth * 2, halfheight, 128);
- return 0;
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 0,
+ 0, 10);
}
-static void CopyPlane2(const uint8_t* src,
- int src_stride_0,
- int src_stride_1,
- uint8_t* dst,
- int dst_stride,
- int width,
- int height) {
- int y;
- void (*CopyRow)(const uint8_t* src, uint8_t* dst, int width) = CopyRow_C;
-#if defined(HAS_COPYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
- }
-#endif
-#if defined(HAS_COPYROW_AVX)
- if (TestCpuFlag(kCpuHasAVX)) {
- CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
- }
-#endif
-#if defined(HAS_COPYROW_ERMS)
- if (TestCpuFlag(kCpuHasERMS)) {
- CopyRow = CopyRow_ERMS;
- }
-#endif
-#if defined(HAS_COPYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
- }
-#endif
-
- // Copy plane
- for (y = 0; y < height - 1; y += 2) {
- CopyRow(src, dst, width);
- CopyRow(src + src_stride_0, dst + dst_stride, width);
- src += src_stride_0 + src_stride_1;
- dst += dst_stride * 2;
- }
- if (height & 1) {
- CopyRow(src, dst, width);
- }
-}
-
-// Support converting from FOURCC_M420
-// Useful for bandwidth constrained transports like USB 1.0 and 2.0 and for
-// easy conversion to I420.
-// M420 format description:
-// M420 is row biplanar 420: 2 rows of Y and 1 row of UV.
-// Chroma is half width / half height. (420)
-// src_stride_m420 is row planar. Normally this will be the width in pixels.
-// The UV plane is half width, but 2 values, so src_stride_m420 applies to
-// this as well as the two Y planes.
-static int X420ToI420(const uint8_t* src_y,
- int src_stride_y0,
- int src_stride_y1,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_uv || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- halfheight = (height + 1) >> 1;
- if (dst_y) {
- dst_y = dst_y + (height - 1) * dst_stride_y;
- }
- dst_u = dst_u + (halfheight - 1) * dst_stride_u;
- dst_v = dst_v + (halfheight - 1) * dst_stride_v;
- dst_stride_y = -dst_stride_y;
- dst_stride_u = -dst_stride_u;
- dst_stride_v = -dst_stride_v;
- }
- // Coalesce rows.
- if (src_stride_y0 == width && src_stride_y1 == width &&
- dst_stride_y == width) {
- width *= height;
- height = 1;
- src_stride_y0 = src_stride_y1 = dst_stride_y = 0;
- }
- // Coalesce rows.
- if (src_stride_uv == halfwidth * 2 && dst_stride_u == halfwidth &&
- dst_stride_v == halfwidth) {
- halfwidth *= halfheight;
- halfheight = 1;
- src_stride_uv = dst_stride_u = dst_stride_v = 0;
- }
-
- if (dst_y) {
- if (src_stride_y0 == src_stride_y1) {
- CopyPlane(src_y, src_stride_y0, dst_y, dst_stride_y, width, height);
- } else {
- CopyPlane2(src_y, src_stride_y0, src_stride_y1, dst_y, dst_stride_y,
- width, height);
- }
- }
-
- // Split UV plane - NV12 / NV21
- SplitUVPlane(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v, dst_stride_v,
- halfwidth, halfheight);
-
- return 0;
+LIBYUV_API
+int I012ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 1, 12);
}
-// Convert NV12 to I420.
LIBYUV_API
-int NV12ToI420(const uint8_t* src_y,
+int I212ToI422(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -654,17 +688,19 @@ int NV12ToI420(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- return X420ToI420(src_y, src_stride_y, src_stride_y, src_uv, src_stride_uv,
- dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
- dst_stride_v, width, height);
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 0, 12);
}
-// Convert NV21 to I420. Same as NV12 but u and v pointers swapped.
LIBYUV_API
-int NV21ToI420(const uint8_t* src_y,
+int I212ToI420(const uint16_t* src_y,
int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -673,15 +709,18 @@ int NV21ToI420(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- return X420ToI420(src_y, src_stride_y, src_stride_y, src_vu, src_stride_vu,
- dst_y, dst_stride_y, dst_v, dst_stride_v, dst_u,
- dst_stride_u, width, height);
+ return I21xToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, 12);
}
-// Convert M420 to I420.
LIBYUV_API
-int M420ToI420(const uint8_t* src_m420,
- int src_stride_m420,
+int I412ToI444(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -690,16 +729,19 @@ int M420ToI420(const uint8_t* src_m420,
int dst_stride_v,
int width,
int height) {
- return X420ToI420(src_m420, src_stride_m420, src_stride_m420 * 2,
- src_m420 + src_stride_m420 * 2, src_stride_m420 * 3, dst_y,
- dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
- width, height);
+ return Planar16bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 0,
+ 0, 12);
}
-// Convert YUY2 to I420.
LIBYUV_API
-int YUY2ToI420(const uint8_t* src_yuy2,
- int src_stride_yuy2,
+int I412ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -708,91 +750,2091 @@ int YUY2ToI420(const uint8_t* src_yuy2,
int dst_stride_v,
int width,
int height) {
- int y;
- void (*YUY2ToUVRow)(const uint8_t* src_yuy2, int src_stride_yuy2,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- YUY2ToUVRow_C;
- void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
- YUY2ToYRow_C;
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2;
- src_stride_yuy2 = -src_stride_yuy2;
+ return I41xToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, 12);
+}
+
+// Any Ix10 To I010 format
+static int Ix10ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y) {
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
}
-#if defined(HAS_YUY2TOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- YUY2ToUVRow = YUY2ToUVRow_Any_SSE2;
- YUY2ToYRow = YUY2ToYRow_Any_SSE2;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToUVRow = YUY2ToUVRow_SSE2;
- YUY2ToYRow = YUY2ToYRow_SSE2;
- }
+ const int dst_y_width = width;
+ const int dst_y_height = Abs(height);
+ const int src_uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ const int src_uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ const int dst_uv_width = SUBSAMPLE(dst_y_width, 1, 1);
+ const int dst_uv_height = SUBSAMPLE(dst_y_height, 1, 1);
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
-#endif
-#if defined(HAS_YUY2TOYROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- YUY2ToUVRow = YUY2ToUVRow_Any_AVX2;
- YUY2ToYRow = YUY2ToYRow_Any_AVX2;
+ r = ScalePlane_12(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
+ dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, src_uv_width, src_uv_height, dst_v,
+ dst_stride_v, dst_uv_width, dst_uv_height, kFilterBilinear);
+ return r;
+}
+
+LIBYUV_API
+int I410ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Ix10ToI010(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, 0, 0);
+}
+
+LIBYUV_API
+int I210ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Ix10ToI010(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, 1, 0);
+}
+
+// Any I[420]1[02] to P[420]1[02] format
+static int IxxxToPxxx(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int depth) {
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ const int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ const int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+
+ ConvertToMSBPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height,
+ depth);
+ MergeUVPlane_16(src_u, src_stride_u, src_v, src_stride_v, dst_uv,
+ dst_stride_uv, uv_width, uv_height, depth);
+ return 0;
+}
+
+LIBYUV_API
+int I010ToP010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 1, 10);
+}
+
+LIBYUV_API
+int I010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int y;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ const int scale = 16385; // 16384 for 10 bits
+ void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
+ int width) = Convert16To8Row_C;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Convert16To8Row = Convert16To8Row_Any_SSSE3;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ Convert16To8Row = Convert16To8Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ Convert16To8Row = Convert16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert16To8Row = Convert16To8Row_HVX;
+ }
+#endif
+
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+
+ // Convert Y plane.
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
+ }
+
+ {
+ // Allocate a row of uv.
+ align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
+ uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
+ if (!row_u)
+ return 1;
+
+ for (y = 0; y < halfheight; ++y) {
+ Convert16To8Row(src_u, row_u, scale, halfwidth);
+ Convert16To8Row(src_v, row_v, scale, halfwidth);
+ MergeUVRow(row_u, row_v, dst_uv, halfwidth);
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row_u);
+ }
+ return 0;
+}
+
+LIBYUV_API
+int I210ToP210(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 0, 10);
+}
+
+LIBYUV_API
+int I410ToP410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 0, 0, 10);
+}
+
+LIBYUV_API
+int I012ToP012(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 1, 12);
+}
+
+LIBYUV_API
+int I212ToP212(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 0, 12);
+}
+
+LIBYUV_API
+int I412ToP412(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return IxxxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 0, 0, 12);
+}
+
+// 422 chroma is 1/2 width, 1x height
+// 420 chroma is 1/2 width, 1/2 height
+LIBYUV_API
+int I422ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ const int src_uv_width = SUBSAMPLE(width, 1, 1);
+ return I4xxToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, src_uv_width, height);
+}
+
+LIBYUV_API
+int I422ToI210(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ // Convert Y plane.
+ Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width,
+ height);
+ // Convert UV planes.
+ Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 1024, halfwidth,
+ height);
+ Convert8To16Plane(src_v, src_stride_v, dst_v, dst_stride_v, 1024, halfwidth,
+ height);
+ return 0;
+}
+
+// TODO(fbarchard): Implement row conversion.
+LIBYUV_API
+int I422ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ int r;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ // Allocate u and v buffers
+ const uint64_t plane_size = (uint64_t)halfwidth * halfheight;
+ if (plane_size > SIZE_MAX / 2)
+ return 1;
+ align_buffer_64(plane_u, (size_t)plane_size * 2);
+ if (!plane_u)
+ return 1;
+ uint8_t* plane_v = plane_u + (size_t)plane_size;
+
+ r = I422ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth,
+ width, height);
+ if (r != 0) {
+ free_aligned_buffer_64(plane_u);
+ return r;
+ }
+ MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu,
+ halfwidth, halfheight);
+ free_aligned_buffer_64(plane_u);
+ return 0;
+}
+
+LIBYUV_API
+int MM21ToNV12(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ if (!src_uv || !dst_uv || width <= 0) {
+ return -1;
+ }
+
+ int sign = height < 0 ? -1 : 1;
+
+ if (dst_y) {
+ DetilePlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height, 32);
+ }
+ DetilePlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, (width + 1) & ~1,
+ (height + sign) / 2, 16);
+
+ return 0;
+}
+
+LIBYUV_API
+int MM21ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int sign = height < 0 ? -1 : 1;
+
+ if (!src_uv || !dst_u || !dst_v || width <= 0) {
+ return -1;
+ }
+
+ if (dst_y) {
+ DetilePlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height, 32);
+ }
+ DetileSplitUVPlane(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, (width + 1) & ~1, (height + sign) / 2, 16);
+
+ return 0;
+}
+
+LIBYUV_API
+int MM21ToYUY2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ int width,
+ int height) {
+ if (!src_y || !src_uv || !dst_yuy2 || width <= 0) {
+ return -1;
+ }
+
+ DetileToYUY2(src_y, src_stride_y, src_uv, src_stride_uv, dst_yuy2,
+ dst_stride_yuy2, width, height, 32);
+
+ return 0;
+}
+
+// Convert MT2T into P010. See tinyurl.com/mtk-10bit-video-format for format
+// documentation.
+// TODO(greenjustin): Add an MT2T to I420 conversion.
+LIBYUV_API
+int MT2TToP010(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ if (width <= 0 || height == 0 || height == INT_MIN || !src_uv || !dst_uv) {
+ return -1;
+ }
+
+ {
+ int uv_width = (width + 1) & ~1;
+ int uv_height = (height + 1) / 2;
+ int y = 0;
+ const int tile_width = 16;
+ const int y_tile_height = 32;
+ const int uv_tile_height = 16;
+ int padded_width = (width + tile_width - 1) & ~(tile_width - 1);
+ int y_tile_row_size = padded_width * y_tile_height * 10 / 8;
+ int uv_tile_row_size = padded_width * uv_tile_height * 10 / 8;
+ size_t row_buf_size = padded_width * y_tile_height * sizeof(uint16_t);
+ void (*UnpackMT2T)(const uint8_t* src, uint16_t* dst, size_t size) =
+ UnpackMT2T_C;
+ align_buffer_64(row_buf, row_buf_size);
+ if (!row_buf)
+ return 1;
+
+#if defined(HAS_UNPACKMT2T_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ UnpackMT2T = UnpackMT2T_AVX2;
+ }
+#endif
+#if defined(HAS_UNPACKMT2T_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ UnpackMT2T = UnpackMT2T_NEON;
+ }
+#endif
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ uv_height = (height + 1) / 2;
+ if (dst_y) {
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
+ dst_uv = dst_uv + (ptrdiff_t)(uv_height - 1) * dst_stride_uv;
+ dst_stride_uv = -dst_stride_uv;
+ }
+
+ // Unpack and detile Y in rows of tiles
+ if (src_y && dst_y) {
+ for (y = 0; y < (height & ~(y_tile_height - 1)); y += y_tile_height) {
+ UnpackMT2T(src_y, (uint16_t*)row_buf, y_tile_row_size);
+ DetilePlane_16((uint16_t*)row_buf, padded_width, dst_y, dst_stride_y,
+ width, y_tile_height, y_tile_height);
+ src_y += src_stride_y * y_tile_height;
+ dst_y += dst_stride_y * y_tile_height;
+ }
+ if (height & (y_tile_height - 1)) {
+ UnpackMT2T(src_y, (uint16_t*)row_buf, y_tile_row_size);
+ DetilePlane_16((uint16_t*)row_buf, padded_width, dst_y, dst_stride_y,
+ width, height & (y_tile_height - 1), y_tile_height);
+ }
+ }
+
+ // Unpack and detile UV plane
+ for (y = 0; y < (uv_height & ~(uv_tile_height - 1)); y += uv_tile_height) {
+ UnpackMT2T(src_uv, (uint16_t*)row_buf, uv_tile_row_size);
+ DetilePlane_16((uint16_t*)row_buf, padded_width, dst_uv, dst_stride_uv,
+ uv_width, uv_tile_height, uv_tile_height);
+ src_uv += src_stride_uv * uv_tile_height;
+ dst_uv += dst_stride_uv * uv_tile_height;
+ }
+ if (uv_height & (uv_tile_height - 1)) {
+ UnpackMT2T(src_uv, (uint16_t*)row_buf, uv_tile_row_size);
+ DetilePlane_16((uint16_t*)row_buf, padded_width, dst_uv, dst_stride_uv,
+ uv_width, uv_height & (uv_tile_height - 1),
+ uv_tile_height);
+ }
+ free_aligned_buffer_64(row_buf);
+ }
+ return 0;
+}
+
+#ifdef I422TONV21_ROW_VERSION
+// Unittest fails for this version.
+// 422 chroma is 1/2 width, 1x height
+// 420 chroma is 1/2 width, 1/2 height
+// Swap src_u and src_v to implement I422ToNV12
+LIBYUV_API
+int I422ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ int y;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_C;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_vu || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ InterpolateRow = InterpolateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow = InterpolateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
+ }
+ }
+#endif
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, halfwidth, height);
+ }
+ {
+ // Allocate 2 rows of vu.
+ int awidth = halfwidth * 2;
+ align_buffer_64(row_vu_0, awidth * 2);
+ uint8_t* row_vu_1 = row_vu_0 + awidth;
+ if (!row_vu_0)
+ return 1;
+
+ for (y = 0; y < height - 1; y += 2) {
+ MergeUVRow(src_v, src_u, row_vu_0, halfwidth);
+ MergeUVRow(src_v + src_stride_v, src_u + src_stride_u, row_vu_1,
+ halfwidth);
+ InterpolateRow(dst_vu, row_vu_0, awidth, awidth, 128);
+ src_u += src_stride_u * 2;
+ src_v += src_stride_v * 2;
+ dst_vu += dst_stride_vu;
+ }
+ if (height & 1) {
+ MergeUVRow(src_v, src_u, dst_vu, halfwidth);
+ }
+ free_aligned_buffer_64(row_vu_0);
+ }
+ return 0;
+}
+#endif // I422TONV21_ROW_VERSION
+
+// 444 chroma is 1x width, 1x height
+// 420 chroma is 1/2 width, 1/2 height
+LIBYUV_API
+int I444ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return I4xxToI420(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height, width, height);
+}
+
+LIBYUV_API
+int I444ToNV12(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ HalfMergeUVPlane(src_u, src_stride_u, src_v, src_stride_v, dst_uv,
+ dst_stride_uv, width, height);
+ return 0;
+}
+
+LIBYUV_API
+int I444ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return I444ToNV12(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_y, dst_stride_y, dst_vu, dst_stride_vu,
+ width, height);
+}
+
+// I400 is greyscale typically used in MJPG
+LIBYUV_API
+int I400ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ SetPlane(dst_u, dst_stride_u, halfwidth, halfheight, 128);
+ SetPlane(dst_v, dst_stride_v, halfwidth, halfheight, 128);
+ return 0;
+}
+
+// I400 is greyscale typically used in MJPG
+LIBYUV_API
+int I400ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !dst_vu || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ SetPlane(dst_vu, dst_stride_vu, halfwidth * 2, halfheight, 128);
+ return 0;
+}
+
+// Convert NV12 to I420.
+// TODO(fbarchard): Consider inverting destination. Faster on ARM with prfm.
+LIBYUV_API
+int NV12ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(halfheight - 1) * src_stride_uv;
+ src_stride_y = -src_stride_y;
+ src_stride_uv = -src_stride_uv;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = dst_stride_y = 0;
+ }
+ // Coalesce rows.
+ if (src_stride_uv == halfwidth * 2 && dst_stride_u == halfwidth &&
+ dst_stride_v == halfwidth &&
+ (ptrdiff_t)halfwidth * halfheight <= INT_MAX) {
+ halfwidth *= halfheight;
+ halfheight = 1;
+ src_stride_uv = dst_stride_u = dst_stride_v = 0;
+ }
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+
+ // Split UV plane - NV12 / NV21
+ SplitUVPlane(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ halfwidth, halfheight);
+
+ return 0;
+}
+
+// Convert NV21 to I420. Same as NV12 but u and v pointers swapped.
+LIBYUV_API
+int NV21ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return NV12ToI420(src_y, src_stride_y, src_vu, src_stride_vu, dst_y,
+ dst_stride_y, dst_v, dst_stride_v, dst_u, dst_stride_u,
+ width, height);
+}
+
+LIBYUV_API
+int NV12ToNV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = UVScale(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1),
+ SUBSAMPLE(height, 1, 1), dst_uv, dst_stride_uv, Abs(width),
+ Abs(height), kFilterBilinear);
+ return r;
+}
+
+LIBYUV_API
+int NV16ToNV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = UVScale(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1), height, dst_uv,
+ dst_stride_uv, Abs(width), Abs(height), kFilterBilinear);
+ return r;
+}
+
+// Any P[420]1[02] to I[420]1[02] format
+static int PxxxToIxxx(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int depth) {
+ const int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ const int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ if (!src_y || !dst_y || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ ConvertToLSBPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height,
+ depth);
+ SplitUVPlane_16(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, uv_width, uv_height, depth);
+ return 0;
+}
+
+LIBYUV_API
+int P010ToI010(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return PxxxToIxxx(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 1, 1, 10);
+}
+
+LIBYUV_API
+int P210ToI210(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return PxxxToIxxx(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 1, 0, 10);
+}
+
+LIBYUV_API
+int P410ToI410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return PxxxToIxxx(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 0, 0, 10);
+}
+
+static void InvertPx10(const uint16_t** src_y,
+ int* src_stride_y,
+ const uint16_t** src_uv,
+ int* src_stride_uv,
+ int* height,
+ int subsample_y) {
+ const int abs_height = Abs(*height);
+ const int src_uv_height = SUBSAMPLE(abs_height, subsample_y, subsample_y);
+ if (*src_y) {
+ *src_y = *src_y + (ptrdiff_t)(abs_height - 1) * *src_stride_y;
+ *src_stride_y = -*src_stride_y;
+ }
+ *src_uv = *src_uv + (ptrdiff_t)(src_uv_height - 1) * *src_stride_uv;
+ *src_stride_uv = -*src_stride_uv;
+ *height = abs_height;
+}
+
+static int Px10ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y) {
+ const int scale16 = 256; // P*10 stores 10-bit in the MSBs.
+ int src_uv_width;
+ int dst_uv_width;
+ if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ InvertPx10(&src_y, &src_stride_y, &src_uv, &src_stride_uv, &height,
+ subsample_y);
+ }
+ src_uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ dst_uv_width = SUBSAMPLE(width, 1, 1);
+
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale16, width,
+ height);
+ }
+
+ if (subsample_x && subsample_y) {
+ int y;
+ const int uv_packed = src_uv_width * 2;
+ const int uv_height = SUBSAMPLE(height, 1, 1);
+ void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
+ int width) = Convert16To8Row_C;
+ void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
+ int width) = SplitUVRow_C;
+
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(uv_packed, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Convert16To8Row = Convert16To8Row_SVE2;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Convert16To8Row = Convert16To8Row_Any_SSSE3;
+ if (IS_ALIGNED(uv_packed, 16)) {
+ Convert16To8Row = Convert16To8Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX2;
+ if (IS_ALIGNED(uv_packed, 32)) {
+ Convert16To8Row = Convert16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert16To8Row = Convert16To8Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert16To8Row = Convert16To8Row_HVX;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SplitUVRow = SplitUVRow_Any_SSE2;
+ if (IS_ALIGNED(src_uv_width, 16)) {
+ SplitUVRow = SplitUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitUVRow = SplitUVRow_Any_AVX2;
+ if (IS_ALIGNED(src_uv_width, 32)) {
+ SplitUVRow = SplitUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SplitUVRow = SplitUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitUVRow = SplitUVRow_Any_NEON;
+ if (IS_ALIGNED(src_uv_width, 16)) {
+ SplitUVRow = SplitUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SplitUVRow = SplitUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SplitUVRow = SplitUVRow_Any_LSX;
+ if (IS_ALIGNED(src_uv_width, 16)) {
+ SplitUVRow = SplitUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SplitUVRow = SplitUVRow_Any_LASX;
+ if (IS_ALIGNED(src_uv_width, 32)) {
+ SplitUVRow = SplitUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitUVRow = SplitUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow = SplitUVRow_HVX;
+ }
+#endif
+
+ align_buffer_64(row_uv, ((uv_packed + 63) & ~63));
+ if (!row_uv) {
+ return 1;
+ }
+ for (y = 0; y < uv_height; ++y) {
+ Convert16To8Row(src_uv, row_uv, scale16, uv_packed);
+ SplitUVRow(row_uv, dst_u, dst_v, src_uv_width);
+ src_uv += src_stride_uv;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ free_aligned_buffer_64(row_uv);
+ return 0;
+ }
+
+ {
+ int y;
+ const int scale = 16384; // 10-bit LSB after SplitUVRow_16.
+ int row_stride;
+ void (*SplitUVRow_16)(const uint16_t* src_uv, uint16_t* dst_u,
+ uint16_t* dst_v, int depth, int width) =
+ SplitUVRow_16_C;
+ void (*Half16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv, int scale, int width) =
+ subsample_x
+ ? HalfRow_16To8_C
+ : ((width & 1) ? HalfWidthRow_16To8_Odd_C : HalfWidthRow_16To8_C);
+
+#if defined(HAS_SPLITUVROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitUVRow_16 = SplitUVRow_16_Any_AVX2;
+ if (IS_ALIGNED(src_uv_width, 16)) {
+ SplitUVRow_16 = SplitUVRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitUVRow_16 = SplitUVRow_16_Any_NEON;
+ if (IS_ALIGNED(src_uv_width, 8)) {
+ SplitUVRow_16 = SplitUVRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow_16 = SplitUVRow_16_HVX;
+ }
+#endif
+ if (subsample_x) {
+#if defined(HAS_HALFROW_16TO8_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Half16To8 = HalfRow_16To8_Any_NEON;
+ if (IS_ALIGNED(dst_uv_width, 16)) {
+ Half16To8 = HalfRow_16To8_NEON;
+ }
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Half16To8 = HalfRow_16To8_SVE2;
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Half16To8 = HalfRow_16To8_SME;
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Half16To8 = HalfRow_16To8_Any_SSSE3;
+ if (IS_ALIGNED(dst_uv_width, 16)) {
+ Half16To8 = HalfRow_16To8_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Half16To8 = HalfRow_16To8_Any_AVX2;
+ if (IS_ALIGNED(dst_uv_width, 32)) {
+ Half16To8 = HalfRow_16To8_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Half16To8 = HalfRow_16To8_AVX512BW;
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Half16To8 = HalfRow_16To8_RVV;
+ }
+#endif
+#if defined(HAS_HALFROW_16TO8_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Half16To8 = HalfRow_16To8_HVX;
+ }
+#endif
+ } else if (IS_ALIGNED(width, 2)) {
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Half16To8 = HalfWidthRow_16To8_Any_NEON;
+ if (IS_ALIGNED(dst_uv_width, 8)) {
+ Half16To8 = HalfWidthRow_16To8_NEON;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Half16To8 = HalfWidthRow_16To8_SVE2;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Half16To8 = HalfWidthRow_16To8_SME;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Half16To8 = HalfWidthRow_16To8_Any_SSSE3;
+ if (IS_ALIGNED(dst_uv_width, 8)) {
+ Half16To8 = HalfWidthRow_16To8_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Half16To8 = HalfWidthRow_16To8_Any_AVX2;
+ if (IS_ALIGNED(dst_uv_width, 16)) {
+ Half16To8 = HalfWidthRow_16To8_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Half16To8 = HalfWidthRow_16To8_AVX512BW;
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Half16To8 = HalfWidthRow_16To8_RVV;
+ }
+#endif
+ } else {
+#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Half16To8 = HalfWidthRow_16To8_Odd_AVX2;
+ }
+#endif
+ }
+
+ row_stride = (src_uv_width + 31) & ~31;
+ align_buffer_64(row_u, (size_t)row_stride * sizeof(uint16_t) * 4u);
+ if (!row_u) {
+ return 1;
+ }
+ {
+ uint16_t* row_u0 = (uint16_t*)row_u;
+ uint16_t* row_u1 = row_u0 + row_stride;
+ uint16_t* row_v0 = row_u1 + row_stride;
+ uint16_t* row_v1 = row_v0 + row_stride;
+ for (y = 0; y < height - 1; y += 2) {
+ SplitUVRow_16(src_uv, row_u0, row_v0, 10, src_uv_width);
+ SplitUVRow_16(src_uv + src_stride_uv, row_u1, row_v1, 10, src_uv_width);
+ Half16To8(row_u0, row_stride, dst_u, scale, dst_uv_width);
+ Half16To8(row_v0, row_stride, dst_v, scale, dst_uv_width);
+ src_uv += src_stride_uv * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ SplitUVRow_16(src_uv, row_u0, row_v0, 10, src_uv_width);
+ Half16To8(row_u0, 0, dst_u, scale, dst_uv_width);
+ Half16To8(row_v0, 0, dst_v, scale, dst_uv_width);
+ }
+ }
+ free_aligned_buffer_64(row_u);
+ return 0;
+ }
+}
+
+LIBYUV_API
+int P010ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Px10ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 1, 1);
+}
+
+LIBYUV_API
+int P210ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Px10ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 1, 0);
+}
+
+LIBYUV_API
+int P410ToI420(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Px10ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 0, 0);
+}
+
+LIBYUV_API
+int P012ToI012(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return PxxxToIxxx(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ width, height, 1, 1, 12);
+}
+
+LIBYUV_API
+int P010ToP410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = UVScale_16(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1),
+ SUBSAMPLE(height, 1, 1), dst_uv, dst_stride_uv, Abs(width),
+ Abs(height), kFilterBilinear);
+ return r;
+}
+
+LIBYUV_API
+int P210ToP410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = UVScale_16(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1), height, dst_uv,
+ dst_stride_uv, Abs(width), Abs(height), kFilterBilinear);
+ return r;
+}
+
+// Convert YUY2 to I420.
+LIBYUV_API
+int YUY2ToI420(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*YUY2ToUVRow)(const uint8_t* src_yuy2, int src_stride_yuy2,
+ uint8_t* dst_u, uint8_t* dst_v, int width) =
+ YUY2ToUVRow_C;
+ void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
+ YUY2ToYRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_yuy2 = src_yuy2 + (ptrdiff_t)(height - 1) * src_stride_yuy2;
+ src_stride_yuy2 = -src_stride_yuy2;
+ }
+#if defined(HAS_YUY2TOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ YUY2ToUVRow = YUY2ToUVRow_Any_SSE2;
+ YUY2ToYRow = YUY2ToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToUVRow = YUY2ToUVRow_SSE2;
+ YUY2ToYRow = YUY2ToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ YUY2ToUVRow = YUY2ToUVRow_Any_AVX2;
+ YUY2ToYRow = YUY2ToYRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
YUY2ToUVRow = YUY2ToUVRow_AVX2;
YUY2ToYRow = YUY2ToYRow_AVX2;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- YUY2ToYRow = YUY2ToYRow_Any_NEON;
- YUY2ToUVRow = YUY2ToUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToYRow = YUY2ToYRow_NEON;
- YUY2ToUVRow = YUY2ToUVRow_NEON;
- }
+#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ YUY2ToUVRow = YUY2ToUVRow_AVX512BW;
+ YUY2ToYRow = YUY2ToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToYRow = YUY2ToYRow_Any_NEON;
+ YUY2ToUVRow = YUY2ToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_NEON;
+ YUY2ToUVRow = YUY2ToUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_SVE2) && defined(HAS_YUY2TOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToYRow = YUY2ToYRow_SVE2;
+ YUY2ToUVRow = YUY2ToUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LSX;
+ YUY2ToUVRow = YUY2ToUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_LSX;
+ YUY2ToUVRow = YUY2ToUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_LASX) && defined(HAS_YUY2TOUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LASX;
+ YUY2ToUVRow = YUY2ToUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ YUY2ToYRow = YUY2ToYRow_LASX;
+ YUY2ToUVRow = YUY2ToUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_HVX) && defined(HAS_YUY2TOUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ YUY2ToYRow = YUY2ToYRow_HVX;
+ YUY2ToUVRow = YUY2ToUVRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height - 1; y += 2) {
+ YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width);
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ YUY2ToYRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, width);
+ src_yuy2 += src_stride_yuy2 * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ YUY2ToUVRow(src_yuy2, 0, dst_u, dst_v, width);
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ }
+ return 0;
+}
+
+// Convert UYVY to I420.
+LIBYUV_API
+int UYVYToI420(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*UYVYToUVRow)(const uint8_t* src_uyvy, int src_stride_uyvy,
+ uint8_t* dst_u, uint8_t* dst_v, int width) =
+ UYVYToUVRow_C;
+ void (*UYVYToYRow)(const uint8_t* src_uyvy, uint8_t* dst_y, int width) =
+ UYVYToYRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uyvy = src_uyvy + (ptrdiff_t)(height - 1) * src_stride_uyvy;
+ src_stride_uyvy = -src_stride_uyvy;
+ }
+#if defined(HAS_UYVYTOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ UYVYToUVRow = UYVYToUVRow_Any_SSE2;
+ UYVYToYRow = UYVYToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToUVRow = UYVYToUVRow_SSE2;
+ UYVYToYRow = UYVYToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ UYVYToUVRow = UYVYToUVRow_Any_AVX2;
+ UYVYToYRow = UYVYToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToUVRow = UYVYToUVRow_AVX2;
+ UYVYToYRow = UYVYToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ UYVYToUVRow = UYVYToUVRow_AVX512BW;
+ UYVYToYRow = UYVYToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ UYVYToYRow = UYVYToYRow_Any_NEON;
+ UYVYToUVRow = UYVYToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_NEON;
+ UYVYToUVRow = UYVYToUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_SVE2) && defined(HAS_UYVYTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ UYVYToYRow = UYVYToYRow_SVE2;
+ UYVYToUVRow = UYVYToUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ UYVYToYRow = UYVYToYRow_Any_LSX;
+ UYVYToUVRow = UYVYToUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_LSX;
+ UYVYToUVRow = UYVYToUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ UYVYToYRow = UYVYToYRow_Any_LASX;
+ UYVYToUVRow = UYVYToUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToYRow = UYVYToYRow_LASX;
+ UYVYToUVRow = UYVYToUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_HVX) && defined(HAS_UYVYTOUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ UYVYToYRow = UYVYToYRow_HVX;
+ UYVYToUVRow = UYVYToUVRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height - 1; y += 2) {
+ UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width);
+ UYVYToYRow(src_uyvy, dst_y, width);
+ UYVYToYRow(src_uyvy + src_stride_uyvy, dst_y + dst_stride_y, width);
+ src_uyvy += src_stride_uyvy * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ UYVYToUVRow(src_uyvy, 0, dst_u, dst_v, width);
+ UYVYToYRow(src_uyvy, dst_y, width);
+ }
+ return 0;
+}
+
+// Convert AYUV to NV12.
+LIBYUV_API
+int AYUVToNV12(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int y;
+ void (*AYUVToUVRow)(const uint8_t* src_ayuv, int src_stride_ayuv,
+ uint8_t* dst_uv, int width) = AYUVToUVRow_C;
+ void (*AYUVToYRow)(const uint8_t* src_ayuv, uint8_t* dst_y, int width) =
+ AYUVToYRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv;
+ src_stride_ayuv = -src_stride_ayuv;
+ }
+#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AYUVToYRow = AYUVToYRow_Any_AVX2;
+ AYUVToUVRow = AYUVToUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ AYUVToYRow = AYUVToYRow_AVX2;
+ AYUVToUVRow = AYUVToUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AYUVTOYROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ AYUVToUVRow = AYUVToUVRow_AVX512VBMI;
+ AYUVToYRow = AYUVToYRow_AVX512VBMI;
+ }
+#endif
+
+#if defined(HAS_AYUVTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AYUVToYRow = AYUVToYRow_Any_NEON;
+ AYUVToUVRow = AYUVToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ AYUVToYRow = AYUVToYRow_NEON;
+ AYUVToUVRow = AYUVToUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AYUVTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ AYUVToUVRow = AYUVToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ AYUVToUVRow = AYUVToUVRow_SVE2;
+ }
+ }
+#endif
+
+ for (y = 0; y < height - 1; y += 2) {
+ AYUVToUVRow(src_ayuv, src_stride_ayuv, dst_uv, width);
+ AYUVToYRow(src_ayuv, dst_y, width);
+ AYUVToYRow(src_ayuv + src_stride_ayuv, dst_y + dst_stride_y, width);
+ src_ayuv += src_stride_ayuv * 2;
+ dst_y += dst_stride_y * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ AYUVToUVRow(src_ayuv, 0, dst_uv, width);
+ AYUVToYRow(src_ayuv, dst_y, width);
+ }
+ return 0;
+}
+
+// Convert AYUV to NV21.
+LIBYUV_API
+int AYUVToNV21(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ int y;
+ void (*AYUVToVURow)(const uint8_t* src_ayuv, int src_stride_ayuv,
+ uint8_t* dst_vu, int width) = AYUVToVURow_C;
+ void (*AYUVToYRow)(const uint8_t* src_ayuv, uint8_t* dst_y, int width) =
+ AYUVToYRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv;
+ src_stride_ayuv = -src_stride_ayuv;
+ }
+#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOVUROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AYUVToYRow = AYUVToYRow_Any_AVX2;
+ AYUVToVURow = AYUVToVURow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ AYUVToYRow = AYUVToYRow_AVX2;
+ AYUVToVURow = AYUVToVURow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AYUVTOYROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ AYUVToVURow = AYUVToVURow_AVX512VBMI;
+ AYUVToYRow = AYUVToYRow_AVX512VBMI;
}
#endif
-#if defined(HAS_YUY2TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- YUY2ToYRow = YUY2ToYRow_Any_MSA;
- YUY2ToUVRow = YUY2ToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- YUY2ToYRow = YUY2ToYRow_MSA;
- YUY2ToUVRow = YUY2ToUVRow_MSA;
+
+#if defined(HAS_AYUVTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AYUVToYRow = AYUVToYRow_Any_NEON;
+ AYUVToVURow = AYUVToVURow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ AYUVToYRow = AYUVToYRow_NEON;
+ AYUVToVURow = AYUVToVURow_NEON;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- YUY2ToYRow = YUY2ToYRow_Any_MMI;
- YUY2ToUVRow = YUY2ToUVRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- YUY2ToYRow = YUY2ToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToUVRow = YUY2ToUVRow_MMI;
- }
+#if defined(HAS_AYUVTOVUROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ AYUVToVURow = AYUVToVURow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ AYUVToVURow = AYUVToVURow_SVE2;
}
}
#endif
for (y = 0; y < height - 1; y += 2) {
- YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width);
- YUY2ToYRow(src_yuy2, dst_y, width);
- YUY2ToYRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, width);
- src_yuy2 += src_stride_yuy2 * 2;
+ AYUVToVURow(src_ayuv, src_stride_ayuv, dst_vu, width);
+ AYUVToYRow(src_ayuv, dst_y, width);
+ AYUVToYRow(src_ayuv + src_stride_ayuv, dst_y + dst_stride_y, width);
+ src_ayuv += src_stride_ayuv * 2;
dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+ dst_vu += dst_stride_vu;
}
if (height & 1) {
- YUY2ToUVRow(src_yuy2, 0, dst_u, dst_v, width);
- YUY2ToYRow(src_yuy2, dst_y, width);
+ AYUVToVURow(src_ayuv, 0, dst_vu, width);
+ AYUVToYRow(src_ayuv, dst_y, width);
}
return 0;
}
-// Convert UYVY to I420.
+// Convert ARGB to I420.
LIBYUV_API
-int UYVYToI420(const uint8_t* src_uyvy,
- int src_stride_uyvy,
+int ARGBToI420(const uint8_t* src_argb,
+ int src_stride_argb,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -801,339 +2843,910 @@ int UYVYToI420(const uint8_t* src_uyvy,
int dst_stride_v,
int width,
int height) {
+ return ARGBToI420Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbI601Constants, width, height);
+}
+
+LIBYUV_API
+int ARGBToI420Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- void (*UYVYToUVRow)(const uint8_t* src_uyvy, int src_stride_uyvy,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- UYVYToUVRow_C;
- void (*UYVYToYRow)(const uint8_t* src_uyvy, uint8_t* dst_y, int width) =
- UYVYToYRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+ if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy;
- src_stride_uyvy = -src_stride_uyvy;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
}
-#if defined(HAS_UYVYTOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- UYVYToUVRow = UYVYToUVRow_Any_SSE2;
- UYVYToYRow = UYVYToYRow_Any_SSE2;
+
+ for (y = 0; y < height - 1; y += 2) {
+ ARGBToUVMatrixRow(src_argb, src_stride_argb, dst_u, dst_v, width,
+ argbconstants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ ARGBToYMatrixRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width,
+ argbconstants);
+ src_argb += src_stride_argb * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ ARGBToUVMatrixRow(src_argb, 0, dst_u, dst_v, width, argbconstants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ }
+ return 0;
+}
+
+#ifdef USE_EXTRACTALPHA
+// Convert ARGB to I420 with Alpha
+// The following version calls ARGBExtractAlpha on the full image.
+LIBYUV_API
+int ARGBToI420Alpha(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height) {
+ int r = ARGBToI420(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height);
+ if (r == 0) {
+ r = ARGBExtractAlpha(src_argb, src_stride_argb, dst_a, dst_stride_a, width,
+ height);
+ }
+ return r;
+}
+#else // USE_EXTRACTALPHA
+// Convert ARGB to I420 with Alpha
+LIBYUV_API
+int ARGBToI420Alpha(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+ void (*ARGBExtractAlphaRow)(const uint8_t* src_argb, uint8_t* dst_a,
+ int width) = ARGBExtractAlphaRow_C;
+ if (!src_argb || !dst_y || !dst_u || !dst_v || !dst_a || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- UYVYToUVRow = UYVYToUVRow_SSE2;
- UYVYToYRow = UYVYToYRow_SSE2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_UYVYTOYROW_AVX2)
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+
+#if defined(HAS_ARGBEXTRACTALPHAROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_SSE2
+ : ARGBExtractAlphaRow_Any_SSE2;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- UYVYToUVRow = UYVYToUVRow_Any_AVX2;
- UYVYToYRow = UYVYToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- UYVYToUVRow = UYVYToUVRow_AVX2;
- UYVYToYRow = UYVYToYRow_AVX2;
- }
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 32) ? ARGBExtractAlphaRow_AVX2
+ : ARGBExtractAlphaRow_Any_AVX2;
}
#endif
-#if defined(HAS_UYVYTOYROW_NEON)
+#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- UYVYToYRow = UYVYToYRow_Any_NEON;
- UYVYToUVRow = UYVYToUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- UYVYToYRow = UYVYToYRow_NEON;
- UYVYToUVRow = UYVYToUVRow_NEON;
- }
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON
+ : ARGBExtractAlphaRow_Any_NEON;
}
#endif
-#if defined(HAS_UYVYTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- UYVYToYRow = UYVYToYRow_Any_MSA;
- UYVYToUVRow = UYVYToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- UYVYToYRow = UYVYToYRow_MSA;
- UYVYToUVRow = UYVYToUVRow_MSA;
- }
+#if defined(HAS_ARGBEXTRACTALPHAROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_LSX
+ : ARGBExtractAlphaRow_Any_LSX;
}
#endif
-#if defined(HAS_UYVYTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- UYVYToYRow = UYVYToYRow_Any_MMI;
- UYVYToUVRow = UYVYToUVRow_Any_MMI;
- if (IS_ALIGNED(width, 16)) {
- UYVYToYRow = UYVYToYRow_MMI;
- UYVYToUVRow = UYVYToUVRow_MMI;
- }
+#if defined(HAS_ARGBEXTRACTALPHAROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LASX
+ : ARGBExtractAlphaRow_Any_LASX;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 16)
+ ? ARGBExtractAlphaRow_WASMSIMD
+ : ARGBExtractAlphaRow_Any_WASMSIMD;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_HVX;
}
#endif
for (y = 0; y < height - 1; y += 2) {
- UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width);
- UYVYToYRow(src_uyvy, dst_y, width);
- UYVYToYRow(src_uyvy + src_stride_uyvy, dst_y + dst_stride_y, width);
- src_uyvy += src_stride_uyvy * 2;
+ ARGBToUVMatrixRow(src_argb, src_stride_argb, dst_u, dst_v, width,
+ &kArgbI601Constants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, &kArgbI601Constants);
+ ARGBToYMatrixRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width,
+ &kArgbI601Constants);
+ ARGBExtractAlphaRow(src_argb, dst_a, width);
+ ARGBExtractAlphaRow(src_argb + src_stride_argb, dst_a + dst_stride_a,
+ width);
+ src_argb += src_stride_argb * 2;
dst_y += dst_stride_y * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
+ dst_a += dst_stride_a * 2;
}
if (height & 1) {
- UYVYToUVRow(src_uyvy, 0, dst_u, dst_v, width);
- UYVYToYRow(src_uyvy, dst_y, width);
+ ARGBToUVMatrixRow(src_argb, 0, dst_u, dst_v, width, &kArgbI601Constants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, &kArgbI601Constants);
+ ARGBExtractAlphaRow(src_argb, dst_a, width);
}
return 0;
}
+#endif // USE_EXTRACTALPHA
-// Convert AYUV to NV12.
+// Convert BGRA to I420.
LIBYUV_API
-int AYUVToNV12(const uint8_t* src_ayuv,
- int src_stride_ayuv,
+int BGRAToI420(const uint8_t* src_bgra,
+ int src_stride_bgra,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_uv,
- int dst_stride_uv,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI420Matrix(src_bgra, src_stride_bgra, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kBgraI601Constants, width, height);
+}
+
+// Convert BGRA to I422.
+LIBYUV_API
+int BGRAToI422(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI422Matrix(src_bgra, src_stride_bgra, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kBgraI601Constants, width, height);
+}
+
+// Convert ABGR to I422.
+LIBYUV_API
+int ABGRToI422(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI422Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kAbgrI601Constants, width, height);
+}
+
+// Convert RGBA to I422.
+LIBYUV_API
+int RGBAToI422(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI422Matrix(src_rgba, src_stride_rgba, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kRgbaI601Constants, width, height);
+}
+
+// Convert ABGR to I420.
+LIBYUV_API
+int ABGRToI420(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI420Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kAbgrI601Constants, width, height);
+}
+
+// Convert RGBA to I420.
+LIBYUV_API
+int RGBAToI420(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
int width,
int height) {
+ return ARGBToI420Matrix(src_rgba, src_stride_rgba, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kRgbaI601Constants, width, height);
+}
+
+static int AR30ToI420Matrix(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- void (*AYUVToUVRow)(const uint8_t* src_ayuv, int src_stride_ayuv,
- uint8_t* dst_uv, int width) = AYUVToUVRow_C;
- void (*AYUVToYRow)(const uint8_t* src_ayuv, uint8_t* dst_y, int width) =
- AYUVToYRow_C;
+ void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) =
+ AR30ToARGBRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+ if (!src_ar30 || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_ayuv = src_ayuv + (height - 1) * src_stride_ayuv;
- src_stride_ayuv = -src_stride_ayuv;
+ src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
+ src_stride_ar30 = -src_stride_ar30;
}
-// place holders for future intel code
-#if defined(HAS_AYUVTOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- AYUVToUVRow = AYUVToUVRow_Any_SSE2;
- AYUVToYRow = AYUVToYRow_Any_SSE2;
- if (IS_ALIGNED(width, 16)) {
- AYUVToUVRow = AYUVToUVRow_SSE2;
- AYUVToYRow = AYUVToYRow_SSE2;
+#if defined(HAS_AR30TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToARGBRow = AR30ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_AYUVTOYROW_AVX2)
+#if defined(HAS_AR30TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- AYUVToUVRow = AYUVToUVRow_Any_AVX2;
- AYUVToYRow = AYUVToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- AYUVToUVRow = AYUVToUVRow_AVX2;
- AYUVToYRow = AYUVToYRow_AVX2;
+ AR30ToARGBRow = AR30ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToARGBRow = AR30ToARGBRow_AVX2;
}
}
#endif
-
-#if defined(HAS_AYUVTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- AYUVToYRow = AYUVToYRow_Any_NEON;
- AYUVToUVRow = AYUVToUVRow_Any_NEON;
+#if defined(HAS_AR30TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- AYUVToYRow = AYUVToYRow_NEON;
- AYUVToUVRow = AYUVToUVRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-
- for (y = 0; y < height - 1; y += 2) {
- AYUVToUVRow(src_ayuv, src_stride_ayuv, dst_uv, width);
- AYUVToYRow(src_ayuv, dst_y, width);
- AYUVToYRow(src_ayuv + src_stride_ayuv, dst_y + dst_stride_y, width);
- src_ayuv += src_stride_ayuv * 2;
- dst_y += dst_stride_y * 2;
- dst_uv += dst_stride_uv;
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
}
- if (height & 1) {
- AYUVToUVRow(src_ayuv, 0, dst_uv, width);
- AYUVToYRow(src_ayuv, dst_y, width);
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
}
- return 0;
-}
-
-// Convert AYUV to NV21.
-LIBYUV_API
-int AYUVToNV21(const uint8_t* src_ayuv,
- int src_stride_ayuv,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
- int y;
- void (*AYUVToVURow)(const uint8_t* src_ayuv, int src_stride_ayuv,
- uint8_t* dst_vu, int width) = AYUVToVURow_C;
- void (*AYUVToYRow)(const uint8_t* src_ayuv, uint8_t* dst_y, int width) =
- AYUVToYRow_C;
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_ayuv = src_ayuv + (height - 1) * src_stride_ayuv;
- src_stride_ayuv = -src_stride_ayuv;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
}
-// place holders for future intel code
-#if defined(HAS_AYUVTOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- AYUVToVURow = AYUVToVURow_Any_SSE2;
- AYUVToYRow = AYUVToYRow_Any_SSE2;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- AYUVToVURow = AYUVToVURow_SSE2;
- AYUVToYRow = AYUVToYRow_SSE2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_AYUVTOYROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- AYUVToVURow = AYUVToVURow_Any_AVX2;
- AYUVToYRow = AYUVToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- AYUVToVURow = AYUVToVURow_AVX2;
- AYUVToYRow = AYUVToYRow_AVX2;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-
-#if defined(HAS_AYUVTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- AYUVToYRow = AYUVToYRow_Any_NEON;
- AYUVToVURow = AYUVToVURow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- AYUVToYRow = AYUVToYRow_NEON;
- AYUVToVURow = AYUVToVURow_NEON;
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-
- for (y = 0; y < height - 1; y += 2) {
- AYUVToVURow(src_ayuv, src_stride_ayuv, dst_vu, width);
- AYUVToYRow(src_ayuv, dst_y, width);
- AYUVToYRow(src_ayuv + src_stride_ayuv, dst_y + dst_stride_y, width);
- src_ayuv += src_stride_ayuv * 2;
- dst_y += dst_stride_y * 2;
- dst_vu += dst_stride_vu;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
}
- if (height & 1) {
- AYUVToVURow(src_ayuv, 0, dst_vu, width);
- AYUVToYRow(src_ayuv, dst_y, width);
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
}
- return 0;
-}
-
-// Convert ARGB to I420.
-LIBYUV_API
-int ARGBToI420(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
}
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+ int row_stride = width * 4;
+ align_buffer_64(row, row_stride * 2);
+ if (!row) {
+ return -1;
+ }
for (y = 0; y < height - 1; y += 2) {
- ARGBToUVRow(src_argb, src_stride_argb, dst_u, dst_v, width);
- ARGBToYRow(src_argb, dst_y, width);
- ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width);
- src_argb += src_stride_argb * 2;
+ AR30ToARGBRow(src_ar30, row, width);
+ AR30ToARGBRow(src_ar30 + src_stride_ar30, row + row_stride, width);
+ ARGBToUVMatrixRow(row, row_stride, dst_u, dst_v, width, argbconstants);
+ ARGBToYMatrixRow(row, dst_y, width, argbconstants);
+ ARGBToYMatrixRow(row + row_stride, dst_y + dst_stride_y, width,
+ argbconstants);
+ src_ar30 += src_stride_ar30 * 2;
dst_y += dst_stride_y * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
- ARGBToUVRow(src_argb, 0, dst_u, dst_v, width);
- ARGBToYRow(src_argb, dst_y, width);
+ AR30ToARGBRow(src_ar30, row, width);
+ ARGBToUVMatrixRow(row, 0, dst_u, dst_v, width, argbconstants);
+ ARGBToYMatrixRow(row, dst_y, width, argbconstants);
}
+ free_aligned_buffer_64(row);
return 0;
}
-// Convert BGRA to I420.
+// Convert AR30 to I420.
LIBYUV_API
-int BGRAToI420(const uint8_t* src_bgra,
- int src_stride_bgra,
+int AR30ToI420(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return AR30ToI420Matrix(src_ar30, src_stride_ar30, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbI601Constants, width, height);
+}
+
+// Convert AB30 to I420.
+LIBYUV_API
+int AB30ToI420(const uint8_t* src_ab30,
+ int src_stride_ab30,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
@@ -1142,744 +3755,805 @@ int BGRAToI420(const uint8_t* src_bgra,
int dst_stride_v,
int width,
int height) {
+ return AR30ToI420Matrix(src_ab30, src_stride_ab30, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kAbgrI601Constants, width, height);
+}
+
+// Convert RGB24 to I420.
+LIBYUV_API
+int RGB24ToI420(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
int y;
- void (*BGRAToUVRow)(const uint8_t* src_bgra0, int src_stride_bgra,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- BGRAToUVRow_C;
- void (*BGRAToYRow)(const uint8_t* src_bgra, uint8_t* dst_y, int width) =
- BGRAToYRow_C;
- if (!src_bgra || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+ void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUVMatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
+ }
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_bgra = src_bgra + (height - 1) * src_stride_bgra;
- src_stride_bgra = -src_stride_bgra;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
}
-#if defined(HAS_BGRATOYROW_SSSE3) && defined(HAS_BGRATOUVROW_SSSE3)
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- BGRAToUVRow = BGRAToUVRow_Any_SSSE3;
- BGRAToYRow = BGRAToYRow_Any_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- BGRAToUVRow = BGRAToUVRow_SSSE3;
- BGRAToYRow = BGRAToYRow_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_BGRATOYROW_NEON)
+#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- BGRAToYRow = BGRAToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- BGRAToYRow = BGRAToYRow_NEON;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_BGRATOUVROW_NEON)
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- BGRAToUVRow = BGRAToUVRow_Any_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- BGRAToUVRow = BGRAToUVRow_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_BGRATOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- BGRAToYRow = BGRAToYRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- BGRAToYRow = BGRAToYRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_BGRATOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- BGRAToUVRow = BGRAToUVRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- BGRAToUVRow = BGRAToUVRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_BGRATOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- BGRAToYRow = BGRAToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- BGRAToYRow = BGRAToYRow_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_BGRATOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- BGRAToUVRow = BGRAToUVRow_Any_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- BGRAToUVRow = BGRAToUVRow_MMI;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
+ if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb24 = src_rgb24 + (ptrdiff_t)(height - 1) * src_stride_rgb24;
+ src_stride_rgb24 = -src_stride_rgb24;
+ }
+
for (y = 0; y < height - 1; y += 2) {
- BGRAToUVRow(src_bgra, src_stride_bgra, dst_u, dst_v, width);
- BGRAToYRow(src_bgra, dst_y, width);
- BGRAToYRow(src_bgra + src_stride_bgra, dst_y + dst_stride_y, width);
- src_bgra += src_stride_bgra * 2;
+ RGBToUVMatrixRow(src_rgb24, src_stride_rgb24, dst_u, dst_v, width,
+ &kArgbI601Constants);
+ RGBToYMatrixRow(src_rgb24, dst_y, width, &kArgbI601Constants);
+ RGBToYMatrixRow(src_rgb24 + src_stride_rgb24, dst_y + dst_stride_y, width,
+ &kArgbI601Constants);
+ src_rgb24 += src_stride_rgb24 * 2;
dst_y += dst_stride_y * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
- BGRAToUVRow(src_bgra, 0, dst_u, dst_v, width);
- BGRAToYRow(src_bgra, dst_y, width);
+ RGBToYMatrixRow(src_rgb24, dst_y, width, &kArgbI601Constants);
+ RGBToUVMatrixRow(src_rgb24, 0, dst_u, dst_v, width, &kArgbI601Constants);
}
return 0;
}
+#undef HAS_RGB24TOYROW
-// Convert ABGR to I420.
+/// Convert RGB24 to J420.
LIBYUV_API
-int ABGRToI420(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
+int RGB24ToJ420(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
int y;
- void (*ABGRToUVRow)(const uint8_t* src_abgr0, int src_stride_abgr,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ABGRToUVRow_C;
- void (*ABGRToYRow)(const uint8_t* src_abgr, uint8_t* dst_y, int width) =
- ABGRToYRow_C;
- if (!src_abgr || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+ void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUVMatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
+ }
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_abgr = src_abgr + (height - 1) * src_stride_abgr;
- src_stride_abgr = -src_stride_abgr;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
}
-#if defined(HAS_ABGRTOYROW_SSSE3) && defined(HAS_ABGRTOUVROW_SSSE3)
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ABGRToUVRow = ABGRToUVRow_Any_SSSE3;
- ABGRToYRow = ABGRToYRow_Any_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_SSSE3;
- ABGRToYRow = ABGRToYRow_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_NEON)
+#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToYRow = ABGRToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_NEON;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ABGRTOUVROW_NEON)
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToUVRow = ABGRToUVRow_Any_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToYRow = ABGRToYRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ABGRToYRow = ABGRToYRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ABGRTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToUVRow = ABGRToUVRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToYRow = ABGRToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ABGRTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToUVRow = ABGRToUVRow_Any_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_MMI;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
+ if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb24 = src_rgb24 + (ptrdiff_t)(height - 1) * src_stride_rgb24;
+ src_stride_rgb24 = -src_stride_rgb24;
+ }
+
for (y = 0; y < height - 1; y += 2) {
- ABGRToUVRow(src_abgr, src_stride_abgr, dst_u, dst_v, width);
- ABGRToYRow(src_abgr, dst_y, width);
- ABGRToYRow(src_abgr + src_stride_abgr, dst_y + dst_stride_y, width);
- src_abgr += src_stride_abgr * 2;
+ RGBToUVMatrixRow(src_rgb24, src_stride_rgb24, dst_u, dst_v, width,
+ &kArgbJPEGConstants);
+ RGBToYMatrixRow(src_rgb24, dst_y, width, &kArgbJPEGConstants);
+ RGBToYMatrixRow(src_rgb24 + src_stride_rgb24, dst_y + dst_stride_y, width,
+ &kArgbJPEGConstants);
+ src_rgb24 += src_stride_rgb24 * 2;
dst_y += dst_stride_y * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
- ABGRToUVRow(src_abgr, 0, dst_u, dst_v, width);
- ABGRToYRow(src_abgr, dst_y, width);
+ RGBToYMatrixRow(src_rgb24, dst_y, width, &kArgbJPEGConstants);
+ RGBToUVMatrixRow(src_rgb24, 0, dst_u, dst_v, width, &kArgbJPEGConstants);
}
return 0;
}
-// Convert RGBA to I420.
+// Convert RAW to I420.
LIBYUV_API
-int RGBAToI420(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
+int RAWToI420(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
int y;
- void (*RGBAToUVRow)(const uint8_t* src_rgba0, int src_stride_rgba,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- RGBAToUVRow_C;
- void (*RGBAToYRow)(const uint8_t* src_rgba, uint8_t* dst_y, int width) =
- RGBAToYRow_C;
- if (!src_rgba || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+ void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUVMatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
+ }
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_rgba = src_rgba + (height - 1) * src_stride_rgba;
- src_stride_rgba = -src_stride_rgba;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
+ }
}
-#if defined(HAS_RGBATOYROW_SSSE3) && defined(HAS_RGBATOUVROW_SSSE3)
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- RGBAToUVRow = RGBAToUVRow_Any_SSSE3;
- RGBAToYRow = RGBAToYRow_Any_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- RGBAToUVRow = RGBAToUVRow_SSSE3;
- RGBAToYRow = RGBAToYRow_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_RGBATOYROW_NEON)
+#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGBAToYRow = RGBAToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RGBAToYRow = RGBAToYRow_NEON;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_RGBATOUVROW_NEON)
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGBAToUVRow = RGBAToUVRow_Any_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- RGBAToUVRow = RGBAToUVRow_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_RGBATOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGBAToYRow = RGBAToYRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- RGBAToYRow = RGBAToYRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_RGBATOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGBAToUVRow = RGBAToUVRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- RGBAToUVRow = RGBAToUVRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_RGBATOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGBAToYRow = RGBAToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RGBAToYRow = RGBAToYRow_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_RGBATOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGBAToUVRow = RGBAToUVRow_Any_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- RGBAToUVRow = RGBAToUVRow_MMI;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
+ if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+
for (y = 0; y < height - 1; y += 2) {
- RGBAToUVRow(src_rgba, src_stride_rgba, dst_u, dst_v, width);
- RGBAToYRow(src_rgba, dst_y, width);
- RGBAToYRow(src_rgba + src_stride_rgba, dst_y + dst_stride_y, width);
- src_rgba += src_stride_rgba * 2;
+ RGBToUVMatrixRow(src_raw, src_stride_raw, dst_u, dst_v, width,
+ &kAbgrI601Constants);
+ RGBToYMatrixRow(src_raw, dst_y, width, &kAbgrI601Constants);
+ RGBToYMatrixRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width,
+ &kAbgrI601Constants);
+ src_raw += src_stride_raw * 2;
dst_y += dst_stride_y * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
- RGBAToUVRow(src_rgba, 0, dst_u, dst_v, width);
- RGBAToYRow(src_rgba, dst_y, width);
+ RGBToYMatrixRow(src_raw, dst_y, width, &kAbgrI601Constants);
+ RGBToUVMatrixRow(src_raw, 0, dst_u, dst_v, width, &kAbgrI601Constants);
}
return 0;
}
+#undef HAS_RAWTOYROW
-// Convert RGB24 to I420.
+// Convert RAW to J420.
LIBYUV_API
-int RGB24ToI420(const uint8_t* src_rgb24,
- int src_stride_rgb24,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
-#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_MSA) || \
- defined(HAS_RGB24TOYROW_MMI))
- void (*RGB24ToUVRow)(const uint8_t* src_rgb24, int src_stride_rgb24,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- RGB24ToUVRow_C;
- void (*RGB24ToYRow)(const uint8_t* src_rgb24, uint8_t* dst_y, int width) =
- RGB24ToYRow_C;
-#else
- void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RGB24ToARGBRow_C;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
-#endif
- if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24;
- src_stride_rgb24 = -src_stride_rgb24;
- }
-
-// Neon version does direct RGB24 to YUV.
-#if defined(HAS_RGB24TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- RGB24ToUVRow = RGB24ToUVRow_Any_NEON;
- RGB24ToYRow = RGB24ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYRow = RGB24ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToUVRow = RGB24ToUVRow_NEON;
- }
- }
- }
-#elif defined(HAS_RGB24TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB24ToUVRow = RGB24ToUVRow_Any_MSA;
- RGB24ToYRow = RGB24ToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToYRow = RGB24ToYRow_MSA;
- RGB24ToUVRow = RGB24ToUVRow_MSA;
- }
- }
-#elif defined(HAS_RGB24TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB24ToUVRow = RGB24ToUVRow_Any_MMI;
- RGB24ToYRow = RGB24ToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYRow = RGB24ToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToUVRow = RGB24ToUVRow_MMI;
- }
- }
- }
-// Other platforms do intermediate conversion from RGB24 to ARGB.
-#else
-#if defined(HAS_RGB24TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToARGBRow = RGB24ToARGBRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+int RAWToJ420(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUVMatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
}
}
#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
+ }
#endif
-
- {
-#if !(defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_MSA) || \
- defined(HAS_RGB24TOYROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
-
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_MSA) || \
- defined(HAS_RGB24TOYROW_MMI))
- RGB24ToUVRow(src_rgb24, src_stride_rgb24, dst_u, dst_v, width);
- RGB24ToYRow(src_rgb24, dst_y, width);
- RGB24ToYRow(src_rgb24 + src_stride_rgb24, dst_y + dst_stride_y, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- RGB24ToARGBRow(src_rgb24 + src_stride_rgb24, row + kRowSize, width);
- ARGBToUVRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
- ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_rgb24 += src_stride_rgb24 * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
}
- if (height & 1) {
-#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_MSA) || \
- defined(HAS_RGB24TOYROW_MMI))
- RGB24ToUVRow(src_rgb24, 0, dst_u, dst_v, width);
- RGB24ToYRow(src_rgb24, dst_y, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- ARGBToUVRow(row, 0, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
+ }
#endif
+#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
-#if !(defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_MSA) || \
- defined(HAS_RGB24TOYROW_MMI))
- free_aligned_buffer_64(row);
-#endif
- }
- return 0;
-}
-
-// TODO(fbarchard): Use Matrix version to implement I420 and J420.
-// Convert RGB24 to J420.
-LIBYUV_API
-int RGB24ToJ420(const uint8_t* src_rgb24,
- int src_stride_rgb24,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- void (*RGB24ToUVJRow)(const uint8_t* src_rgb24, int src_stride_rgb24,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- RGB24ToUVJRow_C;
- void (*RGB24ToYJRow)(const uint8_t* src_rgb24, uint8_t* dst_y, int width) =
- RGB24ToYJRow_C;
-#else
- void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RGB24ToARGBRow_C;
- void (*ARGBToUVJRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVJRow_C;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYJRow_C;
-#endif
- if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24;
- src_stride_rgb24 = -src_stride_rgb24;
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
-
-// Neon version does direct RGB24 to YUV.
-#if defined(HAS_RGB24TOYJROW_NEON)
+#endif
+#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGB24ToUVJRow = RGB24ToUVJRow_Any_NEON;
- RGB24ToYJRow = RGB24ToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYJRow = RGB24ToYJRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToUVJRow = RGB24ToUVJRow_NEON;
- }
- }
- }
-#elif defined(HAS_RGB24TOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB24ToUVJRow = RGB24ToUVJRow_Any_MSA;
- RGB24ToYJRow = RGB24ToYJRow_Any_MSA;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- RGB24ToYJRow = RGB24ToYJRow_MSA;
- RGB24ToUVJRow = RGB24ToUVJRow_MSA;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
-#elif defined(HAS_RGB24TOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB24ToUVJRow = RGB24ToUVJRow_Any_MMI;
- RGB24ToYJRow = RGB24ToYJRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYJRow = RGB24ToYJRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToUVJRow = RGB24ToUVJRow_MMI;
- }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
-// Other platforms do intermediate conversion from RGB24 to ARGB.
-#else
-#if defined(HAS_RGB24TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- RGB24ToARGBRow = RGB24ToARGBRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_SSSE3) && defined(HAS_ARGBTOUVJROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_SSSE3;
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_SSSE3;
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2) && defined(HAS_ARGBTOUVJROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_AVX2;
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVJRow = ARGBToUVJRow_AVX2;
- ARGBToYJRow = ARGBToYJRow_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
}
}
#endif
-#endif
-
- {
-#if !(defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
-
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- RGB24ToUVJRow(src_rgb24, src_stride_rgb24, dst_u, dst_v, width);
- RGB24ToYJRow(src_rgb24, dst_y, width);
- RGB24ToYJRow(src_rgb24 + src_stride_rgb24, dst_y + dst_stride_y, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- RGB24ToARGBRow(src_rgb24 + src_stride_rgb24, row + kRowSize, width);
- ARGBToUVJRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYJRow(row, dst_y, width);
- ARGBToYJRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_rgb24 += src_stride_rgb24 * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD;
}
- if (height & 1) {
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- RGB24ToUVJRow(src_rgb24, 0, dst_u, dst_v, width);
- RGB24ToYJRow(src_rgb24, dst_y, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- ARGBToUVJRow(row, 0, dst_u, dst_v, width);
- ARGBToYJRow(row, dst_y, width);
+ }
#endif
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
-#if !(defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- free_aligned_buffer_64(row);
-#endif
}
- return 0;
-}
-
-// Convert RAW to I420.
-LIBYUV_API
-int RAWToI420(const uint8_t* src_raw,
- int src_stride_raw,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
-#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_MSA) || \
- defined(HAS_RAWTOYROW_MMI))
- void (*RAWToUVRow)(const uint8_t* src_raw, int src_stride_raw, uint8_t* dst_u,
- uint8_t* dst_v, int width) = RAWToUVRow_C;
- void (*RAWToYRow)(const uint8_t* src_raw, uint8_t* dst_y, int width) =
- RAWToYRow_C;
-#else
- void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RAWToARGBRow_C;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
#endif
- if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+
+ if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_raw = src_raw + (height - 1) * src_stride_raw;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
src_stride_raw = -src_stride_raw;
}
-// Neon version does direct RAW to YUV.
-#if defined(HAS_RAWTOYROW_NEON)
+ for (y = 0; y < height - 1; y += 2) {
+ RGBToUVMatrixRow(src_raw, src_stride_raw, dst_u, dst_v, width,
+ &kAbgrJPEGConstants);
+ RGBToYMatrixRow(src_raw, dst_y, width, &kAbgrJPEGConstants);
+ RGBToYMatrixRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width,
+ &kAbgrJPEGConstants);
+ src_raw += src_stride_raw * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ RGBToYMatrixRow(src_raw, dst_y, width, &kAbgrJPEGConstants);
+ RGBToUVMatrixRow(src_raw, 0, dst_u, dst_v, width, &kAbgrJPEGConstants);
+ }
+ return 0;
+}
+
+// Convert RAW to I444 with matrix.
+static int RAWToI444Matrix(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ int y;
+ void (*RGBToUV444MatrixRow)(
+ const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUV444MatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RAWToUVRow = RAWToUVRow_Any_NEON;
- RAWToYRow = RAWToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RAWToYRow = RAWToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- RAWToUVRow = RAWToUVRow_NEON;
- }
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
-#elif defined(HAS_RAWTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RAWToUVRow = RAWToUVRow_Any_MSA;
- RAWToYRow = RAWToYRow_Any_MSA;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- RAWToYRow = RAWToYRow_MSA;
- RAWToUVRow = RAWToUVRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
-#elif defined(HAS_RAWTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RAWToUVRow = RAWToUVRow_Any_MMI;
- RAWToYRow = RAWToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RAWToYRow = RAWToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- RAWToUVRow = RAWToUVRow_MMI;
- }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
-// Other platforms do intermediate conversion from RAW to ARGB.
-#else
-#if defined(HAS_RAWTOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- RAWToARGBRow = RAWToARGBRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+
+#if defined(HAS_RGBTOUV444MATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_RGBTOUV444MATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX2;
}
}
#endif
+#if defined(HAS_RGBTOUV444MATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW;
+ }
#endif
-
- {
-#if !(defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_MSA) || \
- defined(HAS_RAWTOYROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
-
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_MSA) || \
- defined(HAS_RAWTOYROW_MMI))
- RAWToUVRow(src_raw, src_stride_raw, dst_u, dst_v, width);
- RAWToYRow(src_raw, dst_y, width);
- RAWToYRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width);
-#else
- RAWToARGBRow(src_raw, row, width);
- RAWToARGBRow(src_raw + src_stride_raw, row + kRowSize, width);
- ARGBToUVRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
- ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_raw += src_stride_raw * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+#if defined(HAS_RGBTOUV444MATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_NEON;
}
- if (height & 1) {
-#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_MSA) || \
- defined(HAS_RAWTOYROW_MMI))
- RAWToUVRow(src_raw, 0, dst_u, dst_v, width);
- RAWToYRow(src_raw, dst_y, width);
-#else
- RAWToARGBRow(src_raw, row, width);
- ARGBToUVRow(row, 0, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
+ }
+#endif
+#if defined(HAS_RGBTOUV444MATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_RVV;
+ }
#endif
+#if defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToUV444MatrixRow = RGBToUV444MatrixRow_WASMSIMD;
}
-#if !(defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_MSA) || \
- defined(HAS_RAWTOYROW_MMI))
- free_aligned_buffer_64(row);
+ }
#endif
+
+ if (!src_raw || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+
+ for (y = 0; y < height; ++y) {
+ RGBToYMatrixRow(src_raw, dst_y, width, argbconstants);
+ RGBToUV444MatrixRow(src_raw, dst_u, dst_v, width, argbconstants);
+ src_raw += src_stride_raw;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
}
return 0;
}
+// Convert RAW to I444.
+LIBYUV_API
+int RAWToI444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, &kAbgrI601Constants,
+ width, height);
+}
+
+// Convert RAW to J444.
+LIBYUV_API
+int RAWToJ444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, &kAbgrJPEGConstants,
+ width, height);
+}
+
// Convert RGB565 to I420.
LIBYUV_API
int RGB565ToI420(const uint8_t* src_rgb565,
@@ -1893,147 +4567,92 @@ int RGB565ToI420(const uint8_t* src_rgb565,
int width,
int height) {
int y;
-#if (defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
- defined(HAS_RGB565TOYROW_MMI))
- void (*RGB565ToUVRow)(const uint8_t* src_rgb565, int src_stride_rgb565,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- RGB565ToUVRow_C;
- void (*RGB565ToYRow)(const uint8_t* src_rgb565, uint8_t* dst_y, int width) =
- RGB565ToYRow_C;
-#else
- void (*RGB565ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb,
- int width) = RGB565ToARGBRow_C;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
-#endif
- if (!src_rgb565 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_rgb565 = src_rgb565 + (height - 1) * src_stride_rgb565;
- src_stride_rgb565 = -src_stride_rgb565;
- }
+ void (*RGB565ToUVMatrixRow)(const uint8_t* src_rgb565, int src_stride_rgb565,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ RGB565ToUVMatrixRow_C;
+ void (*RGB565ToYMatrixRow)(const uint8_t* src_rgb565, uint8_t* dst_y,
+ int width, const struct ArgbConstants* c) =
+ RGB565ToYMatrixRow_C;
-// Neon version does direct RGB565 to YUV.
-#if defined(HAS_RGB565TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- RGB565ToUVRow = RGB565ToUVRow_Any_NEON;
- RGB565ToYRow = RGB565ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RGB565ToYRow = RGB565ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- RGB565ToUVRow = RGB565ToUVRow_NEON;
- }
- }
- }
-#elif defined(HAS_RGB565TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB565ToUVRow = RGB565ToUVRow_Any_MSA;
- RGB565ToYRow = RGB565ToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- RGB565ToYRow = RGB565ToYRow_MSA;
- RGB565ToUVRow = RGB565ToUVRow_MSA;
- }
- }
-#elif defined(HAS_RGB565TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB565ToUVRow = RGB565ToUVRow_Any_MMI;
- RGB565ToYRow = RGB565ToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RGB565ToYRow = RGB565ToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- RGB565ToUVRow = RGB565ToUVRow_MMI;
- }
- }
- }
-// Other platforms do intermediate conversion from RGB565 to ARGB.
-#else
-#if defined(HAS_RGB565TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_SSE2;
- if (IS_ALIGNED(width, 8)) {
- RGB565ToARGBRow = RGB565ToARGBRow_SSE2;
+#if defined(HAS_RGB565TOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_RGB565TOARGBROW_AVX2)
+#if defined(HAS_RGB565TOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_AVX2;
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- RGB565ToARGBRow = RGB565ToARGBRow_AVX2;
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+#if defined(HAS_RGB565TOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+#if defined(HAS_RGB565TOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_NEON;
}
}
#endif
-#endif
- {
-#if !(defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
- defined(HAS_RGB565TOYROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
- defined(HAS_RGB565TOYROW_MMI))
- RGB565ToUVRow(src_rgb565, src_stride_rgb565, dst_u, dst_v, width);
- RGB565ToYRow(src_rgb565, dst_y, width);
- RGB565ToYRow(src_rgb565 + src_stride_rgb565, dst_y + dst_stride_y, width);
-#else
- RGB565ToARGBRow(src_rgb565, row, width);
- RGB565ToARGBRow(src_rgb565 + src_stride_rgb565, row + kRowSize, width);
- ARGBToUVRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
- ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_rgb565 += src_stride_rgb565 * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+#if defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_WASMSIMD;
}
- if (height & 1) {
-#if (defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
- defined(HAS_RGB565TOYROW_MMI))
- RGB565ToUVRow(src_rgb565, 0, dst_u, dst_v, width);
- RGB565ToYRow(src_rgb565, dst_y, width);
-#else
- RGB565ToARGBRow(src_rgb565, row, width);
- ARGBToUVRow(row, 0, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
+ }
#endif
+#if defined(HAS_RGB565TOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToYMatrixRow = RGB565ToYMatrixRow_WASMSIMD;
}
-#if !(defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
- defined(HAS_RGB565TOYROW_MMI))
- free_aligned_buffer_64(row);
+ }
#endif
+
+ if (!src_rgb565 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb565 = src_rgb565 + (ptrdiff_t)(height - 1) * src_stride_rgb565;
+ src_stride_rgb565 = -src_stride_rgb565;
+ }
+
+ for (y = 0; y < height - 1; y += 2) {
+ RGB565ToUVMatrixRow(src_rgb565, src_stride_rgb565, dst_u, dst_v, width,
+ &kArgbI601Constants);
+ RGB565ToYMatrixRow(src_rgb565, dst_y, width, &kArgbI601Constants);
+ RGB565ToYMatrixRow(src_rgb565 + src_stride_rgb565, dst_y + dst_stride_y,
+ width, &kArgbI601Constants);
+ src_rgb565 += src_stride_rgb565 * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ RGB565ToYMatrixRow(src_rgb565, dst_y, width, &kArgbI601Constants);
+ RGB565ToUVMatrixRow(src_rgb565, 0, dst_u, dst_v, width,
+ &kArgbI601Constants);
}
return 0;
}
-
// Convert ARGB1555 to I420.
LIBYUV_API
int ARGB1555ToI420(const uint8_t* src_argb1555,
@@ -2041,157 +4660,98 @@ int ARGB1555ToI420(const uint8_t* src_argb1555,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
-#if (defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
- defined(HAS_ARGB1555TOYROW_MMI))
- void (*ARGB1555ToUVRow)(const uint8_t* src_argb1555, int src_stride_argb1555,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGB1555ToUVRow_C;
- void (*ARGB1555ToYRow)(const uint8_t* src_argb1555, uint8_t* dst_y,
- int width) = ARGB1555ToYRow_C;
-#else
- void (*ARGB1555ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb,
- int width) = ARGB1555ToARGBRow_C;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
-#endif
- if (!src_argb1555 || !dst_y || !dst_u || !dst_v || width <= 0 ||
- height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb1555 = src_argb1555 + (height - 1) * src_stride_argb1555;
- src_stride_argb1555 = -src_stride_argb1555;
- }
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*ARGB1555ToUVMatrixRow)(
+ const uint8_t* src_argb1555, int src_stride_argb1555, uint8_t* dst_u,
+ uint8_t* dst_v, int width, const struct ArgbConstants* c) =
+ ARGB1555ToUVMatrixRow_C;
+ void (*ARGB1555ToYMatrixRow)(const uint8_t* src_argb1555, uint8_t* dst_y,
+ int width, const struct ArgbConstants* c) =
+ ARGB1555ToYMatrixRow_C;
-// Neon version does direct ARGB1555 to YUV.
-#if defined(HAS_ARGB1555TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_Any_NEON;
- ARGB1555ToYRow = ARGB1555ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGB1555ToYRow = ARGB1555ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_NEON;
- }
+#if defined(HAS_ARGB1555TOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_AVX2;
}
}
-#elif defined(HAS_ARGB1555TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_Any_MSA;
- ARGB1555ToYRow = ARGB1555ToYRow_Any_MSA;
+#endif
+#if defined(HAS_ARGB1555TOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- ARGB1555ToYRow = ARGB1555ToYRow_MSA;
- ARGB1555ToUVRow = ARGB1555ToUVRow_MSA;
- }
- }
-#elif defined(HAS_ARGB1555TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_Any_MMI;
- ARGB1555ToYRow = ARGB1555ToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGB1555ToYRow = ARGB1555ToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_MMI;
- }
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_AVX2;
}
}
-// Other platforms do intermediate conversion from ARGB1555 to ARGB.
-#else
-#if defined(HAS_ARGB1555TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_SSE2;
- if (IS_ALIGNED(width, 8)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_SSE2;
+#endif
+#if defined(HAS_ARGB1555TOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGB1555TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_AVX2;
+#if defined(HAS_ARGB1555TOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX2;
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+#if defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+#if defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_WASMSIMD;
}
}
#endif
-#endif
- {
-#if !(defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
- defined(HAS_ARGB1555TOYROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
- defined(HAS_ARGB1555TOYROW_MMI))
- ARGB1555ToUVRow(src_argb1555, src_stride_argb1555, dst_u, dst_v, width);
- ARGB1555ToYRow(src_argb1555, dst_y, width);
- ARGB1555ToYRow(src_argb1555 + src_stride_argb1555, dst_y + dst_stride_y,
- width);
-#else
- ARGB1555ToARGBRow(src_argb1555, row, width);
- ARGB1555ToARGBRow(src_argb1555 + src_stride_argb1555, row + kRowSize,
- width);
- ARGBToUVRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
- ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_argb1555 += src_stride_argb1555 * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
- }
- if (height & 1) {
-#if (defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
- defined(HAS_ARGB1555TOYROW_MMI))
- ARGB1555ToUVRow(src_argb1555, 0, dst_u, dst_v, width);
- ARGB1555ToYRow(src_argb1555, dst_y, width);
-#else
- ARGB1555ToARGBRow(src_argb1555, row, width);
- ARGBToUVRow(row, 0, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
-#endif
- }
-#if !(defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
- defined(HAS_ARGB1555TOYROW_MMI))
- free_aligned_buffer_64(row);
-#endif
+ if (!src_argb1555 || !dst_y || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb1555 = src_argb1555 + (ptrdiff_t)(height - 1) * src_stride_argb1555;
+ src_stride_argb1555 = -src_stride_argb1555;
+ }
+
+ for (y = 0; y < height - 1; y += 2) {
+ ARGB1555ToUVMatrixRow(src_argb1555, src_stride_argb1555, dst_u, dst_v,
+ width, &kArgbI601Constants);
+ ARGB1555ToYMatrixRow(src_argb1555, dst_y, width, &kArgbI601Constants);
+ ARGB1555ToYMatrixRow(src_argb1555 + src_stride_argb1555,
+ dst_y + dst_stride_y, width, &kArgbI601Constants);
+ src_argb1555 += src_stride_argb1555 * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ ARGB1555ToYMatrixRow(src_argb1555, dst_y, width, &kArgbI601Constants);
+ ARGB1555ToUVMatrixRow(src_argb1555, 0, dst_u, dst_v, width,
+ &kArgbI601Constants);
}
return 0;
}
-
// Convert ARGB4444 to I420.
LIBYUV_API
int ARGB4444ToI420(const uint8_t* src_argb4444,
@@ -2205,170 +4765,92 @@ int ARGB4444ToI420(const uint8_t* src_argb4444,
int width,
int height) {
int y;
-#if (defined(HAS_ARGB4444TOYROW_NEON) || defined(HAS_ARGB4444TOYROW_MMI))
- void (*ARGB4444ToUVRow)(const uint8_t* src_argb4444, int src_stride_argb4444,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGB4444ToUVRow_C;
- void (*ARGB4444ToYRow)(const uint8_t* src_argb4444, uint8_t* dst_y,
- int width) = ARGB4444ToYRow_C;
-#else
- void (*ARGB4444ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb,
- int width) = ARGB4444ToARGBRow_C;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
-#endif
- if (!src_argb4444 || !dst_y || !dst_u || !dst_v || width <= 0 ||
- height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb4444 = src_argb4444 + (height - 1) * src_stride_argb4444;
- src_stride_argb4444 = -src_stride_argb4444;
- }
+ void (*ARGB4444ToUVMatrixRow)(
+ const uint8_t* src_argb4444, int src_stride_argb4444, uint8_t* dst_u,
+ uint8_t* dst_v, int width, const struct ArgbConstants* c) =
+ ARGB4444ToUVMatrixRow_C;
+ void (*ARGB4444ToYMatrixRow)(const uint8_t* src_argb4444, uint8_t* dst_y,
+ int width, const struct ArgbConstants* c) =
+ ARGB4444ToYMatrixRow_C;
-// Neon version does direct ARGB4444 to YUV.
-#if defined(HAS_ARGB4444TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_Any_NEON;
- ARGB4444ToYRow = ARGB4444ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGB4444ToYRow = ARGB4444ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_NEON;
- }
- }
- }
-#elif defined(HAS_ARGB4444TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_Any_MMI;
- ARGB4444ToYRow = ARGB4444ToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGB4444ToYRow = ARGB4444ToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_MMI;
- }
- }
- }
-// Other platforms do intermediate conversion from ARGB4444 to ARGB.
-#else
-#if defined(HAS_ARGB4444TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_SSE2;
- if (IS_ALIGNED(width, 8)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_SSE2;
+#if defined(HAS_ARGB4444TOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGB4444TOARGBROW_AVX2)
+#if defined(HAS_ARGB4444TOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_AVX2;
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_AVX2;
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGB4444TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_MSA;
+#if defined(HAS_ARGB4444TOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_MSA;
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+#if defined(HAS_ARGB4444TOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
- }
+ ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
- }
+#if defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_WASMSIMD;
}
}
-#endif
#endif
- {
-#if !(defined(HAS_ARGB4444TOYROW_NEON) || defined(HAS_ARGB4444TOYROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-#endif
+ if (!src_argb4444 || !dst_y || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb4444 = src_argb4444 + (ptrdiff_t)(height - 1) * src_stride_argb4444;
+ src_stride_argb4444 = -src_stride_argb4444;
+ }
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_ARGB4444TOYROW_NEON) || defined(HAS_ARGB4444TOYROW_MMI))
- ARGB4444ToUVRow(src_argb4444, src_stride_argb4444, dst_u, dst_v, width);
- ARGB4444ToYRow(src_argb4444, dst_y, width);
- ARGB4444ToYRow(src_argb4444 + src_stride_argb4444, dst_y + dst_stride_y,
- width);
-#else
- ARGB4444ToARGBRow(src_argb4444, row, width);
- ARGB4444ToARGBRow(src_argb4444 + src_stride_argb4444, row + kRowSize,
- width);
- ARGBToUVRow(row, kRowSize, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
- ARGBToYRow(row + kRowSize, dst_y + dst_stride_y, width);
-#endif
- src_argb4444 += src_stride_argb4444 * 2;
- dst_y += dst_stride_y * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
- }
- if (height & 1) {
-#if (defined(HAS_ARGB4444TOYROW_NEON) || defined(HAS_ARGB4444TOYROW_MMI))
- ARGB4444ToUVRow(src_argb4444, 0, dst_u, dst_v, width);
- ARGB4444ToYRow(src_argb4444, dst_y, width);
-#else
- ARGB4444ToARGBRow(src_argb4444, row, width);
- ARGBToUVRow(row, 0, dst_u, dst_v, width);
- ARGBToYRow(row, dst_y, width);
-#endif
- }
-#if !(defined(HAS_ARGB4444TOYROW_NEON) || defined(HAS_ARGB4444TOYROW_MMI))
- free_aligned_buffer_64(row);
-#endif
+ for (y = 0; y < height - 1; y += 2) {
+ ARGB4444ToUVMatrixRow(src_argb4444, src_stride_argb4444, dst_u, dst_v,
+ width, &kArgbI601Constants);
+ ARGB4444ToYMatrixRow(src_argb4444, dst_y, width, &kArgbI601Constants);
+ ARGB4444ToYMatrixRow(src_argb4444 + src_stride_argb4444,
+ dst_y + dst_stride_y, width, &kArgbI601Constants);
+ src_argb4444 += src_stride_argb4444 * 2;
+ dst_y += dst_stride_y * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ if (height & 1) {
+ ARGB4444ToYMatrixRow(src_argb4444, dst_y, width, &kArgbI601Constants);
+ ARGB4444ToUVMatrixRow(src_argb4444, 0, dst_u, dst_v, width,
+ &kArgbI601Constants);
}
return 0;
}
-
// Convert RGB24 to J400.
LIBYUV_API
int RGB24ToJ400(const uint8_t* src_rgb24,
@@ -2378,125 +4860,199 @@ int RGB24ToJ400(const uint8_t* src_rgb24,
int width,
int height) {
int y;
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- void (*RGB24ToYJRow)(const uint8_t* src_rgb24, uint8_t* dst_yj, int width) =
- RGB24ToYJRow_C;
-#else
- void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RGB24ToARGBRow_C;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_yj, int width) =
- ARGBToYJRow_C;
-#endif
- if (!src_rgb24 || !dst_yj || width <= 0 || height == 0) {
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+ if (!src_rgb24 || !dst_yj || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
- // Negative height means invert the image.
if (height < 0) {
height = -height;
- src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24;
+ src_rgb24 = src_rgb24 + (ptrdiff_t)(height - 1) * src_stride_rgb24;
src_stride_rgb24 = -src_stride_rgb24;
}
+ // Coalesce rows.
+ if (src_stride_rgb24 == width * 3 && dst_stride_yj == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_rgb24 = dst_stride_yj = 0;
+ }
-// Neon version does direct RGB24 to YUV.
-#if defined(HAS_RGB24TOYJROW_NEON)
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGB24ToYJRow = RGB24ToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYJRow = RGB24ToYJRow_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
-#elif defined(HAS_RGB24TOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB24ToYJRow = RGB24ToYJRow_Any_MSA;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- RGB24ToYJRow = RGB24ToYJRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
-#elif defined(HAS_RGB24TOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB24ToYJRow = RGB24ToYJRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- RGB24ToYJRow = RGB24ToYJRow_MMI;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
-// Other platforms do intermediate conversion from RGB24 to ARGB.
-#else
-#if defined(HAS_RGB24TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3;
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- RGB24ToARGBRow = RGB24ToARGBRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_SSSE3)
+
+ for (y = 0; y < height; ++y) {
+ RGBToYMatrixRow(src_rgb24, dst_yj, width, &kArgbJPEGConstants);
+ src_rgb24 += src_stride_rgb24;
+ dst_yj += dst_stride_yj;
+ }
+ return 0;
+}
+
+// Convert RAW to J400.
+LIBYUV_API
+int RAWToJ400(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height) {
+ int y;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+ if (!src_raw || !dst_yj || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+ // Coalesce rows.
+ if (src_stride_raw == width * 3 && dst_stride_yj == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_raw = dst_stride_yj = 0;
+ }
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2)
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToYJRow = ARGBToYJRow_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
}
}
#endif
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
+ }
#endif
-
- {
-#if !(defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- // Allocate 2 rows of ARGB.
- const int kRowSize = (width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
+ }
+ }
#endif
-
- for (y = 0; y < height - 1; y += 2) {
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- RGB24ToYJRow(src_rgb24, dst_yj, width);
- RGB24ToYJRow(src_rgb24 + src_stride_rgb24, dst_yj + dst_stride_yj, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- RGB24ToARGBRow(src_rgb24 + src_stride_rgb24, row + kRowSize, width);
- ARGBToYJRow(row, dst_yj, width);
- ARGBToYJRow(row + kRowSize, dst_yj + dst_stride_yj, width);
-#endif
- src_rgb24 += src_stride_rgb24 * 2;
- dst_yj += dst_stride_yj * 2;
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
- if (height & 1) {
-#if (defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- RGB24ToYJRow(src_rgb24, dst_yj, width);
-#else
- RGB24ToARGBRow(src_rgb24, row, width);
- ARGBToYJRow(row, dst_yj, width);
+ }
#endif
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
-#if !(defined(HAS_RGB24TOYJROW_NEON) || defined(HAS_RGB24TOYJROW_MSA) || \
- defined(HAS_RGB24TOYJROW_MMI))
- free_aligned_buffer_64(row);
+ }
#endif
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
+ }
}
- return 0;
-}
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
-static void SplitPixels(const uint8_t* src_u,
- int src_pixel_stride_uv,
- uint8_t* dst_u,
- int width) {
- int i;
- for (i = 0; i < width; ++i) {
- *dst_u = *src_u;
- ++dst_u;
- src_u += src_pixel_stride_uv;
+ for (y = 0; y < height; ++y) {
+ RGBToYMatrixRow(src_raw, dst_yj, width, &kAbgrJPEGConstants);
+ src_raw += src_stride_raw;
+ dst_yj += dst_stride_yj;
}
+ return 0;
}
// Convert Android420 to I420.
@@ -2516,60 +5072,145 @@ int Android420ToI420(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- int y;
- const ptrdiff_t vu_off = src_v - src_u;
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+ return Android420ToI420Rotate(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, src_pixel_stride_uv, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, width, height, kRotate0);
+}
+
+// depth is source bits measured from lsb; For msb use 16
+static int Biplanar16bitTo8bit(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int depth) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+ int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ int scale = 1 << (24 - depth);
// Negative height means invert the image.
if (height < 0) {
height = -height;
- halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ uv_height = -uv_height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(uv_height - 1) * src_stride_uv;
src_stride_y = -src_stride_y;
- src_stride_u = -src_stride_u;
- src_stride_v = -src_stride_v;
+ src_stride_uv = -src_stride_uv;
}
+ // Convert Y plane.
if (dst_y) {
- CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
}
+ // Convert UV planes.
+ Convert16To8Plane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, scale,
+ uv_width * 2, uv_height);
+ return 0;
+}
- // Copy UV planes as is - I420
- if (src_pixel_stride_uv == 1) {
- CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, halfheight);
- CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, halfheight);
- return 0;
- // Split UV planes - NV21
- }
- if (src_pixel_stride_uv == 2 && vu_off == -1 &&
- src_stride_u == src_stride_v) {
- SplitUVPlane(src_v, src_stride_v, dst_v, dst_stride_v, dst_u, dst_stride_u,
- halfwidth, halfheight);
- return 0;
- // Split UV planes - NV12
+// Convert 10 bit P010 to 8 bit NV12.
+// Depth set to 16 because P010 uses 10 msb and this function keeps the upper 8
+// bits of the specified number of bits.
+LIBYUV_API
+int P010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return Biplanar16bitTo8bit(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_uv, dst_stride_uv, width, height,
+ 1, 1, 16);
+}
+
+static int Planar8bitTo8bit(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int scale_y,
+ int bias_y,
+ int scale_uv,
+ int bias_uv) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
}
- if (src_pixel_stride_uv == 2 && vu_off == 1 && src_stride_u == src_stride_v) {
- SplitUVPlane(src_u, src_stride_u, dst_u, dst_stride_u, dst_v, dst_stride_v,
- halfwidth, halfheight);
- return 0;
+ int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ uv_height = -uv_height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(uv_height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(uv_height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
}
- for (y = 0; y < halfheight; ++y) {
- SplitPixels(src_u, src_pixel_stride_uv, dst_u, halfwidth);
- SplitPixels(src_v, src_pixel_stride_uv, dst_v, halfwidth);
- src_u += src_stride_u;
- src_v += src_stride_v;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
+ // Convert Y plane.
+ if (dst_y) {
+ Convert8To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale_y, bias_y,
+ width, height);
}
+ // Convert UV planes.
+ Convert8To8Plane(src_u, src_stride_u, dst_u, dst_stride_u, scale_uv, bias_uv,
+ uv_width, uv_height);
+ Convert8To8Plane(src_v, src_stride_v, dst_v, dst_stride_v, scale_uv, bias_uv,
+ uv_width, uv_height);
return 0;
}
+LIBYUV_API
+int J420ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Planar8bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 1, 220, 16, 225, 16);
+}
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/convert_argb.cc b/libs/libyuv/source/convert_argb.cc
index 540503330d..9799e0a791 100644
--- a/libs/libyuv/source/convert_argb.cc
+++ b/libs/libyuv/source/convert_argb.cc
@@ -10,13 +10,16 @@
#include "libyuv/convert_argb.h"
+#include <assert.h>
+#include <limits.h>
+#include <string.h> // For memcpy.
+
+#include "libyuv/convert_from_argb.h"
#include "libyuv/cpu_id.h"
-#ifdef HAVE_JPEG
-#include "libyuv/mjpeg_decoder.h"
-#endif
#include "libyuv/planar_functions.h" // For CopyPlane and ARGBShuffle.
#include "libyuv/rotate_argb.h"
#include "libyuv/row.h"
+#include "libyuv/scale_row.h" // For ScaleRowUp2_Linear and ScaleRowUp2_Bilinear
#include "libyuv/video_common.h"
#ifdef __cplusplus
@@ -32,13 +35,14 @@ int ARGBCopy(const uint8_t* src_argb,
int dst_stride_argb,
int width,
int height) {
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
@@ -47,30 +51,33 @@ int ARGBCopy(const uint8_t* src_argb,
return 0;
}
-// Convert I420 to ARGB with matrix
-static int I420ToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+// Convert I420 to ARGB with matrix.
+LIBYUV_API
+int I420ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
void (*I422ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
const uint8_t* v_buf, uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants, int width) =
I422ToARGBRow_C;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
#if defined(HAS_I422TOARGBROW_SSSE3)
@@ -89,6 +96,12 @@ static int I420ToARGBMatrix(const uint8_t* src_y,
}
}
#endif
+#if defined(HAS_I422TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToARGBRow = I422ToARGBRow_AVX512BW;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I422ToARGBRow = I422ToARGBRow_Any_NEON;
@@ -97,14 +110,50 @@ static int I420ToARGBMatrix(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_I422TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGBRow = I422ToARGBRow_Any_MSA;
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGBRow = I422ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToARGBRow = I422ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
- I422ToARGBRow = I422ToARGBRow_MSA;
+ I422ToARGBRow = I422ToARGBRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_I422TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422ToARGBRow = I422ToARGBRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
I422ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
@@ -226,35 +275,75 @@ int H420ToABGR(const uint8_t* src_y,
width, height);
}
-// Convert I422 to ARGB with matrix
-static int I422ToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+// Convert U420 to ARGB.
+LIBYUV_API
+int U420ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert U420 to ABGR.
+LIBYUV_API
+int U420ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I420ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvu2020Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert I422 to ARGB with matrix.
+LIBYUV_API
+int I422ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
void (*I422ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
const uint8_t* v_buf, uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants, int width) =
I422ToARGBRow_C;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u * 2 == width &&
- src_stride_v * 2 == width && dst_stride_argb == width * 4) {
+ src_stride_v * 2 == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
@@ -275,6 +364,12 @@ static int I422ToARGBMatrix(const uint8_t* src_y,
}
}
#endif
+#if defined(HAS_I422TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToARGBRow = I422ToARGBRow_AVX512BW;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I422ToARGBRow = I422ToARGBRow_Any_NEON;
@@ -283,14 +378,50 @@ static int I422ToARGBMatrix(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_I422TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGBRow = I422ToARGBRow_Any_MSA;
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGBRow = I422ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToARGBRow = I422ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
- I422ToARGBRow = I422ToARGBRow_MSA;
+ I422ToARGBRow = I422ToARGBRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_I422TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422ToARGBRow = I422ToARGBRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
I422ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
@@ -410,325 +541,382 @@ int H422ToABGR(const uint8_t* src_y,
width, height);
}
-// Convert 10 bit YUV to ARGB with matrix
-// TODO(fbarchard): Consider passing scale multiplier to I210ToARGB to
-// multiply 10 bit yuv into high bits to allow any number of bits.
-static int I010ToAR30Matrix(const uint16_t* src_y,
- int src_stride_y,
- const uint16_t* src_u,
- int src_stride_u,
- const uint16_t* src_v,
- int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+// Convert U422 to ARGB.
+LIBYUV_API
+int U422ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I422ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert U422 to ABGR.
+LIBYUV_API
+int U422ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I422ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvu2020Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert I444 to ARGB with matrix.
+LIBYUV_API
+int I444ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*I210ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
- const uint16_t* v_buf, uint8_t* rgb_buf,
+ void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
const struct YuvConstants* yuvconstants, int width) =
- I210ToAR30Row_C;
- if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0) {
+ I444ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
- dst_stride_ar30 = -dst_stride_ar30;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_I210TOAR30ROW_SSSE3)
+ // Coalesce rows.
+ if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
+ }
+#if defined(HAS_I444TOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- I210ToAR30Row = I210ToAR30Row_Any_SSSE3;
+ I444ToARGBRow = I444ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- I210ToAR30Row = I210ToAR30Row_SSSE3;
+ I444ToARGBRow = I444ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_I210TOAR30ROW_AVX2)
+#if defined(HAS_I444TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I210ToAR30Row = I210ToAR30Row_Any_AVX2;
+ I444ToARGBRow = I444ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- I210ToAR30Row = I210ToAR30Row_AVX2;
+ I444ToARGBRow = I444ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444ToARGBRow = I444ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444ToARGBRow = I444ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_LASX;
}
}
#endif
+#if defined(HAS_I444TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToARGBRow = I444ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444ToARGBRow = I444ToARGBRow_HVX;
+ }
+#endif
+
for (y = 0; y < height; ++y) {
- I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
- dst_ar30 += dst_stride_ar30;
+ I444ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
return 0;
}
-// Convert I010 to AR30.
+// Convert I444 to ARGB.
LIBYUV_API
-int I010ToAR30(const uint16_t* src_y,
+int I444ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height) {
- return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_ar30, dst_stride_ar30,
+ return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
&kYuvI601Constants, width, height);
}
-// Convert H010 to AR30.
+// Convert I444 to ABGR.
LIBYUV_API
-int H010ToAR30(const uint16_t* src_y,
+int I444ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height) {
- return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_ar30, dst_stride_ar30,
- &kYuvH709Constants, width, height);
+ return I444ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
}
-// Convert I010 to AB30.
+// Convert J444 to ARGB.
LIBYUV_API
-int I010ToAB30(const uint16_t* src_y,
+int J444ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ab30,
- int dst_stride_ab30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
int width,
int height) {
- return I010ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
- src_stride_u, dst_ab30, dst_stride_ab30,
- &kYvuI601Constants, width, height);
+ return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuvJPEGConstants, width, height);
}
-// Convert H010 to AB30.
+// Convert J444 to ABGR.
LIBYUV_API
-int H010ToAB30(const uint16_t* src_y,
+int J444ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ab30,
- int dst_stride_ab30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height) {
- return I010ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
- src_stride_u, dst_ab30, dst_stride_ab30,
- &kYvuH709Constants, width, height);
-}
-
-// Convert 10 bit YUV to ARGB with matrix
-static int I010ToARGBMatrix(const uint16_t* src_y,
- int src_stride_y,
- const uint16_t* src_u,
- int src_stride_u,
- const uint16_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
- int y;
- void (*I210ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
- const uint16_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I210ToARGBRow_C;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
- }
-#if defined(HAS_I210TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I210ToARGBRow = I210ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I210ToARGBRow = I210ToARGBRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I210TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I210ToARGBRow = I210ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I210ToARGBRow = I210ToARGBRow_AVX2;
- }
- }
-#endif
- for (y = 0; y < height; ++y) {
- I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
- dst_argb += dst_stride_argb;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- return 0;
+ return I444ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuJPEGConstants, // Use Yvu matrix
+ width, height);
}
-// Convert I010 to ARGB.
+// Convert H444 to ARGB.
LIBYUV_API
-int I010ToARGB(const uint16_t* src_y,
+int H444ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
- return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_argb, dst_stride_argb,
- &kYuvI601Constants, width, height);
+ &kYuvH709Constants, width, height);
}
-// Convert I010 to ABGR.
+// Convert H444 to ABGR.
LIBYUV_API
-int I010ToABGR(const uint16_t* src_y,
+int H444ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height) {
- return I010ToARGBMatrix(src_y, src_stride_y, src_v,
+ return I444ToARGBMatrix(src_y, src_stride_y, src_v,
src_stride_v, // Swap U and V
src_u, src_stride_u, dst_abgr, dst_stride_abgr,
- &kYvuI601Constants, // Use Yvu matrix
+ &kYvuH709Constants, // Use Yvu matrix
width, height);
}
-// Convert H010 to ARGB.
+// Convert U444 to ARGB.
LIBYUV_API
-int H010ToARGB(const uint16_t* src_y,
+int U444ToARGB(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
- return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_argb, dst_stride_argb,
- &kYuvH709Constants, width, height);
+ &kYuv2020Constants, width, height);
}
-// Convert H010 to ABGR.
+// Convert U444 to ABGR.
LIBYUV_API
-int H010ToABGR(const uint16_t* src_y,
+int U444ToABGR(const uint8_t* src_y,
int src_stride_y,
- const uint16_t* src_u,
+ const uint8_t* src_u,
int src_stride_u,
- const uint16_t* src_v,
+ const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height) {
- return I010ToARGBMatrix(src_y, src_stride_y, src_v,
+ return I444ToARGBMatrix(src_y, src_stride_y, src_v,
src_stride_v, // Swap U and V
src_u, src_stride_u, dst_abgr, dst_stride_abgr,
- &kYvuH709Constants, // Use Yvu matrix
+ &kYvu2020Constants, // Use Yvu matrix
width, height);
}
-// Convert I444 to ARGB with matrix
-static int I444ToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+// Convert I444 to RGB24 with matrix.
+LIBYUV_API
+int I444ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I444ToARGBRow_C;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
+ void (*I444ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToRGB24Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
- dst_stride_argb == width * 4) {
+ dst_stride_rgb24 == width * 3 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb24 = 0;
}
-#if defined(HAS_I444TOARGBROW_SSSE3)
+#if defined(HAS_I444TORGB24ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- I444ToARGBRow = I444ToARGBRow_Any_SSSE3;
+ I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- I444ToARGBRow = I444ToARGBRow_SSSE3;
+ I444ToRGB24Row = I444ToRGB24Row_SSSE3;
}
}
#endif
-#if defined(HAS_I444TOARGBROW_AVX2)
+#if defined(HAS_I444TORGB24ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I444ToARGBRow = I444ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I444ToARGBRow = I444ToARGBRow_AVX2;
+ I444ToRGB24Row = I444ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ I444ToRGB24Row = I444ToRGB24Row_AVX2;
}
}
#endif
-#if defined(HAS_I444TOARGBROW_NEON)
+#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- I444ToARGBRow = I444ToARGBRow_Any_NEON;
+ I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- I444ToARGBRow = I444ToARGBRow_NEON;
+ I444ToRGB24Row = I444ToRGB24Row_NEON;
}
}
#endif
-#if defined(HAS_I444TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I444ToARGBRow = I444ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I444ToARGBRow = I444ToARGBRow_MSA;
- }
+#if defined(HAS_I444TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToRGB24Row = I444ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToRGB24Row = I444ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToRGB24Row = I444ToRGB24Row_RVV;
}
#endif
for (y = 0; y < height; ++y) {
- I444ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
- dst_argb += dst_stride_argb;
+ I444ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
src_y += src_stride_y;
src_u += src_stride_u;
src_v += src_stride_v;
@@ -736,172 +924,115 @@ static int I444ToARGBMatrix(const uint8_t* src_y,
return 0;
}
-// Convert I444 to ARGB.
+// Convert I444 to RGB24.
LIBYUV_API
-int I444ToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_argb, dst_stride_argb,
- &kYuvI601Constants, width, height);
+int I444ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return I444ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ &kYuvI601Constants, width, height);
}
-// Convert I444 to ABGR.
+// Convert I444 to RAW.
LIBYUV_API
-int I444ToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height) {
- return I444ToARGBMatrix(src_y, src_stride_y, src_v,
- src_stride_v, // Swap U and V
- src_u, src_stride_u, dst_abgr, dst_stride_abgr,
- &kYvuI601Constants, // Use Yvu matrix
- width, height);
+int I444ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return I444ToRGB24Matrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_raw, dst_stride_raw,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
}
-// Convert J444 to ARGB.
+// Convert 10 bit YUV to ARGB with matrix.
+// TODO(fbarchard): Consider passing scale multiplier to I210ToARGB to
+// multiply 10 bit yuv into high bits to allow any number of bits.
LIBYUV_API
-int J444ToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- return I444ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_argb, dst_stride_argb,
- &kYuvJPEGConstants, width, height);
-}
-
-// Convert I420 with Alpha to preattenuated ARGB.
-static int I420AlphaToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- const uint8_t* src_a,
- int src_stride_a,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height,
- int attenuate) {
+int I010ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*I422AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) = I422AlphaToARGBRow_C;
- void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
- int width) = ARGBAttenuateRow_C;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
+ void (*I210ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I210ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_I422ALPHATOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_Any_SSSE3;
+#if defined(HAS_I210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToAR30Row = I210ToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_SSSE3;
+ I210ToAR30Row = I210ToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_I422ALPHATOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_AVX2;
- }
+#if defined(HAS_I210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToAR30Row = I210ToAR30Row_SVE2;
}
#endif
-#if defined(HAS_I422ALPHATOARGBROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_NEON;
- }
+#if defined(HAS_I210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToAR30Row = I210ToAR30Row_SME;
}
#endif
-#if defined(HAS_I422ALPHATOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_Any_MSA;
+#if defined(HAS_I210TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210ToAR30Row = I210ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- I422AlphaToARGBRow = I422AlphaToARGBRow_MSA;
+ I210ToAR30Row = I210ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBATTENUATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
- if (IS_ALIGNED(width, 4)) {
- ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+#if defined(HAS_I210TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210ToAR30Row = I210ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I210ToAR30Row = I210ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_ARGBATTENUATEROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
- if (IS_ALIGNED(width, 8)) {
- ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
- }
- }
-#endif
-#if defined(HAS_ARGBATTENUATEROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBAttenuateRow = ARGBAttenuateRow_NEON;
- }
- }
-#endif
-#if defined(HAS_ARGBATTENUATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBAttenuateRow = ARGBAttenuateRow_MSA;
- }
- }
-#endif
-#if defined(HAS_ARGBATTENUATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBAttenuateRow = ARGBAttenuateRow_MMI;
- }
+#if defined(HAS_I210TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210ToAR30Row = I210ToAR30Row_AVX512BW;
}
#endif
-
for (y = 0; y < height; ++y) {
- I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
- width);
- if (attenuate) {
- ARGBAttenuateRow(dst_argb, dst_argb, width);
- }
- dst_argb += dst_stride_argb;
- src_a += src_stride_a;
+ I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
src_y += src_stride_y;
if (y & 1) {
src_u += src_stride_u;
@@ -911,783 +1042,1061 @@ static int I420AlphaToARGBMatrix(const uint8_t* src_y,
return 0;
}
-// Convert I420 with Alpha to ARGB.
+// Convert I010 to AR30.
LIBYUV_API
-int I420AlphaToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- const uint8_t* src_a,
- int src_stride_a,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height,
- int attenuate) {
- return I420AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, src_a, src_stride_a, dst_argb,
- dst_stride_argb, &kYuvI601Constants, width,
- height, attenuate);
+int I010ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuvI601Constants, width, height);
}
-// Convert I420 with Alpha to ABGR.
+// Convert H010 to AR30.
LIBYUV_API
-int I420AlphaToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- const uint8_t* src_a,
- int src_stride_a,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height,
- int attenuate) {
- return I420AlphaToARGBMatrix(
- src_y, src_stride_y, src_v, src_stride_v, // Swap U and V
- src_u, src_stride_u, src_a, src_stride_a, dst_abgr, dst_stride_abgr,
- &kYvuI601Constants, // Use Yvu matrix
- width, height, attenuate);
+int H010ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuvH709Constants, width, height);
}
-// Convert I400 to ARGB.
+// Convert U010 to AR30.
LIBYUV_API
-int I400ToARGB(const uint8_t* src_y,
+int U010ToAR30(const uint16_t* src_y,
int src_stride_y,
- uint8_t* dst_argb,
- int dst_stride_argb,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert I010 to AB30.
+LIBYUV_API
+int I010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuI601Constants, width, height);
+}
+
+// Convert H010 to AB30.
+LIBYUV_API
+int H010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuH709Constants, width, height);
+}
+
+// Convert U010 to AB30.
+LIBYUV_API
+int U010ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
int width,
int height) {
+ return I010ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert 12 bit YUV to ARGB with matrix.
+// TODO(fbarchard): Consider passing scale multiplier to I212ToARGB to
+// multiply 12 bit yuv into high bits to allow any number of bits.
+LIBYUV_API
+int I012ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*I400ToARGBRow)(const uint8_t* y_buf, uint8_t* rgb_buf, int width) =
- I400ToARGBRow_C;
- if (!src_y || !dst_argb || width <= 0 || height == 0) {
+ void (*I212ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I212ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
- }
- // Coalesce rows.
- if (src_stride_y == width && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_y = dst_stride_argb = 0;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_I400TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- I400ToARGBRow = I400ToARGBRow_Any_SSE2;
+#if defined(HAS_I212TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I212ToAR30Row = I212ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- I400ToARGBRow = I400ToARGBRow_SSE2;
+ I212ToAR30Row = I212ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_I400TOARGBROW_AVX2)
+#if defined(HAS_I212TOAR30ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I400ToARGBRow = I400ToARGBRow_Any_AVX2;
+ I212ToAR30Row = I212ToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- I400ToARGBRow = I400ToARGBRow_AVX2;
+ I212ToAR30Row = I212ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_I400TOARGBROW_NEON)
+#if defined(HAS_I212TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I212ToAR30Row = I212ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I212TOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- I400ToARGBRow = I400ToARGBRow_Any_NEON;
+ I212ToAR30Row = I212ToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- I400ToARGBRow = I400ToARGBRow_NEON;
+ I212ToAR30Row = I212ToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_I400TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I400ToARGBRow = I400ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- I400ToARGBRow = I400ToARGBRow_MSA;
- }
+#if defined(HAS_I212TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I212ToAR30Row = I212ToAR30Row_SVE2;
}
#endif
-#if defined(HAS_I400TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I400ToARGBRow = I400ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- I400ToARGBRow = I400ToARGBRow_MMI;
- }
+#if defined(HAS_I212TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I212ToAR30Row = I212ToAR30Row_SME;
}
#endif
-
for (y = 0; y < height; ++y) {
- I400ToARGBRow(src_y, dst_argb, width);
- dst_argb += dst_stride_argb;
+ I212ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
}
return 0;
}
-// Convert J400 to ARGB.
+// Convert 10 bit YUV to ARGB with matrix.
+// TODO(fbarchard): Consider passing scale multiplier to I210ToARGB to
+// multiply 10 bit yuv into high bits to allow any number of bits.
LIBYUV_API
-int J400ToARGB(const uint8_t* src_y,
- int src_stride_y,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I210ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*J400ToARGBRow)(const uint8_t* src_y, uint8_t* dst_argb, int width) =
- J400ToARGBRow_C;
- if (!src_y || !dst_argb || width <= 0 || height == 0) {
+ void (*I210ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I210ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
- }
- // Coalesce rows.
- if (src_stride_y == width && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_y = dst_stride_argb = 0;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_J400TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- J400ToARGBRow = J400ToARGBRow_Any_SSE2;
+#if defined(HAS_I210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToAR30Row = I210ToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- J400ToARGBRow = J400ToARGBRow_SSE2;
+ I210ToAR30Row = I210ToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_J400TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- J400ToARGBRow = J400ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- J400ToARGBRow = J400ToARGBRow_AVX2;
- }
+#if defined(HAS_I210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToAR30Row = I210ToAR30Row_SVE2;
}
#endif
-#if defined(HAS_J400TOARGBROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- J400ToARGBRow = J400ToARGBRow_Any_NEON;
+#if defined(HAS_I210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToAR30Row = I210ToAR30Row_SME;
+ }
+#endif
+#if defined(HAS_I210TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210ToAR30Row = I210ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- J400ToARGBRow = J400ToARGBRow_NEON;
+ I210ToAR30Row = I210ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_J400TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- J400ToARGBRow = J400ToARGBRow_Any_MSA;
+#if defined(HAS_I210TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210ToAR30Row = I210ToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- J400ToARGBRow = J400ToARGBRow_MSA;
+ I210ToAR30Row = I210ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_J400TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- J400ToARGBRow = J400ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- J400ToARGBRow = J400ToARGBRow_MMI;
- }
+#if defined(HAS_I210TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210ToAR30Row = I210ToAR30Row_AVX512BW;
}
#endif
for (y = 0; y < height; ++y) {
- J400ToARGBRow(src_y, dst_argb, width);
+ I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
src_y += src_stride_y;
- dst_argb += dst_stride_argb;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
return 0;
}
-// Shuffle table for converting BGRA to ARGB.
-static const uvec8 kShuffleMaskBGRAToARGB = {
- 3u, 2u, 1u, 0u, 7u, 6u, 5u, 4u, 11u, 10u, 9u, 8u, 15u, 14u, 13u, 12u};
-
-// Shuffle table for converting ABGR to ARGB.
-static const uvec8 kShuffleMaskABGRToARGB = {
- 2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
-
-// Shuffle table for converting RGBA to ARGB.
-static const uvec8 kShuffleMaskRGBAToARGB = {
- 1u, 2u, 3u, 0u, 5u, 6u, 7u, 4u, 9u, 10u, 11u, 8u, 13u, 14u, 15u, 12u};
+// Convert I210 to AR30.
+LIBYUV_API
+int I210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I210ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuvI601Constants, width, height);
+}
-// Convert BGRA to ARGB.
+// Convert H210 to AR30.
LIBYUV_API
-int BGRAToARGB(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int H210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
int width,
int height) {
- return ARGBShuffle(src_bgra, src_stride_bgra, dst_argb, dst_stride_argb,
- (const uint8_t*)(&kShuffleMaskBGRAToARGB), width, height);
+ return I210ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuvH709Constants, width, height);
}
-// Convert ARGB to BGRA (same as BGRAToARGB).
+// Convert U210 to AR30.
LIBYUV_API
-int ARGBToBGRA(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int U210ToAR30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
int width,
int height) {
- return ARGBShuffle(src_bgra, src_stride_bgra, dst_argb, dst_stride_argb,
- (const uint8_t*)(&kShuffleMaskBGRAToARGB), width, height);
+ return I210ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuv2020Constants, width, height);
}
-// Convert ABGR to ARGB.
+// Convert I210 to AB30.
LIBYUV_API
-int ABGRToARGB(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int I210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
int width,
int height) {
- return ARGBShuffle(src_abgr, src_stride_abgr, dst_argb, dst_stride_argb,
- (const uint8_t*)(&kShuffleMaskABGRToARGB), width, height);
+ return I210ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuI601Constants, width, height);
}
-// Convert ARGB to ABGR to (same as ABGRToARGB).
+// Convert H210 to AB30.
LIBYUV_API
-int ARGBToABGR(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int H210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
int width,
int height) {
- return ARGBShuffle(src_abgr, src_stride_abgr, dst_argb, dst_stride_argb,
- (const uint8_t*)(&kShuffleMaskABGRToARGB), width, height);
+ return I210ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuH709Constants, width, height);
}
-// Convert RGBA to ARGB.
+// Convert U210 to AB30.
LIBYUV_API
-int RGBAToARGB(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_argb,
- int dst_stride_argb,
+int U210ToAB30(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
int width,
int height) {
- return ARGBShuffle(src_rgba, src_stride_rgba, dst_argb, dst_stride_argb,
- (const uint8_t*)(&kShuffleMaskRGBAToARGB), width, height);
+ return I210ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYuv2020Constants, width, height);
}
-// Convert RGB24 to ARGB.
LIBYUV_API
-int RGB24ToARGB(const uint8_t* src_rgb24,
- int src_stride_rgb24,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I410ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RGB24ToARGBRow_C;
- if (!src_rgb24 || !dst_argb || width <= 0 || height == 0) {
+ void (*I410ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_rgb24 = src_rgb24 + (height - 1) * src_stride_rgb24;
- src_stride_rgb24 = -src_stride_rgb24;
- }
- // Coalesce rows.
- if (src_stride_rgb24 == width * 3 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_rgb24 = dst_stride_argb = 0;
- }
-#if defined(HAS_RGB24TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToARGBRow = RGB24ToARGBRow_SSSE3;
- }
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#endif
-#if defined(HAS_RGB24TOARGBROW_NEON)
+#if defined(HAS_I410TOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_NEON;
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- RGB24ToARGBRow = RGB24ToARGBRow_NEON;
+ I410ToAR30Row = I410ToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_RGB24TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- RGB24ToARGBRow = RGB24ToARGBRow_MSA;
- }
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
}
#endif
-#if defined(HAS_RGB24TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB24ToARGBRow = RGB24ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- RGB24ToARGBRow = RGB24ToARGBRow_MMI;
- }
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
}
#endif
-
- for (y = 0; y < height; ++y) {
- RGB24ToARGBRow(src_rgb24, dst_argb, width);
- src_rgb24 += src_stride_rgb24;
- dst_argb += dst_stride_argb;
- }
- return 0;
-}
-
-// Convert RAW to ARGB.
-LIBYUV_API
-int RAWToARGB(const uint8_t* src_raw,
- int src_stride_raw,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- int y;
- void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
- RAWToARGBRow_C;
- if (!src_raw || !dst_argb || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_raw = src_raw + (height - 1) * src_stride_raw;
- src_stride_raw = -src_stride_raw;
- }
- // Coalesce rows.
- if (src_stride_raw == width * 3 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_raw = dst_stride_argb = 0;
- }
-#if defined(HAS_RAWTOARGBROW_SSSE3)
+#if defined(HAS_I410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- RAWToARGBRow = RAWToARGBRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_RAWTOARGBROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- RAWToARGBRow = RAWToARGBRow_Any_NEON;
+ I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- RAWToARGBRow = RAWToARGBRow_NEON;
+ I410ToAR30Row = I410ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_RAWTOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RAWToARGBRow = RAWToARGBRow_Any_MSA;
+#if defined(HAS_I410TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToAR30Row = I410ToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- RAWToARGBRow = RAWToARGBRow_MSA;
+ I410ToAR30Row = I410ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_RAWTOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RAWToARGBRow = RAWToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- RAWToARGBRow = RAWToARGBRow_MMI;
- }
+#if defined(HAS_I410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToAR30Row = I410ToAR30Row_AVX512BW;
}
#endif
-
for (y = 0; y < height; ++y) {
- RAWToARGBRow(src_raw, dst_argb, width);
- src_raw += src_stride_raw;
- dst_argb += dst_stride_argb;
+ I410ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
return 0;
}
-// Convert RGB565 to ARGB.
+// Convert 10 bit YUV to ARGB with matrix.
LIBYUV_API
-int RGB565ToARGB(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I010ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*RGB565ToARGBRow)(const uint8_t* src_rgb565, uint8_t* dst_argb,
- int width) = RGB565ToARGBRow_C;
- if (!src_rgb565 || !dst_argb || width <= 0 || height == 0) {
+ void (*I210ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I210ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_rgb565 = src_rgb565 + (height - 1) * src_stride_rgb565;
- src_stride_rgb565 = -src_stride_rgb565;
- }
- // Coalesce rows.
- if (src_stride_rgb565 == width * 2 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_rgb565 = dst_stride_argb = 0;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_RGB565TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_SSE2;
+#if defined(HAS_I210TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210ToARGBRow = I210ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- RGB565ToARGBRow = RGB565ToARGBRow_SSE2;
- }
- }
-#endif
-#if defined(HAS_RGB565TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- RGB565ToARGBRow = RGB565ToARGBRow_AVX2;
+ I210ToARGBRow = I210ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_RGB565TOARGBROW_NEON)
+#if defined(HAS_I210TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_NEON;
+ I210ToARGBRow = I210ToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- RGB565ToARGBRow = RGB565ToARGBRow_NEON;
+ I210ToARGBRow = I210ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_RGB565TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_MSA;
+#if defined(HAS_I210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToARGBRow = I210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToARGBRow = I210ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210ToARGBRow = I210ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- RGB565ToARGBRow = RGB565ToARGBRow_MSA;
+ I210ToARGBRow = I210ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_RGB565TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RGB565ToARGBRow = RGB565ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- RGB565ToARGBRow = RGB565ToARGBRow_MMI;
- }
+#if defined(HAS_I210TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210ToARGBRow = I210ToARGBRow_AVX512BW;
}
#endif
-
for (y = 0; y < height; ++y) {
- RGB565ToARGBRow(src_rgb565, dst_argb, width);
- src_rgb565 += src_stride_rgb565;
+ I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
}
return 0;
}
-// Convert ARGB1555 to ARGB.
+// Convert I010 to ARGB.
LIBYUV_API
-int ARGB1555ToARGB(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I010ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert I010 to ABGR.
+LIBYUV_API
+int I010ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert H010 to ARGB.
+LIBYUV_API
+int H010ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuvH709Constants, width, height);
+}
+
+// Convert H010 to ABGR.
+LIBYUV_API
+int H010ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuH709Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert U010 to ARGB.
+LIBYUV_API
+int U010ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert U010 to ABGR.
+LIBYUV_API
+int U010ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I010ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvu2020Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert 12 bit YUV to ARGB with matrix.
+LIBYUV_API
+int I012ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*ARGB1555ToARGBRow)(const uint8_t* src_argb1555, uint8_t* dst_argb,
- int width) = ARGB1555ToARGBRow_C;
- if (!src_argb1555 || !dst_argb || width <= 0 || height == 0) {
+ void (*I212ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I212ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb1555 = src_argb1555 + (height - 1) * src_stride_argb1555;
- src_stride_argb1555 = -src_stride_argb1555;
- }
- // Coalesce rows.
- if (src_stride_argb1555 == width * 2 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_argb1555 = dst_stride_argb = 0;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_ARGB1555TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_SSE2;
+#if defined(HAS_I212TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I212ToARGBRow = I212ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_SSE2;
+ I212ToARGBRow = I212ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGB1555TOARGBROW_AVX2)
+#if defined(HAS_I212TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_AVX2;
+ I212ToARGBRow = I212ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX2;
+ I212ToARGBRow = I212ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGB1555TOARGBROW_NEON)
+#if defined(HAS_I212TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I212ToARGBRow = I212ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I212TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON;
+ I212ToARGBRow = I212ToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_NEON;
+ I212ToARGBRow = I212ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_ARGB1555TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_MSA;
- }
+#if defined(HAS_I212TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I212ToARGBRow = I212ToARGBRow_SVE2;
}
#endif
-#if defined(HAS_ARGB1555TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGB1555ToARGBRow = ARGB1555ToARGBRow_MMI;
- }
+#if defined(HAS_I212TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I212ToARGBRow = I212ToARGBRow_SME;
}
#endif
-
for (y = 0; y < height; ++y) {
- ARGB1555ToARGBRow(src_argb1555, dst_argb, width);
- src_argb1555 += src_stride_argb1555;
+ I212ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
}
return 0;
}
-// Convert ARGB4444 to ARGB.
+// Convert 10 bit 422 YUV to ARGB with matrix.
LIBYUV_API
-int ARGB4444ToARGB(const uint8_t* src_argb4444,
- int src_stride_argb4444,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I210ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*ARGB4444ToARGBRow)(const uint8_t* src_argb4444, uint8_t* dst_argb,
- int width) = ARGB4444ToARGBRow_C;
- if (!src_argb4444 || !dst_argb || width <= 0 || height == 0) {
+ void (*I210ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I210ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb4444 = src_argb4444 + (height - 1) * src_stride_argb4444;
- src_stride_argb4444 = -src_stride_argb4444;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb4444 == width * 2 && dst_stride_argb == width * 4) {
+ if (src_stride_y == width && src_stride_u * 2 == width &&
+ src_stride_v * 2 == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_argb4444 = dst_stride_argb = 0;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
}
-#if defined(HAS_ARGB4444TOARGBROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_SSE2;
+#if defined(HAS_I210TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210ToARGBRow = I210ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_SSE2;
- }
- }
-#endif
-#if defined(HAS_ARGB4444TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_AVX2;
+ I210ToARGBRow = I210ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGB4444TOARGBROW_NEON)
+#if defined(HAS_I210TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_NEON;
+ I210ToARGBRow = I210ToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_NEON;
+ I210ToARGBRow = I210ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_ARGB4444TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_MSA;
+#if defined(HAS_I210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToARGBRow = I210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToARGBRow = I210ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210ToARGBRow = I210ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_MSA;
+ I210ToARGBRow = I210ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGB4444TOARGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGB4444ToARGBRow = ARGB4444ToARGBRow_MMI;
- }
+#if defined(HAS_I210TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210ToARGBRow = I210ToARGBRow_AVX512BW;
}
#endif
-
for (y = 0; y < height; ++y) {
- ARGB4444ToARGBRow(src_argb4444, dst_argb, width);
- src_argb4444 += src_stride_argb4444;
+ I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
return 0;
}
-// Convert AR30 to ARGB.
+// Convert I210 to ARGB.
LIBYUV_API
-int AR30ToARGB(const uint8_t* src_ar30,
- int src_stride_ar30,
+int I210ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
- int y;
- if (!src_ar30 || !dst_argb || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_ar30 = src_ar30 + (height - 1) * src_stride_ar30;
- src_stride_ar30 = -src_stride_ar30;
- }
- // Coalesce rows.
- if (src_stride_ar30 == width * 4 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_ar30 = dst_stride_argb = 0;
- }
- for (y = 0; y < height; ++y) {
- AR30ToARGBRow_C(src_ar30, dst_argb, width);
- src_ar30 += src_stride_ar30;
- dst_argb += dst_stride_argb;
- }
- return 0;
+ return I210ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
}
-// Convert AR30 to ABGR.
+// Convert I210 to ABGR.
LIBYUV_API
-int AR30ToABGR(const uint8_t* src_ar30,
- int src_stride_ar30,
+int I210ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
uint8_t* dst_abgr,
int dst_stride_abgr,
int width,
int height) {
- int y;
- if (!src_ar30 || !dst_abgr || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_ar30 = src_ar30 + (height - 1) * src_stride_ar30;
- src_stride_ar30 = -src_stride_ar30;
- }
- // Coalesce rows.
- if (src_stride_ar30 == width * 4 && dst_stride_abgr == width * 4) {
- width *= height;
- height = 1;
- src_stride_ar30 = dst_stride_abgr = 0;
- }
- for (y = 0; y < height; ++y) {
- AR30ToABGRRow_C(src_ar30, dst_abgr, width);
- src_ar30 += src_stride_ar30;
- dst_abgr += dst_stride_abgr;
- }
- return 0;
-}
-
-// Convert AR30 to AB30.
+ return I210ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert H210 to ARGB.
LIBYUV_API
-int AR30ToAB30(const uint8_t* src_ar30,
- int src_stride_ar30,
- uint8_t* dst_ab30,
- int dst_stride_ab30,
+int H210ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I210ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuvH709Constants, width, height);
+}
+
+// Convert H210 to ABGR.
+LIBYUV_API
+int H210ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return I210ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvuH709Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert U210 to ARGB.
+LIBYUV_API
+int U210ToARGB(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I210ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ &kYuv2020Constants, width, height);
+}
+
+// Convert U210 to ABGR.
+LIBYUV_API
+int U210ToABGR(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
int width,
int height) {
+ return I210ToARGBMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_abgr, dst_stride_abgr,
+ &kYvu2020Constants, // Use Yvu matrix
+ width, height);
+}
+
+LIBYUV_API
+int I410ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- if (!src_ar30 || !dst_ab30 || width <= 0 || height == 0) {
+ void (*I410ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_ar30 = src_ar30 + (height - 1) * src_stride_ar30;
- src_stride_ar30 = -src_stride_ar30;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
- if (src_stride_ar30 == width * 4 && dst_stride_ab30 == width * 4) {
+ if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_ar30 = dst_stride_ab30 = 0;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
+ }
+#if defined(HAS_I410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410ToARGBRow = I410ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToARGBRow = I410ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410ToARGBRow = I410ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToARGBRow = I410ToARGBRow_AVX512BW;
}
+#endif
for (y = 0; y < height; ++y) {
- AR30ToAB30Row_C(src_ar30, dst_ab30, width);
- src_ar30 += src_stride_ar30;
- dst_ab30 += dst_stride_ab30;
+ I410ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
return 0;
}
-// Convert NV12 to ARGB with matrix
-static int NV12ToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+LIBYUV_API
+int P010ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV12ToARGBRow)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV12ToARGBRow_C;
- if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0) {
+ void (*P210ToARGBRow)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P210ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_NV12TOARGBROW_SSSE3)
+#if defined(HAS_P210TOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3;
+ P210ToARGBRow = P210ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_SSSE3;
+ P210ToARGBRow = P210ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_AVX2)
+#if defined(HAS_P210TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_AVX2;
+ P210ToARGBRow = P210ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- NV12ToARGBRow = NV12ToARGBRow_AVX2;
+ P210ToARGBRow = P210ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_NEON)
+#if defined(HAS_P210TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P210ToARGBRow = P210ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_NEON;
+ P210ToARGBRow = P210ToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_NEON;
+ P210ToARGBRow = P210ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_MSA;
- }
+#if defined(HAS_P210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToARGBRow = P210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToARGBRow = P210ToARGBRow_SME;
}
#endif
-
for (y = 0; y < height; ++y) {
- NV12ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
+ P210ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
src_y += src_stride_y;
if (y & 1) {
@@ -1697,672 +2106,8106 @@ static int NV12ToARGBMatrix(const uint8_t* src_y,
return 0;
}
-// Convert NV21 to ARGB with matrix
-static int NV21ToARGBMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_argb,
- int dst_stride_argb,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+LIBYUV_API
+int P210ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV21ToARGBRow)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV21ToARGBRow_C;
- if (!src_y || !src_vu || !dst_argb || width <= 0 || height == 0) {
+ void (*P210ToARGBRow)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P210ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_NV21TOARGBROW_SSSE3)
+#if defined(HAS_P210TOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- NV21ToARGBRow = NV21ToARGBRow_Any_SSSE3;
+ P210ToARGBRow = P210ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- NV21ToARGBRow = NV21ToARGBRow_SSSE3;
+ P210ToARGBRow = P210ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_NV21TOARGBROW_AVX2)
+#if defined(HAS_P210TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- NV21ToARGBRow = NV21ToARGBRow_Any_AVX2;
+ P210ToARGBRow = P210ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- NV21ToARGBRow = NV21ToARGBRow_AVX2;
+ P210ToARGBRow = P210ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_NV21TOARGBROW_NEON)
+#if defined(HAS_P210TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P210ToARGBRow = P210ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- NV21ToARGBRow = NV21ToARGBRow_Any_NEON;
+ P210ToARGBRow = P210ToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- NV21ToARGBRow = NV21ToARGBRow_NEON;
+ P210ToARGBRow = P210ToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_NV21TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- NV21ToARGBRow = NV21ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- NV21ToARGBRow = NV21ToARGBRow_MSA;
- }
+#if defined(HAS_P210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToARGBRow = P210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToARGBRow = P210ToARGBRow_SME;
}
#endif
-
for (y = 0; y < height; ++y) {
- NV21ToARGBRow(src_y, src_vu, dst_argb, yuvconstants, width);
+ P210ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
src_y += src_stride_y;
- if (y & 1) {
- src_vu += src_stride_vu;
- }
+ src_uv += src_stride_uv;
}
return 0;
}
-// Convert NV12 to ARGB.
-LIBYUV_API
-int NV12ToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- return NV12ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_argb,
- dst_stride_argb, &kYuvI601Constants, width, height);
-}
-
-// Convert NV21 to ARGB.
-LIBYUV_API
-int NV21ToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- return NV21ToARGBMatrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_argb,
- dst_stride_argb, &kYuvI601Constants, width, height);
-}
-
-// Convert NV12 to ABGR.
-// To output ABGR instead of ARGB swap the UV and use a mirrored yuv matrix.
-// To swap the UV use NV12 instead of NV21.LIBYUV_API
-LIBYUV_API
-int NV12ToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height) {
- return NV21ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_abgr,
- dst_stride_abgr, &kYvuI601Constants, width, height);
-}
-
-// Convert NV21 to ABGR.
LIBYUV_API
-int NV21ToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height) {
- return NV12ToARGBMatrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_abgr,
- dst_stride_abgr, &kYvuI601Constants, width, height);
-}
-
-// TODO(fbarchard): Consider SSSE3 2 step conversion.
-// Convert NV12 to RGB24 with matrix
-static int NV12ToRGB24Matrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+int P410ToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV12ToRGB24Row)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV12ToRGB24Row_C;
- if (!src_y || !src_uv || !dst_rgb24 || width <= 0 || height == 0) {
+ void (*P410ToARGBRow)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_rgb24 = dst_rgb24 + (height - 1) * dst_stride_rgb24;
- dst_stride_rgb24 = -dst_stride_rgb24;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_NV12TORGB24ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- NV12ToRGB24Row = NV12ToRGB24Row_Any_NEON;
+#if defined(HAS_P410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P410ToARGBRow = P410ToARGBRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- NV12ToRGB24Row = NV12ToRGB24Row_NEON;
+ P410ToARGBRow = P410ToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_NV12TORGB24ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- NV12ToRGB24Row = NV12ToRGB24Row_Any_SSSE3;
+#if defined(HAS_P410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P410ToARGBRow = P410ToARGBRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- NV12ToRGB24Row = NV12ToRGB24Row_SSSE3;
+ P410ToARGBRow = P410ToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_NV12TORGB24ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- NV12ToRGB24Row = NV12ToRGB24Row_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- NV12ToRGB24Row = NV12ToRGB24Row_AVX2;
+#if defined(HAS_P410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToARGBRow = P410ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToARGBRow = P410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_NEON;
}
}
#endif
-
+#if defined(HAS_P410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToARGBRow = P410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToARGBRow = P410ToARGBRow_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
- NV12ToRGB24Row(src_y, src_uv, dst_rgb24, yuvconstants, width);
- dst_rgb24 += dst_stride_rgb24;
+ P410ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
src_y += src_stride_y;
- if (y & 1) {
- src_uv += src_stride_uv;
- }
+ src_uv += src_stride_uv;
}
return 0;
}
-// Convert NV21 to RGB24 with matrix
-static int NV21ToRGB24Matrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+LIBYUV_API
+int P010ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV21ToRGB24Row)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV21ToRGB24Row_C;
- if (!src_y || !src_vu || !dst_rgb24 || width <= 0 || height == 0) {
+ void (*P210ToAR30Row)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P210ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_rgb24 = dst_rgb24 + (height - 1) * dst_stride_rgb24;
- dst_stride_rgb24 = -dst_stride_rgb24;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_NV21TORGB24ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- NV21ToRGB24Row = NV21ToRGB24Row_Any_NEON;
+#if defined(HAS_P210TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P210ToAR30Row = P210ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- NV21ToRGB24Row = NV21ToRGB24Row_NEON;
+ P210ToAR30Row = P210ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_NV21TORGB24ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- NV21ToRGB24Row = NV21ToRGB24Row_Any_SSSE3;
+#if defined(HAS_P210TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P210ToAR30Row = P210ToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- NV21ToRGB24Row = NV21ToRGB24Row_SSSE3;
+ P210ToAR30Row = P210ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_NV21TORGB24ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- NV21ToRGB24Row = NV21ToRGB24Row_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- NV21ToRGB24Row = NV21ToRGB24Row_AVX2;
+#if defined(HAS_P210TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P210ToAR30Row = P210ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToAR30Row = P210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToAR30Row = P210ToAR30Row_NEON;
}
}
#endif
-
+#if defined(HAS_P210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToAR30Row = P210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToAR30Row = P210ToAR30Row_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
- NV21ToRGB24Row(src_y, src_vu, dst_rgb24, yuvconstants, width);
- dst_rgb24 += dst_stride_rgb24;
+ P210ToAR30Row(src_y, src_uv, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
src_y += src_stride_y;
if (y & 1) {
- src_vu += src_stride_vu;
+ src_uv += src_stride_uv;
}
}
return 0;
}
-// Convert NV12 to RGB24.
-LIBYUV_API
-int NV12ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height) {
- return NV12ToRGB24Matrix(src_y, src_stride_y, src_uv, src_stride_uv,
- dst_rgb24, dst_stride_rgb24, &kYuvI601Constants,
- width, height);
-}
-
-// Convert NV21 to RGB24.
-LIBYUV_API
-int NV21ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height) {
- return NV21ToRGB24Matrix(src_y, src_stride_y, src_vu, src_stride_vu,
- dst_rgb24, dst_stride_rgb24, &kYuvI601Constants,
- width, height);
-}
-
-// Convert NV12 to RAW.
-LIBYUV_API
-int NV12ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height) {
- return NV21ToRGB24Matrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_raw,
- dst_stride_raw, &kYvuI601Constants, width, height);
-}
-
-// Convert NV21 to RAW.
LIBYUV_API
-int NV21ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height) {
- return NV12ToRGB24Matrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_raw,
- dst_stride_raw, &kYvuI601Constants, width, height);
-}
-
-// Convert NV21 to YUV24
-int NV21ToYUV24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_yuv24,
- int dst_stride_yuv24,
- int width,
- int height) {
+int P210ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV21ToYUV24Row)(const uint8_t* src_y, const uint8_t* src_vu,
- uint8_t* dst_yuv24, int width) = NV21ToYUV24Row_C;
- if (!src_y || !src_vu || !dst_yuv24 || width <= 0 || height == 0) {
+ void (*P210ToAR30Row)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P210ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_yuv24 = dst_yuv24 + (height - 1) * dst_stride_yuv24;
- dst_stride_yuv24 = -dst_stride_yuv24;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_NV21TOYUV24ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- NV21ToYUV24Row = NV21ToYUV24Row_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- NV21ToYUV24Row = NV21ToYUV24Row_NEON;
+#if defined(HAS_P210TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P210ToAR30Row = P210ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToAR30Row = P210ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_NV21TOYUV24ROW_AVX2)
+#if defined(HAS_P210TOAR30ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- NV21ToYUV24Row = NV21ToYUV24Row_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- NV21ToYUV24Row = NV21ToYUV24Row_AVX2;
+ P210ToAR30Row = P210ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ P210ToAR30Row = P210ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P210ToAR30Row = P210ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToAR30Row = P210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToAR30Row = P210ToAR30Row_NEON;
}
}
+#endif
+#if defined(HAS_P210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToAR30Row = P210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToAR30Row = P210ToAR30Row_SME;
+ }
#endif
for (y = 0; y < height; ++y) {
- NV21ToYUV24Row(src_y, src_vu, dst_yuv24, width);
- dst_yuv24 += dst_stride_yuv24;
+ P210ToAR30Row(src_y, src_uv, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
src_y += src_stride_y;
- if (y & 1) {
- src_vu += src_stride_vu;
- }
+ src_uv += src_stride_uv;
}
return 0;
}
-// Convert M420 to ARGB.
LIBYUV_API
-int M420ToARGB(const uint8_t* src_m420,
- int src_stride_m420,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int P410ToAR30Matrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
- void (*NV12ToARGBRow)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV12ToARGBRow_C;
- if (!src_m420 || !dst_argb || width <= 0 || height == 0) {
+ void (*P410ToAR30Row)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToAR30Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
}
-#if defined(HAS_NV12TOARGBROW_SSSE3)
+#if defined(HAS_P410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3;
+ P410ToAR30Row = P410ToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_SSSE3;
+ P410ToAR30Row = P410ToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_AVX2)
+#if defined(HAS_P410TOAR30ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_AVX2;
+ P410ToAR30Row = P410ToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- NV12ToARGBRow = NV12ToARGBRow_AVX2;
+ P410ToAR30Row = P410ToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_NEON)
+#if defined(HAS_P410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToAR30Row = P410ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_NEON;
+ P410ToAR30Row = P410ToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_NEON;
+ P410ToAR30Row = P410ToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_NV12TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- NV12ToARGBRow = NV12ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- NV12ToARGBRow = NV12ToARGBRow_MSA;
- }
+#if defined(HAS_P410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToAR30Row = P410ToAR30Row_SVE2;
}
#endif
-
- for (y = 0; y < height - 1; y += 2) {
- NV12ToARGBRow(src_m420, src_m420 + src_stride_m420 * 2, dst_argb,
- &kYuvI601Constants, width);
- NV12ToARGBRow(src_m420 + src_stride_m420, src_m420 + src_stride_m420 * 2,
- dst_argb + dst_stride_argb, &kYuvI601Constants, width);
- dst_argb += dst_stride_argb * 2;
- src_m420 += src_stride_m420 * 3;
+#if defined(HAS_P410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToAR30Row = P410ToAR30Row_SME;
}
- if (height & 1) {
- NV12ToARGBRow(src_m420, src_m420 + src_stride_m420 * 2, dst_argb,
- &kYuvI601Constants, width);
+#endif
+ for (y = 0; y < height; ++y) {
+ P410ToAR30Row(src_y, src_uv, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
}
return 0;
}
-// Convert YUY2 to ARGB.
+// Convert I420 with Alpha to preattenuated ARGB with matrix.
LIBYUV_API
-int YUY2ToARGB(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int I420AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
int y;
- void (*YUY2ToARGBRow)(const uint8_t* src_yuy2, uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants, int width) =
- YUY2ToARGBRow_C;
- if (!src_yuy2 || !dst_argb || width <= 0 || height == 0) {
+ void (*I422AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I422AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2;
- src_stride_yuy2 = -src_stride_yuy2;
- }
- // Coalesce rows.
- if (src_stride_yuy2 == width * 2 && dst_stride_argb == width * 4) {
- width *= height;
- height = 1;
- src_stride_yuy2 = dst_stride_argb = 0;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
-#if defined(HAS_YUY2TOARGBROW_SSSE3)
+#if defined(HAS_I422ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- YUY2ToARGBRow = YUY2ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToARGBRow = YUY2ToARGBRow_SSSE3;
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SSSE3;
}
}
#endif
-#if defined(HAS_YUY2TOARGBROW_AVX2)
+#if defined(HAS_I422ALPHATOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- YUY2ToARGBRow = YUY2ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- YUY2ToARGBRow = YUY2ToARGBRow_AVX2;
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_AVX2;
}
}
#endif
-#if defined(HAS_YUY2TOARGBROW_NEON)
+#if defined(HAS_I422ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON;
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- YUY2ToARGBRow = YUY2ToARGBRow_NEON;
+ I422AlphaToARGBRow = I422AlphaToARGBRow_NEON;
}
}
#endif
-#if defined(HAS_YUY2TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- YUY2ToARGBRow = YUY2ToARGBRow_Any_MSA;
+#if defined(HAS_I422ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- YUY2ToARGBRow = YUY2ToARGBRow_MSA;
+ I422AlphaToARGBRow = I422AlphaToARGBRow_LSX;
}
}
#endif
- for (y = 0; y < height; ++y) {
- YUY2ToARGBRow(src_yuy2, dst_argb, &kYuvI601Constants, width);
- src_yuy2 += src_stride_yuy2;
- dst_argb += dst_stride_argb;
+#if defined(HAS_I422ALPHATOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_LASX;
+ }
}
- return 0;
-}
-
-// Convert UYVY to ARGB.
-LIBYUV_API
-int UYVYToARGB(const uint8_t* src_uyvy,
- int src_stride_uyvy,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- int y;
- void (*UYVYToARGBRow)(const uint8_t* src_uyvy, uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants, int width) =
- UYVYToARGBRow_C;
- if (!src_uyvy || !dst_argb || width <= 0 || height == 0) {
- return -1;
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I422 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I422AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I422AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I422AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy;
- src_stride_uyvy = -src_stride_uyvy;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
- if (src_stride_uyvy == width * 2 && dst_stride_argb == width * 4) {
+ if (src_stride_y == width && src_stride_u * 2 == width &&
+ src_stride_v * 2 == width && src_stride_a == width &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_uyvy = dst_stride_argb = 0;
+ src_stride_y = src_stride_u = src_stride_v = src_stride_a =
+ dst_stride_argb = 0;
+ }
+#if defined(HAS_I422ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I444 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I444AlphaToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I444AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I444AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
+ src_stride_a == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = src_stride_u = src_stride_v = src_stride_a =
+ dst_stride_argb = 0;
+ }
+#if defined(HAS_I444ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I444AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I420 with Alpha to ARGB.
+LIBYUV_API
+int I420AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate) {
+ return I420AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, src_a, src_stride_a, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, attenuate);
+}
+
+// Convert I420 with Alpha to ABGR.
+LIBYUV_API
+int I420AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate) {
+ return I420AlphaToARGBMatrix(
+ src_y, src_stride_y, src_v, src_stride_v, // Swap U and V
+ src_u, src_stride_u, src_a, src_stride_a, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height, attenuate);
+}
+
+// Convert I422 with Alpha to ARGB.
+LIBYUV_API
+int I422AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate) {
+ return I422AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, src_a, src_stride_a, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, attenuate);
+}
+
+// Convert I422 with Alpha to ABGR.
+LIBYUV_API
+int I422AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate) {
+ return I422AlphaToARGBMatrix(
+ src_y, src_stride_y, src_v, src_stride_v, // Swap U and V
+ src_u, src_stride_u, src_a, src_stride_a, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height, attenuate);
+}
+
+// Convert I444 with Alpha to ARGB.
+LIBYUV_API
+int I444AlphaToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int attenuate) {
+ return I444AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, src_a, src_stride_a, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, attenuate);
+}
+
+// Convert I444 with Alpha to ABGR.
+LIBYUV_API
+int I444AlphaToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height,
+ int attenuate) {
+ return I444AlphaToARGBMatrix(
+ src_y, src_stride_y, src_v, src_stride_v, // Swap U and V
+ src_u, src_stride_u, src_a, src_stride_a, dst_abgr, dst_stride_abgr,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height, attenuate);
+}
+
+// Convert I010 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I010AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I210AlphaToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I210AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I210ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I210 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I210AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I210AlphaToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I210AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I210ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I410 with Alpha to preattenuated ARGB with matrix.
+LIBYUV_API
+int I410AlphaToARGBMatrix(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I410AlphaToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I410AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I410AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I400 to ARGB with matrix.
+LIBYUV_API
+int I400ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I400ToARGBRow)(const uint8_t* y_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I400ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !dst_argb || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = dst_stride_argb = 0;
+ }
+#if defined(HAS_I400TOARGBROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ I400ToARGBRow = I400ToARGBRow_Any_SSE2;
+ if (IS_ALIGNED(width, 8)) {
+ I400ToARGBRow = I400ToARGBRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I400ToARGBRow = I400ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I400ToARGBRow = I400ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I400ToARGBRow = I400ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I400ToARGBRow = I400ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I400ToARGBRow = I400ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I400ToARGBRow = I400ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I400ToARGBRow = I400ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I400ToARGBRow = I400ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I400ToARGBRow = I400ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I400ToARGBRow = I400ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I400ToARGBRow = I400ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I400ToARGBRow = I400ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I400ToARGBRow = I400ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I400ToARGBRow(src_y, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ }
+ return 0;
+}
+
+// Convert I400 to ARGB.
+LIBYUV_API
+int I400ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return I400ToARGBMatrix(src_y, src_stride_y, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert J400 to ARGB.
+LIBYUV_API
+int J400ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*J400ToARGBRow)(const uint8_t* src_y, uint8_t* dst_argb, int width) =
+ J400ToARGBRow_C;
+ if (!src_y || !dst_argb || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = dst_stride_argb = 0;
+ }
+
+#if defined(HAS_J400TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ J400ToARGBRow = J400ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ J400ToARGBRow = J400ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ J400ToARGBRow = J400ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ J400ToARGBRow = J400ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ J400ToARGBRow = J400ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ J400ToARGBRow = J400ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ J400ToARGBRow = J400ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ J400ToARGBRow = J400ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ J400ToARGBRow = J400ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ J400ToARGBRow = J400ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ J400ToARGBRow = J400ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ J400ToARGBRow = J400ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_J400TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ J400ToARGBRow = J400ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ J400ToARGBRow(src_y, dst_argb, width);
+ src_y += src_stride_y;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+#if !defined(__riscv) && !defined(__hexagon__)
+// Shuffle table for converting BGRA to ARGB.
+static const uvec8 kShuffleMaskBGRAToARGB = {
+ 3u, 2u, 1u, 0u, 7u, 6u, 5u, 4u, 11u, 10u, 9u, 8u, 15u, 14u, 13u, 12u};
+
+// Shuffle table for converting ABGR to ARGB.
+static const uvec8 kShuffleMaskABGRToARGB = {
+ 2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
+
+// Shuffle table for converting RGBA to ARGB.
+static const uvec8 kShuffleMaskRGBAToARGB = {
+ 1u, 2u, 3u, 0u, 5u, 6u, 7u, 4u, 9u, 10u, 11u, 8u, 13u, 14u, 15u, 12u};
+
+// Shuffle table for converting AR64 to AB64.
+static const uvec8 kShuffleMaskAR64ToAB64 = {
+ 4u, 5u, 2u, 3u, 0u, 1u, 6u, 7u, 12u, 13u, 10u, 11u, 8u, 9u, 14u, 15u};
+
+// Convert BGRA to ARGB.
+LIBYUV_API
+int BGRAToARGB(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return ARGBShuffle(src_bgra, src_stride_bgra, dst_argb, dst_stride_argb,
+ (const uint8_t*)&kShuffleMaskBGRAToARGB, width, height);
+}
+
+// Convert ARGB to BGRA (same as BGRAToARGB).
+LIBYUV_API
+int ARGBToBGRA(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height) {
+ return ARGBShuffle(src_argb, src_stride_argb, dst_bgra, dst_stride_bgra,
+ (const uint8_t*)&kShuffleMaskBGRAToARGB, width, height);
+}
+
+// Convert ABGR to ARGB.
+LIBYUV_API
+int ABGRToARGB(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return ARGBShuffle(src_abgr, src_stride_abgr, dst_argb, dst_stride_argb,
+ (const uint8_t*)&kShuffleMaskABGRToARGB, width, height);
+}
+
+// Convert ARGB to ABGR to (same as ABGRToARGB).
+LIBYUV_API
+int ARGBToABGR(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return ARGBShuffle(src_argb, src_stride_argb, dst_abgr, dst_stride_abgr,
+ (const uint8_t*)&kShuffleMaskABGRToARGB, width, height);
+}
+
+// Convert RGBA to ARGB.
+LIBYUV_API
+int RGBAToARGB(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return ARGBShuffle(src_rgba, src_stride_rgba, dst_argb, dst_stride_argb,
+ (const uint8_t*)&kShuffleMaskRGBAToARGB, width, height);
+}
+
+// Convert AR64 To AB64.
+LIBYUV_API
+int AR64ToAB64(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint16_t* dst_ab64,
+ int dst_stride_ab64,
+ int width,
+ int height) {
+ return AR64Shuffle(src_ar64, src_stride_ar64, dst_ab64, dst_stride_ab64,
+ (const uint8_t*)&kShuffleMaskAR64ToAB64, width, height);
+}
+#else
+// Convert BGRA to ARGB (same as ARGBToBGRA).
+LIBYUV_API
+int BGRAToARGB(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return ARGBToBGRA(src_bgra, src_stride_bgra, dst_argb, dst_stride_argb, width,
+ height);
+}
+
+// Convert ARGB to BGRA.
+LIBYUV_API
+int ARGBToBGRA(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToBGRARow)(const uint8_t* src_argb, uint8_t* dst_bgra, int width) =
+ ARGBToBGRARow_C;
+ if (!src_argb || !dst_bgra || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_bgra == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_bgra = 0;
+ }
+
+#if defined(HAS_ARGBTOBGRAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToBGRARow = ARGBToBGRARow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOBGRAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToBGRARow = ARGBToBGRARow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToBGRARow(src_argb, dst_bgra, width);
+ src_argb += src_stride_argb;
+ dst_bgra += dst_stride_bgra;
+ }
+ return 0;
+}
+
+// Convert ARGB to ABGR.
+LIBYUV_API
+int ARGBToABGR(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToABGRRow)(const uint8_t* src_argb, uint8_t* dst_abgr, int width) =
+ ARGBToABGRRow_C;
+ if (!src_argb || !dst_abgr || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_abgr == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_abgr = 0;
+ }
+
+#if defined(HAS_ARGBTOABGRROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToABGRRow = ARGBToABGRRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOABGRROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToABGRRow = ARGBToABGRRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToABGRRow(src_argb, dst_abgr, width);
+ src_argb += src_stride_argb;
+ dst_abgr += dst_stride_abgr;
+ }
+ return 0;
+}
+
+// Convert ABGR to ARGB (same as ARGBToABGR).
+LIBYUV_API
+int ABGRToARGB(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return ARGBToABGR(src_abgr, src_stride_abgr, dst_argb, dst_stride_argb, width,
+ height);
+}
+
+// Convert RGBA to ARGB.
+LIBYUV_API
+int RGBAToARGB(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*RGBAToARGBRow)(const uint8_t* src_rgba, uint8_t* dst_argb, int width) =
+ RGBAToARGBRow_C;
+ if (!src_rgba || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgba = src_rgba + (ptrdiff_t)(height - 1) * src_stride_rgba;
+ src_stride_rgba = -src_stride_rgba;
+ }
+ // Coalesce rows.
+ if (src_stride_rgba == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_rgba = dst_stride_argb = 0;
+ }
+
+#if defined(HAS_RGBATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBAToARGBRow = RGBAToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ RGBAToARGBRow = RGBAToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ RGBAToARGBRow(src_rgba, dst_argb, width);
+ src_rgba += src_stride_rgba;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert AR64 To AB64.
+LIBYUV_API
+int AR64ToAB64(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint16_t* dst_ab64,
+ int dst_stride_ab64,
+ int width,
+ int height) {
+ int y;
+ void (*AR64ToAB64Row)(const uint16_t* src_ar64, uint16_t* dst_ab64,
+ int width) = AR64ToAB64Row_C;
+ if (!src_ar64 || !dst_ab64 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar64 = src_ar64 + (ptrdiff_t)(height - 1) * src_stride_ar64;
+ src_stride_ar64 = -src_stride_ar64;
+ }
+ // Coalesce rows.
+ if (src_stride_ar64 == width * 4 && dst_stride_ab64 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar64 = dst_stride_ab64 = 0;
+ }
+
+#if defined(HAS_AR64TOAB64ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ AR64ToAB64Row = AR64ToAB64Row_RVV;
+ }
+#endif
+#if defined(HAS_AR64TOAB64ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ AR64ToAB64Row = AR64ToAB64Row_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ AR64ToAB64Row(src_ar64, dst_ab64, width);
+ src_ar64 += src_stride_ar64;
+ dst_ab64 += dst_stride_ab64;
+ }
+ return 0;
+}
+#endif
+
+// Convert RGB24 to ARGB.
+LIBYUV_API
+int RGB24ToARGB(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
+ RGB24ToARGBRow_C;
+ if (!src_rgb24 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb24 = src_rgb24 + (ptrdiff_t)(height - 1) * src_stride_rgb24;
+ src_stride_rgb24 = -src_stride_rgb24;
+ }
+ // Coalesce rows.
+ if (src_stride_rgb24 == width * 3 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_rgb24 = dst_stride_argb = 0;
+ }
+#if defined(HAS_RGB24TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_RGB24TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ RGB24ToARGBRow(src_rgb24, dst_argb, width);
+ src_rgb24 += src_stride_rgb24;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert RAW to ARGB.
+LIBYUV_API
+int RAWToARGB(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
+ RAWToARGBRow_C;
+ if (!src_raw || !dst_argb || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+ // Coalesce rows.
+ if (src_stride_raw == width * 3 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_raw = dst_stride_argb = 0;
+ }
+#if defined(HAS_RAWTOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RAWToARGBRow = RAWToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RAWToARGBRow = RAWToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RAWToARGBRow = RAWToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToARGBRow = RAWToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RAWToARGBRow = RAWToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RAWToARGBRow = RAWToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ RAWToARGBRow(src_raw, dst_argb, width);
+ src_raw += src_stride_raw;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert RAW to RGBA.
+LIBYUV_API
+int RAWToRGBA(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height) {
+ int y;
+ void (*RAWToRGBARow)(const uint8_t* src_rgb, uint8_t* dst_rgba, int width) =
+ RAWToRGBARow_C;
+ if (!src_raw || !dst_rgba || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+ // Coalesce rows.
+ if (src_stride_raw == width * 3 && dst_stride_rgba == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_raw = dst_stride_rgba = 0;
+ }
+#if defined(HAS_RAWTORGBAROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToRGBARow = RAWToRGBARow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToRGBARow = RAWToRGBARow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RAWTORGBAROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ RAWToRGBARow = RAWToRGBARow_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_RAWTORGBAROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RAWToRGBARow = RAWToRGBARow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RAWToRGBARow = RAWToRGBARow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RAWTORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToRGBARow = RAWToRGBARow_SVE2;
+ }
+#endif
+#if defined(HAS_RAWTORGBAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RAWToRGBARow = RAWToRGBARow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ RAWToRGBARow(src_raw, dst_rgba, width);
+ src_raw += src_stride_raw;
+ dst_rgba += dst_stride_rgba;
+ }
+ return 0;
+}
+
+// Convert RGB565 to ARGB.
+LIBYUV_API
+int RGB565ToARGB(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*RGB565ToARGBRow)(const uint8_t* src_rgb565, uint8_t* dst_argb,
+ int width) = RGB565ToARGBRow_C;
+ if (!src_rgb565 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb565 = src_rgb565 + (ptrdiff_t)(height - 1) * src_stride_rgb565;
+ src_stride_rgb565 = -src_stride_rgb565;
+ }
+ // Coalesce rows.
+ if (src_stride_rgb565 == width * 2 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_rgb565 = dst_stride_argb = 0;
+ }
+
+#if defined(HAS_RGB565TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RGB565TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RGB565TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_RGB565TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RGB565TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ RGB565ToARGBRow = RGB565ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ RGB565ToARGBRow(src_rgb565, dst_argb, width);
+ src_rgb565 += src_stride_rgb565;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert ARGB1555 to ARGB.
+LIBYUV_API
+int ARGB1555ToARGB(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*ARGB1555ToARGBRow)(const uint8_t* src_argb1555, uint8_t* dst_argb,
+ int width) = ARGB1555ToARGBRow_C;
+ if (!src_argb1555 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb1555 = src_argb1555 + (ptrdiff_t)(height - 1) * src_stride_argb1555;
+ src_stride_argb1555 = -src_stride_argb1555;
+ }
+ // Coalesce rows.
+ if (src_stride_argb1555 == width * 2 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb1555 = dst_stride_argb = 0;
+ }
+
+#if defined(HAS_ARGB1555TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGB1555TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGB1555ToARGBRow(src_argb1555, dst_argb, width);
+ src_argb1555 += src_stride_argb1555;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert ARGB4444 to ARGB.
+LIBYUV_API
+int ARGB4444ToARGB(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*ARGB4444ToARGBRow)(const uint8_t* src_argb4444, uint8_t* dst_argb,
+ int width) = ARGB4444ToARGBRow_C;
+ if (!src_argb4444 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb4444 = src_argb4444 + (ptrdiff_t)(height - 1) * src_stride_argb4444;
+ src_stride_argb4444 = -src_stride_argb4444;
+ }
+ // Coalesce rows.
+ if (src_stride_argb4444 == width * 2 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb4444 = dst_stride_argb = 0;
+ }
+
+#if defined(HAS_ARGB4444TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGB4444TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGB4444TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGB4444TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGB4444TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGB4444ToARGBRow = ARGB4444ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGB4444ToARGBRow(src_argb4444, dst_argb, width);
+ src_argb4444 += src_stride_argb4444;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert AR30 to ARGB.
+LIBYUV_API
+int AR30ToARGB(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ if (!src_ar30 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
+ src_stride_ar30 = -src_stride_ar30;
+ }
+ // Coalesce rows.
+ if (src_stride_ar30 == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar30 = dst_stride_argb = 0;
+ }
+ void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) =
+ AR30ToARGBRow_C;
+#if defined(HAS_AR30TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToARGBRow = AR30ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AR30ToARGBRow = AR30ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToARGBRow = AR30ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ AR30ToARGBRow(src_ar30, dst_argb, width);
+ src_ar30 += src_stride_ar30;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert AR30 to ABGR.
+LIBYUV_API
+int AR30ToABGR(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ int y;
+ void (*AR30ToABGRRow)(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) =
+ AR30ToABGRRow_C;
+ if (!src_ar30 || !dst_abgr || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
+ src_stride_ar30 = -src_stride_ar30;
+ }
+ // Coalesce rows.
+ if (src_stride_ar30 == width * 4 && dst_stride_abgr == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar30 = dst_stride_abgr = 0;
+ }
+#if defined(HAS_AR30TOABGRROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR30ToABGRRow = AR30ToABGRRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToABGRRow = AR30ToABGRRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOABGRROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AR30ToABGRRow = AR30ToABGRRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToABGRRow = AR30ToABGRRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOABGRROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ AR30ToABGRRow = AR30ToABGRRow_AVX512BW;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ AR30ToABGRRow(src_ar30, dst_abgr, width);
+ src_ar30 += src_stride_ar30;
+ dst_abgr += dst_stride_abgr;
+ }
+ return 0;
+}
+
+// Convert AR30 to AB30.
+LIBYUV_API
+int AR30ToAB30(const uint8_t* src_ar30,
+ int src_stride_ar30,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height) {
+ int y;
+ void (*AR30ToAB30Row)(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) =
+ AR30ToAB30Row_C;
+ if (!src_ar30 || !dst_ab30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
+ src_stride_ar30 = -src_stride_ar30;
+ }
+ // Coalesce rows.
+ if (src_stride_ar30 == width * 4 && dst_stride_ab30 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar30 = dst_stride_ab30 = 0;
+ }
+#if defined(HAS_AR30TOAB30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR30ToAB30Row = AR30ToAB30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToAB30Row = AR30ToAB30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOAB30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AR30ToAB30Row = AR30ToAB30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AR30ToAB30Row = AR30ToAB30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AR30TOAB30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ AR30ToAB30Row = AR30ToAB30Row_AVX512BW;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ AR30ToAB30Row(src_ar30, dst_ab30, width);
+ src_ar30 += src_stride_ar30;
+ dst_ab30 += dst_stride_ab30;
+ }
+ return 0;
+}
+
+// Convert AR64 to ARGB.
+LIBYUV_API
+int AR64ToARGB(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*AR64ToARGBRow)(const uint16_t* src_ar64, uint8_t* dst_argb,
+ int width) = AR64ToARGBRow_C;
+ if (!src_ar64 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar64 = src_ar64 + (ptrdiff_t)(height - 1) * src_stride_ar64;
+ src_stride_ar64 = -src_stride_ar64;
+ }
+ // Coalesce rows.
+ if (src_stride_ar64 == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar64 = dst_stride_argb = 0;
+ }
+#if defined(HAS_AR64TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ AR64ToARGBRow = AR64ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ AR64ToARGBRow = AR64ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_AR64TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AR64ToARGBRow = AR64ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AR64ToARGBRow = AR64ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AR64TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR64ToARGBRow = AR64ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AR64ToARGBRow = AR64ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AR64TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ AR64ToARGBRow = AR64ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_AR64TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ AR64ToARGBRow = AR64ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ AR64ToARGBRow(src_ar64, dst_argb, width);
+ src_ar64 += src_stride_ar64;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert AB64 to ARGB.
+LIBYUV_API
+int AB64ToARGB(const uint16_t* src_ab64,
+ int src_stride_ab64,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*AB64ToARGBRow)(const uint16_t* src_ar64, uint8_t* dst_argb,
+ int width) = AB64ToARGBRow_C;
+ if (!src_ab64 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ab64 = src_ab64 + (ptrdiff_t)(height - 1) * src_stride_ab64;
+ src_stride_ab64 = -src_stride_ab64;
+ }
+ // Coalesce rows.
+ if (src_stride_ab64 == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ab64 = dst_stride_argb = 0;
+ }
+#if defined(HAS_AB64TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ AB64ToARGBRow = AB64ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ AB64ToARGBRow = AB64ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_AB64TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AB64ToARGBRow = AB64ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ AB64ToARGBRow = AB64ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_AB64TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AB64ToARGBRow = AB64ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ AB64ToARGBRow = AB64ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_AB64TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ AB64ToARGBRow = AB64ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_AB64TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ AB64ToARGBRow = AB64ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ AB64ToARGBRow(src_ab64, dst_argb, width);
+ src_ab64 += src_stride_ab64;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert NV12 to ARGB with matrix.
+LIBYUV_API
+int NV12ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*NV12ToARGBRow)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV12ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_NV12TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToARGBRow = NV12ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ NV12ToARGBRow = NV12ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV12ToARGBRow = NV12ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV12ToARGBRow = NV12ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToARGBRow = NV12ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ NV12ToARGBRow = NV12ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ NV12ToARGBRow = NV12ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV12ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_uv += src_stride_uv;
+ }
+ }
+ return 0;
+}
+
+// Convert NV21 to ARGB with matrix.
+LIBYUV_API
+int NV21ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*NV21ToARGBRow)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV21ToARGBRow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_vu || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_NV21TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ NV21ToARGBRow = NV21ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ NV21ToARGBRow = NV21ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ NV21ToARGBRow = NV21ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV21ToARGBRow = NV21ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV21ToARGBRow = NV21ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV21ToARGBRow = NV21ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ NV21ToARGBRow = NV21ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ NV21ToARGBRow = NV21ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ NV21ToARGBRow = NV21ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ NV21ToARGBRow = NV21ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ NV21ToARGBRow = NV21ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ NV21ToARGBRow = NV21ToARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV21ToARGBRow(src_y, src_vu, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_vu += src_stride_vu;
+ }
+ }
+ return 0;
+}
+
+// Convert NV12 to ARGB.
+LIBYUV_API
+int NV12ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return NV12ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width, height);
+}
+
+// Convert NV21 to ARGB.
+LIBYUV_API
+int NV21ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return NV21ToARGBMatrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width, height);
+}
+
+// Convert NV12 to ABGR.
+// To output ABGR instead of ARGB swap the UV and use a mirrored yuv matrix.
+// To swap the UV use NV12 instead of NV21.LIBYUV_API
+LIBYUV_API
+int NV12ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return NV21ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_abgr,
+ dst_stride_abgr, &kYvuI601Constants, width, height);
+}
+
+// Convert NV21 to ABGR.
+LIBYUV_API
+int NV21ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return NV12ToARGBMatrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_abgr,
+ dst_stride_abgr, &kYvuI601Constants, width, height);
+}
+
+// TODO(fbarchard): Consider SSSE3 2 step conversion.
+// Convert NV12 to RGB24 with matrix.
+LIBYUV_API
+int NV12ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*NV12ToRGB24Row)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV12ToRGB24Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+#if defined(HAS_NV12TORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV12ToRGB24Row(src_y, src_uv, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_uv += src_stride_uv;
+ }
+ }
+ return 0;
+}
+
+// Convert NV21 to RGB24 with matrix.
+LIBYUV_API
+int NV21ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*NV21ToRGB24Row)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV21ToRGB24Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_vu || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+#if defined(HAS_NV21TORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV21ToRGB24Row(src_y, src_vu, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_vu += src_stride_vu;
+ }
+ }
+ return 0;
+}
+
+// Convert NV12 to RGB24.
+LIBYUV_API
+int NV12ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return NV12ToRGB24Matrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_rgb24, dst_stride_rgb24, &kYuvI601Constants,
+ width, height);
+}
+
+// Convert NV21 to RGB24.
+LIBYUV_API
+int NV21ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return NV21ToRGB24Matrix(src_y, src_stride_y, src_vu, src_stride_vu,
+ dst_rgb24, dst_stride_rgb24, &kYuvI601Constants,
+ width, height);
+}
+
+// Convert NV12 to RAW.
+LIBYUV_API
+int NV12ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return NV21ToRGB24Matrix(src_y, src_stride_y, src_uv, src_stride_uv, dst_raw,
+ dst_stride_raw, &kYvuI601Constants, width, height);
+}
+
+// Convert NV21 to RAW.
+LIBYUV_API
+int NV21ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return NV12ToRGB24Matrix(src_y, src_stride_y, src_vu, src_stride_vu, dst_raw,
+ dst_stride_raw, &kYvuI601Constants, width, height);
+}
+
+// Convert NV21 to YUV24
+int NV21ToYUV24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_yuv24,
+ int dst_stride_yuv24,
+ int width,
+ int height) {
+ int y;
+ void (*NV21ToYUV24Row)(const uint8_t* src_y, const uint8_t* src_vu,
+ uint8_t* dst_yuv24, int width) = NV21ToYUV24Row_C;
+ if (!src_y || !src_vu || !dst_yuv24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_yuv24 = dst_yuv24 + (ptrdiff_t)(height - 1) * dst_stride_yuv24;
+ dst_stride_yuv24 = -dst_stride_yuv24;
+ }
+#if defined(HAS_NV21TOYUV24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOYUV24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOYUV24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV21TOYUV24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ NV21ToYUV24Row = NV21ToYUV24Row_AVX512VBMI;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ NV21ToYUV24Row(src_y, src_vu, dst_yuv24, width);
+ dst_yuv24 += dst_stride_yuv24;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_vu += src_stride_vu;
+ }
+ }
+ return 0;
+}
+
+// Convert YUY2 to ARGB with matrix.
+LIBYUV_API
+int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*YUY2ToARGBRow)(const uint8_t* src_yuy2, uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants, int width) =
+ YUY2ToARGBRow_C;
+ if (!src_yuy2 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_yuy2 = src_yuy2 + (ptrdiff_t)(height - 1) * src_stride_yuy2;
+ src_stride_yuy2 = -src_stride_yuy2;
+ }
+ // Coalesce rows.
+ if (src_stride_yuy2 == width * 2 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_yuy2 = dst_stride_argb = 0;
+ }
+#if defined(HAS_YUY2TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_LASX;
+ }
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ YUY2ToARGBRow(src_yuy2, dst_argb, yuvconstants, width);
+ src_yuy2 += src_stride_yuy2;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert YUY2 to ARGB.
+LIBYUV_API
+int YUY2ToARGB(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return YUY2ToARGBMatrix(src_yuy2, src_stride_yuy2, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert UYVY to ARGB with matrix.
+LIBYUV_API
+int UYVYToARGBMatrix(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*UYVYToARGBRow)(const uint8_t* src_uyvy, uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants, int width) =
+ UYVYToARGBRow_C;
+ if (!src_uyvy || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uyvy = src_uyvy + (ptrdiff_t)(height - 1) * src_stride_uyvy;
+ src_stride_uyvy = -src_stride_uyvy;
+ }
+ // Coalesce rows.
+ if (src_stride_uyvy == width * 2 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_uyvy = dst_stride_argb = 0;
+ }
+#if defined(HAS_UYVYTOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ UYVYToARGBRow = UYVYToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ UYVYToARGBRow = UYVYToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ UYVYToARGBRow = UYVYToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToARGBRow = UYVYToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ UYVYToARGBRow = UYVYToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ UYVYToARGBRow = UYVYToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ UYVYToARGBRow = UYVYToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ UYVYToARGBRow = UYVYToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ UYVYToARGBRow = UYVYToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ UYVYToARGBRow = UYVYToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ UYVYToARGBRow = UYVYToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ UYVYToARGBRow = UYVYToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToARGBRow = UYVYToARGBRow_LASX;
+ }
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ UYVYToARGBRow(src_uyvy, dst_argb, yuvconstants, width);
+ src_uyvy += src_stride_uyvy;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Convert UYVY to ARGB.
+LIBYUV_API
+int UYVYToARGB(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return UYVYToARGBMatrix(src_uyvy, src_stride_uyvy, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
+static void WeavePixels(const uint8_t* src_u,
+ const uint8_t* src_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_uv,
+ int width) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ dst_uv[0] = *src_u;
+ dst_uv[1] = *src_v;
+ dst_uv += 2;
+ src_u += src_pixel_stride_uv;
+ src_v += src_pixel_stride_uv;
+ }
+}
+
+// Convert Android420 to ARGB with matrix.
+LIBYUV_API
+int Android420ToARGBMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ uint8_t* dst_uv;
+ const ptrdiff_t vu_off = src_v - src_u;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+
+ // I420
+ if (src_pixel_stride_uv == 1) {
+ return I420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ // NV21
+ }
+ if (src_pixel_stride_uv == 2 && vu_off == -1 &&
+ src_stride_u == src_stride_v) {
+ return NV21ToARGBMatrix(src_y, src_stride_y, src_v, src_stride_v, dst_argb,
+ dst_stride_argb, yuvconstants, width, height);
+ // NV12
+ }
+ if (src_pixel_stride_uv == 2 && vu_off == 1 && src_stride_u == src_stride_v) {
+ return NV12ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, dst_argb,
+ dst_stride_argb, yuvconstants, width, height);
+ }
+
+ // General case fallback creates NV12
+ const uint64_t uv_size = (uint64_t)halfwidth * 2 * halfheight;
+ if (uv_size > SIZE_MAX)
+ return 1;
+ align_buffer_64(plane_uv, (size_t)uv_size);
+ if (!plane_uv)
+ return 1;
+ dst_uv = plane_uv;
+ for (y = 0; y < halfheight; ++y) {
+ WeavePixels(src_u, src_v, src_pixel_stride_uv, dst_uv, halfwidth);
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ dst_uv += halfwidth * 2;
+ }
+ NV12ToARGBMatrix(src_y, src_stride_y, plane_uv, halfwidth * 2, dst_argb,
+ dst_stride_argb, yuvconstants, width, height);
+ free_aligned_buffer_64(plane_uv);
+ return 0;
+}
+
+// Convert Android420 to ARGB.
+LIBYUV_API
+int Android420ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return Android420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, src_pixel_stride_uv, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height);
+}
+
+// Convert Android420 to ABGR.
+LIBYUV_API
+int Android420ToABGR(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_abgr,
+ int dst_stride_abgr,
+ int width,
+ int height) {
+ return Android420ToARGBMatrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, src_pixel_stride_uv, dst_abgr,
+ dst_stride_abgr, &kYvuI601Constants, width,
+ height);
+}
+
+// Convert I422 to RGBA with matrix.
+LIBYUV_API
+int I422ToRGBAMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGBARow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGBARow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgba || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgba = dst_rgba + (ptrdiff_t)(height - 1) * dst_stride_rgba;
+ dst_stride_rgba = -dst_stride_rgba;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && src_stride_u * 2 == width &&
+ src_stride_v * 2 == width && dst_stride_rgba == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_rgba = 0;
+ }
+#if defined(HAS_I422TORGBAROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGBARow = I422ToRGBARow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGBARow = I422ToRGBARow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422ToRGBARow = I422ToRGBARow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGBARow = I422ToRGBARow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGBARow = I422ToRGBARow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGBARow = I422ToRGBARow_SME;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGBARow = I422ToRGBARow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGBARow = I422ToRGBARow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGBARow = I422ToRGBARow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToRGBARow = I422ToRGBARow_RVV;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422ToRGBARow = I422ToRGBARow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
+ dst_rgba += dst_stride_rgba;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I422 to RGBA.
+LIBYUV_API
+int I422ToRGBA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height) {
+ return I422ToRGBAMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgba, dst_stride_rgba,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert I422 to BGRA.
+LIBYUV_API
+int I422ToBGRA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height) {
+ return I422ToRGBAMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_bgra, dst_stride_bgra,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert NV12 to RGB565 with matrix.
+LIBYUV_API
+int NV12ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*NV12ToRGB565Row)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV12ToRGB565Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565;
+ dst_stride_rgb565 = -dst_stride_rgb565;
+ }
+#if defined(HAS_NV12TORGB565ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB565ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB565ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB565ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB565ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TORGB565ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToRGB565Row = NV12ToRGB565Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV12ToRGB565Row(src_y, src_uv, dst_rgb565, yuvconstants, width);
+ dst_rgb565 += dst_stride_rgb565;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_uv += src_stride_uv;
+ }
+ }
+ return 0;
+}
+
+// Convert NV12 to RGB565.
+LIBYUV_API
+int NV12ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ return NV12ToRGB565Matrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_rgb565, dst_stride_rgb565, &kYuvI601Constants,
+ width, height);
+}
+
+// Convert I422 to RGBA with matrix.
+LIBYUV_API
+int I420ToRGBAMatrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGBARow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGBARow_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgba || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgba = dst_rgba + (ptrdiff_t)(height - 1) * dst_stride_rgba;
+ dst_stride_rgba = -dst_stride_rgba;
+ }
+#if defined(HAS_I422TORGBAROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGBARow = I422ToRGBARow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGBARow = I422ToRGBARow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422ToRGBARow = I422ToRGBARow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGBARow = I422ToRGBARow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGBARow = I422ToRGBARow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGBARow = I422ToRGBARow_SME;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGBARow = I422ToRGBARow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGBARow = I422ToRGBARow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGBARow = I422ToRGBARow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGBARow = I422ToRGBARow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToRGBARow = I422ToRGBARow_RVV;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422ToRGBARow = I422ToRGBARow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
+ dst_rgba += dst_stride_rgba;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to RGBA.
+LIBYUV_API
+int I420ToRGBA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height) {
+ return I420ToRGBAMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgba, dst_stride_rgba,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert I420 to BGRA.
+LIBYUV_API
+int I420ToBGRA(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_bgra,
+ int dst_stride_bgra,
+ int width,
+ int height) {
+ return I420ToRGBAMatrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_bgra, dst_stride_bgra,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert I420 to RGB24 with matrix.
+LIBYUV_API
+int I420ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGB24Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+#if defined(HAS_I422TORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB24Row = I422ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB24Row = I422ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToRGB24Row = I422ToRGB24Row_RVV;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to RGB24.
+LIBYUV_API
+int I420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert I420 to RAW.
+LIBYUV_API
+int I420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_raw, dst_stride_raw,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert J420 to RGB24.
+LIBYUV_API
+int J420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ &kYuvJPEGConstants, width, height);
+}
+
+// Convert J420 to RAW.
+LIBYUV_API
+int J420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_raw, dst_stride_raw,
+ &kYvuJPEGConstants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert H420 to RGB24.
+LIBYUV_API
+int H420ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ &kYuvH709Constants, width, height);
+}
+
+// Convert H420 to RAW.
+LIBYUV_API
+int H420ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_raw, dst_stride_raw,
+ &kYvuH709Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert I422 to RGB24 with matrix.
+LIBYUV_API
+int I422ToRGB24Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGB24Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && src_stride_u * 2 == width &&
+ src_stride_v * 2 == width && dst_stride_rgb24 == width * 3 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb24 = 0;
+ }
+#if defined(HAS_I422TORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB24Row = I422ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB24Row = I422ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB24Row = I422ToRGB24Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToRGB24Row = I422ToRGB24Row_RVV;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB24Row = I422ToRGB24Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I422 to RGB24.
+LIBYUV_API
+int I422ToRGB24(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ return I422ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert I422 to RAW.
+LIBYUV_API
+int I422ToRAW(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ return I422ToRGB24Matrix(src_y, src_stride_y, src_v,
+ src_stride_v, // Swap U and V
+ src_u, src_stride_u, dst_raw, dst_stride_raw,
+ &kYvuI601Constants, // Use Yvu matrix
+ width, height);
+}
+
+// Convert I420 to ARGB1555.
+LIBYUV_API
+int I420ToARGB1555(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb1555,
+ int dst_stride_argb1555,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToARGB1555Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) = I422ToARGB1555Row_C;
+ if (!src_y || !src_u || !src_v || !dst_argb1555 || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb1555 = dst_argb1555 + (ptrdiff_t)(height - 1) * dst_stride_argb1555;
+ dst_stride_argb1555 = -dst_stride_argb1555;
+ }
+#if defined(HAS_I422TOARGB1555ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_SME;
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToARGB1555Row(src_y, src_u, src_v, dst_argb1555, &kYuvI601Constants,
+ width);
+ dst_argb1555 += dst_stride_argb1555;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to ARGB4444.
+LIBYUV_API
+int I420ToARGB4444(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb4444,
+ int dst_stride_argb4444,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToARGB4444Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) = I422ToARGB4444Row_C;
+ if (!src_y || !src_u || !src_v || !dst_argb4444 || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb4444 = dst_argb4444 + (ptrdiff_t)(height - 1) * dst_stride_argb4444;
+ dst_stride_argb4444 = -dst_stride_argb4444;
+ }
+#if defined(HAS_I422TOARGB4444ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_SME;
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToARGB4444Row(src_y, src_u, src_v, dst_argb4444, &kYuvI601Constants,
+ width);
+ dst_argb4444 += dst_stride_argb4444;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to RGB565 with specified color matrix.
+LIBYUV_API
+int I420ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGB565Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGB565Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565;
+ dst_stride_rgb565 = -dst_stride_rgb565;
+ }
+#if defined(HAS_I422TORGB565ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB565Row = I422ToRGB565Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToRGB565Row = I422ToRGB565Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB565Row = I422ToRGB565Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB565Row = I422ToRGB565Row_SME;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB565Row = I422ToRGB565Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+ dst_rgb565 += dst_stride_rgb565;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to RGB565.
+LIBYUV_API
+int I420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb565, dst_stride_rgb565,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert J420 to RGB565.
+LIBYUV_API
+int J420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb565, dst_stride_rgb565,
+ &kYuvJPEGConstants, width, height);
+}
+
+// Convert H420 to RGB565.
+LIBYUV_API
+int H420ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb565, dst_stride_rgb565,
+ &kYuvH709Constants, width, height);
+}
+
+// Convert I422 to RGB565 with specified color matrix.
+LIBYUV_API
+int I422ToRGB565Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToRGB565Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToRGB565Row_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565;
+ dst_stride_rgb565 = -dst_stride_rgb565;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && src_stride_u * 2 == width &&
+ src_stride_v * 2 == width && dst_stride_rgb565 == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb565 = 0;
+ }
+#if defined(HAS_I422TORGB565ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB565Row = I422ToRGB565Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToRGB565Row = I422ToRGB565Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB565Row = I422ToRGB565Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB565Row = I422ToRGB565Row_SME;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToRGB565Row = I422ToRGB565Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToRGB565Row = I422ToRGB565Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToRGB565Row = I422ToRGB565Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+ dst_rgb565 += dst_stride_rgb565;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ return 0;
+}
+
+// Convert I422 to RGB565.
+LIBYUV_API
+int I422ToRGB565(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ return I422ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb565, dst_stride_rgb565,
+ &kYuvI601Constants, width, height);
+}
+
+// Ordered 8x8 dither for 888 to 565. Values from 0 to 7.
+static const uint8_t kDither565_4x4[16] = {
+ 0, 4, 1, 5, 6, 2, 7, 3, 1, 5, 0, 4, 7, 3, 6, 2,
+};
+
+// Convert I420 to RGB565 with dithering.
+LIBYUV_API
+int I420ToRGB565Dither(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ const uint8_t* dither4x4,
+ int width,
+ int height) {
+ int y;
+ uint32_t dither4[4];
+ void (*I422ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToARGBRow_C;
+ void (*ARGBToRGB565DitherRow)(const uint8_t* src_argb, uint8_t* dst_rgb,
+ uint32_t dither4, int width) =
+ ARGBToRGB565DitherRow_C;
+ if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565;
+ dst_stride_rgb565 = -dst_stride_rgb565;
+ }
+ if (!dither4x4) {
+ dither4x4 = kDither565_4x4;
+ }
+ memcpy(dither4, dither4x4, 16);
+#if defined(HAS_I422TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToARGBRow = I422ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToARGBRow = I422ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGBRow = I422ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToARGBRow = I422ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToARGBRow = I422ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToARGBRow = I422ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToARGBRow = I422ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToARGBRow = I422ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToARGBRow = I422ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I422ToARGBRow = I422ToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_WASMSIMD;
+ }
+ }
+#endif
+ {
+ // Allocate a row of argb.
+ align_buffer_64(row_argb, width * 4);
+ if (!row_argb)
+ return 1;
+ for (y = 0; y < height; ++y) {
+ I422ToARGBRow(src_y, src_u, src_v, row_argb, &kYuvI601Constants, width);
+ ARGBToRGB565DitherRow(row_argb, dst_rgb565, dither4[y & 3], width);
+ dst_rgb565 += dst_stride_rgb565;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ free_aligned_buffer_64(row_argb);
+ }
+ return 0;
+}
+
+// Convert I420 to AR30 with matrix.
+LIBYUV_API
+int I420ToAR30Matrix(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I422ToAR30Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I422ToAR30Row_C;
+
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+
+#if defined(HAS_I422TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I422ToAR30Row = I422ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToAR30Row = I422ToAR30Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToAR30Row = I422ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToAR30Row = I422ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ I422ToAR30Row = I422ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToAR30Row = I422ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToAR30Row = I422ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToAR30Row = I422ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToAR30Row = I422ToAR30Row_SME;
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I422ToAR30Row = I422ToAR30Row_RVV;
+ }
+#endif
+#if defined(HAS_I422TOAR30ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ I422ToAR30Row = I422ToAR30Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToAR30Row = I422ToAR30Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ I422ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ if (y & 1) {
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ }
+ return 0;
+}
+
+// Convert I420 to AR30.
+LIBYUV_API
+int I420ToAR30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I420ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert H420 to AR30.
+LIBYUV_API
+int H420ToAR30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ return I420ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ &kYvuH709Constants, width, height);
+}
+
+// Convert I420 to AB30.
+LIBYUV_API
+int I420ToAB30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height) {
+ return I420ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuI601Constants, width, height);
+}
+
+// Convert H420 to AB30.
+LIBYUV_API
+int H420ToAB30(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ab30,
+ int dst_stride_ab30,
+ int width,
+ int height) {
+ return I420ToAR30Matrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
+ src_stride_u, dst_ab30, dst_stride_ab30,
+ &kYvuH709Constants, width, height);
+}
+
+static int I420ToARGBMatrixBilinear(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToARGBRow_C;
+ void (*Scale2RowUp_Bilinear)(const uint8_t* src_ptr, ptrdiff_t src_stride,
+ uint8_t* dst_ptr, ptrdiff_t dst_stride,
+ int dst_width) = ScaleRowUp2_Bilinear_Any_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I444TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444ToARGBRow = I444ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444ToARGBRow = I444ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444ToARGBRow = I444ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444ToARGBRow = I444ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToARGBRow = I444ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444ToARGBRow = I444ToARGBRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSE2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSSE3;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_AVX2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_NEON;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_BILINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_RVV;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4);
+ uint8_t* temp_u_1 = row;
+ uint8_t* temp_u_2 = row + row_size;
+ uint8_t* temp_v_1 = row + row_size * 2;
+ uint8_t* temp_v_2 = row + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear(src_v, src_stride_v, temp_v_1, row_size, width);
+ I444ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ I444ToARGBRow(src_y, temp_u_2, temp_v_2, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I422ToARGBMatrixLinear(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToARGBRow_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I444TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444ToARGBRow = I444ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444ToARGBRow = I444ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444ToARGBRow = I444ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444ToARGBRow = I444ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToARGBRow = I444ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444ToARGBRow = I444ToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2);
+ uint8_t* temp_u = row;
+ uint8_t* temp_v = row + row_size;
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear(src_u, temp_u, width);
+ ScaleRowUp2_Linear(src_v, temp_v, width);
+ I444ToARGBRow(src_y, temp_u, temp_v, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I420ToRGB24MatrixBilinear(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I444ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToRGB24Row_C;
+ void (*Scale2RowUp_Bilinear)(const uint8_t* src_ptr, ptrdiff_t src_stride,
+ uint8_t* dst_ptr, ptrdiff_t dst_stride,
+ int dst_width) = ScaleRowUp2_Bilinear_Any_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+#if defined(HAS_I444TORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToRGB24Row = I444ToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444ToRGB24Row = I444ToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ I444ToRGB24Row = I444ToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToRGB24Row = I444ToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToRGB24Row = I444ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToRGB24Row = I444ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToRGB24Row = I444ToRGB24Row_RVV;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSE2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSSE3;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_AVX2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_NEON;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_BILINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_RVV;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4);
+ uint8_t* temp_u_1 = row;
+ uint8_t* temp_u_2 = row + row_size;
+ uint8_t* temp_v_1 = row + row_size * 2;
+ uint8_t* temp_v_2 = row + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444ToRGB24Row(src_y, temp_u_1, temp_v_1, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear(src_v, src_stride_v, temp_v_1, row_size, width);
+ I444ToRGB24Row(src_y, temp_u_1, temp_v_1, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ I444ToRGB24Row(src_y, temp_u_2, temp_v_2, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444ToRGB24Row(src_y, temp_u_1, temp_v_1, dst_rgb24, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I010ToAR30MatrixBilinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I410ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToAR30Row_C;
+ void (*Scale2RowUp_Bilinear_12)(
+ const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr,
+ ptrdiff_t dst_stride, int dst_width) = ScaleRowUp2_Bilinear_16_Any_C;
+ void (*ScaleRowUp2_Linear_12)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+#if defined(HAS_I410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToAR30Row = I410ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410ToAR30Row = I410ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToAR30Row = I410ToAR30Row_AVX512BW;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_SSSE3;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_AVX2;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_NEON;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4 * sizeof(uint16_t));
+ uint16_t* temp_u_1 = (uint16_t*)(row);
+ uint16_t* temp_u_2 = (uint16_t*)(row) + row_size;
+ uint16_t* temp_v_1 = (uint16_t*)(row) + row_size * 2;
+ uint16_t* temp_v_2 = (uint16_t*)(row) + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410ToAR30Row(src_y, temp_u_1, temp_v_1, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear_12(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear_12(src_v, src_stride_v, temp_v_1, row_size, width);
+ I410ToAR30Row(src_y, temp_u_1, temp_v_1, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ I410ToAR30Row(src_y, temp_u_2, temp_v_2, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410ToAR30Row(src_y, temp_u_1, temp_v_1, dst_ar30, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+
+ return 0;
+}
+
+static int I210ToAR30MatrixLinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I410ToAR30Row)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToAR30Row_C;
+ void (*ScaleRowUp2_Linear_12)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+#if defined(HAS_I410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToAR30Row = I410ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410ToAR30Row = I410ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToAR30Row = I410ToAR30Row_AVX512BW;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2 * sizeof(uint16_t));
+ uint16_t* temp_u = (uint16_t*)(row);
+ uint16_t* temp_v = (uint16_t*)(row) + row_size;
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear_12(src_u, temp_u, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v, width);
+ I410ToAR30Row(src_y, temp_u, temp_v, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I010ToARGBMatrixBilinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I410ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToARGBRow_C;
+ void (*Scale2RowUp_Bilinear_12)(
+ const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr,
+ ptrdiff_t dst_stride, int dst_width) = ScaleRowUp2_Bilinear_16_Any_C;
+ void (*ScaleRowUp2_Linear_12)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410ToARGBRow = I410ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToARGBRow = I410ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410ToARGBRow = I410ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToARGBRow = I410ToARGBRow_AVX512BW;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_SSSE3;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_AVX2;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_NEON;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4 * sizeof(uint16_t));
+ uint16_t* temp_u_1 = (uint16_t*)(row);
+ uint16_t* temp_u_2 = (uint16_t*)(row) + row_size;
+ uint16_t* temp_v_1 = (uint16_t*)(row) + row_size * 2;
+ uint16_t* temp_v_2 = (uint16_t*)(row) + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear_12(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear_12(src_v, src_stride_v, temp_v_1, row_size, width);
+ I410ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ I410ToARGBRow(src_y, temp_u_2, temp_v_2, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410ToARGBRow(src_y, temp_u_1, temp_v_1, dst_argb, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I210ToARGBMatrixLinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I410ToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I410ToARGBRow_C;
+ void (*ScaleRowUp2_Linear_12)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410ToARGBRow = I410ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410ToARGBRow = I410ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410ToARGBRow = I410ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410ToARGBRow = I410ToARGBRow_AVX512BW;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2 * sizeof(uint16_t));
+ uint16_t* temp_u = (uint16_t*)(row);
+ uint16_t* temp_v = (uint16_t*)(row) + row_size;
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear_12(src_u, temp_u, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v, width);
+ I410ToARGBRow(src_y, temp_u, temp_v, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I420AlphaToARGBMatrixBilinear(
+ const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I444AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I444AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ void (*Scale2RowUp_Bilinear)(const uint8_t* src_ptr, ptrdiff_t src_stride,
+ uint8_t* dst_ptr, ptrdiff_t dst_stride,
+ int dst_width) = ScaleRowUp2_Bilinear_Any_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I444ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSE2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_SSSE3;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_AVX2;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_NEON;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_BILINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_RVV;
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4);
+ uint8_t* temp_u_1 = row;
+ uint8_t* temp_u_2 = row + row_size;
+ uint8_t* temp_v_1 = row + row_size * 2;
+ uint8_t* temp_v_2 = row + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_a += src_stride_a;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear(src_v, src_stride_v, temp_v_1, row_size, width);
+ I444AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_a += src_stride_a;
+ I444AlphaToARGBRow(src_y, temp_u_2, temp_v_2, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ src_a += src_stride_a;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear(src_v, temp_v_1, width);
+ I444AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I444AlphaToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I444AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I444ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_LINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2);
+ uint8_t* temp_u = row;
+ uint8_t* temp_v = row + row_size;
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear(src_u, temp_u, width);
+ ScaleRowUp2_Linear(src_v, temp_v, width);
+ I444AlphaToARGBRow(src_y, temp_u, temp_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I010AlphaToARGBMatrixBilinear(
+ const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I410AlphaToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I410AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ void (*Scale2RowUp_Bilinear_12)(
+ const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr,
+ ptrdiff_t dst_stride, int dst_width) = ScaleRowUp2_Bilinear_16_Any_C;
+ void (*ScaleRowUp2_Linear_12)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_SSSE3;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_AVX2;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_BILINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear_12 = ScaleRowUp2_Bilinear_12_Any_NEON;
+ ScaleRowUp2_Linear_12 = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 4 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 4 * sizeof(uint16_t));
+ uint16_t* temp_u_1 = (uint16_t*)(row);
+ uint16_t* temp_u_2 = (uint16_t*)(row) + row_size;
+ uint16_t* temp_v_1 = (uint16_t*)(row) + row_size * 2;
+ uint16_t* temp_v_2 = (uint16_t*)(row) + row_size * 3;
+ if (!row)
+ return 1;
+
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_a += src_stride_a;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear_12(src_u, src_stride_u, temp_u_1, row_size, width);
+ Scale2RowUp_Bilinear_12(src_v, src_stride_v, temp_v_1, row_size, width);
+ I410AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_a += src_stride_a;
+ I410AlphaToARGBRow(src_y, temp_u_2, temp_v_2, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_a += src_stride_a;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+
+ if (!(height & 1)) {
+ ScaleRowUp2_Linear_12(src_u, temp_u_1, width);
+ ScaleRowUp2_Linear_12(src_v, temp_v_1, width);
+ I410AlphaToARGBRow(src_y, temp_u_1, temp_v_1, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate) {
+ int y;
+ void (*I410AlphaToARGBRow)(const uint16_t* y_buf, const uint16_t* u_buf,
+ const uint16_t* v_buf, const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) = I410AlphaToARGBRow_C;
+ void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
+ int width) = ARGBAttenuateRow_C;
+ void (*ScaleRowUp2_Linear)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !src_a || !dst_argb || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_12_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2 * sizeof(uint16_t));
+ uint16_t* temp_u = (uint16_t*)(row);
+ uint16_t* temp_v = (uint16_t*)(row) + row_size;
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear(src_u, temp_u, width);
+ ScaleRowUp2_Linear(src_v, temp_v, width);
+ I410AlphaToARGBRow(src_y, temp_u, temp_v, src_a, dst_argb, yuvconstants,
+ width);
+ if (attenuate) {
+ ARGBAttenuateRow(dst_argb, dst_argb, width);
+ }
+ dst_argb += dst_stride_argb;
+ src_a += src_stride_a;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int P010ToARGBMatrixBilinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*P410ToARGBRow)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToARGBRow_C;
+ void (*Scale2RowUp_Bilinear_16)(
+ const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr,
+ ptrdiff_t dst_stride, int dst_width) = ScaleUVRowUp2_Bilinear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_P410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P410ToARGBRow = P410ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P410ToARGBRow = P410ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ P410ToARGBRow = P410ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToARGBRow = P410ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToARGBRow = P410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToARGBRow = P410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToARGBRow = P410ToARGBRow_SME;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_NEON;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (2 * width + 31) & ~31;
+ align_buffer_64(row, row_size * 2 * sizeof(uint16_t));
+ uint16_t* temp_uv_1 = (uint16_t*)(row);
+ uint16_t* temp_uv_2 = (uint16_t*)(row) + row_size;
+ if (!row)
+ return 1;
+
+ Scale2RowUp_Bilinear_16(src_uv, 0, temp_uv_1, row_size, width);
+ P410ToARGBRow(src_y, temp_uv_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear_16(src_uv, src_stride_uv, temp_uv_1, row_size, width);
+ P410ToARGBRow(src_y, temp_uv_1, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ P410ToARGBRow(src_y, temp_uv_2, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+
+ if (!(height & 1)) {
+ Scale2RowUp_Bilinear_16(src_uv, 0, temp_uv_1, row_size, width);
+ P410ToARGBRow(src_y, temp_uv_1, dst_argb, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int P210ToARGBMatrixLinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*P410ToARGBRow)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToARGBRow_C;
+ void (*ScaleRowUp2_Linear)(const uint16_t* src_uv, uint16_t* dst_uv,
+ int dst_width) = ScaleUVRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_P410TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P410ToARGBRow = P410ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P410ToARGBRow = P410ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ P410ToARGBRow = P410ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToARGBRow = P410ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToARGBRow = P410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToARGBRow = P410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToARGBRow = P410ToARGBRow_SME;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_NEON;
+ }
+#endif
+
+ const int row_size = (2 * width + 31) & ~31;
+ align_buffer_64(row, row_size * sizeof(uint16_t));
+ uint16_t* temp_uv = (uint16_t*)(row);
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear(src_uv, temp_uv, width);
+ P410ToARGBRow(src_y, temp_uv, dst_argb, yuvconstants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int P010ToAR30MatrixBilinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*P410ToAR30Row)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToAR30Row_C;
+ void (*Scale2RowUp_Bilinear_16)(
+ const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr,
+ ptrdiff_t dst_stride, int dst_width) = ScaleUVRowUp2_Bilinear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+#if defined(HAS_P410TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P410ToAR30Row = P410ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P410ToAR30Row = P410ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ P410ToAR30Row = P410ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToAR30Row = P410ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToAR30Row = P410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToAR30Row = P410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToAR30Row = P410ToAR30Row_SME;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_NEON;
+ }
+#endif
+
+ // alloc 2 lines temp
+ const int row_size = (2 * width + 31) & ~31;
+ align_buffer_64(row, row_size * 2 * sizeof(uint16_t));
+ uint16_t* temp_uv_1 = (uint16_t*)(row);
+ uint16_t* temp_uv_2 = (uint16_t*)(row) + row_size;
+ if (!row)
+ return 1;
+
+ Scale2RowUp_Bilinear_16(src_uv, 0, temp_uv_1, row_size, width);
+ P410ToAR30Row(src_y, temp_uv_1, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+
+ for (y = 0; y < height - 2; y += 2) {
+ Scale2RowUp_Bilinear_16(src_uv, src_stride_uv, temp_uv_1, row_size, width);
+ P410ToAR30Row(src_y, temp_uv_1, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ P410ToAR30Row(src_y, temp_uv_2, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+
+ if (!(height & 1)) {
+ Scale2RowUp_Bilinear_16(src_uv, 0, temp_uv_1, row_size, width);
+ P410ToAR30Row(src_y, temp_uv_1, dst_ar30, yuvconstants, width);
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int P210ToAR30MatrixLinear(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*P410ToAR30Row)(
+ const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = P410ToAR30Row_C;
+ void (*ScaleRowUp2_Linear)(const uint16_t* src_uv, uint16_t* dst_uv,
+ int dst_width) = ScaleUVRowUp2_Linear_16_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_uv || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+#if defined(HAS_P410TOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ P410ToAR30Row = P410ToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ P410ToAR30Row = P410ToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ P410ToAR30Row = P410ToAR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ P410ToAR30Row = P410ToAR30Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToAR30Row = P410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToAR30Row = P410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToAR30Row = P410ToAR30Row_SME;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_NEON;
+ }
+#endif
+
+ const int row_size = (2 * width + 31) & ~31;
+ align_buffer_64(row, row_size * sizeof(uint16_t));
+ uint16_t* temp_uv = (uint16_t*)(row);
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ScaleRowUp2_Linear(src_uv, temp_uv, width);
+ P410ToAR30Row(src_y, temp_uv, dst_ar30, yuvconstants, width);
+ dst_ar30 += dst_stride_ar30;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int I422ToRGB24MatrixLinear(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
+ int y;
+ void (*I444ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToRGB24Row_C;
+ void (*ScaleRowUp2_Linear)(const uint8_t* src_ptr, uint8_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_Any_C;
+ assert(yuvconstants);
+ if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
}
-#if defined(HAS_UYVYTOARGBROW_SSSE3)
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
+ dst_stride_rgb24 = -dst_stride_rgb24;
+ }
+#if defined(HAS_I444TORGB24ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- UYVYToARGBRow = UYVYToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- UYVYToARGBRow = UYVYToARGBRow_SSSE3;
+ I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToRGB24Row = I444ToRGB24Row_SSSE3;
}
}
#endif
-#if defined(HAS_UYVYTOARGBROW_AVX2)
+#if defined(HAS_I444TORGB24ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- UYVYToARGBRow = UYVYToARGBRow_Any_AVX2;
+ I444ToRGB24Row = I444ToRGB24Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- UYVYToARGBRow = UYVYToARGBRow_AVX2;
+ I444ToRGB24Row = I444ToRGB24Row_AVX2;
}
}
#endif
-#if defined(HAS_UYVYTOARGBROW_NEON)
+#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- UYVYToARGBRow = UYVYToARGBRow_Any_NEON;
+ I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- UYVYToARGBRow = UYVYToARGBRow_NEON;
+ I444ToRGB24Row = I444ToRGB24Row_NEON;
}
}
#endif
-#if defined(HAS_UYVYTOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- UYVYToARGBRow = UYVYToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- UYVYToARGBRow = UYVYToARGBRow_MSA;
- }
+#if defined(HAS_I444TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToRGB24Row = I444ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToRGB24Row = I444ToRGB24Row_SME;
+ }
+#endif
+#if defined(HAS_I444TORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToRGB24Row = I444ToRGB24Row_RVV;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWUP2_LINEAR_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV;
}
#endif
+
+ // alloc 2 lines temp
+ const int row_size = (width + 31) & ~31;
+ align_buffer_64(row, row_size * 2);
+ uint8_t* temp_u = row;
+ uint8_t* temp_v = row + row_size;
+ if (!row)
+ return 1;
+
for (y = 0; y < height; ++y) {
- UYVYToARGBRow(src_uyvy, dst_argb, &kYuvI601Constants, width);
- src_uyvy += src_stride_uyvy;
- dst_argb += dst_stride_argb;
+ ScaleRowUp2_Linear(src_u, temp_u, width);
+ ScaleRowUp2_Linear(src_v, temp_v, width);
+ I444ToRGB24Row(src_y, temp_u, temp_v, dst_rgb24, yuvconstants, width);
+ dst_rgb24 += dst_stride_rgb24;
+ src_y += src_stride_y;
+ src_u += src_stride_u;
+ src_v += src_stride_v;
}
+
+ free_aligned_buffer_64(row);
return 0;
}
-static void WeavePixels(const uint8_t* src_u,
- const uint8_t* src_v,
- int src_pixel_stride_uv,
- uint8_t* dst_uv,
- int width) {
- int i;
- for (i = 0; i < width; ++i) {
- dst_uv[0] = *src_u;
- dst_uv[1] = *src_v;
- dst_uv += 2;
- src_u += src_pixel_stride_uv;
- src_v += src_pixel_stride_uv;
+
+LIBYUV_API
+int I422ToRGB24MatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I422ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ yuvconstants, width, height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I422ToRGB24MatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_rgb24, dst_stride_rgb24, yuvconstants, width, height);
}
+
+ return -1;
}
-// Convert Android420 to ARGB.
LIBYUV_API
-int Android420ToARGBMatrix(const uint8_t* src_y,
+int I420ToARGBMatrixFilter(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- int src_pixel_stride_uv,
uint8_t* dst_argb,
int dst_stride_argb,
const struct YuvConstants* yuvconstants,
int width,
- int height) {
- int y;
- uint8_t* dst_uv;
- const ptrdiff_t vu_off = src_v - src_u;
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_y || !src_u || !src_v || !dst_argb || width <= 0 || height == 0) {
- return -1;
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ case kFilterBilinear:
+ case kFilterBox:
+ return I420ToARGBMatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_argb, dst_stride_argb, yuvconstants, width, height);
+ case kFilterLinear:
+ // Actually we can do this, but probably there's no usage.
+ return -1;
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- halfheight = (height + 1) >> 1;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
- dst_stride_argb = -dst_stride_argb;
+
+ return -1;
+}
+
+LIBYUV_API
+int I422ToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I422ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I422ToARGBMatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_argb, dst_stride_argb, yuvconstants, width, height);
}
- // I420
- if (src_pixel_stride_uv == 1) {
- return I420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_argb, dst_stride_argb,
- yuvconstants, width, height);
- // NV21
+ return -1;
+}
+
+LIBYUV_API
+int I420ToRGB24MatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_rgb24, dst_stride_rgb24,
+ yuvconstants, width, height);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return I420ToRGB24MatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_rgb24, dst_stride_rgb24, yuvconstants, width, height);
}
- if (src_pixel_stride_uv == 2 && vu_off == -1 &&
- src_stride_u == src_stride_v) {
- return NV21ToARGBMatrix(src_y, src_stride_y, src_v, src_stride_v, dst_argb,
- dst_stride_argb, yuvconstants, width, height);
- // NV12
+
+ return -1;
+}
+
+LIBYUV_API
+int I010ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I010ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ yuvconstants, width, height);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return I010ToAR30MatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_ar30, dst_stride_ar30, yuvconstants, width, height);
}
- if (src_pixel_stride_uv == 2 && vu_off == 1 && src_stride_u == src_stride_v) {
- return NV12ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, dst_argb,
- dst_stride_argb, yuvconstants, width, height);
+
+ return -1;
+}
+
+LIBYUV_API
+int I210ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I210ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_ar30, dst_stride_ar30,
+ yuvconstants, width, height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I210ToAR30MatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_ar30, dst_stride_ar30, yuvconstants, width, height);
}
- // General case fallback creates NV12
- align_buffer_64(plane_uv, halfwidth * 2 * halfheight);
- dst_uv = plane_uv;
- for (y = 0; y < halfheight; ++y) {
- WeavePixels(src_u, src_v, src_pixel_stride_uv, dst_uv, halfwidth);
- src_u += src_stride_u;
- src_v += src_stride_v;
- dst_uv += halfwidth * 2;
+ return -1;
+}
+
+LIBYUV_API
+int I010ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I010ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return I010ToARGBMatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_argb, dst_stride_argb, yuvconstants, width, height);
}
- NV12ToARGBMatrix(src_y, src_stride_y, plane_uv, halfwidth * 2, dst_argb,
- dst_stride_argb, yuvconstants, width, height);
- free_aligned_buffer_64(plane_uv);
- return 0;
+
+ return -1;
}
-// Convert Android420 to ARGB.
LIBYUV_API
-int Android420ToARGB(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- int src_pixel_stride_uv,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- return Android420ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, src_pixel_stride_uv, dst_argb,
- dst_stride_argb, &kYuvI601Constants, width,
- height);
+int I210ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I210ToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I210ToARGBMatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
+ dst_argb, dst_stride_argb, yuvconstants, width, height);
+ }
+
+ return -1;
}
-// Convert Android420 to ABGR.
LIBYUV_API
-int Android420ToABGR(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- int src_pixel_stride_uv,
- uint8_t* dst_abgr,
- int dst_stride_abgr,
- int width,
- int height) {
- return Android420ToARGBMatrix(src_y, src_stride_y, src_v, src_stride_v, src_u,
- src_stride_u, src_pixel_stride_uv, dst_abgr,
- dst_stride_abgr, &kYvuI601Constants, width,
- height);
+int I420AlphaToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I420AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u,
+ src_v, src_stride_v, src_a, src_stride_a,
+ dst_argb, dst_stride_argb, yuvconstants,
+ width, height, attenuate);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return I420AlphaToARGBMatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, src_a,
+ src_stride_a, dst_argb, dst_stride_argb, yuvconstants, width, height,
+ attenuate);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int I422AlphaToARGBMatrixFilter(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I422AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u,
+ src_v, src_stride_v, src_a, src_stride_a,
+ dst_argb, dst_stride_argb, yuvconstants,
+ width, height, attenuate);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I422AlphaToARGBMatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, src_a,
+ src_stride_a, dst_argb, dst_stride_argb, yuvconstants, width, height,
+ attenuate);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int I010AlphaToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I010AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u,
+ src_v, src_stride_v, src_a, src_stride_a,
+ dst_argb, dst_stride_argb, yuvconstants,
+ width, height, attenuate);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return I010AlphaToARGBMatrixBilinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, src_a,
+ src_stride_a, dst_argb, dst_stride_argb, yuvconstants, width, height,
+ attenuate);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int I210AlphaToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ int attenuate,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return I210AlphaToARGBMatrix(src_y, src_stride_y, src_u, src_stride_u,
+ src_v, src_stride_v, src_a, src_stride_a,
+ dst_argb, dst_stride_argb, yuvconstants,
+ width, height, attenuate);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return I210AlphaToARGBMatrixLinear(
+ src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, src_a,
+ src_stride_a, dst_argb, dst_stride_argb, yuvconstants, width, height,
+ attenuate);
+ }
+
+ return -1;
+}
+
+// TODO(fb): Verify this function works correctly. P010 is like NV12 but 10 bit
+// UV is biplanar.
+LIBYUV_API
+int P010ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return P010ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_argb, dst_stride_argb, yuvconstants, width,
+ height);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return P010ToARGBMatrixBilinear(src_y, src_stride_y, src_uv,
+ src_stride_uv, dst_argb, dst_stride_argb,
+ yuvconstants, width, height);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int P210ToARGBMatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return P210ToARGBMatrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_argb, dst_stride_argb, yuvconstants, width,
+ height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return P210ToARGBMatrixLinear(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_argb, dst_stride_argb, yuvconstants,
+ width, height);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int P010ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return P010ToAR30Matrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_ar30, dst_stride_ar30, yuvconstants, width,
+ height);
+ case kFilterLinear: // TODO(fb): Implement Linear using Bilinear stride 0
+ case kFilterBilinear:
+ case kFilterBox:
+ return P010ToAR30MatrixBilinear(src_y, src_stride_y, src_uv,
+ src_stride_uv, dst_ar30, dst_stride_ar30,
+ yuvconstants, width, height);
+ }
+
+ return -1;
+}
+
+LIBYUV_API
+int P210ToAR30MatrixFilter(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height,
+ enum FilterMode filter) {
+ switch (filter) {
+ case kFilterNone:
+ return P210ToAR30Matrix(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_ar30, dst_stride_ar30, yuvconstants, width,
+ height);
+ case kFilterBilinear:
+ case kFilterBox:
+ case kFilterLinear:
+ return P210ToAR30MatrixLinear(src_y, src_stride_y, src_uv, src_stride_uv,
+ dst_ar30, dst_stride_ar30, yuvconstants,
+ width, height);
+ }
+
+ return -1;
}
#ifdef __cplusplus
diff --git a/libs/libyuv/source/convert_from.cc b/libs/libyuv/source/convert_from.cc
index 60140cb4e2..6f5853bc2d 100644
--- a/libs/libyuv/source/convert_from.cc
+++ b/libs/libyuv/source/convert_from.cc
@@ -10,7 +10,9 @@
#include "libyuv/convert_from.h"
-#include "libyuv/basic_types.h"
+#include <limits.h>
+#include <stdint.h>
+
#include "libyuv/convert.h" // For I420Copy
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
@@ -46,23 +48,26 @@ static int I420ToI4xx(const uint8_t* src_y,
int src_y_height,
int dst_uv_width,
int dst_uv_height) {
- const int dst_y_width = Abs(src_y_width);
- const int dst_y_height = Abs(src_y_height);
const int src_uv_width = SUBSAMPLE(src_y_width, 1, 1);
const int src_uv_height = SUBSAMPLE(src_y_height, 1, 1);
- if (src_y_width == 0 || src_y_height == 0 || dst_uv_width <= 0 ||
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v ||
+ src_y_width <= 0 || src_y_height == 0 || dst_uv_width <= 0 ||
dst_uv_height <= 0) {
return -1;
}
if (dst_y) {
- ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, dst_y,
- dst_stride_y, dst_y_width, dst_y_height, kFilterBilinear);
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, src_y_width,
+ src_y_height);
}
- ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
- dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
- ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, dst_v,
- dst_stride_v, dst_uv_width, dst_uv_height, kFilterBilinear);
- return 0;
+ r = ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
+ dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, src_uv_width, src_uv_height, dst_v,
+ dst_stride_v, dst_uv_width, dst_uv_height, kFilterBilinear);
+ return r;
}
// Convert 8 bit YUV to 10 bit.
@@ -83,24 +88,29 @@ int I420ToI010(const uint8_t* src_y,
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
- src_stride_y = -src_stride_y;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
// Convert Y plane.
- Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width,
- height);
+ if (dst_y) {
+ Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width,
+ height);
+ }
// Convert UV planes.
Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 1024, halfwidth,
halfheight);
@@ -109,6 +119,55 @@ int I420ToI010(const uint8_t* src_y,
return 0;
}
+// Convert 8 bit YUV to 12 bit.
+LIBYUV_API
+int I420ToI012(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ // Convert Y plane.
+ if (dst_y) {
+ Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 4096, width,
+ height);
+ }
+ // Convert UV planes.
+ Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 4096, halfwidth,
+ halfheight);
+ Convert8To16Plane(src_v, src_stride_v, dst_v, dst_stride_v, 4096, halfwidth,
+ halfheight);
+ return 0;
+}
+
// 420 chroma is 1/2 width, 1/2 height
// 422 chroma is 1/2 width, 1x height
LIBYUV_API
@@ -159,6 +218,114 @@ int I420ToI444(const uint8_t* src_y,
dst_uv_height);
}
+// 420 chroma to 444 chroma, 10/12 bit version
+LIBYUV_API
+int I010ToI410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = ScalePlane_12(src_u, src_stride_u, SUBSAMPLE(width, 1, 1),
+ SUBSAMPLE(height, 1, 1), dst_u, dst_stride_u, width,
+ Abs(height), kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, SUBSAMPLE(width, 1, 1),
+ SUBSAMPLE(height, 1, 1), dst_v, dst_stride_v, width,
+ Abs(height), kFilterBilinear);
+ return r;
+}
+
+// 422 chroma to 444 chroma, 10/12 bit version
+LIBYUV_API
+int I210ToI410(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = ScalePlane_12(src_u, src_stride_u, SUBSAMPLE(width, 1, 1), height, dst_u,
+ dst_stride_u, width, Abs(height), kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, SUBSAMPLE(width, 1, 1), height, dst_v,
+ dst_stride_v, width, Abs(height), kFilterBilinear);
+ return r;
+}
+
+// 422 chroma is 1/2 width, 1x height
+// 444 chroma is 1x width, 1x height
+LIBYUV_API
+int I422ToI444(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int r;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ r = ScalePlane(src_u, src_stride_u, SUBSAMPLE(width, 1, 1), height, dst_u,
+ dst_stride_u, width, Abs(height), kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, SUBSAMPLE(width, 1, 1), height, dst_v,
+ dst_stride_v, width, Abs(height), kFilterBilinear);
+ return r;
+}
+
// Copy to I400. Source can be I420,422,444,400,NV12,NV21
LIBYUV_API
int I400Copy(const uint8_t* src_y,
@@ -167,15 +334,9 @@ int I400Copy(const uint8_t* src_y,
int dst_stride_y,
int width,
int height) {
- if (!src_y || !dst_y || width <= 0 || height == 0) {
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
- }
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
return 0;
}
@@ -195,18 +356,20 @@ int I422ToYUY2(const uint8_t* src_y,
void (*I422ToYUY2Row)(const uint8_t* src_y, const uint8_t* src_u,
const uint8_t* src_v, uint8_t* dst_yuy2, int width) =
I422ToYUY2Row_C;
- if (!src_y || !src_u || !src_v || !dst_yuy2 || width <= 0 || height == 0) {
+ if (!src_y || !src_u || !src_v || !dst_yuy2 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2;
+ dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
dst_stride_yuy2 = -dst_stride_yuy2;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u * 2 == width &&
- src_stride_v * 2 == width && dst_stride_yuy2 == width * 2) {
+ src_stride_v * 2 == width && dst_stride_yuy2 == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_yuy2 = 0;
@@ -261,13 +424,14 @@ int I420ToYUY2(const uint8_t* src_y,
void (*I422ToYUY2Row)(const uint8_t* src_y, const uint8_t* src_u,
const uint8_t* src_v, uint8_t* dst_yuy2, int width) =
I422ToYUY2Row_C;
- if (!src_y || !src_u || !src_v || !dst_yuy2 || width <= 0 || height == 0) {
+ if (!src_y || !src_u || !src_v || !dst_yuy2 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2;
+ dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
dst_stride_yuy2 = -dst_stride_yuy2;
}
#if defined(HAS_I422TOYUY2ROW_SSE2)
@@ -294,19 +458,19 @@ int I420ToYUY2(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- I422ToYUY2Row = I422ToYUY2Row_MSA;
+#if defined(HAS_I422TOYUY2ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToYUY2Row = I422ToYUY2Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToYUY2Row = I422ToYUY2Row_LSX;
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- I422ToYUY2Row = I422ToYUY2Row_MMI;
+#if defined(HAS_I422TOYUY2ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToYUY2Row = I422ToYUY2Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ I422ToYUY2Row = I422ToYUY2Row_LASX;
}
}
#endif
@@ -341,18 +505,20 @@ int I422ToUYVY(const uint8_t* src_y,
void (*I422ToUYVYRow)(const uint8_t* src_y, const uint8_t* src_u,
const uint8_t* src_v, uint8_t* dst_uyvy, int width) =
I422ToUYVYRow_C;
- if (!src_y || !src_u || !src_v || !dst_uyvy || width <= 0 || height == 0) {
+ if (!src_y || !src_u || !src_v || !dst_uyvy || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy;
+ dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy;
dst_stride_uyvy = -dst_stride_uyvy;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u * 2 == width &&
- src_stride_v * 2 == width && dst_stride_uyvy == width * 2) {
+ src_stride_v * 2 == width && dst_stride_uyvy == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_uyvy = 0;
@@ -381,19 +547,19 @@ int I422ToUYVY(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- I422ToUYVYRow = I422ToUYVYRow_MSA;
+#if defined(HAS_I422TOUYVYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToUYVYRow = I422ToUYVYRow_LSX;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- I422ToUYVYRow = I422ToUYVYRow_MMI;
+#if defined(HAS_I422TOUYVYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ I422ToUYVYRow = I422ToUYVYRow_LASX;
}
}
#endif
@@ -423,13 +589,14 @@ int I420ToUYVY(const uint8_t* src_y,
void (*I422ToUYVYRow)(const uint8_t* src_y, const uint8_t* src_u,
const uint8_t* src_v, uint8_t* dst_uyvy, int width) =
I422ToUYVYRow_C;
- if (!src_y || !src_u || !src_v || !dst_uyvy || width <= 0 || height == 0) {
+ if (!src_y || !src_u || !src_v || !dst_uyvy || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy;
+ dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy;
dst_stride_uyvy = -dst_stride_uyvy;
}
#if defined(HAS_I422TOUYVYROW_SSE2)
@@ -456,19 +623,19 @@ int I420ToUYVY(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- I422ToUYVYRow = I422ToUYVYRow_MSA;
+#if defined(HAS_I422TOUYVYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToUYVYRow = I422ToUYVYRow_LSX;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- I422ToUYVYRow = I422ToUYVYRow_MMI;
+#if defined(HAS_I422TOUYVYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ I422ToUYVYRow = I422ToUYVYRow_LASX;
}
}
#endif
@@ -488,7 +655,6 @@ int I420ToUYVY(const uint8_t* src_y,
return 0;
}
-// TODO(fbarchard): test negative height for invert.
LIBYUV_API
int I420ToNV12(const uint8_t* src_y,
int src_stride_y,
@@ -502,12 +668,23 @@ int I420ToNV12(const uint8_t* src_y,
int dst_stride_uv,
int width,
int height) {
- if (!src_y || !src_u || !src_v || !dst_y || !dst_uv || width <= 0 ||
- height == 0) {
+ int halfwidth = (width + 1) / 2;
+ int halfheight = (height + 1) / 2;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
- int halfwidth = (width + 1) / 2;
- int halfheight = height > 0 ? (height + 1) / 2 : (height - 1) / 2;
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
if (dst_y) {
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
@@ -534,805 +711,314 @@ int I420ToNV21(const uint8_t* src_y,
width, height);
}
-// Convert I422 to RGBA with matrix
-static int I420ToRGBAMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
+static int I4xxToPxxx(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y) {
int y;
- void (*I422ToRGBARow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToRGBARow_C;
- if (!src_y || !src_u || !src_v || !dst_rgba || width <= 0 || height == 0) {
+ int uv_width;
+ int uv_height;
+ int row_stride;
+ void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits,
+ int width) = Convert8To16Row_C;
+ void (*MergeUVRow_16)(const uint16_t* src_u, const uint16_t* src_v,
+ uint16_t* dst_uv, int depth, int width) =
+ MergeUVRow_16_C;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_y || !dst_uv ||
+ width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
- // Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_rgba = dst_rgba + (height - 1) * dst_stride_rgba;
- dst_stride_rgba = -dst_stride_rgba;
- }
-#if defined(HAS_I422TORGBAROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I422TORGBAROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToRGBARow = I422ToRGBARow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGBARow = I422ToRGBARow_AVX2;
- }
- }
-#endif
-#if defined(HAS_I422TORGBAROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToRGBARow = I422ToRGBARow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_NEON;
- }
- }
-#endif
-#if defined(HAS_I422TORGBAROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToRGBARow = I422ToRGBARow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_MSA;
- }
- }
-#endif
-
- for (y = 0; y < height; ++y) {
- I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
- dst_rgba += dst_stride_rgba;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
+ uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(uv_height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(uv_height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ } else {
+ uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
}
- return 0;
-}
+ uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
-// Convert I420 to RGBA.
-LIBYUV_API
-int I420ToRGBA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- int width,
- int height) {
- return I420ToRGBAMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgba, dst_stride_rgba,
- &kYuvI601Constants, width, height);
-}
-
-// Convert I420 to BGRA.
-LIBYUV_API
-int I420ToBGRA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_bgra,
- int dst_stride_bgra,
- int width,
- int height) {
- return I420ToRGBAMatrix(src_y, src_stride_y, src_v,
- src_stride_v, // Swap U and V
- src_u, src_stride_u, dst_bgra, dst_stride_bgra,
- &kYvuI601Constants, // Use Yvu matrix
- width, height);
-}
+ Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 10, width,
+ height);
+ ConvertToMSBPlane_16(dst_y, dst_stride_y, dst_y, dst_stride_y, width, height,
+ 10);
-// Convert I420 to RGB24 with matrix
-static int I420ToRGB24Matrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
- int y;
- void (*I422ToRGB24Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToRGB24Row_C;
- if (!src_y || !src_u || !src_v || !dst_rgb24 || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_rgb24 = dst_rgb24 + (height - 1) * dst_stride_rgb24;
- dst_stride_rgb24 = -dst_stride_rgb24;
- }
-#if defined(HAS_I422TORGB24ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGB24Row = I422ToRGB24Row_SSSE3;
+#if defined(HAS_CONVERT8TO16ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Convert8To16Row = Convert8To16Row_Any_SSE2;
+ if (IS_ALIGNED(uv_width, 16)) {
+ Convert8To16Row = Convert8To16Row_SSE2;
}
}
#endif
-#if defined(HAS_I422TORGB24ROW_AVX2)
+#if defined(HAS_CONVERT8TO16ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- I422ToRGB24Row = I422ToRGB24Row_AVX2;
- }
- }
-#endif
-#if defined(HAS_I422TORGB24ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToRGB24Row = I422ToRGB24Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB24Row = I422ToRGB24Row_NEON;
- }
- }
-#endif
-#if defined(HAS_I422TORGB24ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToRGB24Row = I422ToRGB24Row_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGB24Row = I422ToRGB24Row_MSA;
- }
- }
-#endif
-
- for (y = 0; y < height; ++y) {
- I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
- dst_rgb24 += dst_stride_rgb24;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- return 0;
-}
-
-// Convert I420 to RGB24.
-LIBYUV_API
-int I420ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height) {
- return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgb24, dst_stride_rgb24,
- &kYuvI601Constants, width, height);
-}
-
-// Convert I420 to RAW.
-LIBYUV_API
-int I420ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height) {
- return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
- src_stride_v, // Swap U and V
- src_u, src_stride_u, dst_raw, dst_stride_raw,
- &kYvuI601Constants, // Use Yvu matrix
- width, height);
-}
-
-// Convert H420 to RGB24.
-LIBYUV_API
-int H420ToRGB24(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height) {
- return I420ToRGB24Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgb24, dst_stride_rgb24,
- &kYuvH709Constants, width, height);
-}
-
-// Convert H420 to RAW.
-LIBYUV_API
-int H420ToRAW(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height) {
- return I420ToRGB24Matrix(src_y, src_stride_y, src_v,
- src_stride_v, // Swap U and V
- src_u, src_stride_u, dst_raw, dst_stride_raw,
- &kYvuH709Constants, // Use Yvu matrix
- width, height);
-}
-
-// Convert I420 to ARGB1555.
-LIBYUV_API
-int I420ToARGB1555(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb1555,
- int dst_stride_argb1555,
- int width,
- int height) {
- int y;
- void (*I422ToARGB1555Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants,
- int width) = I422ToARGB1555Row_C;
- if (!src_y || !src_u || !src_v || !dst_argb1555 || width <= 0 ||
- height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_argb1555 = dst_argb1555 + (height - 1) * dst_stride_argb1555;
- dst_stride_argb1555 = -dst_stride_argb1555;
- }
-#if defined(HAS_I422TOARGB1555ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToARGB1555Row = I422ToARGB1555Row_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB1555Row = I422ToARGB1555Row_SSSE3;
+ Convert8To16Row = Convert8To16Row_Any_AVX2;
+ if (IS_ALIGNED(uv_width, 32)) {
+ Convert8To16Row = Convert8To16Row_AVX2;
}
}
#endif
-#if defined(HAS_I422TOARGB1555ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToARGB1555Row = I422ToARGB1555Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToARGB1555Row = I422ToARGB1555Row_AVX2;
- }
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To16Row = Convert8To16Row_AVX512BW;
}
#endif
-#if defined(HAS_I422TOARGB1555ROW_NEON)
+#if defined(HAS_CONVERT8TO16ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB1555Row = I422ToARGB1555Row_NEON;
+ Convert8To16Row = Convert8To16Row_Any_NEON;
+ if (IS_ALIGNED(uv_width, 16)) {
+ Convert8To16Row = Convert8To16Row_NEON;
}
}
#endif
-#if defined(HAS_I422TOARGB1555ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGB1555Row = I422ToARGB1555Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB1555Row = I422ToARGB1555Row_MSA;
- }
+#if defined(HAS_CONVERT8TO16ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To16Row = Convert8To16Row_SME;
}
#endif
-
- for (y = 0; y < height; ++y) {
- I422ToARGB1555Row(src_y, src_u, src_v, dst_argb1555, &kYuvI601Constants,
- width);
- dst_argb1555 += dst_stride_argb1555;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- return 0;
-}
-
-// Convert I420 to ARGB4444.
-LIBYUV_API
-int I420ToARGB4444(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_argb4444,
- int dst_stride_argb4444,
- int width,
- int height) {
- int y;
- void (*I422ToARGB4444Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants,
- int width) = I422ToARGB4444Row_C;
- if (!src_y || !src_u || !src_v || !dst_argb4444 || width <= 0 ||
- height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_argb4444 = dst_argb4444 + (height - 1) * dst_stride_argb4444;
- dst_stride_argb4444 = -dst_stride_argb4444;
- }
-#if defined(HAS_I422TOARGB4444ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToARGB4444Row = I422ToARGB4444Row_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB4444Row = I422ToARGB4444Row_SSSE3;
- }
+#if defined(HAS_CONVERT8TO16ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert8To16Row = Convert8To16Row_RVV;
}
#endif
-#if defined(HAS_I422TOARGB4444ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToARGB4444Row = I422ToARGB4444Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToARGB4444Row = I422ToARGB4444Row_AVX2;
- }
+#if defined(HAS_CONVERT8TO16ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To16Row = Convert8To16Row_HVX;
}
#endif
-#if defined(HAS_I422TOARGB4444ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB4444Row = I422ToARGB4444Row_NEON;
+#if defined(HAS_CONVERT8TO16ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Convert8To16Row = Convert8To16Row_Any_WASMSIMD;
+ if (IS_ALIGNED(uv_width, 16)) {
+ Convert8To16Row = Convert8To16Row_WASMSIMD;
}
}
#endif
-#if defined(HAS_I422TOARGB4444ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGB4444Row = I422ToARGB4444Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGB4444Row = I422ToARGB4444Row_MSA;
+#if defined(HAS_MERGEUVROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow_16 = MergeUVRow_16_Any_AVX2;
+ if (IS_ALIGNED(uv_width, 8)) {
+ MergeUVRow_16 = MergeUVRow_16_AVX2;
}
}
#endif
-
- for (y = 0; y < height; ++y) {
- I422ToARGB4444Row(src_y, src_u, src_v, dst_argb4444, &kYuvI601Constants,
- width);
- dst_argb4444 += dst_stride_argb4444;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- return 0;
-}
-
-// Convert I420 to RGB565 with specified color matrix.
-LIBYUV_API
-int I420ToRGB565Matrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
- int y;
- void (*I422ToRGB565Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToRGB565Row_C;
- if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565;
- dst_stride_rgb565 = -dst_stride_rgb565;
- }
-#if defined(HAS_I422TORGB565ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_SSSE3;
- }
+#if defined(HAS_MERGEUVROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow_16 = MergeUVRow_16_AVX512BW;
}
#endif
-#if defined(HAS_I422TORGB565ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGB565Row = I422ToRGB565Row_AVX2;
+#if defined(HAS_MERGEUVROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow_16 = MergeUVRow_16_Any_NEON;
+ if (IS_ALIGNED(uv_width, 8)) {
+ MergeUVRow_16 = MergeUVRow_16_NEON;
}
}
#endif
-#if defined(HAS_I422TORGB565ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_NEON;
- }
+#if defined(HAS_MERGEUVROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_16 = MergeUVRow_16_SME;
}
#endif
-#if defined(HAS_I422TORGB565ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_MSA;
- }
+#if defined(HAS_MERGEUVROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow_16 = MergeUVRow_16_HVX;
}
#endif
- for (y = 0; y < height; ++y) {
- I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
- dst_rgb565 += dst_stride_rgb565;
- src_y += src_stride_y;
- if (y & 1) {
+ row_stride = (uv_width + 31) & ~31;
+ align_buffer_64(row_u, (size_t)row_stride * sizeof(uint16_t) * 2u);
+ if (!row_u) {
+ return 1;
+ }
+ {
+ uint16_t* row_u16 = (uint16_t*)row_u;
+ uint16_t* row_v16 = row_u16 + row_stride;
+ for (y = 0; y < uv_height; ++y) {
+ Convert8To16Row(src_u, row_u16, 10, uv_width);
+ Convert8To16Row(src_v, row_v16, 10, uv_width);
+ MergeUVRow_16(row_u16, row_v16, dst_uv, 10, uv_width);
src_u += src_stride_u;
src_v += src_stride_v;
+ dst_uv += dst_stride_uv;
}
}
+ free_aligned_buffer_64(row_u);
return 0;
}
-// Convert I420 to RGB565.
LIBYUV_API
-int I420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
- return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgb565, dst_stride_rgb565,
- &kYuvI601Constants, width, height);
-}
-
-// Convert J420 to RGB565.
-LIBYUV_API
-int J420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
- return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgb565, dst_stride_rgb565,
- &kYuvJPEGConstants, width, height);
+int I420ToP010(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return I4xxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 1);
}
-// Convert H420 to RGB565.
LIBYUV_API
-int H420ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
- return I420ToRGB565Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgb565, dst_stride_rgb565,
- &kYuvH709Constants, width, height);
+int I422ToP210(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return I4xxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 1, 0);
}
-// Convert I422 to RGB565.
LIBYUV_API
-int I422ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
- int y;
- void (*I422ToRGB565Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToRGB565Row_C;
- if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565;
- dst_stride_rgb565 = -dst_stride_rgb565;
- }
-#if defined(HAS_I422TORGB565ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I422TORGB565ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGB565Row = I422ToRGB565Row_AVX2;
- }
- }
-#endif
-#if defined(HAS_I422TORGB565ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_NEON;
- }
- }
-#endif
-#if defined(HAS_I422TORGB565ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToRGB565Row = I422ToRGB565Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGB565Row = I422ToRGB565Row_MSA;
- }
- }
-#endif
-
- for (y = 0; y < height; ++y) {
- I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, &kYuvI601Constants, width);
- dst_rgb565 += dst_stride_rgb565;
- src_y += src_stride_y;
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- return 0;
+int I444ToP410(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return I4xxToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, 0, 0);
}
-// Ordered 8x8 dither for 888 to 565. Values from 0 to 7.
-static const uint8_t kDither565_4x4[16] = {
- 0, 4, 1, 5, 6, 2, 7, 3, 1, 5, 0, 4, 7, 3, 6, 2,
-};
-
-// Convert I420 to RGB565 with dithering.
-LIBYUV_API
-int I420ToRGB565Dither(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- const uint8_t* dither4x4,
- int width,
- int height) {
- int y;
- void (*I422ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToARGBRow_C;
- void (*ARGBToRGB565DitherRow)(const uint8_t* src_argb, uint8_t* dst_rgb,
- const uint32_t dither4, int width) =
- ARGBToRGB565DitherRow_C;
- if (!src_y || !src_u || !src_v || !dst_rgb565 || width <= 0 || height == 0) {
+static int I420ToPxxx(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int dst_uv_width,
+ int (*i4xx_to_pxxx)(const uint8_t*,
+ int,
+ const uint8_t*,
+ int,
+ const uint8_t*,
+ int,
+ uint16_t*,
+ int,
+ uint16_t*,
+ int,
+ int,
+ int)) {
+ int r;
+ int abs_height;
+ uint64_t uv_size;
+ uint64_t total;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_y || !dst_uv ||
+ width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
- // Negative height means invert the image.
+ abs_height = Abs(height);
if (height < 0) {
- height = -height;
- dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565;
- dst_stride_rgb565 = -dst_stride_rgb565;
- }
- if (!dither4x4) {
- dither4x4 = kDither565_4x4;
- }
-#if defined(HAS_I422TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToARGBRow = I422ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGBRow = I422ToARGBRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToARGBRow = I422ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToARGBRow = I422ToARGBRow_AVX2;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToARGBRow = I422ToARGBRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGBRow = I422ToARGBRow_NEON;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGBRow = I422ToARGBRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- I422ToARGBRow = I422ToARGBRow_MSA;
- }
- }
-#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2;
- }
- }
-#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
- if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2;
- }
+ const int src_uv_height = SUBSAMPLE(abs_height, 1, 1);
+ src_y = src_y + (ptrdiff_t)(abs_height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(src_uv_height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(src_uv_height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ height = abs_height;
}
-#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON;
- }
+ uv_size = (uint64_t)dst_uv_width * (uint64_t)abs_height;
+ total = uv_size * 2u;
+ if (total > SIZE_MAX) {
+ return 1;
}
-#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_MSA;
- }
+ align_buffer_64(plane_u, (size_t)total);
+ if (!plane_u) {
+ return 1;
}
-#endif
{
- // Allocate a row of argb.
- align_buffer_64(row_argb, width * 4);
- for (y = 0; y < height; ++y) {
- I422ToARGBRow(src_y, src_u, src_v, row_argb, &kYuvI601Constants, width);
- ARGBToRGB565DitherRow(row_argb, dst_rgb565,
- *(const uint32_t*)(dither4x4 + ((y & 3) << 2)),
- width);
- dst_rgb565 += dst_stride_rgb565;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- free_aligned_buffer_64(row_argb);
- }
- return 0;
-}
-
-// Convert I420 to AR30 with matrix
-static int I420ToAR30Matrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
- int y;
- void (*I422ToAR30Row)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToAR30Row_C;
-
- if (!src_y || !src_u || !src_v || !dst_ar30 || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
- dst_stride_ar30 = -dst_stride_ar30;
- }
-
-#if defined(HAS_I422TOAR30ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToAR30Row = I422ToAR30Row_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToAR30Row = I422ToAR30Row_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I422TOAR30ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToAR30Row = I422ToAR30Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToAR30Row = I422ToAR30Row_AVX2;
- }
- }
-#endif
-
- for (y = 0; y < height; ++y) {
- I422ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
- dst_ar30 += dst_stride_ar30;
- src_y += src_stride_y;
- if (y & 1) {
- src_u += src_stride_u;
- src_v += src_stride_v;
- }
- }
- return 0;
+ uint8_t* tmp_u = plane_u;
+ uint8_t* tmp_v = plane_u + uv_size;
+ r = I420ToI4xx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, NULL, 0, tmp_u, dst_uv_width, tmp_v,
+ dst_uv_width, width, height, dst_uv_width, abs_height);
+ if (!r) {
+ r = i4xx_to_pxxx(src_y, src_stride_y, tmp_u, dst_uv_width, tmp_v,
+ dst_uv_width, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height);
+ }
+ }
+ free_aligned_buffer_64(plane_u);
+ return r;
}
-// Convert I420 to AR30.
LIBYUV_API
-int I420ToAR30(const uint8_t* src_y,
+int I420ToP210(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height) {
- return I420ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_ar30, dst_stride_ar30,
- &kYuvI601Constants, width, height);
+ return I420ToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, (Abs(width) + 1) >> 1, I422ToP210);
}
-// Convert H420 to AR30.
LIBYUV_API
-int H420ToAR30(const uint8_t* src_y,
+int I420ToP410(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
int width,
int height) {
- return I420ToAR30Matrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_ar30, dst_stride_ar30,
- &kYvuH709Constants, width, height);
+ return I420ToPxxx(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ width, height, Abs(width), I444ToP410);
}
// Convert I420 to specified format
@@ -1350,9 +1036,11 @@ int ConvertFromI420(const uint8_t* y,
uint32_t fourcc) {
uint32_t format = CanonicalFourCC(fourcc);
int r = 0;
- if (!y || !u || !v || !dst_sample || width <= 0 || height == 0) {
+ if (!y || !u || !v || !dst_sample || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+ const int abs_height = Abs(height);
switch (format) {
// Single plane formats
case FOURCC_YUY2:
@@ -1421,7 +1109,8 @@ int ConvertFromI420(const uint8_t* y,
height);
break;
case FOURCC_NV12: {
- uint8_t* dst_uv = dst_sample + width * height;
+ int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
+ uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV12(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_uv,
dst_sample_stride ? dst_sample_stride : width, width,
@@ -1429,28 +1118,121 @@ int ConvertFromI420(const uint8_t* y,
break;
}
case FOURCC_NV21: {
- uint8_t* dst_vu = dst_sample + width * height;
+ int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
+ uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV21(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_vu,
dst_sample_stride ? dst_sample_stride : width, width,
height);
break;
}
- // TODO(fbarchard): Add M420.
+ case FOURCC_NV16: {
+ int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
+ int dst_uv_stride = dst_sample_stride ? dst_sample_stride : width;
+ uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
+ int halfwidth = (width + 1) / 2;
+ int halfheight = SUBSAMPLE(height, 1, 1);
+ CopyPlane(y, y_stride, dst_sample, dst_y_stride, width, height);
+ MergeUVPlane(u, u_stride, v, v_stride, dst_uv, dst_uv_stride * 2,
+ halfwidth, halfheight);
+ if (abs_height > 1) {
+ MergeUVPlane(u, u_stride, v, v_stride, dst_uv + dst_uv_stride,
+ dst_uv_stride * 2, halfwidth, height / 2);
+ }
+ r = 0;
+ break;
+ }
+ case FOURCC_NV24: {
+ int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
+ int dst_uv_stride = dst_sample_stride ? dst_sample_stride * 2 : width * 2;
+ uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
+ align_buffer_64(temp_u, (size_t)width * abs_height);
+ align_buffer_64(temp_v, (size_t)width * abs_height);
+ if (!temp_u || !temp_v) {
+ free_aligned_buffer_64(temp_u);
+ free_aligned_buffer_64(temp_v);
+ return 1;
+ }
+ r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample,
+ dst_y_stride, temp_u, width, temp_v, width, width, height);
+ if (r == 0) {
+ MergeUVPlane(temp_u, width, temp_v, width, dst_uv, dst_uv_stride, width,
+ abs_height);
+ }
+ free_aligned_buffer_64(temp_u);
+ free_aligned_buffer_64(temp_v);
+ break;
+ }
+ case FOURCC_P010:
+ case FOURCC_P210: {
+ const int uv_min = ((width + 1) / 2) * 2;
+ int dst_stride_y;
+ int dst_stride_uv;
+ uint16_t* dst_y16;
+ uint16_t* dst_uv16;
+ if (dst_sample_stride == 0) {
+ dst_stride_y = width;
+ dst_stride_uv = uv_min;
+ } else {
+ if (dst_sample_stride & 1) {
+ return -1;
+ }
+ dst_stride_y = dst_sample_stride / 2;
+ dst_stride_uv = dst_sample_stride / 2;
+ if (Abs(dst_stride_uv) < uv_min) {
+ return -1;
+ }
+ }
+ dst_y16 = (uint16_t*)dst_sample;
+ dst_uv16 = dst_y16 + (ptrdiff_t)dst_stride_y * abs_height;
+ if (format == FOURCC_P010) {
+ r = I420ToP010(y, y_stride, u, u_stride, v, v_stride, dst_y16,
+ dst_stride_y, dst_uv16, dst_stride_uv, width, height);
+ } else {
+ r = I420ToP210(y, y_stride, u, u_stride, v, v_stride, dst_y16,
+ dst_stride_y, dst_uv16, dst_stride_uv, width, height);
+ }
+ break;
+ }
+ case FOURCC_P410: {
+ const int uv_min = width * 2;
+ int dst_stride_y;
+ int dst_stride_uv;
+ uint16_t* dst_y16;
+ uint16_t* dst_uv16;
+ if (dst_sample_stride == 0) {
+ dst_stride_y = width;
+ dst_stride_uv = uv_min;
+ } else {
+ if (dst_sample_stride & 1) {
+ return -1;
+ }
+ dst_stride_y = dst_sample_stride / 2;
+ dst_stride_uv = dst_sample_stride;
+ if (Abs(dst_stride_uv) < uv_min) {
+ return -1;
+ }
+ }
+ dst_y16 = (uint16_t*)dst_sample;
+ dst_uv16 = dst_y16 + (ptrdiff_t)dst_stride_y * abs_height;
+ r = I420ToP410(y, y_stride, u, u_stride, v, v_stride, dst_y16,
+ dst_stride_y, dst_uv16, dst_stride_uv, width, height);
+ break;
+ }
// Triplanar formats
case FOURCC_I420:
case FOURCC_YV12: {
dst_sample_stride = dst_sample_stride ? dst_sample_stride : width;
int halfstride = (dst_sample_stride + 1) / 2;
- int halfheight = (height + 1) / 2;
+ int halfheight = (abs_height + 1) / 2;
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV12) {
- dst_v = dst_sample + dst_sample_stride * height;
- dst_u = dst_v + halfstride * halfheight;
+ dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_u = dst_v + (ptrdiff_t)halfstride * halfheight;
} else {
- dst_u = dst_sample + dst_sample_stride * height;
- dst_v = dst_u + halfstride * halfheight;
+ dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_v = dst_u + (ptrdiff_t)halfstride * halfheight;
}
r = I420Copy(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, halfstride, dst_v, halfstride,
@@ -1464,11 +1246,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV16) {
- dst_v = dst_sample + dst_sample_stride * height;
- dst_u = dst_v + halfstride * height;
+ dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_u = dst_v + (ptrdiff_t)halfstride * abs_height;
} else {
- dst_u = dst_sample + dst_sample_stride * height;
- dst_v = dst_u + halfstride * height;
+ dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_v = dst_u + (ptrdiff_t)halfstride * abs_height;
}
r = I420ToI422(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, halfstride, dst_v, halfstride,
@@ -1481,11 +1263,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV24) {
- dst_v = dst_sample + dst_sample_stride * height;
- dst_u = dst_v + dst_sample_stride * height;
+ dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_u = dst_v + (ptrdiff_t)dst_sample_stride * abs_height;
} else {
- dst_u = dst_sample + dst_sample_stride * height;
- dst_v = dst_u + dst_sample_stride * height;
+ dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
+ dst_v = dst_u + (ptrdiff_t)dst_sample_stride * abs_height;
}
r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, dst_sample_stride, dst_v,
diff --git a/libs/libyuv/source/convert_from_argb.cc b/libs/libyuv/source/convert_from_argb.cc
index b7b5643a18..8f71a0ad32 100644
--- a/libs/libyuv/source/convert_from_argb.cc
+++ b/libs/libyuv/source/convert_from_argb.cc
@@ -10,7 +10,11 @@
#include "libyuv/convert_from_argb.h"
-#include "libyuv/basic_types.h"
+#include <limits.h>
+#include <stdint.h>
+#include <string.h> // For memcpy.
+
+#include "libyuv/convert.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/row.h"
@@ -32,102 +36,201 @@ int ARGBToI444(const uint8_t* src_argb,
int dst_stride_v,
int width,
int height) {
+ return ARGBToI444Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbI601Constants, width, height);
+}
+
+LIBYUV_API
+int ARGBToI444Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- void (*ARGBToUV444Row)(const uint8_t* src_argb, uint8_t* dst_u,
- uint8_t* dst_v, int width) = ARGBToUV444Row_C;
- if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUV444MatrixRow)(
+ const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = ARGBToUV444MatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
}
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
}
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_y == width &&
- dst_stride_u == width && dst_stride_v == width) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
}
-#if defined(HAS_ARGBTOUV444ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUV444Row = ARGBToUV444Row_Any_SSSE3;
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUV444Row = ARGBToUV444Row_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOUV444ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUV444Row = ARGBToUV444Row_Any_NEON;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToUV444Row = ARGBToUV444Row_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUV444ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUV444Row = ARGBToUV444Row_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
- ARGBToUV444Row = ARGBToUV444Row_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
}
#endif
-#if defined(HAS_ARGBTOUV444ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUV444Row = ARGBToUV444Row_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToUV444Row = ARGBToUV444Row_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_SSSE3)
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2)
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+#if defined(HAS_ARGBTOUV444MATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD;
}
}
#endif
+ if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
for (y = 0; y < height; ++y) {
- ARGBToUV444Row(src_argb, dst_u, dst_v, width);
- ARGBToYRow(src_argb, dst_y, width);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ ARGBToUV444MatrixRow(src_argb, dst_u, dst_v, width, argbconstants);
src_argb += src_stride_argb;
dst_y += dst_stride_y;
dst_u += dst_stride_u;
@@ -148,102 +251,216 @@ int ARGBToI422(const uint8_t* src_argb,
int dst_stride_v,
int width,
int height) {
+ return ARGBToI422Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbI601Constants, width, height);
+}
+
+LIBYUV_API
+int ARGBToI422Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_y == width &&
- dst_stride_u * 2 == width && dst_stride_v * 2 == width) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0;
- }
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
}
}
#endif
-
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
}
}
#endif
+ if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
for (y = 0; y < height; ++y) {
- ARGBToUVRow(src_argb, 0, dst_u, dst_v, width);
- ARGBToYRow(src_argb, dst_y, width);
+ ARGBToUVMatrixRow(src_argb, 0, dst_u, dst_v, width, argbconstants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
src_argb += src_stride_argb;
dst_y += dst_stride_y;
dst_u += dst_stride_u;
@@ -261,1197 +478,3345 @@ int ARGBToNV12(const uint8_t* src_argb,
int dst_stride_uv,
int width,
int height) {
+ return ARGBToNV12Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToNV12Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
int halfwidth = (width + 1) >> 1;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- void (*MergeUVRow_)(const uint8_t* src_u, const uint8_t* src_v,
- uint8_t* dst_uv, int width) = MergeUVRow_C;
- if (!src_argb || !dst_y || !dst_uv || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
- }
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
}
#endif
-#if defined(HAS_MERGEUVROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- MergeUVRow_ = MergeUVRow_Any_SSE2;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_SSE2;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
}
#endif
-#if defined(HAS_MERGEUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- MergeUVRow_ = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(halfwidth, 32)) {
- MergeUVRow_ = MergeUVRow_AVX2;
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_MERGEUVROW_NEON)
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeUVRow_ = MergeUVRow_Any_NEON;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_NEON;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow_ = MergeUVRow_Any_MSA;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow_ = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- MergeUVRow_ = MergeUVRow_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
}
}
#endif
- {
- // Allocate a rows of uv.
- align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
- uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
-
- for (y = 0; y < height - 1; y += 2) {
- ARGBToUVRow(src_argb, src_stride_argb, row_u, row_v, width);
- MergeUVRow_(row_u, row_v, dst_uv, halfwidth);
- ARGBToYRow(src_argb, dst_y, width);
- ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width);
- src_argb += src_stride_argb * 2;
- dst_y += dst_stride_y * 2;
- dst_uv += dst_stride_uv;
- }
- if (height & 1) {
- ARGBToUVRow(src_argb, 0, row_u, row_v, width);
- MergeUVRow_(row_u, row_v, dst_uv, halfwidth);
- ARGBToYRow(src_argb, dst_y, width);
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
}
- free_aligned_buffer_64(row_u);
- }
- return 0;
-}
-
-// Same as NV12 but U and V swapped.
-LIBYUV_API
-int ARGBToNV21(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
- int y;
- int halfwidth = (width + 1) >> 1;
- void (*ARGBToUVRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- void (*MergeUVRow_)(const uint8_t* src_u, const uint8_t* src_v,
- uint8_t* dst_vu, int width) = MergeUVRow_C;
- if (!src_argb || !dst_y || !dst_vu || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
}
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
#if defined(HAS_MERGEUVROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
- MergeUVRow_ = MergeUVRow_Any_SSE2;
+ MergeUVRow = MergeUVRow_Any_SSE2;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_SSE2;
+ MergeUVRow = MergeUVRow_SSE2;
}
}
#endif
#if defined(HAS_MERGEUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- MergeUVRow_ = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(halfwidth, 32)) {
- MergeUVRow_ = MergeUVRow_AVX2;
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
#if defined(HAS_MERGEUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeUVRow_ = MergeUVRow_Any_NEON;
+ MergeUVRow = MergeUVRow_Any_NEON;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_NEON;
+ MergeUVRow = MergeUVRow_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow_ = MergeUVRow_Any_MSA;
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_MSA;
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow_ = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- MergeUVRow_ = MergeUVRow_MMI;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
}
}
#endif
- {
- // Allocate a rows of uv.
- align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
- uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
-
- for (y = 0; y < height - 1; y += 2) {
- ARGBToUVRow(src_argb, src_stride_argb, row_u, row_v, width);
- MergeUVRow_(row_v, row_u, dst_vu, halfwidth);
- ARGBToYRow(src_argb, dst_y, width);
- ARGBToYRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width);
- src_argb += src_stride_argb * 2;
- dst_y += dst_stride_y * 2;
- dst_vu += dst_stride_vu;
- }
- if (height & 1) {
- ARGBToUVRow(src_argb, 0, row_u, row_v, width);
- MergeUVRow_(row_v, row_u, dst_vu, halfwidth);
- ARGBToYRow(src_argb, dst_y, width);
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
}
- free_aligned_buffer_64(row_u);
}
- return 0;
-}
+#endif
+
+ // Allocate a rows of uv.
+ align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
+ uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
+ if (!row_u)
+ return 1;
+
+ for (y = 0; y < height - 1; y += 2) {
+ ARGBToUVMatrixRow(src_argb, src_stride_argb, row_u, row_v, width,
+ argbconstants);
+ MergeUVRow(row_u, row_v, dst_uv, halfwidth);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ ARGBToYMatrixRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width,
+ argbconstants);
+ src_argb += src_stride_argb * 2;
+ dst_y += dst_stride_y * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, argbconstants);
+ MergeUVRow(row_u, row_v, dst_uv, halfwidth);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ }
+ free_aligned_buffer_64(row_u);
+ return 0;
+}
LIBYUV_API
-int ABGRToNV12(const uint8_t* src_abgr,
- int src_stride_abgr,
+int ARGBToNV16(const uint8_t* src_argb,
+ int src_stride_argb,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_uv,
int dst_stride_uv,
int width,
int height) {
+ return ARGBToNV16Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToNV16Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- int halfwidth = (width + 1) >> 1;
- void (*ABGRToUVRow)(const uint8_t* src_abgr0, int src_stride_abgr,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ABGRToUVRow_C;
- void (*ABGRToYRow)(const uint8_t* src_abgr, uint8_t* dst_y, int width) =
- ABGRToYRow_C;
- void (*MergeUVRow_)(const uint8_t* src_u, const uint8_t* src_v,
- uint8_t* dst_uv, int width) = MergeUVRow_C;
- if (!src_abgr || !dst_y || !dst_uv || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_abgr = src_abgr + (height - 1) * src_stride_abgr;
- src_stride_abgr = -src_stride_abgr;
- }
-#if defined(HAS_ABGRTOYROW_SSSE3) && defined(HAS_ABGRTOUVROW_SSSE3)
+ int halfwidth = (width >> 1) + (width & 1);
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ABGRToUVRow = ABGRToUVRow_Any_SSSE3;
- ABGRToYRow = ABGRToYRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_SSSE3;
- ABGRToYRow = ABGRToYRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_AVX2) && defined(HAS_ABGRTOUVROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ABGRToUVRow = ABGRToUVRow_Any_AVX2;
- ABGRToYRow = ABGRToYRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ABGRToUVRow = ABGRToUVRow_AVX2;
- ABGRToYRow = ABGRToYRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToYRow = ABGRToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_NEON;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ABGRTOUVROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToUVRow = ABGRToUVRow_Any_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToYRow = ABGRToYRow_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ABGRToYRow = ABGRToYRow_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ABGRTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToUVRow = ABGRToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ABGRToUVRow = ABGRToUVRow_MSA;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
-#if defined(HAS_ABGRTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToYRow = ABGRToYRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ABGRTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToUVRow = ABGRToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
}
#endif
-#if defined(HAS_MERGEUVROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- MergeUVRow_ = MergeUVRow_Any_SSE2;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_SSE2;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
}
#endif
-#if defined(HAS_MERGEUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- MergeUVRow_ = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(halfwidth, 32)) {
- MergeUVRow_ = MergeUVRow_AVX2;
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_MERGEUVROW_NEON)
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeUVRow_ = MergeUVRow_Any_NEON;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_NEON;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow_ = MergeUVRow_Any_MSA;
- if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_MSA;
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow_ = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- MergeUVRow_ = MergeUVRow_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
}
}
#endif
- {
- // Allocate a rows of uv.
- align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
- uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
-
- for (y = 0; y < height - 1; y += 2) {
- ABGRToUVRow(src_abgr, src_stride_abgr, row_u, row_v, width);
- MergeUVRow_(row_u, row_v, dst_uv, halfwidth);
- ABGRToYRow(src_abgr, dst_y, width);
- ABGRToYRow(src_abgr + src_stride_abgr, dst_y + dst_stride_y, width);
- src_abgr += src_stride_abgr * 2;
- dst_y += dst_stride_y * 2;
- dst_uv += dst_stride_uv;
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
}
- if (height & 1) {
- ABGRToUVRow(src_abgr, 0, row_u, row_v, width);
- MergeUVRow_(row_u, row_v, dst_uv, halfwidth);
- ABGRToYRow(src_abgr, dst_y, width);
- }
- free_aligned_buffer_64(row_u);
- }
- return 0;
-}
-
-// Same as NV12 but U and V swapped.
-LIBYUV_API
-int ABGRToNV21(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
- int y;
- int halfwidth = (width + 1) >> 1;
- void (*ABGRToUVRow)(const uint8_t* src_abgr0, int src_stride_abgr,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ABGRToUVRow_C;
- void (*ABGRToYRow)(const uint8_t* src_abgr, uint8_t* dst_y, int width) =
- ABGRToYRow_C;
- void (*MergeUVRow_)(const uint8_t* src_u, const uint8_t* src_v,
- uint8_t* dst_vu, int width) = MergeUVRow_C;
- if (!src_abgr || !dst_y || !dst_vu || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_abgr = src_abgr + (height - 1) * src_stride_abgr;
- src_stride_abgr = -src_stride_abgr;
}
-#if defined(HAS_ABGRTOYROW_SSSE3) && defined(HAS_ABGRTOUVROW_SSSE3)
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ABGRToUVRow = ABGRToUVRow_Any_SSSE3;
- ABGRToYRow = ABGRToYRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_SSSE3;
- ABGRToYRow = ABGRToYRow_SSSE3;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_AVX2) && defined(HAS_ABGRTOUVROW_AVX2)
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ABGRToUVRow = ABGRToUVRow_Any_AVX2;
- ABGRToYRow = ABGRToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ABGRToUVRow = ABGRToUVRow_AVX2;
- ABGRToYRow = ABGRToYRow_AVX2;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ABGRTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToYRow = ABGRToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_NEON;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_ABGRTOUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ABGRToUVRow = ABGRToUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_NEON;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
}
#endif
-#if defined(HAS_ABGRTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToYRow = ABGRToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ABGRToYRow = ABGRToYRow_MSA;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ABGRTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ABGRToUVRow = ABGRToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ABGRToUVRow = ABGRToUVRow_MSA;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
}
#endif
-#if defined(HAS_ABGRTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToYRow = ABGRToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ABGRToYRow = ABGRToYRow_MMI;
- }
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
}
#endif
-#if defined(HAS_ABGRTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ABGRToUVRow = ABGRToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ABGRToUVRow = ABGRToUVRow_MMI;
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
#if defined(HAS_MERGEUVROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
- MergeUVRow_ = MergeUVRow_Any_SSE2;
+ MergeUVRow = MergeUVRow_Any_SSE2;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_SSE2;
+ MergeUVRow = MergeUVRow_SSE2;
}
}
#endif
#if defined(HAS_MERGEUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- MergeUVRow_ = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(halfwidth, 32)) {
- MergeUVRow_ = MergeUVRow_AVX2;
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
#if defined(HAS_MERGEUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeUVRow_ = MergeUVRow_Any_NEON;
+ MergeUVRow = MergeUVRow_Any_NEON;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_NEON;
+ MergeUVRow = MergeUVRow_NEON;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow_ = MergeUVRow_Any_MSA;
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
if (IS_ALIGNED(halfwidth, 16)) {
- MergeUVRow_ = MergeUVRow_MSA;
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow_ = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- MergeUVRow_ = MergeUVRow_MMI;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
}
}
#endif
- {
- // Allocate a rows of uv.
- align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
- uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
-
- for (y = 0; y < height - 1; y += 2) {
- ABGRToUVRow(src_abgr, src_stride_abgr, row_u, row_v, width);
- MergeUVRow_(row_v, row_u, dst_vu, halfwidth);
- ABGRToYRow(src_abgr, dst_y, width);
- ABGRToYRow(src_abgr + src_stride_abgr, dst_y + dst_stride_y, width);
- src_abgr += src_stride_abgr * 2;
- dst_y += dst_stride_y * 2;
- dst_vu += dst_stride_vu;
- }
- if (height & 1) {
- ABGRToUVRow(src_abgr, 0, row_u, row_v, width);
- MergeUVRow_(row_v, row_u, dst_vu, halfwidth);
- ABGRToYRow(src_abgr, dst_y, width);
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
}
- free_aligned_buffer_64(row_u);
}
+#endif
+
+ size_t row_size =
+ (static_cast<size_t>(halfwidth) + 31) & ~static_cast<size_t>(31);
+ if (row_size > SIZE_MAX / 2) {
+ return -1;
+ }
+ align_buffer_64(row_u, row_size * 2);
+ if (!row_u)
+ return 1;
+ uint8_t* row_v = row_u + row_size;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, argbconstants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ MergeUVRow(row_u, row_v, dst_uv, halfwidth);
+ src_argb += src_stride_argb;
+ dst_y += dst_stride_y;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row_u);
return 0;
}
-// Convert ARGB to YUY2.
LIBYUV_API
-int ARGBToYUY2(const uint8_t* src_argb,
+int ARGBToNV24(const uint8_t* src_argb,
int src_stride_argb,
- uint8_t* dst_yuy2,
- int dst_stride_yuy2,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
int width,
int height) {
+ return ARGBToNV24Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToNV24Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
int y;
- void (*ARGBToUVRow)(const uint8_t* src_argb, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- void (*I422ToYUY2Row)(const uint8_t* src_y, const uint8_t* src_u,
- const uint8_t* src_v, uint8_t* dst_yuy2, int width) =
- I422ToYUY2Row_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUV444MatrixRow)(
+ const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = ARGBToUV444MatrixRow_C;
- if (!src_argb || !dst_yuy2 || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_yuy2 = dst_yuy2 + (height - 1) * dst_stride_yuy2;
- dst_stride_yuy2 = -dst_stride_yuy2;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_yuy2 == width * 2) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_yuy2 = 0;
- }
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_SSE2;
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- I422ToYUY2Row = I422ToYUY2Row_SSE2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_AVX2)
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_AVX2;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- I422ToYUY2Row = I422ToYUY2Row_AVX2;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- I422ToYUY2Row = I422ToYUY2Row_NEON;
- }
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_I422TOYUY2ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- I422ToYUY2Row = I422ToYUY2Row_MSA;
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON;
}
}
#endif
-#if defined(HAS_I422TOYUY2ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I422ToYUY2Row = I422ToYUY2Row_Any_MMI;
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 8)) {
- I422ToYUY2Row = I422ToYUY2Row_MMI;
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM;
}
}
#endif
-
- {
- // Allocate a rows of yuv.
- align_buffer_64(row_y, ((width + 63) & ~63) * 2);
- uint8_t* row_u = row_y + ((width + 63) & ~63);
- uint8_t* row_v = row_u + ((width + 63) & ~63) / 2;
-
- for (y = 0; y < height; ++y) {
- ARGBToUVRow(src_argb, 0, row_u, row_v, width);
- ARGBToYRow(src_argb, row_y, width);
- I422ToYUY2Row(row_y, row_u, row_v, dst_yuy2, width);
- src_argb += src_stride_argb;
- dst_yuy2 += dst_stride_yuy2;
+#if defined(HAS_ARGBTOUV444MATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD;
}
-
- free_aligned_buffer_64(row_y);
}
- return 0;
-}
-
-// Convert ARGB to UYVY.
-LIBYUV_API
-int ARGBToUYVY(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_uyvy,
- int dst_stride_uyvy,
- int width,
- int height) {
- int y;
- void (*ARGBToUVRow)(const uint8_t* src_argb, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVRow_C;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- void (*I422ToUYVYRow)(const uint8_t* src_y, const uint8_t* src_u,
- const uint8_t* src_v, uint8_t* dst_uyvy, int width) =
- I422ToUYVYRow_C;
-
- if (!src_argb || !dst_uyvy || width <= 0 || height == 0) {
+#endif
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_uyvy = dst_uyvy + (height - 1) * dst_stride_uyvy;
- dst_stride_uyvy = -dst_stride_uyvy;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_uyvy == width * 2) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_uyvy = 0;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
}
-#if defined(HAS_ARGBTOYROW_SSSE3) && defined(HAS_ARGBTOUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- ARGBToYRow = ARGBToYRow_SSSE3;
+ MergeUVRow = MergeUVRow_SSE2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2) && defined(HAS_ARGBTOUVROW_AVX2)
+#if defined(HAS_MERGEUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToUVRow = ARGBToUVRow_Any_AVX2;
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_AVX2;
- ARGBToYRow = ARGBToYRow_AVX2;
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
- }
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
}
#endif
-#if defined(HAS_ARGBTOUVROW_NEON)
+#if defined(HAS_MERGEUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVRow = ARGBToUVRow_Any_NEON;
+ MergeUVRow = MergeUVRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_NEON;
+ MergeUVRow = MergeUVRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVRow = ARGBToUVRow_Any_MSA;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVRow = ARGBToUVRow_MSA;
+ MergeUVRow = MergeUVRow_LASX;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
- }
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
}
#endif
-#if defined(HAS_ARGBTOUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVRow = ARGBToUVRow_Any_MMI;
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_MMI;
+ MergeUVRow = MergeUVRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_SSE2;
+
+ size_t row_size =
+ (static_cast<size_t>(width) + 31) & ~static_cast<size_t>(31);
+ if (row_size > SIZE_MAX / 2) {
+ return -1;
+ }
+ align_buffer_64(row_u, row_size * 2);
+ if (!row_u)
+ return 1;
+ uint8_t* row_v = row_u + row_size;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUV444MatrixRow(src_argb, row_u, row_v, width, argbconstants);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ MergeUVRow(row_u, row_v, dst_uv, width);
+ src_argb += src_stride_argb;
+ dst_y += dst_stride_y;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row_u);
+ return 0;
+}
+
+LIBYUV_API
+int ARGBToNV21Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ int y;
+ int halfwidth = (width + 1) >> 1;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- I422ToUYVYRow = I422ToUYVYRow_SSE2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- I422ToUYVYRow = I422ToUYVYRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- I422ToUYVYRow = I422ToUYVYRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- I422ToUYVYRow = I422ToUYVYRow_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_I422TOUYVYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- I422ToUYVYRow = I422ToUYVYRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- I422ToUYVYRow = I422ToUYVYRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-
- {
- // Allocate a rows of yuv.
- align_buffer_64(row_y, ((width + 63) & ~63) * 2);
- uint8_t* row_u = row_y + ((width + 63) & ~63);
- uint8_t* row_v = row_u + ((width + 63) & ~63) / 2;
-
- for (y = 0; y < height; ++y) {
- ARGBToUVRow(src_argb, 0, row_u, row_v, width);
- ARGBToYRow(src_argb, row_y, width);
- I422ToUYVYRow(row_y, row_u, row_v, dst_uyvy, width);
- src_argb += src_stride_argb;
- dst_uyvy += dst_stride_uyvy;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
-
- free_aligned_buffer_64(row_y);
}
- return 0;
-}
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
-// Convert ARGB to I400.
-LIBYUV_API
-int ARGBToI400(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_y,
- int dst_stride_y,
- int width,
- int height) {
- int y;
- void (*ARGBToYRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
- ARGBToYRow_C;
- if (!src_argb || !dst_y || width <= 0 || height == 0) {
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_vu, int width) = MergeUVRow_C;
+ if (!src_argb || !dst_y || !dst_vu || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
+ // Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_y == width) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_y = 0;
- }
-#if defined(HAS_ARGBTOYROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToYRow = ARGBToYRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_SSSE3;
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_AVX2)
+#if defined(HAS_MERGEUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYRow = ARGBToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToYRow = ARGBToYRow_AVX2;
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_NEON)
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYRow = ARGBToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_NEON;
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYRow = ARGBToYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYRow = ARGBToYRow_MSA;
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYRow = ARGBToYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYRow = ARGBToYRow_MMI;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
}
}
#endif
-
- for (y = 0; y < height; ++y) {
- ARGBToYRow(src_argb, dst_y, width);
- src_argb += src_stride_argb;
- dst_y += dst_stride_y;
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
}
- return 0;
-}
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
-// Shuffle table for converting ARGB to RGBA.
-static const uvec8 kShuffleMaskARGBToRGBA = {
- 3u, 0u, 1u, 2u, 7u, 4u, 5u, 6u, 11u, 8u, 9u, 10u, 15u, 12u, 13u, 14u};
+ // Allocate a rows of uv.
+ align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
+ uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
+ if (!row_u)
+ return 1;
-// Convert ARGB to RGBA.
-LIBYUV_API
-int ARGBToRGBA(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- int width,
- int height) {
- return ARGBShuffle(src_argb, src_stride_argb, dst_rgba, dst_stride_rgba,
- (const uint8_t*)(&kShuffleMaskARGBToRGBA), width, height);
+ for (y = 0; y < height - 1; y += 2) {
+ ARGBToUVMatrixRow(src_argb, src_stride_argb, row_u, row_v, width,
+ argbconstants);
+ MergeUVRow(row_v, row_u, dst_vu, halfwidth);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ ARGBToYMatrixRow(src_argb + src_stride_argb, dst_y + dst_stride_y, width,
+ argbconstants);
+ src_argb += src_stride_argb * 2;
+ dst_y += dst_stride_y * 2;
+ dst_vu += dst_stride_vu;
+ }
+ if (height & 1) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, argbconstants);
+ MergeUVRow(row_v, row_u, dst_vu, halfwidth);
+ ARGBToYMatrixRow(src_argb, dst_y, width, argbconstants);
+ }
+ free_aligned_buffer_64(row_u);
+ return 0;
}
-
-// Convert ARGB To RGB24.
LIBYUV_API
-int ARGBToRGB24(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_rgb24,
- int dst_stride_rgb24,
- int width,
- int height) {
+int ARGBToI400Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ const struct ArgbConstants* constants,
+ int width,
+ int height) {
int y;
- void (*ARGBToRGB24Row)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
- ARGBToRGB24Row_C;
- if (!src_argb || !dst_rgb24 || width <= 0 || height == 0) {
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ if (!src_argb || !dst_y || !constants || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_rgb24 == width * 3) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_rgb24 = 0;
- }
-#if defined(HAS_ARGBTORGB24ROW_SSSE3)
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToRGB24Row = ARGBToRGB24Row_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTORGB24ROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToRGB24Row = ARGBToRGB24Row_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTORGB24ROW_AVX512VBMI)
- if (TestCpuFlag(kCpuHasAVX512VBMI)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_AVX512VBMI;
- if (IS_ALIGNED(width, 32)) {
- ARGBToRGB24Row = ARGBToRGB24Row_AVX512VBMI;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_ARGBTORGB24ROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToRGB24Row = ARGBToRGB24Row_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTORGB24ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToRGB24Row = ARGBToRGB24Row_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTORGB24ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToRGB24Row = ARGBToRGB24Row_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB24Row = ARGBToRGB24Row_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
- ARGBToRGB24Row(src_argb, dst_rgb24, width);
+ ARGBToYMatrixRow(src_argb, dst_y, width, constants);
src_argb += src_stride_argb;
- dst_rgb24 += dst_stride_rgb24;
+ dst_y += dst_stride_y;
}
return 0;
}
-
-// Convert ARGB To RAW.
LIBYUV_API
-int ARGBToRAW(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_raw,
- int dst_stride_raw,
- int width,
- int height) {
+int ARGBToYUY2Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ const struct ArgbConstants* constants,
+ int width,
+ int height) {
int y;
- void (*ARGBToRAWRow)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
- ARGBToRAWRow_C;
- if (!src_argb || !dst_raw || width <= 0 || height == 0) {
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*I422ToYUY2Row)(const uint8_t* src_y, const uint8_t* src_u,
+ const uint8_t* src_v, uint8_t* dst_yuy2, int width) =
+ I422ToYUY2Row_C;
+
+ if (!src_argb || !dst_yuy2 || !constants || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_raw == width * 3) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_raw = 0;
+ dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
+ dst_stride_yuy2 = -dst_stride_yuy2;
}
-#if defined(HAS_ARGBTORAWROW_SSSE3)
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToRAWRow = ARGBToRAWRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToRAWRow = ARGBToRAWRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTORAWROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToRAWRow = ARGBToRAWRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToRAWRow = ARGBToRAWRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTORAWROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToRAWRow = ARGBToRAWRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToRAWRow = ARGBToRAWRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTORAWROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToRAWRow = ARGBToRAWRow_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToRAWRow = ARGBToRAWRow_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTORAWROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToRAWRow = ARGBToRAWRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRAWRow = ARGBToRAWRow_MMI;
- }
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
-
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_I422TOYUY2ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ I422ToYUY2Row = I422ToYUY2Row_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToYUY2Row = I422ToYUY2Row_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOYUY2ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToYUY2Row = I422ToYUY2Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ I422ToYUY2Row = I422ToYUY2Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOYUY2ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToYUY2Row = I422ToYUY2Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToYUY2Row = I422ToYUY2Row_NEON;
+ }
+ }
+#endif
+
+ {
+ align_buffer_64(row_y, ((width + 63) & ~63) * 2);
+ uint8_t* row_u = row_y + ((width + 63) & ~63);
+ uint8_t* row_v = row_u + ((width + 63) & ~63) / 2;
+ if (!row_y)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, constants);
+ ARGBToYMatrixRow(src_argb, row_y, width, constants);
+ I422ToYUY2Row(row_y, row_u, row_v, dst_yuy2, width);
+ src_argb += src_stride_argb;
+ dst_yuy2 += dst_stride_yuy2;
+ }
+
+ free_aligned_buffer_64(row_y);
+ }
+ return 0;
+}
+
+LIBYUV_API
+int ARGBToUYVYMatrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_uyvy,
+ int dst_stride_uyvy,
+ const struct ArgbConstants* constants,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*I422ToUYVYRow)(const uint8_t* src_y, const uint8_t* src_u,
+ const uint8_t* src_v, uint8_t* dst_uyvy, int width) =
+ I422ToUYVYRow_C;
+
+ if (!src_argb || !dst_uyvy || !constants || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy;
+ dst_stride_uyvy = -dst_stride_uyvy;
+ }
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_I422TOUYVYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToUYVYRow = I422ToUYVYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOUYVYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ I422ToUYVYRow = I422ToUYVYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I422TOUYVYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToUYVYRow = I422ToUYVYRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ I422ToUYVYRow = I422ToUYVYRow_NEON;
+ }
+ }
+#endif
+
+ {
+ align_buffer_64(row_y, ((width + 63) & ~63) * 2);
+ uint8_t* row_u = row_y + ((width + 63) & ~63);
+ uint8_t* row_v = row_u + ((width + 63) & ~63) / 2;
+ if (!row_y)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, constants);
+ ARGBToYMatrixRow(src_argb, row_y, width, constants);
+ I422ToUYVYRow(row_y, row_u, row_v, dst_uyvy, width);
+ src_argb += src_stride_argb;
+ dst_uyvy += dst_stride_uyvy;
+ }
+
+ free_aligned_buffer_64(row_y);
+ }
+ return 0;
+}
+
+// Same as NV12 but U and V swapped.
+LIBYUV_API
+int ARGBToNV21(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return ARGBToNV21Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_vu, dst_stride_vu, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToP010(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return ARGBToP010Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToP010Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ width > (INT_MAX / 2) || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ int y;
+ int halfwidth = (width >> 1) + (width & 1);
+ void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits,
+ int width) = Convert8To16Row_C;
+ void (*MultiplyRow_16)(const uint16_t* src_y, uint16_t* dst_y, int scale,
+ int width) = MultiplyRow_16_C;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+#if defined(HAS_CONVERT8TO16ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Convert8To16Row = Convert8To16Row_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert8To16Row = Convert8To16Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To16Row = Convert8To16Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To16Row = Convert8To16Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To16Row = Convert8To16Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To16Row = Convert8To16Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert8To16Row = Convert8To16Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To16Row = Convert8To16Row_HVX;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Convert8To16Row = Convert8To16Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MultiplyRow_16 = MultiplyRow_16_SME;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MultiplyRow_16 = MultiplyRow_16_RVV;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MultiplyRow_16 = MultiplyRow_16_HVX;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
+
+ size_t row_y_size =
+ (static_cast<size_t>(width) + 31) & ~static_cast<size_t>(31);
+ size_t row_uv_size =
+ (static_cast<size_t>(halfwidth) + 31) & ~static_cast<size_t>(31);
+ size_t row_uv_merged_size =
+ (static_cast<size_t>(halfwidth) * 2 + 31) & ~static_cast<size_t>(31);
+ if (row_uv_size > (SIZE_MAX - row_y_size) / 2) {
+ return -1;
+ }
+ size_t row_size = row_y_size + row_uv_size * 2;
+ if (row_uv_merged_size > SIZE_MAX - row_size) {
+ return -1;
+ }
+ align_buffer_64(row, row_size + row_uv_merged_size);
+ if (!row)
+ return 1;
+ uint8_t* row_y = row;
+ uint8_t* row_u = row_y + row_y_size;
+ uint8_t* row_v = row_u + row_uv_size;
+ uint8_t* row_uv = row_v + row_uv_size;
+
+ for (y = 0; y < height - 1; y += 2) {
+ ARGBToUVMatrixRow(src_argb, src_stride_argb, row_u, row_v, width,
+ argbconstants);
+ MergeUVRow(row_u, row_v, row_uv, halfwidth);
+ Convert8To16Row(row_uv, dst_uv, 10, halfwidth * 2);
+ MultiplyRow_16(dst_uv, dst_uv, 64, halfwidth * 2);
+ ARGBToYMatrixRow(src_argb, row_y, width, argbconstants);
+ Convert8To16Row(row_y, dst_y, 10, width);
+ MultiplyRow_16(dst_y, dst_y, 64, width);
+ ARGBToYMatrixRow(src_argb + src_stride_argb, row_y, width, argbconstants);
+ Convert8To16Row(row_y, dst_y + dst_stride_y, 10, width);
+ MultiplyRow_16(dst_y + dst_stride_y, dst_y + dst_stride_y, 64, width);
+ src_argb += src_stride_argb * 2;
+ dst_y += dst_stride_y * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, argbconstants);
+ MergeUVRow(row_u, row_v, row_uv, halfwidth);
+ Convert8To16Row(row_uv, dst_uv, 10, halfwidth * 2);
+ MultiplyRow_16(dst_uv, dst_uv, 64, halfwidth * 2);
+ ARGBToYMatrixRow(src_argb, row_y, width, argbconstants);
+ Convert8To16Row(row_y, dst_y, 10, width);
+ MultiplyRow_16(dst_y, dst_y, 64, width);
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+LIBYUV_API
+int ARGBToP210(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return ARGBToP210Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToP210Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ width > (INT_MAX / 2) || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ int y;
+ int halfwidth = (width >> 1) + (width & 1);
+ void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits,
+ int width) = Convert8To16Row_C;
+ void (*MultiplyRow_16)(const uint16_t* src_y, uint16_t* dst_y, int scale,
+ int width) = MultiplyRow_16_C;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) =
+ ARGBToUVMatrixRow_C;
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+#if defined(HAS_CONVERT8TO16ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Convert8To16Row = Convert8To16Row_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert8To16Row = Convert8To16Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To16Row = Convert8To16Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To16Row = Convert8To16Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To16Row = Convert8To16Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To16Row = Convert8To16Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert8To16Row = Convert8To16Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To16Row = Convert8To16Row_HVX;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Convert8To16Row = Convert8To16Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MultiplyRow_16 = MultiplyRow_16_SME;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MultiplyRow_16 = MultiplyRow_16_RVV;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MultiplyRow_16 = MultiplyRow_16_HVX;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
+
+ size_t row_y_size =
+ (static_cast<size_t>(width) + 31) & ~static_cast<size_t>(31);
+ size_t row_uv_size =
+ (static_cast<size_t>(halfwidth) + 31) & ~static_cast<size_t>(31);
+ size_t row_uv_merged_size =
+ (static_cast<size_t>(halfwidth) * 2 + 31) & ~static_cast<size_t>(31);
+ if (row_uv_size > (SIZE_MAX - row_y_size) / 2) {
+ return -1;
+ }
+ size_t row_size = row_y_size + row_uv_size * 2;
+ if (row_uv_merged_size > SIZE_MAX - row_size) {
+ return -1;
+ }
+ align_buffer_64(row, row_size + row_uv_merged_size);
+ if (!row)
+ return 1;
+ uint8_t* row_y = row;
+ uint8_t* row_u = row_y + row_y_size;
+ uint8_t* row_v = row_u + row_uv_size;
+ uint8_t* row_uv = row_v + row_uv_size;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUVMatrixRow(src_argb, 0, row_u, row_v, width, argbconstants);
+ MergeUVRow(row_u, row_v, row_uv, halfwidth);
+ Convert8To16Row(row_uv, dst_uv, 10, halfwidth * 2);
+ MultiplyRow_16(dst_uv, dst_uv, 64, halfwidth * 2);
+ ARGBToYMatrixRow(src_argb, row_y, width, argbconstants);
+ Convert8To16Row(row_y, dst_y, 10, width);
+ MultiplyRow_16(dst_y, dst_y, 64, width);
+ src_argb += src_stride_argb;
+ dst_y += dst_stride_y;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+LIBYUV_API
+int ARGBToP410(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return ARGBToP410Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kArgbI601Constants, width,
+ height);
+}
+
+LIBYUV_API
+int ARGBToP410Matrix(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ if (!src_argb || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ width > (INT_MAX / 2) || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ int y;
+ void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits,
+ int width) = Convert8To16Row_C;
+ void (*MultiplyRow_16)(const uint16_t* src_y, uint16_t* dst_y, int scale,
+ int width) = MultiplyRow_16_C;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
+ void (*ARGBToUV444MatrixRow)(
+ const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = ARGBToUV444MatrixRow_C;
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
+ }
+#endif
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD;
+ }
+ }
+#endif
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+#if defined(HAS_CONVERT8TO16ROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Convert8To16Row = Convert8To16Row_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert8To16Row = Convert8To16Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To16Row = Convert8To16Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To16Row = Convert8To16Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To16Row = Convert8To16Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To16Row = Convert8To16Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert8To16Row = Convert8To16Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To16Row = Convert8To16Row_HVX;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Convert8To16Row = Convert8To16Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MultiplyRow_16 = MultiplyRow_16_SME;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MultiplyRow_16 = MultiplyRow_16_RVV;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MultiplyRow_16 = MultiplyRow_16_HVX;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
+ }
+ }
+#endif
+
+ size_t row_size =
+ (static_cast<size_t>(width) + 31) & ~static_cast<size_t>(31);
+ size_t row_uv_merged_size =
+ (static_cast<size_t>(width) * 2 + 31) & ~static_cast<size_t>(31);
+ if (row_size > SIZE_MAX / 3) {
+ return -1;
+ }
+ size_t rows = row_size * 3;
+ if (row_uv_merged_size > SIZE_MAX - rows) {
+ return -1;
+ }
+ align_buffer_64(row, rows + row_uv_merged_size);
+ if (!row)
+ return 1;
+ uint8_t* row_y = row;
+ uint8_t* row_u = row_y + row_size;
+ uint8_t* row_v = row_u + row_size;
+ uint8_t* row_uv = row_v + row_size;
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUV444MatrixRow(src_argb, row_u, row_v, width, argbconstants);
+ MergeUVRow(row_u, row_v, row_uv, width);
+ Convert8To16Row(row_uv, dst_uv, 10, width * 2);
+ MultiplyRow_16(dst_uv, dst_uv, 64, width * 2);
+ ARGBToYMatrixRow(src_argb, row_y, width, argbconstants);
+ Convert8To16Row(row_y, dst_y, 10, width);
+ MultiplyRow_16(dst_y, dst_y, 64, width);
+ src_argb += src_stride_argb;
+ dst_y += dst_stride_y;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+LIBYUV_API
+int ABGRToNV12(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return ARGBToNV12Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y,
+ dst_uv, dst_stride_uv, &kAbgrI601Constants, width,
+ height);
+}
+
+// Same as NV12 but U and V swapped.
+LIBYUV_API
+int ABGRToNV21(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return ARGBToNV21Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y,
+ dst_vu, dst_stride_vu, &kAbgrI601Constants, width,
+ height);
+}
+
+// Convert ARGB to YUY2.
+LIBYUV_API
+int ARGBToYUY2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ int width,
+ int height) {
+ return ARGBToYUY2Matrix(src_argb, src_stride_argb, dst_yuy2, dst_stride_yuy2,
+ &kArgbI601Constants, width, height);
+}
+
+// Convert ARGB to UYVY.
+LIBYUV_API
+int ARGBToUYVY(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_uyvy,
+ int dst_stride_uyvy,
+ int width,
+ int height) {
+ return ARGBToUYVYMatrix(src_argb, src_stride_argb, dst_uyvy, dst_stride_uyvy,
+ &kArgbI601Constants, width, height);
+}
+
+// Convert ARGB to I400.
+LIBYUV_API
+int ARGBToI400(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
+ return ARGBToI400Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ &kArgbI601Constants, width, height);
+}
+
+#if !defined(__riscv) && !defined(__hexagon__)
+// Shuffle table for converting ARGB to RGBA.
+static const uvec8 kShuffleMaskARGBToRGBA = {
+ 3u, 0u, 1u, 2u, 7u, 4u, 5u, 6u, 11u, 8u, 9u, 10u, 15u, 12u, 13u, 14u};
+
+// Convert ARGB to RGBA.
+LIBYUV_API
+int ARGBToRGBA(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height) {
+ return ARGBShuffle(src_argb, src_stride_argb, dst_rgba, dst_stride_rgba,
+ (const uint8_t*)(&kShuffleMaskARGBToRGBA), width, height);
+}
+#else
+// Convert ARGB to RGBA.
+LIBYUV_API
+int ARGBToRGBA(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_rgba,
+ int dst_stride_rgba,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToRGBARow)(const uint8_t* src_argb, uint8_t* dst_rgba, int width) =
+ ARGBToRGBARow_C;
+ if (!src_argb || !dst_rgba || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_rgba == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_rgba = 0;
+ }
+
+#if defined(HAS_ARGBTORGBAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToRGBARow = ARGBToRGBARow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTORGBAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToRGBARow = ARGBToRGBARow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToRGBARow(src_argb, dst_rgba, width);
+ src_argb += src_stride_argb;
+ dst_rgba += dst_stride_rgba;
+ }
+ return 0;
+}
+#endif
+
+// Convert ARGB To RGB24.
+LIBYUV_API
+int ARGBToRGB24(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToRGB24Row)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
+ ARGBToRGB24Row_C;
+ if (!src_argb || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_rgb24 == width * 3 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_rgb24 = 0;
+ }
+#if defined(HAS_ARGBTORGB24ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToRGB24Row(src_argb, dst_rgb24, width);
+ src_argb += src_stride_argb;
+ dst_rgb24 += dst_stride_rgb24;
+ }
+ return 0;
+}
+
+// Convert ARGB To RAW.
+LIBYUV_API
+int ARGBToRAW(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_raw,
+ int dst_stride_raw,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToRAWRow)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
+ ARGBToRAWRow_C;
+ if (!src_argb || !dst_raw || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_raw == width * 3 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_raw = 0;
+ }
+#if defined(HAS_ARGBTORAWROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToRAWRow = ARGBToRAWRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRAWRow = ARGBToRAWRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToRAWRow = ARGBToRAWRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToRAWRow = ARGBToRAWRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToRAWRow = ARGBToRAWRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToRAWRow = ARGBToRAWRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRAWRow = ARGBToRAWRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRAWRow = ARGBToRAWRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToRAWRow = ARGBToRAWRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRAWRow = ARGBToRAWRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToRAWRow = ARGBToRAWRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToRAWRow = ARGBToRAWRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORAWROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToRAWRow = ARGBToRAWRow_RVV;
+ }
+#endif
+
for (y = 0; y < height; ++y) {
ARGBToRAWRow(src_argb, dst_raw, width);
src_argb += src_stride_argb;
@@ -1475,688 +3840,1016 @@ int ARGBToRGB565Dither(const uint8_t* src_argb,
int width,
int height) {
int y;
+ uint32_t dither4[4];
void (*ARGBToRGB565DitherRow)(const uint8_t* src_argb, uint8_t* dst_rgb,
- const uint32_t dither4, int width) =
+ uint32_t dither4, int width) =
ARGBToRGB565DitherRow_C;
- if (!src_argb || !dst_rgb565 || width <= 0 || height == 0) {
+ if (!src_argb || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ if (!dither4x4) {
+ dither4x4 = kDither565_4x4;
+ }
+ memcpy(dither4, dither4x4, 16);
+
+#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToRGB565DitherRow(src_argb, dst_rgb565, dither4[y & 3], width);
+ src_argb += src_stride_argb;
+ dst_rgb565 += dst_stride_rgb565;
+ }
+ return 0;
+}
+
+// Convert ARGB To RGB565.
+// TODO(fbarchard): Consider using dither function low level with zeros.
+LIBYUV_API
+int ARGBToRGB565(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_rgb565,
+ int dst_stride_rgb565,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToRGB565Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
+ int width) = ARGBToRGB565Row_C;
+ if (!src_argb || !dst_rgb565 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_rgb565 == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_rgb565 = 0;
+ }
+
+#if defined(HAS_ARGBTORGB565ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTORGB565ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToRGB565Row(src_argb, dst_rgb565, width);
+ src_argb += src_stride_argb;
+ dst_rgb565 += dst_stride_rgb565;
+ }
+ return 0;
+}
+
+// Convert ARGB To ARGB1555.
+LIBYUV_API
+int ARGBToARGB1555(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb1555,
+ int dst_stride_argb1555,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToARGB1555Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
+ int width) = ARGBToARGB1555Row_C;
+ if (!src_argb || !dst_argb1555 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
- if (!dither4x4) {
- dither4x4 = kDither565_4x4;
- }
-#if defined(HAS_ARGBTORGB565DITHERROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2;
- }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_argb1555 == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_argb1555 = 0;
}
-#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
+
+#if defined(HAS_ARGBTOARGB1555ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
+ ARGBToARGB1555Row = ARGBToARGB1555Row_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX2;
+ ARGBToARGB1555Row = ARGBToARGB1555Row_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
+#if defined(HAS_ARGBTOARGB1555ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
+ ARGBToARGB1555Row = ARGBToARGB1555Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_NEON;
+ ARGBToARGB1555Row = ARGBToARGB1555Row_NEON;
}
}
#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_MSA;
+#if defined(HAS_ARGBTOARGB1555ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToARGB1555Row = ARGBToARGB1555Row_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_MSA;
+ ARGBToARGB1555Row = ARGBToARGB1555Row_LSX;
}
}
#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_MMI;
+#if defined(HAS_ARGBTOARGB1555ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToARGB1555Row = ARGBToARGB1555Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToARGB1555Row = ARGBToARGB1555Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOARGB1555ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToARGB1555Row = ARGBToARGB1555Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToARGB1555Row = ARGBToARGB1555Row_WASMSIMD;
}
}
#endif
for (y = 0; y < height; ++y) {
- ARGBToRGB565DitherRow(src_argb, dst_rgb565,
- *(const uint32_t*)(dither4x4 + ((y & 3) << 2)),
- width);
+ ARGBToARGB1555Row(src_argb, dst_argb1555, width);
src_argb += src_stride_argb;
- dst_rgb565 += dst_stride_rgb565;
+ dst_argb1555 += dst_stride_argb1555;
}
return 0;
}
-// Convert ARGB To RGB565.
-// TODO(fbarchard): Consider using dither function low level with zeros.
+// Convert ARGB To ARGB4444.
LIBYUV_API
-int ARGBToRGB565(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
+int ARGBToARGB4444(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb4444,
+ int dst_stride_argb4444,
+ int width,
+ int height) {
int y;
- void (*ARGBToRGB565Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
- int width) = ARGBToRGB565Row_C;
- if (!src_argb || !dst_rgb565 || width <= 0 || height == 0) {
+ void (*ARGBToARGB4444Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
+ int width) = ARGBToARGB4444Row_C;
+ if (!src_argb || !dst_argb4444 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_rgb565 == width * 2) {
+ if (src_stride_argb == width * 4 && dst_stride_argb4444 == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_argb = dst_stride_rgb565 = 0;
- }
-#if defined(HAS_ARGBTORGB565ROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBToRGB565Row = ARGBToRGB565Row_Any_SSE2;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB565Row = ARGBToRGB565Row_SSE2;
- }
+ src_stride_argb = dst_stride_argb4444 = 0;
}
-#endif
-#if defined(HAS_ARGBTORGB565ROW_AVX2)
+
+#if defined(HAS_ARGBTOARGB4444ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToRGB565Row = ARGBToRGB565Row_Any_AVX2;
+ ARGBToARGB4444Row = ARGBToARGB4444Row_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565Row = ARGBToRGB565Row_AVX2;
+ ARGBToARGB4444Row = ARGBToARGB4444Row_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTORGB565ROW_NEON)
+#if defined(HAS_ARGBTOARGB4444ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON;
+ ARGBToARGB4444Row = ARGBToARGB4444Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565Row = ARGBToRGB565Row_NEON;
+ ARGBToARGB4444Row = ARGBToARGB4444Row_NEON;
}
}
#endif
-#if defined(HAS_ARGBTORGB565ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToRGB565Row = ARGBToRGB565Row_Any_MSA;
+#if defined(HAS_ARGBTOARGB4444ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToARGB4444Row = ARGBToARGB4444Row_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToRGB565Row = ARGBToRGB565Row_MSA;
+ ARGBToARGB4444Row = ARGBToARGB4444Row_LSX;
}
}
#endif
-#if defined(HAS_ARGBTORGB565ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToRGB565Row = ARGBToRGB565Row_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToRGB565Row = ARGBToRGB565Row_MMI;
+#if defined(HAS_ARGBTOARGB4444ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToARGB4444Row = ARGBToARGB4444Row_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToARGB4444Row = ARGBToARGB4444Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOARGB4444ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToARGB4444Row = ARGBToARGB4444Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToARGB4444Row = ARGBToARGB4444Row_WASMSIMD;
}
}
#endif
for (y = 0; y < height; ++y) {
- ARGBToRGB565Row(src_argb, dst_rgb565, width);
+ ARGBToARGB4444Row(src_argb, dst_argb4444, width);
src_argb += src_stride_argb;
- dst_rgb565 += dst_stride_rgb565;
+ dst_argb4444 += dst_stride_argb4444;
}
return 0;
}
-// Convert ARGB To ARGB1555.
+// Convert ABGR To AR30.
LIBYUV_API
-int ARGBToARGB1555(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb1555,
- int dst_stride_argb1555,
- int width,
- int height) {
+int ABGRToAR30(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
int y;
- void (*ARGBToARGB1555Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
- int width) = ARGBToARGB1555Row_C;
- if (!src_argb || !dst_argb1555 || width <= 0 || height == 0) {
+ void (*ABGRToAR30Row)(const uint8_t* src_abgr, uint8_t* dst_rgb, int width) =
+ ABGRToAR30Row_C;
+ if (!src_abgr || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
+ src_abgr = src_abgr + (ptrdiff_t)(height - 1) * src_stride_abgr;
+ src_stride_abgr = -src_stride_abgr;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb1555 == width * 2) {
+ if (src_stride_abgr == width * 4 && dst_stride_ar30 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_argb = dst_stride_argb1555 = 0;
+ src_stride_abgr = dst_stride_ar30 = 0;
}
-#if defined(HAS_ARGBTOARGB1555ROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_Any_SSE2;
+#if defined(HAS_ABGRTOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ABGRToAR30Row = ABGRToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ABGRToAR30Row = ABGRToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ABGRTOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ABGRToAR30Row = ABGRToAR30Row_Any_SSSE3;
if (IS_ALIGNED(width, 4)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_SSE2;
+ ABGRToAR30Row = ABGRToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOARGB1555ROW_AVX2)
+#if defined(HAS_ABGRTOAR30ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_Any_AVX2;
+ ABGRToAR30Row = ABGRToAR30Row_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_AVX2;
+ ABGRToAR30Row = ABGRToAR30Row_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOARGB1555ROW_NEON)
+#if defined(HAS_ABGRTOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ABGRToAR30Row = ABGRToAR30Row_AVX512BW;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ ABGRToAR30Row(src_abgr, dst_ar30, width);
+ src_abgr += src_stride_abgr;
+ dst_ar30 += dst_stride_ar30;
+ }
+ return 0;
+}
+
+// Convert ARGB To AR30.
+LIBYUV_API
+int ARGBToAR30(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToAR30Row)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
+ ARGBToAR30Row_C;
+ if (!src_argb || !dst_ar30 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_ar30 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_ar30 = 0;
+ }
+#if defined(HAS_ARGBTOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_Any_NEON;
+ ARGBToAR30Row = ARGBToAR30Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_NEON;
+ ARGBToAR30Row = ARGBToAR30Row_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOARGB1555ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_MSA;
+#if defined(HAS_ARGBTOAR30ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToAR30Row = ARGBToAR30Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBToAR30Row = ARGBToAR30Row_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOARGB1555ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToARGB1555Row = ARGBToARGB1555Row_MMI;
+#if defined(HAS_ARGBTOAR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToAR30Row = ARGBToAR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToAR30Row = ARGBToAR30Row_AVX2;
}
}
#endif
-
+#if defined(HAS_ARGBTOAR30ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToAR30Row = ARGBToAR30Row_AVX512BW;
+ }
+#endif
for (y = 0; y < height; ++y) {
- ARGBToARGB1555Row(src_argb, dst_argb1555, width);
+ ARGBToAR30Row(src_argb, dst_ar30, width);
src_argb += src_stride_argb;
- dst_argb1555 += dst_stride_argb1555;
+ dst_ar30 += dst_stride_ar30;
}
return 0;
}
-// Convert ARGB To ARGB4444.
+// ARGB little endian (bgra in memory) to J444
+LIBYUV_API
+int ARGBToJ444(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI444Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbJPEGConstants, width, height);
+}
+
+// Convert ARGB to J420. (JPeg full range I420).
+LIBYUV_API
+int ARGBToJ420(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI420Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbJPEGConstants, width, height);
+}
+
+// Convert ARGB to J422. (JPeg full range I422).
+LIBYUV_API
+int ARGBToJ422(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI422Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kArgbJPEGConstants, width, height);
+}
+
+// Convert ARGB to J400.
+LIBYUV_API
+int ARGBToJ400(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
+ return ARGBToI400Matrix(src_argb, src_stride_argb, dst_y, dst_stride_y,
+ &kArgbJPEGConstants, width, height);
+}
+
+// Convert RGBA to J400.
+LIBYUV_API
+int RGBAToJ400(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height) {
+ return ARGBToI400Matrix(src_rgba, src_stride_rgba, dst_yj, dst_stride_yj,
+ &kRgbaJPEGConstants, width, height);
+}
+
+// Convert ABGR to J420. (JPeg full range I420).
+LIBYUV_API
+int ABGRToJ420(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI420Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kAbgrJPEGConstants, width, height);
+}
+
+// Convert ABGR to J422. (JPeg full range I422).
+LIBYUV_API
+int ABGRToJ422(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return ARGBToI422Matrix(src_abgr, src_stride_abgr, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v,
+ &kAbgrJPEGConstants, width, height);
+}
+
+// Convert ABGR to J400.
LIBYUV_API
-int ARGBToARGB4444(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb4444,
- int dst_stride_argb4444,
- int width,
- int height) {
+int ABGRToJ400(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ int width,
+ int height) {
+ return ARGBToI400Matrix(src_abgr, src_stride_abgr, dst_yj, dst_stride_yj,
+ &kAbgrJPEGConstants, width, height);
+}
+
+// Convert ARGB to AR64.
+LIBYUV_API
+int ARGBToAR64(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height) {
int y;
- void (*ARGBToARGB4444Row)(const uint8_t* src_argb, uint8_t* dst_rgb,
- int width) = ARGBToARGB4444Row_C;
- if (!src_argb || !dst_argb4444 || width <= 0 || height == 0) {
+ void (*ARGBToAR64Row)(const uint8_t* src_argb, uint16_t* dst_ar64,
+ int width) = ARGBToAR64Row_C;
+ if (!src_argb || !dst_ar64 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+ // Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb4444 == width * 2) {
+ if (src_stride_argb == width * 4 && dst_stride_ar64 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_argb = dst_stride_argb4444 = 0;
+ src_stride_argb = dst_stride_ar64 = 0;
}
-#if defined(HAS_ARGBTOARGB4444ROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_Any_SSE2;
+#if defined(HAS_ARGBTOAR64ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToAR64Row = ARGBToAR64Row_Any_SSSE3;
if (IS_ALIGNED(width, 4)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_SSE2;
+ ARGBToAR64Row = ARGBToAR64Row_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOARGB4444ROW_AVX2)
+#if defined(HAS_ARGBTOAR64ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_Any_AVX2;
+ ARGBToAR64Row = ARGBToAR64Row_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_AVX2;
+ ARGBToAR64Row = ARGBToAR64Row_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOARGB4444ROW_NEON)
+#if defined(HAS_ARGBTOAR64ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_Any_NEON;
+ ARGBToAR64Row = ARGBToAR64Row_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_NEON;
+ ARGBToAR64Row = ARGBToAR64Row_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOARGB4444ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_MSA;
- }
+#if defined(HAS_ARGBTOAR64ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToAR64Row = ARGBToAR64Row_RVV;
}
#endif
-#if defined(HAS_ARGBTOARGB4444ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- ARGBToARGB4444Row = ARGBToARGB4444Row_MMI;
- }
+#if defined(HAS_ARGBTOAR64ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToAR64Row = ARGBToAR64Row_HVX;
}
#endif
for (y = 0; y < height; ++y) {
- ARGBToARGB4444Row(src_argb, dst_argb4444, width);
+ ARGBToAR64Row(src_argb, dst_ar64, width);
src_argb += src_stride_argb;
- dst_argb4444 += dst_stride_argb4444;
+ dst_ar64 += dst_stride_ar64;
}
return 0;
}
-// Convert ABGR To AR30.
+// Convert ARGB to AB64.
LIBYUV_API
-int ABGRToAR30(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
+int ARGBToAB64(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint16_t* dst_ab64,
+ int dst_stride_ab64,
int width,
int height) {
int y;
- void (*ABGRToAR30Row)(const uint8_t* src_abgr, uint8_t* dst_rgb, int width) =
- ABGRToAR30Row_C;
- if (!src_abgr || !dst_ar30 || width <= 0 || height == 0) {
+ void (*ARGBToAB64Row)(const uint8_t* src_argb, uint16_t* dst_ar64,
+ int width) = ARGBToAB64Row_C;
+ if (!src_argb || !dst_ab64 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+ // Negative height means invert the image.
if (height < 0) {
height = -height;
- src_abgr = src_abgr + (height - 1) * src_stride_abgr;
- src_stride_abgr = -src_stride_abgr;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_abgr == width * 4 && dst_stride_ar30 == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_ab64 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_abgr = dst_stride_ar30 = 0;
+ src_stride_argb = dst_stride_ab64 = 0;
}
-#if defined(HAS_ABGRTOAR30ROW_SSSE3)
+#if defined(HAS_ARGBTOAB64ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ABGRToAR30Row = ABGRToAR30Row_Any_SSSE3;
+ ARGBToAB64Row = ARGBToAB64Row_Any_SSSE3;
if (IS_ALIGNED(width, 4)) {
- ABGRToAR30Row = ABGRToAR30Row_SSSE3;
+ ARGBToAB64Row = ARGBToAB64Row_SSSE3;
}
}
#endif
-#if defined(HAS_ABGRTOAR30ROW_AVX2)
+#if defined(HAS_ARGBTOAB64ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ABGRToAR30Row = ABGRToAR30Row_Any_AVX2;
+ ARGBToAB64Row = ARGBToAB64Row_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- ABGRToAR30Row = ABGRToAR30Row_AVX2;
+ ARGBToAB64Row = ARGBToAB64Row_AVX2;
}
}
#endif
- for (y = 0; y < height; ++y) {
- ABGRToAR30Row(src_abgr, dst_ar30, width);
- src_abgr += src_stride_abgr;
- dst_ar30 += dst_stride_ar30;
- }
- return 0;
-}
-
-// Convert ARGB To AR30.
-LIBYUV_API
-int ARGBToAR30(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_ar30,
- int dst_stride_ar30,
- int width,
- int height) {
- int y;
- void (*ARGBToAR30Row)(const uint8_t* src_argb, uint8_t* dst_rgb, int width) =
- ARGBToAR30Row_C;
- if (!src_argb || !dst_ar30 || width <= 0 || height == 0) {
- return -1;
- }
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_ar30 == width * 4) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_ar30 = 0;
- }
-#if defined(HAS_ARGBTOAR30ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToAR30Row = ARGBToAR30Row_Any_SSSE3;
- if (IS_ALIGNED(width, 4)) {
- ARGBToAR30Row = ARGBToAR30Row_SSSE3;
+#if defined(HAS_ARGBTOAB64ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToAB64Row = ARGBToAB64Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToAB64Row = ARGBToAB64Row_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOAR30ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToAR30Row = ARGBToAR30Row_Any_AVX2;
- if (IS_ALIGNED(width, 8)) {
- ARGBToAR30Row = ARGBToAR30Row_AVX2;
- }
+#if defined(HAS_ARGBTOAB64ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToAB64Row = ARGBToAB64Row_RVV;
+ }
+#endif
+#if defined(HAS_ARGBTOAB64ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToAB64Row = ARGBToAB64Row_HVX;
}
#endif
+
for (y = 0; y < height; ++y) {
- ARGBToAR30Row(src_argb, dst_ar30, width);
+ ARGBToAB64Row(src_argb, dst_ab64, width);
src_argb += src_stride_argb;
- dst_ar30 += dst_stride_ar30;
+ dst_ab64 += dst_stride_ab64;
}
return 0;
}
-// Convert ARGB to J420. (JPeg full range I420).
-LIBYUV_API
-int ARGBToJ420(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_yj,
- int dst_stride_yj,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
+// Convert RAW to NV21 with Matrix.
+static int RAWToNVMatrix(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height,
+ int is_nv21) {
int y;
- void (*ARGBToUVJRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVJRow_C;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_yj, int width) =
- ARGBToYJRow_C;
- if (!src_argb || !dst_yj || !dst_u || !dst_v || width <= 0 || height == 0) {
+ int halfwidth = (width + 1) >> 1;
+ void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
+ uint8_t* dst_u, uint8_t* dst_v, int width,
+ const struct ArgbConstants* c) = RGBToUVMatrixRow_C;
+ void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
+ const struct ArgbConstants* c) = RGBToYMatrixRow_C;
+ void (*MergeUVRow)(const uint8_t* src_uj, const uint8_t* src_vj,
+ uint8_t* dst_vu, int width) = MergeUVRow_C;
+
+ if (!src_raw || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
}
-#if defined(HAS_ARGBTOYJROW_SSSE3) && defined(HAS_ARGBTOUVJROW_SSSE3)
+
+#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_SSSE3;
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_SSSE3;
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+ RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2)
+#if defined(HAS_RGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToYJRow = ARGBToYJRow_AVX2;
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYJRow = ARGBToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_NEON;
- }
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_ARGBTOUVJROW_NEON)
+#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_NEON;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYJRow = ARGBToYJRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYJRow = ARGBToYJRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOUVJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_MSA;
+#if defined(HAS_RGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- ARGBToUVJRow = ARGBToUVJRow_MSA;
+ RGBToYMatrixRow = RGBToYMatrixRow_LASX;
}
}
#endif
-#if defined(HAS_ARGBTOUVJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_MMI;
+#if defined(HAS_RGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_RVV;
+ }
+#endif
+#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_MMI;
+ RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD;
}
}
#endif
- for (y = 0; y < height - 1; y += 2) {
- ARGBToUVJRow(src_argb, src_stride_argb, dst_u, dst_v, width);
- ARGBToYJRow(src_argb, dst_yj, width);
- ARGBToYJRow(src_argb + src_stride_argb, dst_yj + dst_stride_yj, width);
- src_argb += src_stride_argb * 2;
- dst_yj += dst_stride_yj * 2;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
- }
- if (height & 1) {
- ARGBToUVJRow(src_argb, 0, dst_u, dst_v, width);
- ARGBToYJRow(src_argb, dst_yj, width);
- }
- return 0;
-}
-
-// Convert ARGB to J422. (JPeg full range I422).
-LIBYUV_API
-int ARGBToJ422(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_yj,
- int dst_stride_yj,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
- int y;
- void (*ARGBToUVJRow)(const uint8_t* src_argb0, int src_stride_argb,
- uint8_t* dst_u, uint8_t* dst_v, int width) =
- ARGBToUVJRow_C;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_yj, int width) =
- ARGBToYJRow_C;
- if (!src_argb || !dst_yj || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
- }
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
- }
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_yj == width &&
- dst_stride_u * 2 == width && dst_stride_v * 2 == width) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_yj = dst_stride_u = dst_stride_v = 0;
- }
-#if defined(HAS_ARGBTOYJROW_SSSE3) && defined(HAS_ARGBTOUVJROW_SSSE3)
+#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_SSSE3;
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_SSSE3;
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2)
+#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToYJRow = ARGBToYJRow_AVX2;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYJRow = ARGBToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_NEON;
- }
+#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
-#if defined(HAS_ARGBTOUVJROW_NEON)
+#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_NEON;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_NEON;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYJRow = ARGBToYJRow_Any_MSA;
+#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_MSA;
+ RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYJRow = ARGBToYJRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_MMI;
+
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
}
}
#endif
-#if defined(HAS_ARGBTOUVJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- ARGBToUVJRow = ARGBToUVJRow_MSA;
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOUVJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToUVJRow = ARGBToUVJRow_Any_MMI;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVJRow = ARGBToUVJRow_MMI;
- }
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
}
#endif
-
- for (y = 0; y < height; ++y) {
- ARGBToUVJRow(src_argb, 0, dst_u, dst_v, width);
- ARGBToYJRow(src_argb, dst_yj, width);
- src_argb += src_stride_argb;
- dst_yj += dst_stride_yj;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
- }
- return 0;
-}
-
-// Convert ARGB to J400.
-LIBYUV_API
-int ARGBToJ400(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_yj,
- int dst_stride_yj,
- int width,
- int height) {
- int y;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_yj, int width) =
- ARGBToYJRow_C;
- if (!src_argb || !dst_yj || width <= 0 || height == 0) {
- return -1;
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
}
- if (height < 0) {
- height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
- src_stride_argb = -src_stride_argb;
+#endif
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
}
- // Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_yj == width) {
- width *= height;
- height = 1;
- src_stride_argb = dst_stride_yj = 0;
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
}
-#if defined(HAS_ARGBTOYJROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- ARGBToYJRow = ARGBToYJRow_AVX2;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYJRow = ARGBToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_NEON;
- }
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
}
#endif
-#if defined(HAS_ARGBTOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYJRow = ARGBToYJRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_MSA;
- }
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
}
#endif
-#if defined(HAS_ARGBTOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYJRow = ARGBToYJRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_MMI;
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
}
}
#endif
- for (y = 0; y < height; ++y) {
- ARGBToYJRow(src_argb, dst_yj, width);
- src_argb += src_stride_argb;
- dst_yj += dst_stride_yj;
+ {
+ // Allocate 1 row of U and 1 row of V.
+ align_buffer_64(row_u, halfwidth);
+ align_buffer_64(row_v, halfwidth);
+
+ if (!row_u || !row_v) {
+ free_aligned_buffer_64(row_u);
+ free_aligned_buffer_64(row_v);
+ return 1;
+ }
+
+ const uint8_t* first_chroma = is_nv21 ? row_v : row_u;
+ const uint8_t* second_chroma = is_nv21 ? row_u : row_v;
+
+ for (y = 0; y < height - 1; y += 2) {
+ RGBToUVMatrixRow(src_raw, src_stride_raw, row_u, row_v, width,
+ argbconstants);
+ MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth);
+ RGBToYMatrixRow(src_raw, dst_y, width, argbconstants);
+ RGBToYMatrixRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width,
+ argbconstants);
+ src_raw += src_stride_raw * 2;
+ dst_y += dst_stride_y * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ RGBToUVMatrixRow(src_raw, 0, row_u, row_v, width, argbconstants);
+ MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth);
+ RGBToYMatrixRow(src_raw, dst_y, width, argbconstants);
+ }
+ free_aligned_buffer_64(row_v);
+ free_aligned_buffer_64(row_u);
}
return 0;
}
+LIBYUV_API
+int RAWToNV21Matrix(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ const struct ArgbConstants* argbconstants,
+ int width,
+ int height) {
+ return RAWToNVMatrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
+ dst_stride_vu, argbconstants, width, height, 1);
+}
+
+LIBYUV_API
+int RAWToJNV21(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
+ dst_stride_vu, &kAbgrJPEGConstants, width, height);
+}
+
+LIBYUV_API
+int RAWToNV21(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
+ dst_stride_vu, &kAbgrI601Constants, width, height);
+}
+
+LIBYUV_API
+int RGB24ToNV12(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return RAWToNVMatrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y, dst_uv,
+ dst_stride_uv, &kArgbI601Constants, width, height, 0);
+}
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/convert_jpeg.cc b/libs/libyuv/source/convert_jpeg.cc
index f440c7c2e9..d9ce28a923 100644
--- a/libs/libyuv/source/convert_jpeg.cc
+++ b/libs/libyuv/source/convert_jpeg.cc
@@ -94,7 +94,7 @@ int MJPGSize(const uint8_t* src_mjpg,
int* width,
int* height) {
MJpegDecoder mjpeg_decoder;
- LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
+ bool ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
if (ret) {
*width = mjpeg_decoder.GetWidth();
*height = mjpeg_decoder.GetHeight();
@@ -126,7 +126,7 @@ int MJPGToI420(const uint8_t* src_mjpg,
// TODO(fbarchard): Port MJpeg to C.
MJpegDecoder mjpeg_decoder;
- LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
+ bool ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
if (ret && (mjpeg_decoder.GetWidth() != src_width ||
mjpeg_decoder.GetHeight() != src_height)) {
// ERROR: MJPEG frame has unexpected dimensions
@@ -266,7 +266,7 @@ int MJPGToNV21(const uint8_t* src_mjpg,
// TODO(fbarchard): Port MJpeg to C.
MJpegDecoder mjpeg_decoder;
- LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
+ bool ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
if (ret && (mjpeg_decoder.GetWidth() != src_width ||
mjpeg_decoder.GetHeight() != src_height)) {
// ERROR: MJPEG frame has unexpected dimensions
@@ -328,6 +328,140 @@ int MJPGToNV21(const uint8_t* src_mjpg,
return ret ? 0 : 1;
}
+static void JpegI420ToNV12(void* opaque,
+ const uint8_t* const* data,
+ const int* strides,
+ int rows) {
+ NV21Buffers* dest = (NV21Buffers*)(opaque);
+ // Use NV21 with VU swapped.
+ I420ToNV21(data[0], strides[0], data[2], strides[2], data[1], strides[1],
+ dest->y, dest->y_stride, dest->vu, dest->vu_stride, dest->w, rows);
+ dest->y += rows * dest->y_stride;
+ dest->vu += ((rows + 1) >> 1) * dest->vu_stride;
+ dest->h -= rows;
+}
+
+static void JpegI422ToNV12(void* opaque,
+ const uint8_t* const* data,
+ const int* strides,
+ int rows) {
+ NV21Buffers* dest = (NV21Buffers*)(opaque);
+ // Use NV21 with VU swapped.
+ I422ToNV21(data[0], strides[0], data[2], strides[2], data[1], strides[1],
+ dest->y, dest->y_stride, dest->vu, dest->vu_stride, dest->w, rows);
+ dest->y += rows * dest->y_stride;
+ dest->vu += ((rows + 1) >> 1) * dest->vu_stride;
+ dest->h -= rows;
+}
+
+static void JpegI444ToNV12(void* opaque,
+ const uint8_t* const* data,
+ const int* strides,
+ int rows) {
+ NV21Buffers* dest = (NV21Buffers*)(opaque);
+ // Use NV21 with VU swapped.
+ I444ToNV21(data[0], strides[0], data[2], strides[2], data[1], strides[1],
+ dest->y, dest->y_stride, dest->vu, dest->vu_stride, dest->w, rows);
+ dest->y += rows * dest->y_stride;
+ dest->vu += ((rows + 1) >> 1) * dest->vu_stride;
+ dest->h -= rows;
+}
+
+static void JpegI400ToNV12(void* opaque,
+ const uint8_t* const* data,
+ const int* strides,
+ int rows) {
+ NV21Buffers* dest = (NV21Buffers*)(opaque);
+ // Use NV21 since there is no UV plane.
+ I400ToNV21(data[0], strides[0], dest->y, dest->y_stride, dest->vu,
+ dest->vu_stride, dest->w, rows);
+ dest->y += rows * dest->y_stride;
+ dest->vu += ((rows + 1) >> 1) * dest->vu_stride;
+ dest->h -= rows;
+}
+
+// MJPG (Motion JPEG) to NV12.
+LIBYUV_API
+int MJPGToNV12(const uint8_t* sample,
+ size_t sample_size,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height) {
+ if (sample_size == kUnknownDataSize) {
+ // ERROR: MJPEG frame size unknown
+ return -1;
+ }
+
+ // TODO(fbarchard): Port MJpeg to C.
+ MJpegDecoder mjpeg_decoder;
+ bool ret = mjpeg_decoder.LoadFrame(sample, sample_size);
+ if (ret && (mjpeg_decoder.GetWidth() != src_width ||
+ mjpeg_decoder.GetHeight() != src_height)) {
+ // ERROR: MJPEG frame has unexpected dimensions
+ mjpeg_decoder.UnloadFrame();
+ return 1; // runtime failure
+ }
+ if (ret) {
+ // Use NV21Buffers but with UV instead of VU.
+ NV21Buffers bufs = {dst_y, dst_stride_y, dst_uv,
+ dst_stride_uv, dst_width, dst_height};
+ // YUV420
+ if (mjpeg_decoder.GetColorSpace() == MJpegDecoder::kColorSpaceYCbCr &&
+ mjpeg_decoder.GetNumComponents() == 3 &&
+ mjpeg_decoder.GetVertSampFactor(0) == 2 &&
+ mjpeg_decoder.GetHorizSampFactor(0) == 2 &&
+ mjpeg_decoder.GetVertSampFactor(1) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(1) == 1 &&
+ mjpeg_decoder.GetVertSampFactor(2) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(2) == 1) {
+ ret = mjpeg_decoder.DecodeToCallback(&JpegI420ToNV12, &bufs, dst_width,
+ dst_height);
+ // YUV422
+ } else if (mjpeg_decoder.GetColorSpace() ==
+ MJpegDecoder::kColorSpaceYCbCr &&
+ mjpeg_decoder.GetNumComponents() == 3 &&
+ mjpeg_decoder.GetVertSampFactor(0) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(0) == 2 &&
+ mjpeg_decoder.GetVertSampFactor(1) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(1) == 1 &&
+ mjpeg_decoder.GetVertSampFactor(2) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(2) == 1) {
+ ret = mjpeg_decoder.DecodeToCallback(&JpegI422ToNV12, &bufs, dst_width,
+ dst_height);
+ // YUV444
+ } else if (mjpeg_decoder.GetColorSpace() ==
+ MJpegDecoder::kColorSpaceYCbCr &&
+ mjpeg_decoder.GetNumComponents() == 3 &&
+ mjpeg_decoder.GetVertSampFactor(0) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(0) == 1 &&
+ mjpeg_decoder.GetVertSampFactor(1) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(1) == 1 &&
+ mjpeg_decoder.GetVertSampFactor(2) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(2) == 1) {
+ ret = mjpeg_decoder.DecodeToCallback(&JpegI444ToNV12, &bufs, dst_width,
+ dst_height);
+ // YUV400
+ } else if (mjpeg_decoder.GetColorSpace() ==
+ MJpegDecoder::kColorSpaceGrayscale &&
+ mjpeg_decoder.GetNumComponents() == 1 &&
+ mjpeg_decoder.GetVertSampFactor(0) == 1 &&
+ mjpeg_decoder.GetHorizSampFactor(0) == 1) {
+ ret = mjpeg_decoder.DecodeToCallback(&JpegI400ToNV12, &bufs, dst_width,
+ dst_height);
+ } else {
+ // Unknown colorspace.
+ mjpeg_decoder.UnloadFrame();
+ return 1;
+ }
+ }
+ return ret ? 0 : 1;
+}
+
struct ARGBBuffers {
uint8_t* argb;
int argb_stride;
@@ -397,7 +531,7 @@ int MJPGToARGB(const uint8_t* src_mjpg,
// TODO(fbarchard): Port MJpeg to C.
MJpegDecoder mjpeg_decoder;
- LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
+ bool ret = mjpeg_decoder.LoadFrame(src_mjpg, src_size_mjpg);
if (ret && (mjpeg_decoder.GetWidth() != src_width ||
mjpeg_decoder.GetHeight() != src_height)) {
// ERROR: MJPEG frame has unexpected dimensions
diff --git a/libs/libyuv/source/convert_to_argb.cc b/libs/libyuv/source/convert_to_argb.cc
index bde1aa8891..17842d285a 100644
--- a/libs/libyuv/source/convert_to_argb.cc
+++ b/libs/libyuv/source/convert_to_argb.cc
@@ -10,6 +10,11 @@
#include "libyuv/convert_argb.h"
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
#include "libyuv/cpu_id.h"
#ifdef HAVE_JPEG
#include "libyuv/mjpeg_decoder.h"
@@ -23,6 +28,280 @@ namespace libyuv {
extern "C" {
#endif
+static int NV16ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*NV12ToARGBRow)(
+ const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) = NV12ToARGBRow_C;
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_NV12TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToARGBRow = NV12ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ NV12ToARGBRow = NV12ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV12ToARGBRow = NV12ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV12ToARGBRow = NV12ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ NV12ToARGBRow = NV12ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ NV12ToARGBRow = NV12ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ NV12ToARGBRow = NV12ToARGBRow_HVX;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ NV12ToARGBRow = NV12ToARGBRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ NV12ToARGBRow = NV12ToARGBRow_WASMSIMD;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ NV12ToARGBRow(src_y, src_uv, dst_argb, &kYuvI601Constants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+ return 0;
+}
+
+static int NV24ToARGB(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
+ const uint8_t* v_buf, uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants, int width) =
+ I444ToARGBRow_C;
+ void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
+ int width) = SplitUVRow_C;
+ if (!src_y || !src_uv || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+#if defined(HAS_I444TOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ I444ToARGBRow = I444ToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ I444ToARGBRow = I444ToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ I444ToARGBRow = I444ToARGBRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I444ToARGBRow = I444ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ I444ToARGBRow = I444ToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ I444ToARGBRow = I444ToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ I444ToARGBRow = I444ToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ I444ToARGBRow = I444ToARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ I444ToARGBRow = I444ToARGBRow_HVX;
+ }
+#endif
+
+#if defined(HAS_SPLITUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SplitUVRow = SplitUVRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ SplitUVRow = SplitUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitUVRow = SplitUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ SplitUVRow = SplitUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SplitUVRow = SplitUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitUVRow = SplitUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ SplitUVRow = SplitUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SplitUVRow = SplitUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SplitUVRow = SplitUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ SplitUVRow = SplitUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SplitUVRow = SplitUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ SplitUVRow = SplitUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitUVRow = SplitUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow = SplitUVRow_HVX;
+ }
+#endif
+
+ align_buffer_64(row_u, width);
+ align_buffer_64(row_v, width);
+ if (!row_u || !row_v) {
+ free_aligned_buffer_64(row_u);
+ free_aligned_buffer_64(row_v);
+ return 1;
+ }
+ for (y = 0; y < height; ++y) {
+ SplitUVRow(src_uv, row_u, row_v, width);
+ I444ToARGBRow(src_y, row_u, row_v, dst_argb, &kYuvI601Constants, width);
+ dst_argb += dst_stride_argb;
+ src_y += src_stride_y;
+ src_uv += src_stride_uv;
+ }
+ free_aligned_buffer_64(row_u);
+ free_aligned_buffer_64(row_v);
+ return 0;
+}
+
// Convert camera sample to ARGB with cropping, rotation and vertical flip.
// src_width is used for source stride computation
// src_height is used to compute location of planes, and indicate inversion
@@ -32,9 +311,6 @@ extern "C" {
// TODO(fbarchard): Add the following:
// H010ToARGB
// I010ToARGB
-// J400ToARGB
-// J422ToARGB
-// J444ToARGB
LIBYUV_API
int ConvertToARGB(const uint8_t* sample,
@@ -49,12 +325,26 @@ int ConvertToARGB(const uint8_t* sample,
int crop_height,
enum RotationMode rotation,
uint32_t fourcc) {
+ if (src_height == INT_MIN || crop_height == INT_MIN) {
+ return -1;
+ }
+
+ int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
+
+ if (dst_argb == NULL || sample == NULL || src_width <= 0 ||
+ src_width > INT_MAX / 4 || crop_width <= 0 || crop_width > INT_MAX / 4 ||
+ src_height == 0 || crop_height == 0 || crop_x < 0 || crop_y < 0 ||
+ crop_width > src_width || crop_x > src_width - crop_width ||
+ abs_crop_height > abs_src_height ||
+ crop_y > abs_src_height - abs_crop_height) {
+ return -1;
+ }
+
uint32_t format = CanonicalFourCC(fourcc);
int aligned_src_width = (src_width + 1) & ~1;
const uint8_t* src;
const uint8_t* src_uv;
- int abs_src_height = (src_height < 0) ? -src_height : src_height;
- int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height;
int r = 0;
// One pass rotation is available for some formats. For the rest, convert
@@ -62,24 +352,25 @@ int ConvertToARGB(const uint8_t* sample,
// and then rotate the ARGB to the final destination buffer.
// For in-place conversion, if destination dst_argb is same as source sample,
// also enable temporary buffer.
- LIBYUV_BOOL need_buf =
- (rotation && format != FOURCC_ARGB) || dst_argb == sample;
+ int need_buf = (rotation && format != FOURCC_ARGB) || dst_argb == sample;
uint8_t* dest_argb = dst_argb;
int dest_dst_stride_argb = dst_stride_argb;
uint8_t* rotate_buffer = NULL;
- int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
+ int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height;
- if (dst_argb == NULL || sample == NULL || src_width <= 0 || crop_width <= 0 ||
- src_height == 0 || crop_height == 0) {
- return -1;
- }
if (src_height < 0) {
inv_crop_height = -inv_crop_height;
}
if (need_buf) {
- int argb_size = crop_width * 4 * abs_crop_height;
- rotate_buffer = (uint8_t*)malloc(argb_size); /* NOLINT */
+ const uint64_t rotate_buffer_size =
+ (uint64_t)crop_width * 4 * abs_crop_height;
+#if UINT64_MAX > SIZE_MAX
+ if (rotate_buffer_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+#endif
+ rotate_buffer = (uint8_t*)malloc((size_t)rotate_buffer_size);
if (!rotate_buffer) {
return 1; // Out of memory runtime error.
}
@@ -90,146 +381,215 @@ int ConvertToARGB(const uint8_t* sample,
switch (format) {
// Single plane formats
case FOURCC_YUY2:
- src = sample + (aligned_src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2;
r = YUY2ToARGB(src, aligned_src_width * 2, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
break;
case FOURCC_UYVY:
- src = sample + (aligned_src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2;
r = UYVYToARGB(src, aligned_src_width * 2, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
break;
case FOURCC_24BG:
- src = sample + (src_width * crop_y + crop_x) * 3;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3;
r = RGB24ToARGB(src, src_width * 3, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_RAW:
- src = sample + (src_width * crop_y + crop_x) * 3;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3;
r = RAWToARGB(src, src_width * 3, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_ARGB:
if (!need_buf && !rotation) {
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = ARGBToARGB(src, src_width * 4, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
}
break;
case FOURCC_BGRA:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = BGRAToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_ABGR:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = ABGRToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_RGBA:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = RGBAToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_AR30:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = AR30ToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_AB30:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = AB30ToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
case FOURCC_RGBP:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = RGB565ToARGB(src, src_width * 2, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
break;
case FOURCC_RGBO:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = ARGB1555ToARGB(src, src_width * 2, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
break;
case FOURCC_R444:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = ARGB4444ToARGB(src, src_width * 2, dst_argb, dst_stride_argb,
crop_width, inv_crop_height);
break;
case FOURCC_I400:
- src = sample + src_width * crop_y + crop_x;
+ src = sample + (ptrdiff_t)src_width * crop_y + crop_x;
r = I400ToARGB(src, src_width, dst_argb, dst_stride_argb, crop_width,
inv_crop_height);
break;
+ case FOURCC_J400:
+ src = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ r = J400ToARGB(src, src_width, dst_argb, dst_stride_argb, crop_width,
+ inv_crop_height);
+ break;
// Biplanar formats
case FOURCC_NV12:
- src = sample + (src_width * crop_y + crop_x);
- src_uv =
- sample + aligned_src_width * (abs_src_height + crop_y / 2) + crop_x;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample +
+ aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
+ (crop_x & ~1);
r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
break;
case FOURCC_NV21:
- src = sample + (src_width * crop_y + crop_x);
- src_uv =
- sample + aligned_src_width * (abs_src_height + crop_y / 2) + crop_x;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample +
+ aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
+ (crop_x & ~1);
// Call NV12 but with u and v parameters swapped.
r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
break;
- case FOURCC_M420:
- src = sample + (src_width * crop_y) * 12 / 8 + crop_x;
- r = M420ToARGB(src, src_width, dst_argb, dst_stride_argb, crop_width,
- inv_crop_height);
+ case FOURCC_NV16:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample +
+ aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y) +
+ (crop_x & ~1);
+ r = NV16ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
+ dst_stride_argb, crop_width, inv_crop_height);
break;
-
+ case FOURCC_NV24:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample + (ptrdiff_t)aligned_src_width * abs_src_height +
+ (ptrdiff_t)aligned_src_width * 2 * crop_y +
+ (ptrdiff_t)crop_x * 2;
+ r = NV24ToARGB(src, src_width, src_uv, aligned_src_width * 2, dst_argb,
+ dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ case FOURCC_P010: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)(crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2);
+ r = P010ToARGBMatrix(src_y16, src_width, src_uv16, aligned_src_width,
+ dst_argb, dst_stride_argb, &kYuvI601Constants,
+ crop_width, inv_crop_height);
+ break;
+ }
+ case FOURCC_P210: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * aligned_src_width) + ((crop_x / 2) * 2);
+ r = P210ToARGBMatrix(src_y16, src_width, src_uv16, aligned_src_width,
+ dst_argb, dst_stride_argb, &kYuvI601Constants,
+ crop_width, inv_crop_height);
+ break;
+ }
+ case FOURCC_P410: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * src_width * 2) + crop_x * 2;
+ r = P410ToARGBMatrix(src_y16, src_width, src_uv16, src_width * 2,
+ dst_argb, dst_stride_argb, &kYuvI601Constants,
+ crop_width, inv_crop_height);
+ break;
+ }
// Triplanar formats
case FOURCC_I420:
case FOURCC_YV12: {
- const uint8_t* src_y = sample + (src_width * crop_y + crop_x);
+ const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u;
const uint8_t* src_v;
int halfwidth = (src_width + 1) / 2;
int halfheight = (abs_src_height + 1) / 2;
if (format == FOURCC_YV12) {
- src_v = sample + src_width * abs_src_height +
- (halfwidth * crop_y + crop_x) / 2;
- src_u = sample + src_width * abs_src_height +
- halfwidth * (halfheight + crop_y / 2) + crop_x / 2;
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
} else {
- src_u = sample + src_width * abs_src_height +
- (halfwidth * crop_y + crop_x) / 2;
- src_v = sample + src_width * abs_src_height +
- halfwidth * (halfheight + crop_y / 2) + crop_x / 2;
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
}
r = I420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
+ case FOURCC_J420: {
+ int halfwidth = (src_width + 1) / 2;
+ int halfheight = (abs_src_height + 1) / 2;
+ const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
+ crop_x / 2;
+ r = J420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
case FOURCC_H420: {
int halfwidth = (src_width + 1) / 2;
int halfheight = (abs_src_height + 1) / 2;
- const uint8_t* src_y = sample + (src_width * crop_y + crop_x);
- const uint8_t* src_u = sample + src_width * abs_src_height +
- (halfwidth * crop_y + crop_x) / 2;
- const uint8_t* src_v = sample + src_width * abs_src_height +
- halfwidth * (halfheight + crop_y / 2) + crop_x / 2;
+ const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
+ crop_x / 2;
r = H420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
- case FOURCC_J420: {
+ case FOURCC_U420: {
int halfwidth = (src_width + 1) / 2;
int halfheight = (abs_src_height + 1) / 2;
- const uint8_t* src_y = sample + (src_width * crop_y + crop_x);
- const uint8_t* src_u = sample + src_width * abs_src_height +
- (halfwidth * crop_y + crop_x) / 2;
- const uint8_t* src_v = sample + src_width * abs_src_height +
- halfwidth * (halfheight + crop_y / 2) + crop_x / 2;
- r = J420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
+ const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
+ crop_x / 2;
+ r = U420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
@@ -237,53 +597,121 @@ int ConvertToARGB(const uint8_t* sample,
case FOURCC_I422:
case FOURCC_YV16: {
int halfwidth = (src_width + 1) / 2;
- const uint8_t* src_y = sample + src_width * crop_y + crop_x;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
const uint8_t* src_u;
const uint8_t* src_v;
if (format == FOURCC_YV16) {
- src_v = sample + src_width * abs_src_height + halfwidth * crop_y +
- crop_x / 2;
- src_u = sample + src_width * abs_src_height +
- halfwidth * (abs_src_height + crop_y) + crop_x / 2;
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + crop_x / 2;
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2;
} else {
- src_u = sample + src_width * abs_src_height + halfwidth * crop_y +
- crop_x / 2;
- src_v = sample + src_width * abs_src_height +
- halfwidth * (abs_src_height + crop_y) + crop_x / 2;
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + crop_x / 2;
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2;
}
r = I422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
+ case FOURCC_J422: {
+ int halfwidth = (src_width + 1) / 2;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) +
+ crop_x / 2;
+ r = J422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
case FOURCC_H422: {
int halfwidth = (src_width + 1) / 2;
- const uint8_t* src_y = sample + src_width * crop_y + crop_x;
- const uint8_t* src_u =
- sample + src_width * abs_src_height + halfwidth * crop_y + crop_x / 2;
- const uint8_t* src_v = sample + src_width * abs_src_height +
- halfwidth * (abs_src_height + crop_y) + crop_x / 2;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) +
+ crop_x / 2;
r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
+ case FOURCC_U422: {
+ int halfwidth = (src_width + 1) / 2;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + crop_x / 2;
+ const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) +
+ crop_x / 2;
+ r = U422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
case FOURCC_I444:
case FOURCC_YV24: {
- const uint8_t* src_y = sample + src_width * crop_y + crop_x;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
const uint8_t* src_u;
const uint8_t* src_v;
if (format == FOURCC_YV24) {
- src_v = sample + src_width * (abs_src_height + crop_y) + crop_x;
- src_u = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x;
+ src_v =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ src_u = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
} else {
- src_u = sample + src_width * (abs_src_height + crop_y) + crop_x;
- src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x;
+ src_u =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
}
r = I444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}
+
+ case FOURCC_J444: {
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ const uint8_t* src_v =
+ sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
+ r = J444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
+ case FOURCC_H444: {
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ const uint8_t* src_v =
+ sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
+ r = H444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
+ case FOURCC_U444: {
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
+ const uint8_t* src_u =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ const uint8_t* src_v =
+ sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
+ r = U444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width,
+ dst_argb, dst_stride_argb, crop_width, inv_crop_height);
+ break;
+ }
+
#ifdef HAVE_JPEG
case FOURCC_MJPG:
r = MJPGToARGB(sample, sample_size, dst_argb, dst_stride_argb, src_width,
@@ -301,7 +729,7 @@ int ConvertToARGB(const uint8_t* sample,
}
free(rotate_buffer);
} else if (rotation) {
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = ARGBRotate(src, src_width * 4, dst_argb, dst_stride_argb, crop_width,
inv_crop_height, rotation);
}
diff --git a/libs/libyuv/source/convert_to_i420.cc b/libs/libyuv/source/convert_to_i420.cc
index 584be0ac33..e5a094df9b 100644
--- a/libs/libyuv/source/convert_to_i420.cc
+++ b/libs/libyuv/source/convert_to_i420.cc
@@ -8,10 +8,17 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include <stdlib.h>
-
#include "libyuv/convert.h"
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libyuv/cpu_id.h"
+#include "libyuv/planar_functions.h"
+#include "libyuv/row.h"
#include "libyuv/video_common.h"
#ifdef __cplusplus
@@ -19,6 +26,257 @@ namespace libyuv {
extern "C" {
#endif
+static int NV16ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ // UV rows are halfwidth * 2 (even) bytes.
+ int awidth = halfwidth * 2;
+ void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
+ int width) = SplitUVRow_C;
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_C;
+ if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ }
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+#if defined(HAS_SPLITUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SplitUVRow = SplitUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ SplitUVRow = SplitUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitUVRow = SplitUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ SplitUVRow = SplitUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SplitUVRow = SplitUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitUVRow = SplitUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ SplitUVRow = SplitUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SplitUVRow = SplitUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SplitUVRow = SplitUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ SplitUVRow = SplitUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SplitUVRow = SplitUVRow_Any_LASX;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ SplitUVRow = SplitUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_SPLITUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitUVRow = SplitUVRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_Any_AVX2;
+ if (IS_ALIGNED(awidth, 32)) {
+ InterpolateRow = InterpolateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_Any_NEON;
+ if (IS_ALIGNED(awidth, 16)) {
+ InterpolateRow = InterpolateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(awidth, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(awidth, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(awidth, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
+ }
+ }
+#endif
+
+ align_buffer_64(row_uv, awidth);
+ if (!row_uv) {
+ return 1;
+ }
+ for (y = 0; y < halfheight; ++y) {
+ if (y * 2 + 1 < height) {
+ // Average 2 rows of UV with rounding (source_y_fraction 128).
+ InterpolateRow(row_uv, src_uv, src_stride_uv, awidth, 128);
+ SplitUVRow(row_uv, dst_u, dst_v, halfwidth);
+ } else {
+ SplitUVRow(src_uv, dst_u, dst_v, halfwidth);
+ }
+ src_uv += (ptrdiff_t)src_stride_uv * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ free_aligned_buffer_64(row_uv);
+ return 0;
+}
+
+static int NV24ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int x, y;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ void (*HalfRow)(const uint8_t* src_uv, ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv, int width) = HalfRow_C;
+ if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ if (src_y) {
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+ src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ }
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+#if defined(HAS_HALFROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ HalfRow = HalfRow_Any_AVX2;
+ if (IS_ALIGNED(width * 2, 32)) {
+ HalfRow = HalfRow_AVX2;
+ }
+ }
+#endif
+
+ align_buffer_64(row_uv, width * 2);
+ if (!row_uv) {
+ return 1;
+ }
+ for (y = 0; y < halfheight; ++y) {
+ if (y * 2 + 1 < height) {
+ HalfRow(src_uv, src_stride_uv, row_uv, width * 2);
+ } else {
+ memcpy(row_uv, src_uv, (size_t)width * 2);
+ }
+ for (x = 0; x < halfwidth - 1; ++x) {
+ dst_u[x] = (row_uv[x * 4 + 0] + row_uv[x * 4 + 2] + 1) >> 1;
+ dst_v[x] = (row_uv[x * 4 + 1] + row_uv[x * 4 + 3] + 1) >> 1;
+ }
+ if (width & 1) {
+ dst_u[halfwidth - 1] = row_uv[(halfwidth - 1) * 4 + 0];
+ dst_v[halfwidth - 1] = row_uv[(halfwidth - 1) * 4 + 1];
+ } else {
+ dst_u[halfwidth - 1] = (row_uv[(halfwidth - 1) * 4 + 0] +
+ row_uv[(halfwidth - 1) * 4 + 2] + 1) >>
+ 1;
+ dst_v[halfwidth - 1] = (row_uv[(halfwidth - 1) * 4 + 1] +
+ row_uv[(halfwidth - 1) * 4 + 3] + 1) >>
+ 1;
+ }
+ src_uv += (ptrdiff_t)src_stride_uv * 2;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ free_aligned_buffer_64(row_uv);
+ return 0;
+}
+
// Convert camera sample to I420 with cropping, rotation and vertical flip.
// src_width is used for source stride computation
// src_height is used to compute location of planes, and indicate inversion
@@ -41,18 +299,28 @@ int ConvertToI420(const uint8_t* sample,
int crop_height,
enum RotationMode rotation,
uint32_t fourcc) {
+ if (src_height == INT_MIN || crop_height == INT_MIN) {
+ return -1;
+ }
+
+ const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ const int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
+
+ if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 ||
+ src_width > INT_MAX / 4 || crop_width <= 0 || src_height == 0 ||
+ crop_height == 0 || crop_x < 0 || crop_y < 0 || crop_width > src_width ||
+ crop_x > src_width - crop_width || abs_crop_height > abs_src_height ||
+ crop_y > abs_src_height - abs_crop_height) {
+ return -1;
+ }
+
uint32_t format = CanonicalFourCC(fourcc);
- int aligned_src_width = (src_width + 1) & ~1;
const uint8_t* src;
const uint8_t* src_uv;
- const int abs_src_height = (src_height < 0) ? -src_height : src_height;
- // TODO(nisse): Why allow crop_height < 0?
- const int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
int r = 0;
- LIBYUV_BOOL need_buf =
- (rotation && format != FOURCC_I420 && format != FOURCC_NV12 &&
- format != FOURCC_NV21 && format != FOURCC_YV12) ||
- dst_y == sample;
+ int need_buf = (rotation && format != FOURCC_I420 && format != FOURCC_NV12 &&
+ format != FOURCC_NV21 && format != FOURCC_YV12) ||
+ dst_y == sample;
uint8_t* tmp_y = dst_y;
uint8_t* tmp_u = dst_u;
uint8_t* tmp_v = dst_v;
@@ -62,11 +330,7 @@ int ConvertToI420(const uint8_t* sample,
uint8_t* rotate_buffer = NULL;
const int inv_crop_height =
(src_height < 0) ? -abs_crop_height : abs_crop_height;
-
- if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 ||
- crop_width <= 0 || src_height == 0 || crop_height == 0) {
- return -1;
- }
+ int aligned_src_width = (src_width + 1) & ~1;
// One pass rotation is available for some formats. For the rest, convert
// to I420 (with optional vertical flipping) into a temporary I420 buffer,
@@ -74,9 +338,14 @@ int ConvertToI420(const uint8_t* sample,
// For in-place conversion, if destination dst_y is same as source sample,
// also enable temporary buffer.
if (need_buf) {
- int y_size = crop_width * abs_crop_height;
- int uv_size = ((crop_width + 1) / 2) * ((abs_crop_height + 1) / 2);
- rotate_buffer = (uint8_t*)malloc(y_size + uv_size * 2); /* NOLINT */
+ size_t y_size = (size_t)crop_width * abs_crop_height;
+ size_t uv_size =
+ (size_t)((crop_width + 1) / 2) * ((abs_crop_height + 1) / 2);
+ if (uv_size > SIZE_MAX / 2 || y_size > SIZE_MAX - uv_size * 2) {
+ return -1; // Invalid size.
+ }
+ const size_t rotate_buffer_size = y_size + uv_size * 2;
+ rotate_buffer = (uint8_t*)malloc(rotate_buffer_size);
if (!rotate_buffer) {
return 1; // Out of memory runtime error.
}
@@ -89,119 +358,189 @@ int ConvertToI420(const uint8_t* sample,
switch (format) {
// Single plane formats
- case FOURCC_YUY2:
- src = sample + (aligned_src_width * crop_y + crop_x) * 2;
- r = YUY2ToI420(src, aligned_src_width * 2, dst_y, dst_stride_y, dst_u,
- dst_stride_u, dst_v, dst_stride_v, crop_width,
- inv_crop_height);
+ case FOURCC_YUY2: { // TODO(fbarchard): Find better odd crop fix.
+ uint8_t* u = (crop_x & 1) ? dst_v : dst_u;
+ uint8_t* v = (crop_x & 1) ? dst_u : dst_v;
+ int stride_u = (crop_x & 1) ? dst_stride_v : dst_stride_u;
+ int stride_v = (crop_x & 1) ? dst_stride_u : dst_stride_v;
+ src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2;
+ r = YUY2ToI420(src, aligned_src_width * 2, dst_y, dst_stride_y, u,
+ stride_u, v, stride_v, crop_width, inv_crop_height);
break;
- case FOURCC_UYVY:
- src = sample + (aligned_src_width * crop_y + crop_x) * 2;
- r = UYVYToI420(src, aligned_src_width * 2, dst_y, dst_stride_y, dst_u,
- dst_stride_u, dst_v, dst_stride_v, crop_width,
- inv_crop_height);
+ }
+ case FOURCC_UYVY: {
+ uint8_t* u = (crop_x & 1) ? dst_v : dst_u;
+ uint8_t* v = (crop_x & 1) ? dst_u : dst_v;
+ int stride_u = (crop_x & 1) ? dst_stride_v : dst_stride_u;
+ int stride_v = (crop_x & 1) ? dst_stride_u : dst_stride_v;
+ src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2;
+ r = UYVYToI420(src, aligned_src_width * 2, dst_y, dst_stride_y, u,
+ stride_u, v, stride_v, crop_width, inv_crop_height);
break;
+ }
case FOURCC_RGBP:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = RGB565ToI420(src, src_width * 2, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_RGBO:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = ARGB1555ToI420(src, src_width * 2, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_R444:
- src = sample + (src_width * crop_y + crop_x) * 2;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2;
r = ARGB4444ToI420(src, src_width * 2, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_24BG:
- src = sample + (src_width * crop_y + crop_x) * 3;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3;
r = RGB24ToI420(src, src_width * 3, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_RAW:
- src = sample + (src_width * crop_y + crop_x) * 3;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3;
r = RAWToI420(src, src_width * 3, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_ARGB:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = ARGBToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_BGRA:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = BGRAToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_ABGR:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = ABGRToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
case FOURCC_RGBA:
- src = sample + (src_width * crop_y + crop_x) * 4;
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
r = RGBAToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
dst_stride_u, dst_v, dst_stride_v, crop_width,
inv_crop_height);
break;
- // TODO(fbarchard): Add AR30 and AB30
+ case FOURCC_AR30:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
+ r = AR30ToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, crop_width,
+ inv_crop_height);
+ break;
+ case FOURCC_AB30:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4;
+ r = AB30ToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, crop_width,
+ inv_crop_height);
+ break;
case FOURCC_I400:
- src = sample + src_width * crop_y + crop_x;
+ src = sample + (ptrdiff_t)src_width * crop_y + crop_x;
r = I400ToI420(src, src_width, dst_y, dst_stride_y, dst_u, dst_stride_u,
dst_v, dst_stride_v, crop_width, inv_crop_height);
break;
// Biplanar formats
case FOURCC_NV12:
- src = sample + (src_width * crop_y + crop_x);
- src_uv = sample + (src_width * abs_src_height) +
- ((crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2);
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)(crop_y / 2) * aligned_src_width) +
+ ((crop_x / 2) * 2);
r = NV12ToI420Rotate(src, src_width, src_uv, aligned_src_width, dst_y,
dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, crop_width, inv_crop_height, rotation);
break;
case FOURCC_NV21:
- src = sample + (src_width * crop_y + crop_x);
- src_uv = sample + (src_width * abs_src_height) +
- ((crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2);
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)(crop_y / 2) * aligned_src_width) +
+ ((crop_x / 2) * 2);
// Call NV12 but with dst_u and dst_v parameters swapped.
r = NV12ToI420Rotate(src, src_width, src_uv, aligned_src_width, dst_y,
dst_stride_y, dst_v, dst_stride_v, dst_u,
dst_stride_u, crop_width, inv_crop_height, rotation);
break;
- case FOURCC_M420:
- src = sample + (src_width * crop_y) * 12 / 8 + crop_x;
- r = M420ToI420(src, src_width, dst_y, dst_stride_y, dst_u, dst_stride_u,
- dst_v, dst_stride_v, crop_width, inv_crop_height);
+ case FOURCC_NV16:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample + ((ptrdiff_t)aligned_src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * aligned_src_width) + ((crop_x / 2) * 2);
+ r = NV16ToI420(src, src_width, src_uv, aligned_src_width, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ crop_width, inv_crop_height);
+ break;
+ case FOURCC_NV24:
+ src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
+ src_uv = sample + ((ptrdiff_t)aligned_src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * aligned_src_width * 2) + (crop_x * 2);
+ r = NV24ToI420(src, src_width, src_uv, aligned_src_width * 2, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ crop_width, inv_crop_height);
+ break;
+ case FOURCC_P010: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)(crop_y / 2) * aligned_src_width) + ((crop_x / 2) * 2);
+ r = P010ToI420(src_y16, src_width, src_uv16, aligned_src_width, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ crop_width, inv_crop_height);
break;
+ }
+ case FOURCC_P210: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * aligned_src_width) + ((crop_x / 2) * 2);
+ r = P210ToI420(src_y16, src_width, src_uv16, aligned_src_width, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ crop_width, inv_crop_height);
+ break;
+ }
+ case FOURCC_P410: {
+ const uint16_t* src16 = (const uint16_t*)sample;
+ const uint16_t* src_y16 =
+ src16 + ((ptrdiff_t)src_width * crop_y + crop_x);
+ const uint16_t* src_uv16 =
+ src16 + ((ptrdiff_t)src_width * abs_src_height) +
+ ((ptrdiff_t)crop_y * src_width * 2) + crop_x * 2;
+ r = P410ToI420(src_y16, src_width, src_uv16, src_width * 2, dst_y,
+ dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ crop_width, inv_crop_height);
+ break;
+ }
// Triplanar formats
case FOURCC_I420:
case FOURCC_YV12: {
- const uint8_t* src_y = sample + (src_width * crop_y + crop_x);
+ const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u;
const uint8_t* src_v;
int halfwidth = (src_width + 1) / 2;
int halfheight = (abs_src_height + 1) / 2;
if (format == FOURCC_YV12) {
- src_v = sample + src_width * abs_src_height + halfwidth * (crop_y / 2) +
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + (crop_x / 2);
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + (crop_y / 2)) +
(crop_x / 2);
- src_u = sample + src_width * abs_src_height +
- halfwidth * (halfheight + (crop_y / 2)) + (crop_x / 2);
} else {
- src_u = sample + src_width * abs_src_height + halfwidth * (crop_y / 2) +
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * (crop_y / 2) + (crop_x / 2);
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)halfheight + (crop_y / 2)) +
(crop_x / 2);
- src_v = sample + src_width * abs_src_height +
- halfwidth * (halfheight + (crop_y / 2)) + (crop_x / 2);
}
r = I420Rotate(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
@@ -210,20 +549,20 @@ int ConvertToI420(const uint8_t* sample,
}
case FOURCC_I422:
case FOURCC_YV16: {
- const uint8_t* src_y = sample + src_width * crop_y + crop_x;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
const uint8_t* src_u;
const uint8_t* src_v;
int halfwidth = (src_width + 1) / 2;
if (format == FOURCC_YV16) {
- src_v = sample + src_width * abs_src_height + halfwidth * crop_y +
- (crop_x / 2);
- src_u = sample + src_width * abs_src_height +
- halfwidth * (abs_src_height + crop_y) + (crop_x / 2);
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + (crop_x / 2);
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + (crop_x / 2);
} else {
- src_u = sample + src_width * abs_src_height + halfwidth * crop_y +
- (crop_x / 2);
- src_v = sample + src_width * abs_src_height +
- halfwidth * (abs_src_height + crop_y) + (crop_x / 2);
+ src_u = sample + (ptrdiff_t)src_width * abs_src_height +
+ (ptrdiff_t)halfwidth * crop_y + (crop_x / 2);
+ src_v = sample + (ptrdiff_t)src_width * abs_src_height +
+ halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + (crop_x / 2);
}
r = I422ToI420(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
@@ -232,15 +571,19 @@ int ConvertToI420(const uint8_t* sample,
}
case FOURCC_I444:
case FOURCC_YV24: {
- const uint8_t* src_y = sample + src_width * crop_y + crop_x;
+ const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x;
const uint8_t* src_u;
const uint8_t* src_v;
if (format == FOURCC_YV24) {
- src_v = sample + src_width * (abs_src_height + crop_y) + crop_x;
- src_u = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x;
+ src_v =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ src_u = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
} else {
- src_u = sample + src_width * (abs_src_height + crop_y) + crop_x;
- src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x;
+ src_u =
+ sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x;
+ src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) +
+ crop_x;
}
r = I444ToI420(src_y, src_width, src_u, src_width, src_v, src_width,
dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
diff --git a/libs/libyuv/source/cpu_id.cc b/libs/libyuv/source/cpu_id.cc
index 48e2b61526..f78283a972 100644
--- a/libs/libyuv/source/cpu_id.cc
+++ b/libs/libyuv/source/cpu_id.cc
@@ -20,9 +20,25 @@
#endif
// For ArmCpuCaps() but unittested on all platforms
-#include <stdio.h>
+#include <stdio.h> // For fopen()
#include <string.h>
+#if defined(__linux__) && (defined(__aarch64__) || defined(__loongarch__))
+#include <sys/auxv.h> // For getauxval()
+#endif
+
+#if defined(_WIN32) && defined(__aarch64__)
+#undef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#undef WIN32_EXTRA_LEAN
+#define WIN32_EXTRA_LEAN
+#include <windows.h> // For IsProcessorFeaturePresent()
+#endif
+
+#if defined(__APPLE__) && defined(__aarch64__)
+#include <sys/sysctl.h> // For sysctlbyname()
+#endif
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
@@ -40,7 +56,6 @@ extern "C" {
// cpu_info_ variable for SIMD instruction sets detected.
LIBYUV_API int cpu_info_ = 0;
-// TODO(fbarchard): Consider using int for cpuid so casting is not needed.
// Low level cpuid for X86.
#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
defined(__x86_64__)) && \
@@ -75,9 +90,9 @@ void CpuId(int info_eax, int info_ecx, int* cpu_info) {
asm volatile(
#if defined(__i386__) && defined(__PIC__)
// Preserve ebx for fpic 32 bit.
- "mov %%ebx, %%edi \n"
+ "mov %%ebx, %%edi \n"
"cpuid \n"
- "xchg %%edi, %%ebx \n"
+ "xchg %%edi, %%ebx \n"
: "=D"(info_ebx),
#else
"cpuid \n"
@@ -108,14 +123,14 @@ void CpuId(int eax, int ecx, int* cpu_info) {
// }
// For VS2013 and earlier 32 bit, the _xgetbv(0) optimizer produces bad code.
// https://code.google.com/p/libyuv/issues/detail?id=529
-#if defined(_M_IX86) && (_MSC_VER < 1900)
+#if defined(_M_IX86) && defined(_MSC_VER) && (_MSC_VER < 1900)
#pragma optimize("g", off)
#endif
#if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
defined(__x86_64__)) && \
!defined(__pnacl__) && !defined(__CLR_VER) && !defined(__native_client__)
// X86 CPUs have xgetbv to detect OS saves high parts of ymm registers.
-int GetXCR0() {
+static int GetXCR0() {
int xcr0 = 0;
#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
xcr0 = (int)_xgetbv(0); // VS2010 SP1 required. NOLINT
@@ -129,79 +144,257 @@ int GetXCR0() {
#define GetXCR0() 0
#endif // defined(_M_IX86) || defined(_M_X64) ..
// Return optimization to previous setting.
-#if defined(_M_IX86) && (_MSC_VER < 1900)
+#if defined(_M_IX86) && defined(_MSC_VER) && (_MSC_VER < 1900)
#pragma optimize("g", on)
#endif
-// based on libvpx arm_cpudetect.c
+static int cpuinfo_search(const char* cpuinfo_line,
+ const char* needle,
+ int needle_len) {
+ const char* p = strstr(cpuinfo_line, needle);
+ return p && (p[needle_len] == ' ' || p[needle_len] == '\n');
+}
+
+// Based on libvpx arm_cpudetect.c
// For Arm, but public to allow testing on any CPU
LIBYUV_API SAFEBUFFERS int ArmCpuCaps(const char* cpuinfo_name) {
char cpuinfo_line[512];
- FILE* f = fopen(cpuinfo_name, "r");
+ FILE* f = fopen(cpuinfo_name, "re");
if (!f) {
// Assume Neon if /proc/cpuinfo is unavailable.
// This will occur for Chrome sandbox for Pepper or Render process.
return kCpuHasNEON;
}
- while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
+ memset(cpuinfo_line, 0, sizeof(cpuinfo_line));
+ int features = 0;
+ while (fgets(cpuinfo_line, sizeof(cpuinfo_line), f)) {
if (memcmp(cpuinfo_line, "Features", 8) == 0) {
- char* p = strstr(cpuinfo_line, " neon");
- if (p && (p[5] == ' ' || p[5] == '\n')) {
- fclose(f);
- return kCpuHasNEON;
- }
- // aarch64 uses asimd for Neon.
- p = strstr(cpuinfo_line, " asimd");
- if (p) {
- fclose(f);
- return kCpuHasNEON;
+ if (cpuinfo_search(cpuinfo_line, " neon", 5)) {
+ features |= kCpuHasNEON;
}
}
}
fclose(f);
- return 0;
+ return features;
}
-// TODO(fbarchard): Consider read_msa_ir().
-// TODO(fbarchard): Add unittest.
-LIBYUV_API SAFEBUFFERS int MipsCpuCaps(const char* cpuinfo_name,
- const char ase[]) {
- char cpuinfo_line[512];
- FILE* f = fopen(cpuinfo_name, "r");
- if (!f) {
- // ase enabled if /proc/cpuinfo is unavailable.
- if (strcmp(ase, " msa") == 0) {
- return kCpuHasMSA;
+#ifdef __aarch64__
+#ifdef __linux__
+// Define hwcap values ourselves: building with an old auxv header where these
+// hwcap values are not defined should not prevent features from being enabled.
+#define YUV_AARCH64_HWCAP_ASIMDDP (1UL << 20)
+#define YUV_AARCH64_HWCAP_SVE (1UL << 22)
+#define YUV_AARCH64_HWCAP2_SVE2 (1UL << 1)
+#define YUV_AARCH64_HWCAP2_SVEF32MM (1UL << 10)
+#define YUV_AARCH64_HWCAP2_I8MM (1UL << 13)
+#define YUV_AARCH64_HWCAP2_SME (1UL << 23)
+#define YUV_AARCH64_HWCAP2_SME2 (1UL << 37)
+
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps(unsigned long hwcap,
+ unsigned long hwcap2) {
+ // Neon is mandatory on AArch64, so enable regardless of hwcaps.
+ int features = kCpuHasNEON;
+
+ // Don't try to enable later extensions unless earlier extensions are also
+ // reported available. Some of these constraints aren't strictly required by
+ // the architecture, but are satisfied by all micro-architectures of
+ // interest. This also avoids an issue on some emulators where true
+ // architectural constraints are not satisfied, e.g. SVE2 may be reported as
+ // available while SVE is not.
+ if (hwcap & YUV_AARCH64_HWCAP_ASIMDDP) {
+ features |= kCpuHasNeonDotProd;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_I8MM) {
+ features |= kCpuHasNeonI8MM;
+ if (hwcap & YUV_AARCH64_HWCAP_SVE) {
+ features |= kCpuHasSVE;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SVEF32MM) {
+ features |= kCpuHasSVEF32MM;
+ }
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SVE2) {
+ features |= kCpuHasSVE2;
+ }
+ }
+ // SME may be present without SVE
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SME) {
+ features |= kCpuHasSME;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SME2) {
+ features |= kCpuHasSME2;
+ }
+ }
}
- if (strcmp(ase, " mmi") == 0) {
- return kCpuHasMMI;
+ }
+ return features;
+}
+
+#elif defined(_WIN32)
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ // For more information on IsProcessorFeaturePresent(), see:
+ // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent#parameters
+#ifdef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
+ if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)) {
+ features |= kCpuHasNeonDotProd;
+ }
+#endif
+ // No Neon I8MM or SVE feature detection available here at time of writing.
+ return features;
+}
+
+#elif defined(__APPLE__)
+static bool have_feature(const char* feature) {
+ // For more information on sysctlbyname(), see:
+ // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
+ // If feature is "hw.optional.arm.sme_max_svl_b", this function returns
+ // whether the value of sme_max_svl_b is nonzero.
+ int64_t feature_present = 0;
+ size_t size = sizeof(feature_present);
+ if (sysctlbyname(feature, &feature_present, &size, NULL, 0) != 0) {
+ return false;
+ }
+ return feature_present != 0;
+}
+
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ if (have_feature("hw.optional.arm.FEAT_DotProd")) {
+ features |= kCpuHasNeonDotProd;
+ if (have_feature("hw.optional.arm.FEAT_I8MM")) {
+ features |= kCpuHasNeonI8MM;
+ // An Apple processor may have SME hardware (e.g. A18/A19), but the XNU
+ // kernel only enables EL0 execution if sme_max_svl_b is nonzero.
+ // macOS and iPadOS (e.g. on M4) support and enable user-mode SME, but iOS
+ // disables user-mode SME, which causes SMSTART to trigger an
+ // EXC_BAD_INSTRUCTION fault. See
+ // https://github.com/pytorch/cpuinfo/issues/432.
+ if (have_feature("hw.optional.arm.sme_max_svl_b") &&
+ have_feature("hw.optional.arm.FEAT_SME")) {
+ features |= kCpuHasSME;
+ if (have_feature("hw.optional.arm.FEAT_SME2")) {
+ features |= kCpuHasSME2;
+ }
+ }
}
+ }
+ // No SVE feature detection available here at time of writing.
+ return features;
+}
+
+#else // !defined(__linux__) && !defined(_WIN32) && !defined(__APPLE__)
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ // TODO(libyuv:980) support feature detection on other platforms.
+
+ return features;
+}
+#endif
+#endif // defined(__aarch64__)
+
+LIBYUV_API SAFEBUFFERS int RiscvCpuCaps(const char* cpuinfo_name) {
+ char cpuinfo_line[512];
+ int flag = 0;
+ FILE* f = fopen(cpuinfo_name, "re");
+ if (!f) {
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv) && __riscv_xlen == 64
+ // Assume RVV if /proc/cpuinfo is unavailable.
+ // This will occur for Chrome sandbox for Pepper or Render process.
+ return kCpuHasRVV;
+#else
return 0;
+#endif
}
- while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
- if (memcmp(cpuinfo_line, "ASEs implemented", 16) == 0) {
- char* p = strstr(cpuinfo_line, ase);
- if (p) {
- fclose(f);
- if (strcmp(ase, " msa") == 0) {
- return kCpuHasMSA;
+ memset(cpuinfo_line, 0, sizeof(cpuinfo_line));
+ while (fgets(cpuinfo_line, sizeof(cpuinfo_line), f)) {
+ if (memcmp(cpuinfo_line, "isa", 3) == 0) {
+ // ISA string must begin with rv64{i,e,g} for a 64-bit processor.
+ char* isa = strstr(cpuinfo_line, "rv64");
+ if (isa) {
+ size_t isa_len = strlen(isa);
+ char* extensions;
+ size_t extensions_len = 0;
+ size_t std_isa_len;
+ // Remove the new-line character at the end of string
+ if (isa[isa_len - 1] == '\n') {
+ isa[--isa_len] = '\0';
}
- return 0;
- }
- } else if (memcmp(cpuinfo_line, "cpu model", 9) == 0) {
- char* p = strstr(cpuinfo_line, "Loongson-3");
- if (p) {
- fclose(f);
- if (strcmp(ase, " mmi") == 0) {
- return kCpuHasMMI;
+ // 5 ISA characters
+ if (isa_len < 5) {
+ fclose(f);
+ return 0;
+ }
+ // Skip {i,e,g} canonical checking.
+ // Skip rvxxx
+ isa += 5;
+ // Find the very first occurrence of 's', 'x' or 'z'.
+ // To detect multi-letter standard, non-standard, and
+ // supervisor-level extensions.
+ extensions = strpbrk(isa, "zxs");
+ if (extensions) {
+ extensions_len = strlen(extensions);
+ // Multi-letter extensions are seperated by a single underscore
+ // as described in RISC-V User-Level ISA V2.2.
+ char* ext = extensions;
+ while (ext) {
+ char* next = strchr(ext, '_');
+ if (next) {
+ *next = '\0';
+ next++;
+ }
+ // Search for the ZVFH (Vector FP16) extension.
+ if (!strcmp(ext, "zvfh")) {
+ flag |= kCpuHasRVVZVFH;
+ }
+ ext = next;
+ }
+ }
+ std_isa_len = isa_len - extensions_len - 5;
+ // Detect the v in the standard single-letter extensions.
+ if (memchr(isa, 'v', std_isa_len)) {
+ // The RVV implied the F extension.
+ flag |= kCpuHasRVV;
}
- return 0;
}
}
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv) && __riscv_xlen == 64
+ // Assume RVV if /proc/cpuinfo is from x86 host running QEMU.
+ else if ((memcmp(cpuinfo_line, "vendor_id\t: GenuineIntel", 24) == 0) ||
+ (memcmp(cpuinfo_line, "vendor_id\t: AuthenticAMD", 24) == 0)) {
+ fclose(f);
+ return kCpuHasRVV;
+ }
+#endif
}
fclose(f);
- return 0;
+ return flag;
+}
+
+#if defined(__loongarch__) && defined(__linux__)
+// Define hwcap values ourselves: building with an old auxv header where these
+// hwcap values are not defined should not prevent features from being enabled.
+#define YUV_LOONGARCH_HWCAP_LSX (1 << 4)
+#define YUV_LOONGARCH_HWCAP_LASX (1 << 5)
+
+LIBYUV_API SAFEBUFFERS int LoongArchCpuCaps(void) {
+ int flag = 0;
+ unsigned long hwcap = getauxval(AT_HWCAP);
+
+ if (hwcap & YUV_LOONGARCH_HWCAP_LSX)
+ flag |= kCpuHasLSX;
+
+ if (hwcap & YUV_LOONGARCH_HWCAP_LASX)
+ flag |= kCpuHasLASX;
+ return flag;
}
+#endif
static SAFEBUFFERS int GetCpuFlags(void) {
int cpu_info = 0;
@@ -211,62 +404,121 @@ static SAFEBUFFERS int GetCpuFlags(void) {
int cpu_info0[4] = {0, 0, 0, 0};
int cpu_info1[4] = {0, 0, 0, 0};
int cpu_info7[4] = {0, 0, 0, 0};
+ int cpu_einfo7[4] = {0, 0, 0, 0};
+ int cpu_einfo1e[4] = {0, 0, 0, 0};
+ int cpu_info24[4] = {0, 0, 0, 0};
+ int cpu_info21[4] = {0, 0, 0, 0};
+ int cpu_amdinfo21[4] = {0, 0, 0, 0};
+ int cpu_extinfo0[4] = {0, 0, 0, 0};
CpuId(0, 0, cpu_info0);
CpuId(1, 0, cpu_info1);
if (cpu_info0[0] >= 7) {
CpuId(7, 0, cpu_info7);
+ // Leaf 7 EAX reports the maximum supported sub-leaf.
+ if (cpu_info7[0] >= 1) {
+ CpuId(7, 1, cpu_einfo7);
+ }
+ }
+ // Extended leaf 0x80000021 is only valid if the CPU reports it. Querying an
+ // unsupported extended leaf returns data from the highest basic leaf on
+ // Intel CPUs, which could spuriously set bits below.
+ CpuId(0x80000000, 0, cpu_extinfo0);
+ if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) {
+ CpuId(0x80000021, 0, cpu_amdinfo21);
+ }
+ if (cpu_info0[0] >= 0x1e) {
+ CpuId(0x1e, 1, cpu_einfo1e);
+ }
+ if (cpu_info0[0] >= 0x21) {
+ CpuId(0x21, 0, cpu_info21);
+ }
+ if (cpu_info0[0] >= 0x24) {
+ CpuId(0x24, 0, cpu_info24);
}
cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) |
((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) |
- ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0);
+ ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0) |
+ ((cpu_info7[3] & 0x00000010) ? kCpuHasFSMR : 0);
// AVX requires OS saves YMM registers.
if (((cpu_info1[2] & 0x1c000000) == 0x1c000000) && // AVX and OSXSave
((GetXCR0() & 6) == 6)) { // Test OS saves YMM registers
cpu_info |= kCpuHasAVX | ((cpu_info7[1] & 0x00000020) ? kCpuHasAVX2 : 0) |
((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) |
- ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0);
+ ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0) |
+ ((cpu_einfo7[0] & 0x00000010) ? kCpuHasAVXVNNI : 0) |
+ ((cpu_einfo7[3] & 0x00000010) ? kCpuHasAVXVNNIINT8 : 0);
+
+ cpu_info |= ((cpu_amdinfo21[0] & 0x00008000) ? kCpuHasERMS : 0);
// Detect AVX512bw
- if ((GetXCR0() & 0xe0) == 0xe0) {
- cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0;
- cpu_info |= (cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0;
- cpu_info |= (cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0;
- cpu_info |= (cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0;
- cpu_info |= (cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0;
- cpu_info |= (cpu_info7[2] & 0x00004000) ? kCpuHasAVX512VPOPCNTDQ : 0;
- cpu_info |= (cpu_info7[2] & 0x00000100) ? kCpuHasGFNI : 0;
+ if ((GetXCR0() & 0xe0) == 0xe0 && (cpu_info7[1] & 0x00010000)) {
+ cpu_info |= ((cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0) |
+ ((cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0) |
+ ((cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0) |
+ ((cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0) |
+ ((cpu_info7[2] & 0x00000800) ? kCpuHasAVX512VNNI : 0) |
+ ((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) |
+ ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) |
+ ((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0) |
+ ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) |
+ ((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0);
+ if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) {
+ cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0;
+ }
}
}
#endif
-#if defined(__mips__) && defined(__linux__)
-#if defined(__mips_msa)
- cpu_info = MipsCpuCaps("/proc/cpuinfo", " msa");
-#elif defined(_MIPS_ARCH_LOONGSON3A)
- cpu_info = MipsCpuCaps("/proc/cpuinfo", " mmi");
-#endif
- cpu_info |= kCpuHasMIPS;
-#endif
-#if defined(__arm__) || defined(__aarch64__)
-// gcc -mfpu=neon defines __ARM_NEON__
-// __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon.
-// For Linux, /proc/cpuinfo can be tested but without that assume Neon.
-#if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__)
- cpu_info = kCpuHasNEON;
-// For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon
-// flag in it.
-// So for aarch64, neon enabling is hard coded here.
+#if defined(__loongarch__) && defined(__linux__)
+ cpu_info = LoongArchCpuCaps();
+ cpu_info |= kCpuHasLOONGARCH;
#endif
#if defined(__aarch64__)
- cpu_info = kCpuHasNEON;
+#if defined(__linux__)
+ // getauxval is supported since Android SDK version 18, minimum at time of
+ // writing is 21, so should be safe to always use this. If getauxval is
+ // somehow disabled then getauxval returns 0, which will leave Neon enabled
+ // since Neon is mandatory on AArch64.
+ unsigned long hwcap = getauxval(AT_HWCAP);
+ unsigned long hwcap2 = getauxval(AT_HWCAP2);
+ cpu_info = AArch64CpuCaps(hwcap, hwcap2);
#else
+ cpu_info = AArch64CpuCaps();
+#endif
+ cpu_info |= kCpuHasARM;
+#endif // __aarch64__
+#if defined(__arm__)
+ // gcc -mfpu=neon defines __ARM_NEON__
+ // __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon.
+ // For Linux, /proc/cpuinfo can be tested but without that assume Neon.
// Linux arm parse text file for neon detect.
+#if defined(__linux__)
cpu_info = ArmCpuCaps("/proc/cpuinfo");
+#elif defined(__ARM_NEON__)
+ cpu_info = kCpuHasNEON;
+#else
+ cpu_info = 0;
#endif
cpu_info |= kCpuHasARM;
#endif // __arm__
+#if defined(__riscv) && defined(__linux__)
+ cpu_info = RiscvCpuCaps("/proc/cpuinfo");
+ cpu_info |= kCpuHasRISCV;
+#endif // __riscv
+#if defined(__wasm__)
+ cpu_info = kCpuHasWASM;
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+ cpu_info |= kCpuHasWASMSIMD;
+#endif
+#endif // __wasm__
+#if defined(__hexagon__)
+ cpu_info = kCpuHasHexagon;
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+ cpu_info |= kCpuHasHVX;
+#endif
+#endif // __hexagon__
cpu_info |= kCpuInitialized;
return cpu_info;
}
diff --git a/libs/libyuv/source/mjpeg_decoder.cc b/libs/libyuv/source/mjpeg_decoder.cc
index 80e381dd6b..4c6a22f935 100644
--- a/libs/libyuv/source/mjpeg_decoder.cc
+++ b/libs/libyuv/source/mjpeg_decoder.cc
@@ -13,8 +13,7 @@
#ifdef HAVE_JPEG
#include <assert.h>
-#if !defined(__pnacl__) && !defined(__CLR_VER) && \
- !defined(COVERAGE_ENABLED) && !defined(TARGET_IPHONE_SIMULATOR)
+#if !defined(__pnacl__) && !defined(__CLR_VER) && !defined(COVERAGE_ENABLED)
// Must be included before jpeglib.
#include <setjmp.h>
#define HAVE_SETJMP
@@ -26,7 +25,8 @@
#endif
-#include <stdio.h> // For jpeglib.h.
+#include <stdio.h> // For jpeglib.h.
+#include <stdlib.h> // For calloc(), malloc(), and free().
// C++ build requires extern C for jpeg internals.
#ifdef __cplusplus
@@ -66,16 +66,17 @@ void ErrorHandler(jpeg_common_struct* cinfo);
void OutputHandler(jpeg_common_struct* cinfo);
MJpegDecoder::MJpegDecoder()
- : has_scanline_padding_(LIBYUV_FALSE),
+ : has_scanline_padding_(false),
num_outbufs_(0),
scanlines_(NULL),
scanlines_sizes_(NULL),
databuf_(NULL),
databuf_strides_(NULL) {
- decompress_struct_ = new jpeg_decompress_struct;
- source_mgr_ = new jpeg_source_mgr;
+ decompress_struct_ = static_cast<jpeg_decompress_struct*>(
+ malloc(sizeof(jpeg_decompress_struct)));
+ source_mgr_ = static_cast<jpeg_source_mgr*>(malloc(sizeof(jpeg_source_mgr)));
#ifdef HAVE_SETJMP
- error_mgr_ = new SetJmpErrorMgr;
+ error_mgr_ = static_cast<SetJmpErrorMgr*>(malloc(sizeof(SetJmpErrorMgr)));
decompress_struct_->err = jpeg_std_error(&error_mgr_->base);
// Override standard exit()-based error handler.
error_mgr_->base.error_exit = &ErrorHandler;
@@ -95,42 +96,45 @@ MJpegDecoder::MJpegDecoder()
MJpegDecoder::~MJpegDecoder() {
jpeg_destroy_decompress(decompress_struct_);
- delete decompress_struct_;
- delete source_mgr_;
+ free(decompress_struct_);
+ free(source_mgr_);
#ifdef HAVE_SETJMP
- delete error_mgr_;
+ free(error_mgr_);
#endif
DestroyOutputBuffers();
}
-LIBYUV_BOOL MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
+bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
if (!ValidateJpeg(src, src_len)) {
- return LIBYUV_FALSE;
+ return false;
}
buf_.data = src;
- buf_.len = static_cast<int>(src_len);
+ buf_.len = (int)src_len;
buf_vec_.pos = 0;
decompress_struct_->client_data = &buf_vec_;
#ifdef HAVE_SETJMP
if (setjmp(error_mgr_->setjmp_buffer)) {
// We called jpeg_read_header, it experienced an error, and we called
// longjmp() and rewound the stack to here. Return error.
- return LIBYUV_FALSE;
+ return false;
}
#endif
if (jpeg_read_header(decompress_struct_, TRUE) != JPEG_HEADER_OK) {
// ERROR: Bad MJPEG header
- return LIBYUV_FALSE;
+ return false;
}
AllocOutputBuffers(GetNumComponents());
for (int i = 0; i < num_outbufs_; ++i) {
int scanlines_size = GetComponentScanlinesPerImcuRow(i);
- if (scanlines_sizes_[i] != scanlines_size) {
- if (scanlines_[i]) {
- delete scanlines_[i];
- }
- scanlines_[i] = new uint8_t*[scanlines_size];
+ // databuf_[i] is sized by scanlines_size * stride, so a change in either
+ // must trigger reallocation below. Remember the change before
+ // scanlines_sizes_[i] is overwritten.
+ bool scanlines_changed = scanlines_sizes_[i] != scanlines_size;
+ if (scanlines_changed) {
+ free(scanlines_[i]);
+ scanlines_[i] =
+ static_cast<uint8_t**>(malloc(scanlines_size * sizeof(uint8_t*)));
scanlines_sizes_[i] = scanlines_size;
}
@@ -141,20 +145,23 @@ LIBYUV_BOOL MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
// the next scanline) and will be overwritten when jpeglib writes out that
// next scanline.
int databuf_stride = GetComponentStride(i);
+ // Cannot overflow:
+ // - JPEG width is stored as an u16, so `databuf_stride` (width rounded up
+ // to
+ // DCTSIZE) is at most slightly larger than UINT16_MAX.
+ // - Sampling factor is stored in 4 bits, so scanlines_size is < 16.
int databuf_size = scanlines_size * databuf_stride;
- if (databuf_strides_[i] != databuf_stride) {
- if (databuf_[i]) {
- delete databuf_[i];
- }
- databuf_[i] = new uint8_t[databuf_size];
+ if (scanlines_changed || databuf_strides_[i] != databuf_stride) {
+ free(databuf_[i]);
+ databuf_[i] = static_cast<uint8_t*>(malloc(databuf_size));
databuf_strides_[i] = databuf_stride;
}
if (GetComponentStride(i) != GetComponentWidth(i)) {
- has_scanline_padding_ = LIBYUV_TRUE;
+ has_scanline_padding_ = true;
}
}
- return LIBYUV_TRUE;
+ return true;
}
static int DivideAndRoundUp(int numerator, int denominator) {
@@ -231,50 +238,52 @@ int MJpegDecoder::GetComponentSize(int component) {
return GetComponentWidth(component) * GetComponentHeight(component);
}
-LIBYUV_BOOL MJpegDecoder::UnloadFrame() {
+bool MJpegDecoder::UnloadFrame() {
#ifdef HAVE_SETJMP
if (setjmp(error_mgr_->setjmp_buffer)) {
// We called jpeg_abort_decompress, it experienced an error, and we called
// longjmp() and rewound the stack to here. Return error.
- return LIBYUV_FALSE;
+ return false;
}
#endif
jpeg_abort_decompress(decompress_struct_);
- return LIBYUV_TRUE;
+ return true;
}
// TODO(fbarchard): Allow rectangle to be specified: x, y, width, height.
-LIBYUV_BOOL MJpegDecoder::DecodeToBuffers(uint8_t** planes,
- int dst_width,
- int dst_height) {
+bool MJpegDecoder::DecodeToBuffers(uint8_t** planes,
+ int dst_width,
+ int dst_height) {
if (dst_width != GetWidth() || dst_height > GetHeight()) {
// ERROR: Bad dimensions
- return LIBYUV_FALSE;
+ return false;
}
#ifdef HAVE_SETJMP
if (setjmp(error_mgr_->setjmp_buffer)) {
// We called into jpeglib, it experienced an error sometime during this
// function call, and we called longjmp() and rewound the stack to here.
// Return error.
- return LIBYUV_FALSE;
+ return false;
}
#endif
if (!StartDecode()) {
- return LIBYUV_FALSE;
+ return false;
}
SetScanlinePointers(databuf_);
int lines_left = dst_height;
// Compute amount of lines to skip to implement vertical crop.
- // TODO(fbarchard): Ensure skip is a multiple of maximum component
- // subsample. ie 2
+ // Round skip down to a multiple of the maximum vertical sampling factor so
+ // that every component skips a whole number of rows. The rows remaining
+ // below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
+ skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
// There is no API to skip lines in the output data, so we read them
// into the temp buffer.
while (skip >= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
skip -= GetImageScanlinesPerImcuRow();
}
@@ -283,21 +292,25 @@ LIBYUV_BOOL MJpegDecoder::DecodeToBuffers(uint8_t** planes,
// copy the parts we want into the destination.
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
+ }
+ // Image lines wanted from this iMCU row, clamped to the destination.
+ int image_lines_to_copy = GetImageScanlinesPerImcuRow() - skip;
+ if (image_lines_to_copy > lines_left) {
+ image_lines_to_copy = lines_left;
}
for (int i = 0; i < num_outbufs_; ++i) {
- // TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int scanlines_to_copy =
- GetComponentScanlinesPerImcuRow(i) - rows_to_skip;
+ DivideAndRoundUp(image_lines_to_copy, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
CopyPlane(databuf_[i] + data_to_skip, GetComponentStride(i), planes[i],
GetComponentWidth(i), GetComponentWidth(i),
scanlines_to_copy);
planes[i] += scanlines_to_copy * GetComponentWidth(i);
}
- lines_left -= (GetImageScanlinesPerImcuRow() - skip);
+ lines_left -= image_lines_to_copy;
}
}
@@ -306,7 +319,7 @@ LIBYUV_BOOL MJpegDecoder::DecodeToBuffers(uint8_t** planes,
lines_left -= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
for (int i = 0; i < num_outbufs_; ++i) {
int scanlines_to_copy = GetComponentScanlinesPerImcuRow(i);
@@ -320,7 +333,7 @@ LIBYUV_BOOL MJpegDecoder::DecodeToBuffers(uint8_t** planes,
// Have a partial iMCU row left over to decode.
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
for (int i = 0; i < num_outbufs_; ++i) {
int scanlines_to_copy =
@@ -333,34 +346,38 @@ LIBYUV_BOOL MJpegDecoder::DecodeToBuffers(uint8_t** planes,
return FinishDecode();
}
-LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn,
- void* opaque,
- int dst_width,
- int dst_height) {
+bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
+ void* opaque,
+ int dst_width,
+ int dst_height) {
if (dst_width != GetWidth() || dst_height > GetHeight()) {
// ERROR: Bad dimensions
- return LIBYUV_FALSE;
+ return false;
}
#ifdef HAVE_SETJMP
if (setjmp(error_mgr_->setjmp_buffer)) {
// We called into jpeglib, it experienced an error sometime during this
// function call, and we called longjmp() and rewound the stack to here.
// Return error.
- return LIBYUV_FALSE;
+ return false;
}
#endif
if (!StartDecode()) {
- return LIBYUV_FALSE;
+ return false;
}
SetScanlinePointers(databuf_);
int lines_left = dst_height;
- // TODO(fbarchard): Compute amount of lines to skip to implement vertical crop
+ // Compute amount of lines to skip to implement vertical crop.
+ // Round skip down to a multiple of the maximum vertical sampling factor so
+ // that every component skips a whole number of rows. The rows remaining
+ // below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
+ skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
while (skip >= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
skip -= GetImageScanlinesPerImcuRow();
}
@@ -368,10 +385,9 @@ LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn,
// Have a partial iMCU row left over to skip.
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
for (int i = 0; i < num_outbufs_; ++i) {
- // TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
@@ -379,7 +395,11 @@ LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn,
// callback.
databuf_[i] += data_to_skip;
}
+ // Image lines wanted from this iMCU row, clamped to the destination.
int scanlines_to_copy = GetImageScanlinesPerImcuRow() - skip;
+ if (scanlines_to_copy > lines_left) {
+ scanlines_to_copy = lines_left;
+ }
(*fn)(opaque, databuf_, databuf_strides_, scanlines_to_copy);
// Now change them back.
for (int i = 0; i < num_outbufs_; ++i) {
@@ -395,7 +415,7 @@ LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn,
lines_left -= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
(*fn)(opaque, databuf_, databuf_strides_, GetImageScanlinesPerImcuRow());
}
@@ -403,7 +423,7 @@ LIBYUV_BOOL MJpegDecoder::DecodeToCallback(CallbackFunction fn,
// Have a partial iMCU row left over to decode.
if (!DecodeImcuRow()) {
FinishDecode();
- return LIBYUV_FALSE;
+ return false;
}
(*fn)(opaque, databuf_, databuf_strides_, lines_left);
}
@@ -417,7 +437,6 @@ void init_source(j_decompress_ptr cinfo) {
boolean fill_input_buffer(j_decompress_ptr cinfo) {
BufferVector* buf_vec = reinterpret_cast<BufferVector*>(cinfo->client_data);
if (buf_vec->pos >= buf_vec->len) {
- assert(0 && "No more data");
// ERROR: No more data
return FALSE;
}
@@ -429,7 +448,7 @@ boolean fill_input_buffer(j_decompress_ptr cinfo) {
void skip_input_data(j_decompress_ptr cinfo, long num_bytes) { // NOLINT
jpeg_source_mgr* src = cinfo->src;
- size_t bytes = static_cast<size_t>(num_bytes);
+ size_t bytes = (size_t)num_bytes;
if (bytes > src->bytes_in_buffer) {
src->next_input_byte = nullptr;
src->bytes_in_buffer = 0;
@@ -478,17 +497,11 @@ void MJpegDecoder::AllocOutputBuffers(int num_outbufs) {
// it.
DestroyOutputBuffers();
- scanlines_ = new uint8_t**[num_outbufs];
- scanlines_sizes_ = new int[num_outbufs];
- databuf_ = new uint8_t*[num_outbufs];
- databuf_strides_ = new int[num_outbufs];
-
- for (int i = 0; i < num_outbufs; ++i) {
- scanlines_[i] = NULL;
- scanlines_sizes_[i] = 0;
- databuf_[i] = NULL;
- databuf_strides_[i] = 0;
- }
+ scanlines_ =
+ static_cast<uint8_t***>(calloc(num_outbufs, sizeof(uint8_t**)));
+ scanlines_sizes_ = static_cast<int*>(calloc(num_outbufs, sizeof(int)));
+ databuf_ = static_cast<uint8_t**>(calloc(num_outbufs, sizeof(uint8_t*)));
+ databuf_strides_ = static_cast<int*>(calloc(num_outbufs, sizeof(int)));
num_outbufs_ = num_outbufs;
}
@@ -496,13 +509,13 @@ void MJpegDecoder::AllocOutputBuffers(int num_outbufs) {
void MJpegDecoder::DestroyOutputBuffers() {
for (int i = 0; i < num_outbufs_; ++i) {
- delete[] scanlines_[i];
- delete[] databuf_[i];
+ free(scanlines_[i]);
+ free(databuf_[i]);
}
- delete[] scanlines_;
- delete[] databuf_;
- delete[] scanlines_sizes_;
- delete[] databuf_strides_;
+ free(scanlines_);
+ free(databuf_);
+ free(scanlines_sizes_);
+ free(databuf_strides_);
scanlines_ = NULL;
databuf_ = NULL;
scanlines_sizes_ = NULL;
@@ -511,29 +524,29 @@ void MJpegDecoder::DestroyOutputBuffers() {
}
// JDCT_IFAST and do_block_smoothing improve performance substantially.
-LIBYUV_BOOL MJpegDecoder::StartDecode() {
+bool MJpegDecoder::StartDecode() {
decompress_struct_->raw_data_out = TRUE;
decompress_struct_->dct_method = JDCT_IFAST; // JDCT_ISLOW is default
decompress_struct_->dither_mode = JDITHER_NONE;
// Not applicable to 'raw':
- decompress_struct_->do_fancy_upsampling = (boolean)(LIBYUV_FALSE);
+ decompress_struct_->do_fancy_upsampling = (boolean)(0);
// Only for buffered mode:
- decompress_struct_->enable_2pass_quant = (boolean)(LIBYUV_FALSE);
+ decompress_struct_->enable_2pass_quant = (boolean)(0);
// Blocky but fast:
- decompress_struct_->do_block_smoothing = (boolean)(LIBYUV_FALSE);
+ decompress_struct_->do_block_smoothing = (boolean)(0);
if (!jpeg_start_decompress(decompress_struct_)) {
// ERROR: Couldn't start JPEG decompressor";
- return LIBYUV_FALSE;
+ return false;
}
- return LIBYUV_TRUE;
+ return true;
}
-LIBYUV_BOOL MJpegDecoder::FinishDecode() {
+bool MJpegDecoder::FinishDecode() {
// jpeglib considers it an error if we finish without decoding the whole
// image, so we call "abort" rather than "finish".
jpeg_abort_decompress(decompress_struct_);
- return LIBYUV_TRUE;
+ return true;
}
void MJpegDecoder::SetScanlinePointers(uint8_t** data) {
@@ -546,7 +559,7 @@ void MJpegDecoder::SetScanlinePointers(uint8_t** data) {
}
}
-inline LIBYUV_BOOL MJpegDecoder::DecodeImcuRow() {
+inline bool MJpegDecoder::DecodeImcuRow() {
return (unsigned int)(GetImageScanlinesPerImcuRow()) ==
jpeg_read_raw_data(decompress_struct_, scanlines_,
GetImageScanlinesPerImcuRow());
diff --git a/libs/libyuv/source/mjpeg_validate.cc b/libs/libyuv/source/mjpeg_validate.cc
index ba0a03ab9e..91d62ecd54 100644
--- a/libs/libyuv/source/mjpeg_validate.cc
+++ b/libs/libyuv/source/mjpeg_validate.cc
@@ -18,7 +18,7 @@ extern "C" {
#endif
// Helper function to scan for EOI marker (0xff 0xd9).
-static LIBYUV_BOOL ScanEOI(const uint8_t* src_mjpg, size_t src_size_mjpg) {
+static int ScanEOI(const uint8_t* src_mjpg, size_t src_size_mjpg) {
if (src_size_mjpg >= 2) {
const uint8_t* end = src_mjpg + src_size_mjpg - 1;
const uint8_t* it = src_mjpg;
@@ -29,34 +29,34 @@ static LIBYUV_BOOL ScanEOI(const uint8_t* src_mjpg, size_t src_size_mjpg) {
break;
}
if (it[1] == 0xd9) {
- return LIBYUV_TRUE; // Success: Valid jpeg.
+ return 1; // Success: Valid jpeg.
}
++it; // Skip over current 0xff.
}
}
// ERROR: Invalid jpeg end code not found. Size src_size_mjpg
- return LIBYUV_FALSE;
+ return 0;
}
// Helper function to validate the jpeg appears intact.
-LIBYUV_BOOL ValidateJpeg(const uint8_t* src_mjpg, size_t src_size_mjpg) {
+int ValidateJpeg(const uint8_t* src_mjpg, size_t src_size_mjpg) {
// Maximum size that ValidateJpeg will consider valid.
const size_t kMaxJpegSize = 0x7fffffffull;
const size_t kBackSearchSize = 1024;
if (src_size_mjpg < 64 || src_size_mjpg > kMaxJpegSize || !src_mjpg) {
// ERROR: Invalid jpeg size: src_size_mjpg
- return LIBYUV_FALSE;
+ return 0;
}
// SOI marker
if (src_mjpg[0] != 0xff || src_mjpg[1] != 0xd8 || src_mjpg[2] != 0xff) {
// ERROR: Invalid jpeg initial start code
- return LIBYUV_FALSE;
+ return 0;
}
// Look for the End Of Image (EOI) marker near the end of the buffer.
if (src_size_mjpg > kBackSearchSize) {
if (ScanEOI(src_mjpg + src_size_mjpg - kBackSearchSize, kBackSearchSize)) {
- return LIBYUV_TRUE; // Success: Valid jpeg.
+ return 1; // Success: Valid jpeg.
}
// Reduce search size for forward search.
src_size_mjpg = src_size_mjpg - kBackSearchSize + 1;
diff --git a/libs/libyuv/source/planar_functions.cc b/libs/libyuv/source/planar_functions.cc
index c07f0943d3..c307644908 100644
--- a/libs/libyuv/source/planar_functions.cc
+++ b/libs/libyuv/source/planar_functions.cc
@@ -10,12 +10,12 @@
#include "libyuv/planar_functions.h"
+#include <assert.h>
+#include <limits.h>
#include <string.h> // for memset()
+#include "libyuv/convert_from_argb.h" // For ArgbConstants
#include "libyuv/cpu_id.h"
-#ifdef HAVE_JPEG
-#include "libyuv/mjpeg_decoder.h"
-#endif
#include "libyuv/row.h"
#include "libyuv/scale_row.h" // for ScaleRowDown2
@@ -34,14 +34,18 @@ void CopyPlane(const uint8_t* src_y,
int height) {
int y;
void (*CopyRow)(const uint8_t* src, uint8_t* dst, int width) = CopyRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (src_stride_y == width && dst_stride_y == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_y = 0;
@@ -61,6 +65,11 @@ void CopyPlane(const uint8_t* src_y,
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
+#if defined(HAS_COPYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ CopyRow = CopyRow_AVX512BW;
+ }
+#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
@@ -71,6 +80,36 @@ void CopyPlane(const uint8_t* src_y,
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
+#if defined(HAS_COPYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ CopyRow = CopyRow_SVE2;
+ }
+#endif
+#if defined(HAS_COPYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ CopyRow = CopyRow_SME;
+ }
+#endif
+#if defined(HAS_COPYROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ CopyRow = CopyRow_RVV;
+ }
+#endif
+#if defined(HAS_COPYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ CopyRow = IS_ALIGNED(width, 16) ? CopyRow_LSX : CopyRow_Any_LSX;
+ }
+#endif
+#if defined(HAS_COPYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LASX : CopyRow_Any_LASX;
+ }
+#endif
+#if defined(HAS_COPYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ CopyRow = CopyRow_HVX;
+ }
+#endif
// Copy plane
for (y = 0; y < height; ++y) {
@@ -80,8 +119,6 @@ void CopyPlane(const uint8_t* src_y,
}
}
-// TODO(fbarchard): Consider support for negative height.
-// TODO(fbarchard): Consider stride measured in bytes.
LIBYUV_API
void CopyPlane_16(const uint16_t* src_y,
int src_stride_y,
@@ -91,25 +128,39 @@ void CopyPlane_16(const uint16_t* src_y,
int height) {
int y;
void (*CopyRow)(const uint16_t* src, uint16_t* dst, int width) = CopyRow_16_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
// Coalesce rows.
- if (src_stride_y == width && dst_stride_y == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_y = 0;
}
-#if defined(HAS_COPYROW_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 32)) {
- CopyRow = CopyRow_16_SSE2;
+ // Nothing to do.
+ if (src_y == dst_y && src_stride_y == dst_stride_y) {
+ return;
}
-#endif
-#if defined(HAS_COPYROW_16_ERMS)
- if (TestCpuFlag(kCpuHasERMS)) {
- CopyRow = CopyRow_16_ERMS;
+#if defined(HAS_COPYROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ CopyRow = IS_ALIGNED(width, 32) ? CopyRow_16_AVX2 : CopyRow_16_Any_AVX2;
}
#endif
#if defined(HAS_COPYROW_16_NEON)
- if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 32)) {
- CopyRow = CopyRow_16_NEON;
+ if (TestCpuFlag(kCpuHasNEON)) {
+ CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_NEON : CopyRow_16_Any_NEON;
+ }
+#endif
+#if defined(HAS_COPYROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ CopyRow = CopyRow_16_HVX;
}
#endif
@@ -134,18 +185,35 @@ void Convert16To8Plane(const uint16_t* src_y,
void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
int width) = Convert16To8Row_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (src_stride_y == width && dst_stride_y == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_y = 0;
}
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
#if defined(HAS_CONVERT16TO8ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
Convert16To8Row = Convert16To8Row_Any_SSSE3;
@@ -162,6 +230,16 @@ void Convert16To8Plane(const uint16_t* src_y,
}
}
#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert16To8Row = Convert16To8Row_HVX;
+ }
+#endif
// Convert plane
for (y = 0; y < height; ++y) {
@@ -173,25 +251,39 @@ void Convert16To8Plane(const uint16_t* src_y,
// Convert a plane of 8 bit data to 16 bit
LIBYUV_API
-void Convert8To16Plane(const uint8_t* src_y,
- int src_stride_y,
- uint16_t* dst_y,
- int dst_stride_y,
- int scale, // 16384 for 10 bits
- int width,
- int height) {
+void Convert8To16Plane(
+ const uint8_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale)
+ int width,
+ int height) {
int y;
- void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int scale,
+ void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits,
int width) = Convert8To16Row_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Convert legacy scale to bits if needed.
+ if (bits > 16) {
+ int b = 0;
+ while (bits > 1) {
+ bits >>= 1;
+ ++b;
+ }
+ bits = b;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (src_stride_y == width && dst_stride_y == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_y = 0;
@@ -212,10 +304,121 @@ void Convert8To16Plane(const uint8_t* src_y,
}
}
#endif
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To16Row = Convert8To16Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To16Row = Convert8To16Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To16Row = Convert8To16Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Convert8To16Row = Convert8To16Row_RVV;
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ Convert8To16Row = Convert8To16Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ Convert8To16Row = Convert8To16Row_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO16ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To16Row = Convert8To16Row_HVX;
+ }
+#endif
+
+ // Convert plane
+ for (y = 0; y < height; ++y) {
+ Convert8To16Row(src_y, dst_y, bits, width);
+ src_y += src_stride_y;
+ dst_y += dst_stride_y;
+ }
+}
+
+// Convert a plane of 8 bit data to 8 bit
+LIBYUV_API
+void Convert8To8Plane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int scale, // 220 for Y, 225 to UV
+ int bias, // 16
+ int width,
+ int height) {
+ int y;
+ void (*Convert8To8Row)(const uint8_t* src_y, uint8_t* dst_y, int scale,
+ int bias, int width) = Convert8To8Row_C;
+
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_y = dst_stride_y = 0;
+ }
+#if defined(HAS_CONVERT8TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To8Row = Convert8To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To8Row = Convert8To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Convert8To8Row = Convert8To8Row_SVE2;
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To8Row = Convert8To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert8To8Row = Convert8To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To8Row = Convert8To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert8To8Row = Convert8To8Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ Convert8To8Row = Convert8To8Row_HVX;
+ }
+#endif
// Convert plane
for (y = 0; y < height; ++y) {
- Convert8To16Row(src_y, dst_y, scale, width);
+ Convert8To8Row(src_y, dst_y, scale, bias, width);
src_y += src_stride_y;
dst_y += dst_stride_y;
}
@@ -238,15 +441,18 @@ int I422Copy(const uint8_t* src_y,
int width,
int height) {
int halfwidth = (width + 1) >> 1;
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
+
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (height - 1) * src_stride_u;
- src_v = src_v + (height - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -276,15 +482,16 @@ int I444Copy(const uint8_t* src_y,
int dst_stride_v,
int width,
int height) {
- if (!src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (height - 1) * src_stride_u;
- src_v = src_v + (height - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -298,6 +505,88 @@ int I444Copy(const uint8_t* src_y,
return 0;
}
+// Copy I210.
+LIBYUV_API
+int I210Copy(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ // Copy UV planes.
+ CopyPlane_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, height);
+ CopyPlane_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, height);
+ return 0;
+}
+
+// Copy I410.
+LIBYUV_API
+int I410Copy(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ if (dst_y) {
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ CopyPlane_16(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
+ CopyPlane_16(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
+ return 0;
+}
+
// Copy I400.
LIBYUV_API
int I400ToI400(const uint8_t* src_y,
@@ -306,13 +595,13 @@ int I400ToI400(const uint8_t* src_y,
int dst_stride_y,
int width,
int height) {
- if (!src_y || !dst_y || width <= 0 || height == 0) {
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
@@ -335,13 +624,13 @@ int I420ToI400(const uint8_t* src_y,
(void)src_stride_u;
(void)src_v;
(void)src_stride_v;
- if (!src_y || !dst_y || width <= 0 || height == 0) {
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
@@ -349,6 +638,57 @@ int I420ToI400(const uint8_t* src_y,
return 0;
}
+// Copy NV12. Supports inverting.
+LIBYUV_API
+int NV12Copy(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+
+ if (!src_y || !dst_y || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(halfheight - 1) * src_stride_uv;
+ src_stride_y = -src_stride_y;
+ src_stride_uv = -src_stride_uv;
+ }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ CopyPlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, halfwidth * 2,
+ halfheight);
+ return 0;
+}
+
+// Copy NV21. Supports inverting.
+LIBYUV_API
+int NV21Copy(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_vu,
+ int src_stride_vu,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
+ return NV12Copy(src_y, src_stride_y, src_vu, src_stride_vu, dst_y,
+ dst_stride_y, dst_vu, dst_stride_vu, width, height);
+}
+
// Support function for NV12 etc UV channels.
// Width and height are plane sizes (typically half pixel width).
LIBYUV_API
@@ -363,17 +703,20 @@ void SplitUVPlane(const uint8_t* src_uv,
int y;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_u = dst_u + (height - 1) * dst_stride_u;
- dst_v = dst_v + (height - 1) * dst_stride_v;
+ dst_u = dst_u + (ptrdiff_t)(height - 1) * dst_stride_u;
+ dst_v = dst_v + (ptrdiff_t)(height - 1) * dst_stride_v;
dst_stride_u = -dst_stride_u;
dst_stride_v = -dst_stride_v;
}
// Coalesce rows.
if (src_stride_uv == width * 2 && dst_stride_u == width &&
- dst_stride_v == width) {
+ dst_stride_v == width && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_uv = dst_stride_u = dst_stride_v = 0;
@@ -394,6 +737,11 @@ void SplitUVPlane(const uint8_t* src_uv,
}
}
#endif
+#if defined(HAS_SPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SplitUVRow = SplitUVRow_AVX512BW;
+ }
+#endif
#if defined(HAS_SPLITUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SplitUVRow = SplitUVRow_Any_NEON;
@@ -402,22 +750,37 @@ void SplitUVPlane(const uint8_t* src_uv,
}
}
#endif
-#if defined(HAS_SPLITUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SplitUVRow = SplitUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- SplitUVRow = SplitUVRow_MSA;
+#if defined(HAS_SPLITUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SplitUVRow = SplitUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SplitUVRow = SplitUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ SplitUVRow = SplitUVRow_LSX;
}
}
#endif
-#if defined(HAS_SPLITUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SplitUVRow = SplitUVRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SplitUVRow = SplitUVRow_MMI;
+#if defined(HAS_SPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SplitUVRow = SplitUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ SplitUVRow = SplitUVRow_LASX;
}
}
#endif
+#if defined(HAS_SPLITUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitUVRow = SplitUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow = SplitUVRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
// Copy a row of UV.
@@ -440,15 +803,18 @@ void MergeUVPlane(const uint8_t* src_u,
int y;
void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_uv = dst_uv + (height - 1) * dst_stride_uv;
+ dst_uv = dst_uv + (ptrdiff_t)(height - 1) * dst_stride_uv;
dst_stride_uv = -dst_stride_uv;
}
// Coalesce rows.
if (src_stride_u == width && src_stride_v == width &&
- dst_stride_uv == width * 2) {
+ dst_stride_uv == width * 2 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_u = src_stride_v = dst_stride_uv = 0;
@@ -464,11 +830,16 @@ void MergeUVPlane(const uint8_t* src_u,
#if defined(HAS_MERGEUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
MergeUVRow = MergeUVRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
+ if (IS_ALIGNED(width, 16)) {
MergeUVRow = MergeUVRow_AVX2;
}
}
#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+#endif
#if defined(HAS_MERGEUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
MergeUVRow = MergeUVRow_Any_NEON;
@@ -477,22 +848,50 @@ void MergeUVPlane(const uint8_t* src_u,
}
}
#endif
-#if defined(HAS_MERGEUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MergeUVRow = MergeUVRow_Any_MSA;
+#if defined(HAS_MERGEUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ MergeUVRow = MergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- MergeUVRow = MergeUVRow_MSA;
+ MergeUVRow = MergeUVRow_LSX;
}
}
#endif
-#if defined(HAS_MERGEUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeUVRow = MergeUVRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- MergeUVRow = MergeUVRow_MMI;
+#if defined(HAS_MERGEUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MergeUVRow = MergeUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ MergeUVRow = MergeUVRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MergeUVRow = MergeUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MergeUVRow = MergeUVRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_MERGEUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow = MergeUVRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
// Merge a row of U and V into a row of UV.
@@ -503,548 +902,2259 @@ void MergeUVPlane(const uint8_t* src_u,
}
}
-// Swap U and V channels in interleaved UV plane.
+// Support function for P010 etc UV channels.
+// Width and height are plane sizes (typically half pixel width).
LIBYUV_API
-void SwapUVPlane(const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
+void SplitUVPlane_16(const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int depth) {
int y;
- void (*SwapUVRow)(const uint8_t* src_uv, uint8_t* dst_vu, int width) =
- SwapUVRow_C;
+ void (*SplitUVRow_16)(const uint16_t* src_uv, uint16_t* dst_u,
+ uint16_t* dst_v, int depth, int width) =
+ SplitUVRow_16_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_uv = src_uv + (height - 1) * src_stride_uv;
- src_stride_uv = -src_stride_uv;
+ dst_u = dst_u + (ptrdiff_t)(height - 1) * dst_stride_u;
+ dst_v = dst_v + (ptrdiff_t)(height - 1) * dst_stride_v;
+ dst_stride_u = -dst_stride_u;
+ dst_stride_v = -dst_stride_v;
}
// Coalesce rows.
- if (src_stride_uv == width * 2 && dst_stride_vu == width * 2) {
+ if (src_stride_uv == width * 2 && dst_stride_u == width &&
+ dst_stride_v == width && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_uv = dst_stride_vu = 0;
+ src_stride_uv = dst_stride_u = dst_stride_v = 0;
}
-
-#if defined(HAS_SWAPUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- SwapUVRow = SwapUVRow_Any_SSSE3;
+#if defined(HAS_SPLITUVROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitUVRow_16 = SplitUVRow_16_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- SwapUVRow = SwapUVRow_SSSE3;
+ SplitUVRow_16 = SplitUVRow_16_AVX2;
}
}
#endif
-#if defined(HAS_SWAPUVROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- SwapUVRow = SwapUVRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- SwapUVRow = SwapUVRow_AVX2;
+#if defined(HAS_SPLITUVROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitUVRow_16 = SplitUVRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ SplitUVRow_16 = SplitUVRow_16_NEON;
}
}
#endif
-#if defined(HAS_SWAPUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- SwapUVRow = SwapUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- SwapUVRow = SwapUVRow_NEON;
- }
+#if defined(HAS_SPLITUVROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow_16 = SplitUVRow_16_HVX;
}
#endif
for (y = 0; y < height; ++y) {
- SwapUVRow(src_uv, dst_vu, width);
+ // Copy a row of UV.
+ SplitUVRow_16(src_uv, dst_u, dst_v, depth, width);
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
src_uv += src_stride_uv;
- dst_vu += dst_stride_vu;
}
}
-// Convert NV21 to NV12.
LIBYUV_API
-int NV21ToNV12(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_vu,
- int src_stride_vu,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_uv,
- int dst_stride_uv,
- int width,
- int height) {
- int halfwidth = (width + 1) >> 1;
- int halfheight = (height + 1) >> 1;
- if (!src_vu || !dst_uv || width <= 0 || height == 0) {
- return -1;
+void MergeUVPlane_16(const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeUVRow_16)(const uint16_t* src_u, const uint16_t* src_v,
+ uint16_t* dst_uv, int depth, int width) =
+ MergeUVRow_16_C;
+ assert(depth >= 8);
+ assert(depth <= 16);
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
}
- if (dst_y) {
- CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_uv = dst_uv + (ptrdiff_t)(height - 1) * dst_stride_uv;
+ dst_stride_uv = -dst_stride_uv;
+ }
+ // Coalesce rows.
+ if (src_stride_u == width && src_stride_v == width &&
+ dst_stride_uv == width * 2 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_u = src_stride_v = dst_stride_uv = 0;
+ }
+#if defined(HAS_MERGEUVROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow_16 = MergeUVRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ MergeUVRow_16 = MergeUVRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow_16 = MergeUVRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow_16 = MergeUVRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeUVRow_16 = MergeUVRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_16 = MergeUVRow_16_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeUVRow_16 = MergeUVRow_16_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ // Merge a row of U and V into a row of UV.
+ MergeUVRow_16(src_u, src_v, dst_uv, depth, width);
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ dst_uv += dst_stride_uv;
}
- SwapUVPlane(src_vu, src_stride_vu, dst_uv, dst_stride_uv, halfwidth,
- halfheight);
- return 0;
}
-// Support function for NV12 etc RGB channels.
-// Width and height are plane sizes (typically half pixel width).
+// Convert plane from lsb to msb
LIBYUV_API
-void SplitRGBPlane(const uint8_t* src_rgb,
- int src_stride_rgb,
- uint8_t* dst_r,
- int dst_stride_r,
- uint8_t* dst_g,
- int dst_stride_g,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
+void ConvertToMSBPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int depth) {
int y;
- void (*SplitRGBRow)(const uint8_t* src_rgb, uint8_t* dst_r, uint8_t* dst_g,
- uint8_t* dst_b, int width) = SplitRGBRow_C;
+ int scale = 1 << (16 - depth);
+ void (*MultiplyRow_16)(const uint16_t* src_y, uint16_t* dst_y, int scale,
+ int width) = MultiplyRow_16_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_r = dst_r + (height - 1) * dst_stride_r;
- dst_g = dst_g + (height - 1) * dst_stride_g;
- dst_b = dst_b + (height - 1) * dst_stride_b;
- dst_stride_r = -dst_stride_r;
- dst_stride_g = -dst_stride_g;
- dst_stride_b = -dst_stride_b;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (src_stride_rgb == width * 3 && dst_stride_r == width &&
- dst_stride_g == width && dst_stride_b == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_rgb = dst_stride_r = dst_stride_g = dst_stride_b = 0;
+ src_stride_y = dst_stride_y = 0;
}
-#if defined(HAS_SPLITRGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- SplitRGBRow = SplitRGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- SplitRGBRow = SplitRGBRow_SSSE3;
+
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX2;
}
}
#endif
-#if defined(HAS_SPLITRGBROW_NEON)
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MultiplyRow_16 = MultiplyRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- SplitRGBRow = SplitRGBRow_Any_NEON;
+ MultiplyRow_16 = MultiplyRow_16_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- SplitRGBRow = SplitRGBRow_NEON;
+ MultiplyRow_16 = MultiplyRow_16_NEON;
}
}
#endif
-#if defined(HAS_SPLITRGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SplitRGBRow = SplitRGBRow_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- SplitRGBRow = SplitRGBRow_MMI;
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MultiplyRow_16 = MultiplyRow_16_SME;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MultiplyRow_16 = MultiplyRow_16_RVV;
+ }
+#endif
+#if defined(HAS_MULTIPLYROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MultiplyRow_16 = MultiplyRow_16_WASMSIMD;
}
}
#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MultiplyRow_16 = MultiplyRow_16_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
- // Copy a row of RGB.
- SplitRGBRow(src_rgb, dst_r, dst_g, dst_b, width);
- dst_r += dst_stride_r;
- dst_g += dst_stride_g;
- dst_b += dst_stride_b;
- src_rgb += src_stride_rgb;
+ MultiplyRow_16(src_y, dst_y, scale, width);
+ src_y += src_stride_y;
+ dst_y += dst_stride_y;
}
}
+// Convert plane from msb to lsb
LIBYUV_API
-void MergeRGBPlane(const uint8_t* src_r,
- int src_stride_r,
- const uint8_t* src_g,
- int src_stride_g,
- const uint8_t* src_b,
- int src_stride_b,
- uint8_t* dst_rgb,
- int dst_stride_rgb,
- int width,
- int height) {
+void ConvertToLSBPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int depth) {
int y;
- void (*MergeRGBRow)(const uint8_t* src_r, const uint8_t* src_g,
- const uint8_t* src_b, uint8_t* dst_rgb, int width) =
- MergeRGBRow_C;
- // Coalesce rows.
+ int scale = 1 << depth;
+ void (*DivideRow)(const uint16_t* src_y, uint16_t* dst_y, int scale,
+ int width) = DivideRow_16_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_rgb = dst_rgb + (height - 1) * dst_stride_rgb;
- dst_stride_rgb = -dst_stride_rgb;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
- dst_stride_rgb == width * 3) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0;
+ src_stride_y = dst_stride_y = 0;
}
-#if defined(HAS_MERGERGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- MergeRGBRow = MergeRGBRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- MergeRGBRow = MergeRGBRow_SSSE3;
+
+#if defined(HAS_DIVIDEROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ DivideRow = DivideRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ DivideRow = DivideRow_16_AVX2;
}
}
#endif
-#if defined(HAS_MERGERGBROW_NEON)
+#if defined(HAS_DIVIDEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ DivideRow = DivideRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_DIVIDEROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- MergeRGBRow = MergeRGBRow_Any_NEON;
+ DivideRow = DivideRow_16_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- MergeRGBRow = MergeRGBRow_NEON;
+ DivideRow = DivideRow_16_NEON;
}
}
#endif
-#if defined(HAS_MERGERGBROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MergeRGBRow = MergeRGBRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- MergeRGBRow = MergeRGBRow_MMI;
- }
+#if defined(HAS_DIVIDEROW_16_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ DivideRow = DivideRow_16_SVE2;
+ }
+#endif
+#if defined(HAS_DIVIDEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ DivideRow = DivideRow_16_HVX;
}
#endif
for (y = 0; y < height; ++y) {
- // Merge a row of U and V into a row of RGB.
- MergeRGBRow(src_r, src_g, src_b, dst_rgb, width);
- src_r += src_stride_r;
- src_g += src_stride_g;
- src_b += src_stride_b;
- dst_rgb += dst_stride_rgb;
+ DivideRow(src_y, dst_y, scale, width);
+ src_y += src_stride_y;
+ dst_y += dst_stride_y;
}
}
-// Mirror a plane of data.
-void MirrorPlane(const uint8_t* src_y,
- int src_stride_y,
- uint8_t* dst_y,
- int dst_stride_y,
+// Swap U and V channels in interleaved UV plane.
+LIBYUV_API
+void SwapUVPlane(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
int width,
int height) {
int y;
- void (*MirrorRow)(const uint8_t* src, uint8_t* dst, int width) = MirrorRow_C;
+ void (*SwapUVRow)(const uint8_t* src_uv, uint8_t* dst_vu, int width) =
+ SwapUVRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
}
-#if defined(HAS_MIRRORROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- MirrorRow = MirrorRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- MirrorRow = MirrorRow_NEON;
- }
+ // Coalesce rows.
+ if (src_stride_uv == width * 2 && dst_stride_vu == width * 2 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_uv = dst_stride_vu = 0;
}
-#endif
-#if defined(HAS_MIRRORROW_SSSE3)
+
+#if defined(HAS_SWAPUVROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- MirrorRow = MirrorRow_Any_SSSE3;
+ SwapUVRow = SwapUVRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- MirrorRow = MirrorRow_SSSE3;
+ SwapUVRow = SwapUVRow_SSSE3;
}
}
#endif
-#if defined(HAS_MIRRORROW_AVX2)
+#if defined(HAS_SWAPUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- MirrorRow = MirrorRow_Any_AVX2;
+ SwapUVRow = SwapUVRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- MirrorRow = MirrorRow_AVX2;
+ SwapUVRow = SwapUVRow_AVX2;
}
}
#endif
-#if defined(HAS_MIRRORROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MirrorRow = MirrorRow_Any_MSA;
- if (IS_ALIGNED(width, 64)) {
- MirrorRow = MirrorRow_MSA;
+#if defined(HAS_SWAPUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SwapUVRow = SwapUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SWAPUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SwapUVRow = SwapUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ SwapUVRow = SwapUVRow_NEON;
}
}
#endif
-#if defined(HAS_MIRRORROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MirrorRow = MirrorRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- MirrorRow = MirrorRow_MMI;
+#if defined(HAS_SWAPUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SwapUVRow = SwapUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SWAPUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ SwapUVRow = SwapUVRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ SwapUVRow = SwapUVRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_SWAPUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SwapUVRow = SwapUVRow_HVX;
+ }
+#endif
- // Mirror plane
for (y = 0; y < height; ++y) {
- MirrorRow(src_y, dst_y, width);
- src_y += src_stride_y;
- dst_y += dst_stride_y;
+ SwapUVRow(src_uv, dst_vu, width);
+ src_uv += src_stride_uv;
+ dst_vu += dst_stride_vu;
}
}
-// Convert YUY2 to I422.
+// Convert NV12 to NV21.
LIBYUV_API
-int YUY2ToI422(const uint8_t* src_yuy2,
- int src_stride_yuy2,
+int NV12ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
uint8_t* dst_y,
int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
int width,
int height) {
- int y;
- void (*YUY2ToUV422Row)(const uint8_t* src_yuy2, uint8_t* dst_u,
- uint8_t* dst_v, int width) = YUY2ToUV422Row_C;
- void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
- YUY2ToYRow_C;
- if (!src_yuy2 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+
+ if (!src_uv || !dst_vu || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
+
+ if (dst_y) {
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2;
- src_stride_yuy2 = -src_stride_yuy2;
+ halfheight = (height + 1) >> 1;
+ src_uv = src_uv + (ptrdiff_t)(halfheight - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
}
- // Coalesce rows.
- if (src_stride_yuy2 == width * 2 && dst_stride_y == width &&
- dst_stride_u * 2 == width && dst_stride_v * 2 == width &&
- width * height <= 32768) {
- width *= height;
- height = 1;
+
+ SwapUVPlane(src_uv, src_stride_uv, dst_vu, dst_stride_vu, halfwidth,
+ halfheight);
+ return 0;
+}
+
+// Test if tile_height is a power of 2 (16 or 32)
+#define IS_POWEROFTWO(x) (!((x) & ((x) - 1)))
+
+// Detile a plane of data
+// tile width is 16 and assumed.
+// tile_height is 16 or 32 for MM21.
+// src_stride_y is bytes per row of source ignoring tiling. e.g. 640
+// TODO: More detile row functions.
+LIBYUV_API
+int DetilePlane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int tile_height) {
+ const ptrdiff_t src_tile_stride = 16 * tile_height;
+ int y;
+ void (*DetileRow)(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst,
+ int width) = DetileRow_C;
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN ||
+ !IS_POWEROFTWO(tile_height)) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
+
+#if defined(HAS_DETILEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ DetileRow = DetileRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ DetileRow = DetileRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_DETILEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ DetileRow = DetileRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ DetileRow = DetileRow_NEON;
+ }
+ }
+#endif
+
+ // Detile plane
+ for (y = 0; y < height; ++y) {
+ DetileRow(src_y, src_tile_stride, dst_y, width);
+ dst_y += dst_stride_y;
+ src_y += 16;
+ // Advance to next row of tiles.
+ if ((y & (tile_height - 1)) == (tile_height - 1)) {
+ src_y = src_y - src_tile_stride + src_stride_y * tile_height;
+ }
+ }
+ return 0;
+}
+
+// Convert a plane of 16 bit tiles of 16 x H to linear.
+// tile width is 16 and assumed.
+// tile_height is 16 or 32 for MT2T.
+LIBYUV_API
+int DetilePlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height,
+ int tile_height) {
+ const ptrdiff_t src_tile_stride = 16 * tile_height;
+ int y;
+ void (*DetileRow_16)(const uint16_t* src, ptrdiff_t src_tile_stride,
+ uint16_t* dst, int width) = DetileRow_16_C;
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN ||
+ !IS_POWEROFTWO(tile_height)) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
+
+#if defined(HAS_DETILEROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ DetileRow_16 = DetileRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ DetileRow_16 = DetileRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_DETILEROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ DetileRow_16 = DetileRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ DetileRow_16 = DetileRow_16_NEON;
+ }
+ }
+#endif
+
+ // Detile plane
+ for (y = 0; y < height; ++y) {
+ DetileRow_16(src_y, src_tile_stride, dst_y, width);
+ dst_y += dst_stride_y;
+ src_y += 16;
+ // Advance to next row of tiles.
+ if ((y & (tile_height - 1)) == (tile_height - 1)) {
+ src_y = src_y - src_tile_stride + src_stride_y * tile_height;
+ }
+ }
+ return 0;
+}
+
+LIBYUV_API
+void DetileSplitUVPlane(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int tile_height) {
+ const ptrdiff_t src_tile_stride = 16 * tile_height;
+ int y;
+ void (*DetileSplitUVRow)(const uint8_t* src, ptrdiff_t src_tile_stride,
+ uint8_t* dst_u, uint8_t* dst_v, int width) =
+ DetileSplitUVRow_C;
+ assert(src_stride_uv >= 0);
+ assert(tile_height > 0);
+ assert(src_stride_uv > 0);
+
+ // tile_height must be a power of 2 (16 or 32); the row loop below uses it as
+ // a bit mask.
+ if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0 ||
+ !IS_POWEROFTWO(tile_height)) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_u = dst_u + (ptrdiff_t)(height - 1) * dst_stride_u;
+ dst_stride_u = -dst_stride_u;
+ dst_v = dst_v + (ptrdiff_t)(height - 1) * dst_stride_v;
+ dst_stride_v = -dst_stride_v;
+ }
+
+#if defined(HAS_DETILESPLITUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ DetileSplitUVRow = DetileSplitUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ DetileSplitUVRow = DetileSplitUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_DETILESPLITUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ DetileSplitUVRow = DetileSplitUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ DetileSplitUVRow = DetileSplitUVRow_NEON;
+ }
+ }
+#endif
+
+ // Detile plane
+ for (y = 0; y < height; ++y) {
+ DetileSplitUVRow(src_uv, src_tile_stride, dst_u, dst_v, width);
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ src_uv += 16;
+ // Advance to next row of tiles.
+ if ((y & (tile_height - 1)) == (tile_height - 1)) {
+ src_uv = src_uv - src_tile_stride + src_stride_uv * tile_height;
+ }
+ }
+}
+
+LIBYUV_API
+void DetileToYUY2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_yuy2,
+ int dst_stride_yuy2,
+ int width,
+ int height,
+ int tile_height) {
+ const ptrdiff_t src_y_tile_stride = 16 * tile_height;
+ const ptrdiff_t src_uv_tile_stride = src_y_tile_stride / 2;
+ int y;
+ void (*DetileToYUY2)(const uint8_t* src_y, ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2, int width) = DetileToYUY2_C;
+ assert(src_stride_y >= 0);
+ assert(src_stride_y > 0);
+ assert(src_stride_uv >= 0);
+ assert(src_stride_uv > 0);
+ assert(tile_height > 0);
+
+ // tile_height must be a power of 2 and at least 2 (16 or 32); the row loop
+ // uses it as a bit mask and the UV tile is tile_height / 2 rows.
+ if (width <= 0 || height == 0 || height == INT_MIN || tile_height < 2 ||
+ !IS_POWEROFTWO(tile_height)) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
+ dst_stride_yuy2 = -dst_stride_yuy2;
+ }
+
+#if defined(HAS_DETILETOYUY2_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ DetileToYUY2 = DetileToYUY2_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ DetileToYUY2 = DetileToYUY2_NEON;
+ }
+ }
+#endif
+
+#if defined(HAS_DETILETOYUY2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ DetileToYUY2 = DetileToYUY2_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ DetileToYUY2 = DetileToYUY2_AVX2;
+ }
+ }
+#endif
+
+ // Detile plane
+ for (y = 0; y < height; ++y) {
+ DetileToYUY2(src_y, src_y_tile_stride, src_uv, src_uv_tile_stride, dst_yuy2,
+ width);
+ dst_yuy2 += dst_stride_yuy2;
+ src_y += 16;
+
+ if (y & 0x1)
+ src_uv += 16;
+
+ // Advance to next row of tiles.
+ if ((y & (tile_height - 1)) == (tile_height - 1)) {
+ src_y = src_y - src_y_tile_stride + src_stride_y * tile_height;
+ src_uv = src_uv - src_uv_tile_stride + src_stride_uv * (tile_height / 2);
+ }
+ }
+}
+
+// Support function for NV12 etc RGB channels.
+// Width and height are plane sizes (typically half pixel width).
+LIBYUV_API
+void SplitRGBPlane(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_r,
+ int dst_stride_r,
+ uint8_t* dst_g,
+ int dst_stride_g,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ int y;
+ void (*SplitRGBRow)(const uint8_t* src_rgb, uint8_t* dst_r, uint8_t* dst_g,
+ uint8_t* dst_b, int width) = SplitRGBRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r;
+ dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g;
+ dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b;
+ dst_stride_r = -dst_stride_r;
+ dst_stride_g = -dst_stride_g;
+ dst_stride_b = -dst_stride_b;
+ }
+ // Coalesce rows.
+ if (src_stride_rgb == width * 3 && dst_stride_r == width &&
+ dst_stride_g == width && dst_stride_b == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_rgb = dst_stride_r = dst_stride_g = dst_stride_b = 0;
+ }
+#if defined(HAS_SPLITRGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ SplitRGBRow = SplitRGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ SplitRGBRow = SplitRGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_SPLITRGBROW_SSE41)
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ SplitRGBRow = SplitRGBRow_Any_SSE41;
+ if (IS_ALIGNED(width, 16)) {
+ SplitRGBRow = SplitRGBRow_SSE41;
+ }
+ }
+#endif
+#if defined(HAS_SPLITRGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitRGBRow = SplitRGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ SplitRGBRow = SplitRGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITRGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitRGBRow = SplitRGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ SplitRGBRow = SplitRGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITRGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitRGBRow = SplitRGBRow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ // Copy a row of RGB.
+ SplitRGBRow(src_rgb, dst_r, dst_g, dst_b, width);
+ dst_r += dst_stride_r;
+ dst_g += dst_stride_g;
+ dst_b += dst_stride_b;
+ src_rgb += src_stride_rgb;
+ }
+}
+
+LIBYUV_API
+void MergeRGBPlane(const uint8_t* src_r,
+ int src_stride_r,
+ const uint8_t* src_g,
+ int src_stride_g,
+ const uint8_t* src_b,
+ int src_stride_b,
+ uint8_t* dst_rgb,
+ int dst_stride_rgb,
+ int width,
+ int height) {
+ int y;
+ void (*MergeRGBRow)(const uint8_t* src_r, const uint8_t* src_g,
+ const uint8_t* src_b, uint8_t* dst_rgb, int width) =
+ MergeRGBRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Coalesce rows.
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_rgb = dst_rgb + (ptrdiff_t)(height - 1) * dst_stride_rgb;
+ dst_stride_rgb = -dst_stride_rgb;
+ }
+ // Coalesce rows.
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ dst_stride_rgb == width * 3 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0;
+ }
+#if defined(HAS_MERGERGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeRGBRow = MergeRGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ MergeRGBRow = MergeRGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGERGBROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
+ MergeRGBRow = MergeRGBRow_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_MERGERGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeRGBRow = MergeRGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MergeRGBRow = MergeRGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGERGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeRGBRow = MergeRGBRow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ // Merge a row of U and V into a row of RGB.
+ MergeRGBRow(src_r, src_g, src_b, dst_rgb, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ dst_rgb += dst_stride_rgb;
+ }
+}
+
+LIBYUV_NOINLINE
+static void SplitARGBPlaneAlpha(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_r,
+ int dst_stride_r,
+ uint8_t* dst_g,
+ int dst_stride_g,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height) {
+ int y;
+ void (*SplitARGBRow)(const uint8_t* src_rgb, uint8_t* dst_r, uint8_t* dst_g,
+ uint8_t* dst_b, uint8_t* dst_a, int width) =
+ SplitARGBRow_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_argb == width * 4 && dst_stride_r == width &&
+ dst_stride_g == width && dst_stride_b == width && dst_stride_a == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_r = dst_stride_g = dst_stride_b =
+ dst_stride_a = 0;
+ }
+
+#if defined(HAS_SPLITARGBROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SplitARGBRow = SplitARGBRow_Any_SSE2;
+ if (IS_ALIGNED(width, 8)) {
+ SplitARGBRow = SplitARGBRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ SplitARGBRow = SplitARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ SplitARGBRow = SplitARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_SPLITARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitARGBRow = SplitARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ SplitARGBRow = SplitARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitARGBRow = SplitARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ SplitARGBRow = SplitARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitARGBRow = SplitARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitARGBRow = SplitARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ SplitARGBRow(src_argb, dst_r, dst_g, dst_b, dst_a, width);
+ dst_r += dst_stride_r;
+ dst_g += dst_stride_g;
+ dst_b += dst_stride_b;
+ dst_a += dst_stride_a;
+ src_argb += src_stride_argb;
+ }
+}
+
+LIBYUV_NOINLINE
+static void SplitARGBPlaneOpaque(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_r,
+ int dst_stride_r,
+ uint8_t* dst_g,
+ int dst_stride_g,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ int y;
+ void (*SplitXRGBRow)(const uint8_t* src_rgb, uint8_t* dst_r, uint8_t* dst_g,
+ uint8_t* dst_b, int width) = SplitXRGBRow_C;
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_argb == width * 4 && dst_stride_r == width &&
+ dst_stride_g == width && dst_stride_b == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_r = dst_stride_g = dst_stride_b = 0;
+ }
+
+#if defined(HAS_SPLITXRGBROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ SplitXRGBRow = SplitXRGBRow_Any_SSE2;
+ if (IS_ALIGNED(width, 8)) {
+ SplitXRGBRow = SplitXRGBRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITXRGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ SplitXRGBRow = SplitXRGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ SplitXRGBRow = SplitXRGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_SPLITXRGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitXRGBRow = SplitXRGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ SplitXRGBRow = SplitXRGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SPLITXRGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SplitXRGBRow = SplitXRGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ SplitXRGBRow = SplitXRGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SPLITXRGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitXRGBRow = SplitXRGBRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITXRGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitXRGBRow = SplitXRGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ SplitXRGBRow(src_argb, dst_r, dst_g, dst_b, width);
+ dst_r += dst_stride_r;
+ dst_g += dst_stride_g;
+ dst_b += dst_stride_b;
+ src_argb += src_stride_argb;
+ }
+}
+
+LIBYUV_API
+void SplitARGBPlane(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_r,
+ int dst_stride_r,
+ uint8_t* dst_g,
+ int dst_stride_g,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ int width,
+ int height) {
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r;
+ dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g;
+ dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b;
+ dst_stride_r = -dst_stride_r;
+ dst_stride_g = -dst_stride_g;
+ dst_stride_b = -dst_stride_b;
+ if (dst_a) {
+ dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a;
+ dst_stride_a = -dst_stride_a;
+ }
+ }
+
+ if (dst_a == NULL) {
+ SplitARGBPlaneOpaque(src_argb, src_stride_argb, dst_r, dst_stride_r, dst_g,
+ dst_stride_g, dst_b, dst_stride_b, width, height);
+ } else {
+ SplitARGBPlaneAlpha(src_argb, src_stride_argb, dst_r, dst_stride_r, dst_g,
+ dst_stride_g, dst_b, dst_stride_b, dst_a, dst_stride_a,
+ width, height);
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeARGBPlaneAlpha(const uint8_t* src_r,
+ int src_stride_r,
+ const uint8_t* src_g,
+ int src_stride_g,
+ const uint8_t* src_b,
+ int src_stride_b,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*MergeARGBRow)(const uint8_t* src_r, const uint8_t* src_g,
+ const uint8_t* src_b, const uint8_t* src_a,
+ uint8_t* dst_argb, int width) = MergeARGBRow_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ src_stride_a == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = src_stride_a =
+ dst_stride_argb = 0;
+ }
+#if defined(HAS_MERGEARGBROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeARGBRow = MergeARGBRow_Any_SSE2;
+ if (IS_ALIGNED(width, 8)) {
+ MergeARGBRow = MergeARGBRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeARGBRow = MergeARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeARGBRow = MergeARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeARGBRow = MergeARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MergeARGBRow = MergeARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeARGBRow = MergeARGBRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEARGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeARGBRow = MergeARGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeARGBRow(src_r, src_g, src_b, src_a, dst_argb, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ src_a += src_stride_a;
+ dst_argb += dst_stride_argb;
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeARGBPlaneOpaque(const uint8_t* src_r,
+ int src_stride_r,
+ const uint8_t* src_g,
+ int src_stride_g,
+ const uint8_t* src_b,
+ int src_stride_b,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
+ void (*MergeXRGBRow)(const uint8_t* src_r, const uint8_t* src_g,
+ const uint8_t* src_b, uint8_t* dst_argb, int width) =
+ MergeXRGBRow_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = dst_stride_argb = 0;
+ }
+#if defined(HAS_MERGEXRGBROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeXRGBRow = MergeXRGBRow_Any_SSE2;
+ if (IS_ALIGNED(width, 8)) {
+ MergeXRGBRow = MergeXRGBRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXRGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeXRGBRow = MergeXRGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeXRGBRow = MergeXRGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXRGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeXRGBRow = MergeXRGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ MergeXRGBRow = MergeXRGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXRGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeXRGBRow = MergeXRGBRow_RVV;
+ }
+#endif
+#if defined(HAS_MERGEXRGBROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MergeXRGBRow = MergeXRGBRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeXRGBRow(src_r, src_g, src_b, dst_argb, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ dst_argb += dst_stride_argb;
+ }
+}
+
+LIBYUV_API
+void MergeARGBPlane(const uint8_t* src_r,
+ int src_stride_r,
+ const uint8_t* src_g,
+ int src_stride_g,
+ const uint8_t* src_b,
+ int src_stride_b,
+ const uint8_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+
+ if (src_a == NULL) {
+ MergeARGBPlaneOpaque(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, dst_argb, dst_stride_argb, width,
+ height);
+ } else {
+ MergeARGBPlaneAlpha(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, src_a, src_stride_a, dst_argb,
+ dst_stride_argb, width, height);
+ }
+}
+
+// TODO(yuan): Support 2 bit alpha channel.
+LIBYUV_API
+void MergeXR30Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ uint8_t* dst_ar30,
+ int dst_stride_ar30,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeXR30Row)(const uint16_t* src_r, const uint16_t* src_g,
+ const uint16_t* src_b, uint8_t* dst_ar30, int depth,
+ int width) = MergeXR30Row_C;
+
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
+ dst_stride_ar30 = -dst_stride_ar30;
+ }
+ // Coalesce rows.
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = dst_stride_ar30 = 0;
+ }
+#if defined(HAS_MERGEXR30ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeXR30Row = MergeXR30Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeXR30Row = MergeXR30Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXR30ROW_10_AVX512BW)
+ if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
+ (kCpuHasAVX512BW | kCpuHasAVX512VL)) {
+ MergeXR30Row = MergeXR30Row_10_AVX512BW;
+ }
+#endif
+#if defined(HAS_MERGEXR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ if (depth == 10) {
+ MergeXR30Row = MergeXR30Row_10_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeXR30Row = MergeXR30Row_10_NEON;
+ }
+ } else {
+ MergeXR30Row = MergeXR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeXR30Row = MergeXR30Row_NEON;
+ }
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeXR30Row(src_r, src_g, src_b, dst_ar30, depth, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ dst_ar30 += dst_stride_ar30;
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeAR64PlaneAlpha(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeAR64Row)(const uint16_t* src_r, const uint16_t* src_g,
+ const uint16_t* src_b, const uint16_t* src_a,
+ uint16_t* dst_argb, int depth, int width) =
+ MergeAR64Row_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ src_stride_a == width && dst_stride_ar64 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = src_stride_a =
+ dst_stride_ar64 = 0;
+ }
+#if defined(HAS_MERGEAR64ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeAR64Row = MergeAR64Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeAR64Row = MergeAR64Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEAR64ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeAR64Row = MergeAR64Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeAR64Row = MergeAR64Row_NEON;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeAR64Row(src_r, src_g, src_b, src_a, dst_ar64, depth, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ src_a += src_stride_a;
+ dst_ar64 += dst_stride_ar64;
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeAR64PlaneOpaque(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeXR64Row)(const uint16_t* src_r, const uint16_t* src_g,
+ const uint16_t* src_b, uint16_t* dst_argb, int depth,
+ int width) = MergeXR64Row_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ // Coalesce rows.
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ dst_stride_ar64 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = dst_stride_ar64 = 0;
+ }
+#if defined(HAS_MERGEXR64ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeXR64Row = MergeXR64Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeXR64Row = MergeXR64Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXR64ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeXR64Row = MergeXR64Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeXR64Row = MergeXR64Row_NEON;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeXR64Row(src_r, src_g, src_b, dst_ar64, depth, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ dst_ar64 += dst_stride_ar64;
+ }
+}
+
+LIBYUV_API
+void MergeAR64Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ int width,
+ int height,
+ int depth) {
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_ar64 = dst_ar64 + (ptrdiff_t)(height - 1) * dst_stride_ar64;
+ dst_stride_ar64 = -dst_stride_ar64;
+ }
+
+ if (src_a == NULL) {
+ MergeAR64PlaneOpaque(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, dst_ar64, dst_stride_ar64, width, height,
+ depth);
+ } else {
+ MergeAR64PlaneAlpha(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, src_a, src_stride_a, dst_ar64,
+ dst_stride_ar64, width, height, depth);
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeARGB16To8PlaneAlpha(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeARGB16To8Row)(const uint16_t* src_r, const uint16_t* src_g,
+ const uint16_t* src_b, const uint16_t* src_a,
+ uint8_t* dst_argb, int depth, int width) =
+ MergeARGB16To8Row_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ src_stride_a == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = src_stride_a =
+ dst_stride_argb = 0;
+ }
+#if defined(HAS_MERGEARGB16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeARGB16To8Row = MergeARGB16To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeARGB16To8Row = MergeARGB16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEARGB16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeARGB16To8Row = MergeARGB16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeARGB16To8Row = MergeARGB16To8Row_NEON;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeARGB16To8Row(src_r, src_g, src_b, src_a, dst_argb, depth, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ src_a += src_stride_a;
+ dst_argb += dst_stride_argb;
+ }
+}
+
+LIBYUV_NOINLINE
+static void MergeARGB16To8PlaneOpaque(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int depth) {
+ int y;
+ void (*MergeXRGB16To8Row)(const uint16_t* src_r, const uint16_t* src_g,
+ const uint16_t* src_b, uint8_t* dst_argb, int depth,
+ int width) = MergeXRGB16To8Row_C;
+
+ assert(height >= 0);
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ // Coalesce rows.
+ if (src_stride_r == width && src_stride_g == width && src_stride_b == width &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_r = src_stride_g = src_stride_b = dst_stride_argb = 0;
+ }
+#if defined(HAS_MERGEXRGB16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeXRGB16To8Row = MergeXRGB16To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MergeXRGB16To8Row = MergeXRGB16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEXRGB16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeXRGB16To8Row = MergeXRGB16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ MergeXRGB16To8Row = MergeXRGB16To8Row_NEON;
+ }
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ MergeXRGB16To8Row(src_r, src_g, src_b, dst_argb, depth, width);
+ src_r += src_stride_r;
+ src_g += src_stride_g;
+ src_b += src_stride_b;
+ dst_argb += dst_stride_argb;
+ }
+}
+
+LIBYUV_API
+void MergeARGB16To8Plane(const uint16_t* src_r,
+ int src_stride_r,
+ const uint16_t* src_g,
+ int src_stride_g,
+ const uint16_t* src_b,
+ int src_stride_b,
+ const uint16_t* src_a,
+ int src_stride_a,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ int depth) {
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+
+ if (src_a == NULL) {
+ MergeARGB16To8PlaneOpaque(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, dst_argb, dst_stride_argb, width,
+ height, depth);
+ } else {
+ MergeARGB16To8PlaneAlpha(src_r, src_stride_r, src_g, src_stride_g, src_b,
+ src_stride_b, src_a, src_stride_a, dst_argb,
+ dst_stride_argb, width, height, depth);
+ }
+}
+
+// Convert YUY2 to I422.
+LIBYUV_API
+int YUY2ToI422(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*YUY2ToUV422Row)(const uint8_t* src_yuy2, uint8_t* dst_u,
+ uint8_t* dst_v, int width) = YUY2ToUV422Row_C;
+ void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
+ YUY2ToYRow_C;
+ if (!src_yuy2 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_yuy2 = src_yuy2 + (ptrdiff_t)(height - 1) * src_stride_yuy2;
+ src_stride_yuy2 = -src_stride_yuy2;
+ }
+ // Coalesce rows.
+ if (src_stride_yuy2 == width * 2 && dst_stride_y == width &&
+ dst_stride_u * 2 == width && dst_stride_v * 2 == width &&
+ (ptrdiff_t)width * height <= 32768) {
+ width *= height;
+ height = 1;
src_stride_yuy2 = dst_stride_y = dst_stride_u = dst_stride_v = 0;
}
-#if defined(HAS_YUY2TOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- YUY2ToUV422Row = YUY2ToUV422Row_Any_SSE2;
- YUY2ToYRow = YUY2ToYRow_Any_SSE2;
+#if defined(HAS_YUY2TOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ YUY2ToUV422Row = YUY2ToUV422Row_Any_SSE2;
+ YUY2ToYRow = YUY2ToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToUV422Row = YUY2ToUV422Row_SSE2;
+ YUY2ToYRow = YUY2ToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ YUY2ToUV422Row = YUY2ToUV422Row_Any_AVX2;
+ YUY2ToYRow = YUY2ToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ YUY2ToUV422Row = YUY2ToUV422Row_AVX2;
+ YUY2ToYRow = YUY2ToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ YUY2ToYRow = YUY2ToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToYRow = YUY2ToYRow_Any_NEON;
+ YUY2ToUV422Row = YUY2ToUV422Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_NEON;
+ YUY2ToUV422Row = YUY2ToUV422Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToYRow = YUY2ToYRow_SVE2;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LSX;
+ YUY2ToUV422Row = YUY2ToUV422Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_LSX;
+ YUY2ToUV422Row = YUY2ToUV422Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_LASX) && defined(HAS_YUY2TOUV422ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LASX;
+ YUY2ToUV422Row = YUY2ToUV422Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ YUY2ToYRow = YUY2ToYRow_LASX;
+ YUY2ToUV422Row = YUY2ToUV422Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ YUY2ToYRow = YUY2ToYRow_HVX;
+ YUY2ToUV422Row = YUY2ToUV422Row_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ YUY2ToUV422Row(src_yuy2, dst_u, dst_v, width);
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ src_yuy2 += src_stride_yuy2;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ return 0;
+}
+
+// Convert UYVY to I422.
+LIBYUV_API
+int UYVYToI422(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*UYVYToUV422Row)(const uint8_t* src_uyvy, uint8_t* dst_u,
+ uint8_t* dst_v, int width) = UYVYToUV422Row_C;
+ void (*UYVYToYRow)(const uint8_t* src_uyvy, uint8_t* dst_y, int width) =
+ UYVYToYRow_C;
+ if (!src_uyvy || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uyvy = src_uyvy + (ptrdiff_t)(height - 1) * src_stride_uyvy;
+ src_stride_uyvy = -src_stride_uyvy;
+ }
+ // Coalesce rows.
+ if (src_stride_uyvy == width * 2 && dst_stride_y == width &&
+ dst_stride_u * 2 == width && dst_stride_v * 2 == width &&
+ (ptrdiff_t)width * height <= 32768) {
+ width *= height;
+ height = 1;
+ src_stride_uyvy = dst_stride_y = dst_stride_u = dst_stride_v = 0;
+ }
+#if defined(HAS_UYVYTOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ UYVYToUV422Row = UYVYToUV422Row_Any_SSE2;
+ UYVYToYRow = UYVYToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToUV422Row = UYVYToUV422Row_SSE2;
+ UYVYToYRow = UYVYToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ UYVYToUV422Row = UYVYToUV422Row_Any_AVX2;
+ UYVYToYRow = UYVYToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToUV422Row = UYVYToUV422Row_AVX2;
+ UYVYToYRow = UYVYToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ UYVYToYRow = UYVYToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ UYVYToYRow = UYVYToYRow_Any_NEON;
+ UYVYToUV422Row = UYVYToUV422Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_NEON;
+ UYVYToUV422Row = UYVYToUV422Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ UYVYToYRow = UYVYToYRow_SVE2;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LSX) && defined(HAS_UYVYTOUV422ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ UYVYToYRow = UYVYToYRow_Any_LSX;
+ UYVYToUV422Row = UYVYToUV422Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_LSX;
+ UYVYToUV422Row = UYVYToUV422Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LASX) && defined(HAS_UYVYTOUV422ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ UYVYToYRow = UYVYToYRow_Any_LASX;
+ UYVYToUV422Row = UYVYToUV422Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToYRow = UYVYToYRow_LASX;
+ UYVYToUV422Row = UYVYToUV422Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ UYVYToYRow = UYVYToYRow_HVX;
+ UYVYToUV422Row = UYVYToUV422Row_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ UYVYToUV422Row(src_uyvy, dst_u, dst_v, width);
+ UYVYToYRow(src_uyvy, dst_y, width);
+ src_uyvy += src_stride_uyvy;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ return 0;
+}
+
+// Convert YUY2 to Y.
+LIBYUV_API
+int YUY2ToY(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
+ int y;
+ void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
+ YUY2ToYRow_C;
+ if (!src_yuy2 || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_yuy2 = src_yuy2 + (ptrdiff_t)(height - 1) * src_stride_yuy2;
+ src_stride_yuy2 = -src_stride_yuy2;
+ }
+ // Coalesce rows.
+ if (src_stride_yuy2 == width * 2 && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_yuy2 = dst_stride_y = 0;
+ }
+#if defined(HAS_YUY2TOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ YUY2ToYRow = YUY2ToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ YUY2ToYRow = YUY2ToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ YUY2ToYRow = YUY2ToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ YUY2ToYRow = YUY2ToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToYRow = YUY2ToYRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToYRow = YUY2ToYRow_SVE2;
+ }
+#endif
+#if defined(HAS_YUY2TOYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ YUY2ToYRow = YUY2ToYRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ src_yuy2 += src_stride_yuy2;
+ dst_y += dst_stride_y;
+ }
+ return 0;
+}
+
+// Convert UYVY to Y.
+LIBYUV_API
+int UYVYToY(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
+ int y;
+ void (*UYVYToYRow)(const uint8_t* src_uyvy, uint8_t* dst_y, int width) =
+ UYVYToYRow_C;
+ if (!src_uyvy || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uyvy = src_uyvy + (ptrdiff_t)(height - 1) * src_stride_uyvy;
+ src_stride_uyvy = -src_stride_uyvy;
+ }
+ // Coalesce rows.
+ if (src_stride_uyvy == width * 2 && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_uyvy = dst_stride_y = 0;
+ }
+#if defined(HAS_UYVYTOYROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ UYVYToYRow = UYVYToYRow_Any_SSE2;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ UYVYToYRow = UYVYToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToYRow = UYVYToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ UYVYToYRow = UYVYToYRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ UYVYToYRow = UYVYToYRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ UYVYToYRow = UYVYToYRow_SVE2;
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ UYVYToYRow = UYVYToYRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ UYVYToYRow = UYVYToYRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ UYVYToYRow = UYVYToYRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ UYVYToYRow = UYVYToYRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_UYVYTOYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ UYVYToYRow = UYVYToYRow_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ UYVYToYRow(src_uyvy, dst_y, width);
+ src_uyvy += src_stride_uyvy;
+ dst_y += dst_stride_y;
+ }
+ return 0;
+}
+
+// Mirror a plane of data.
+// See Also I400Mirror
+LIBYUV_API
+void MirrorPlane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
+ int y;
+ void (*MirrorRow)(const uint8_t* src, uint8_t* dst, int width) = MirrorRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ }
+#if defined(HAS_MIRRORROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MirrorRow = MirrorRow_Any_NEON;
+ if (IS_ALIGNED(width, 32)) {
+ MirrorRow = MirrorRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MIRRORROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ MirrorRow = MirrorRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- YUY2ToUV422Row = YUY2ToUV422Row_SSE2;
- YUY2ToYRow = YUY2ToYRow_SSE2;
+ MirrorRow = MirrorRow_SSSE3;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_AVX2)
+#if defined(HAS_MIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- YUY2ToUV422Row = YUY2ToUV422Row_Any_AVX2;
- YUY2ToYRow = YUY2ToYRow_Any_AVX2;
+ MirrorRow = MirrorRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- YUY2ToUV422Row = YUY2ToUV422Row_AVX2;
- YUY2ToYRow = YUY2ToYRow_AVX2;
+ MirrorRow = MirrorRow_AVX2;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- YUY2ToYRow = YUY2ToYRow_Any_NEON;
- YUY2ToUV422Row = YUY2ToUV422Row_Any_NEON;
+#if defined(HAS_MIRRORROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MirrorRow = MirrorRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MIRRORROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MirrorRow = MirrorRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- YUY2ToYRow = YUY2ToYRow_NEON;
- YUY2ToUV422Row = YUY2ToUV422Row_NEON;
+ MirrorRow = MirrorRow_LSX;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- YUY2ToYRow = YUY2ToYRow_Any_MSA;
- YUY2ToUV422Row = YUY2ToUV422Row_Any_MSA;
+#if defined(HAS_MIRRORROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MirrorRow = MirrorRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- YUY2ToYRow = YUY2ToYRow_MSA;
- YUY2ToUV422Row = YUY2ToUV422Row_MSA;
+ MirrorRow = MirrorRow_LASX;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- YUY2ToYRow = YUY2ToYRow_Any_MMI;
- YUY2ToUV422Row = YUY2ToUV422Row_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- YUY2ToYRow = YUY2ToYRow_MMI;
- YUY2ToUV422Row = YUY2ToUV422Row_MMI;
+#if defined(HAS_MIRRORROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MirrorRow = MirrorRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorRow = MirrorRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_MIRRORROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MirrorRow = MirrorRow_HVX;
+ }
+#endif
+ // Mirror plane
for (y = 0; y < height; ++y) {
- YUY2ToUV422Row(src_yuy2, dst_u, dst_v, width);
- YUY2ToYRow(src_yuy2, dst_y, width);
- src_yuy2 += src_stride_yuy2;
+ MirrorRow(src_y, dst_y, width);
+ src_y += src_stride_y;
dst_y += dst_stride_y;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
}
- return 0;
}
-// Convert UYVY to I422.
+// Mirror a plane of 16-bit data.
LIBYUV_API
-int UYVYToI422(const uint8_t* src_uyvy,
- int src_stride_uyvy,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_u,
- int dst_stride_u,
- uint8_t* dst_v,
- int dst_stride_v,
- int width,
- int height) {
+void MirrorPlane_16(const uint16_t* src_y,
+ int src_stride_y,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ int width,
+ int height) {
int y;
- void (*UYVYToUV422Row)(const uint8_t* src_uyvy, uint8_t* dst_u,
- uint8_t* dst_v, int width) = UYVYToUV422Row_C;
- void (*UYVYToYRow)(const uint8_t* src_uyvy, uint8_t* dst_y, int width) =
- UYVYToYRow_C;
- if (!src_uyvy || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
- return -1;
+ void (*MirrorRow_16)(const uint16_t* src, uint16_t* dst, int width) =
+ MirrorRow_16_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy;
- src_stride_uyvy = -src_stride_uyvy;
- }
- // Coalesce rows.
- if (src_stride_uyvy == width * 2 && dst_stride_y == width &&
- dst_stride_u * 2 == width && dst_stride_v * 2 == width &&
- width * height <= 32768) {
- width *= height;
- height = 1;
- src_stride_uyvy = dst_stride_y = dst_stride_u = dst_stride_v = 0;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
}
-#if defined(HAS_UYVYTOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- UYVYToUV422Row = UYVYToUV422Row_Any_SSE2;
- UYVYToYRow = UYVYToYRow_Any_SSE2;
+#if defined(HAS_MIRRORROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MirrorRow_16 = MirrorRow_16_Any_NEON;
if (IS_ALIGNED(width, 16)) {
- UYVYToUV422Row = UYVYToUV422Row_SSE2;
- UYVYToYRow = UYVYToYRow_SSE2;
+ MirrorRow_16 = MirrorRow_16_NEON;
}
}
#endif
-#if defined(HAS_UYVYTOYROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- UYVYToUV422Row = UYVYToUV422Row_Any_AVX2;
- UYVYToYRow = UYVYToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- UYVYToUV422Row = UYVYToUV422Row_AVX2;
- UYVYToYRow = UYVYToYRow_AVX2;
+#if defined(HAS_MIRRORUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_Any_SSSE3);
+ if (IS_ALIGNED(width, 8)) {
+ MirrorRow_16 =
+ reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_SSSE3);
}
}
#endif
-#if defined(HAS_UYVYTOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- UYVYToYRow = UYVYToYRow_Any_NEON;
- UYVYToUV422Row = UYVYToUV422Row_Any_NEON;
+#if defined(HAS_MIRRORUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_Any_AVX2);
if (IS_ALIGNED(width, 16)) {
- UYVYToYRow = UYVYToYRow_NEON;
- UYVYToUV422Row = UYVYToUV422Row_NEON;
+ MirrorRow_16 =
+ reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_AVX2);
}
}
#endif
-#if defined(HAS_UYVYTOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- UYVYToYRow = UYVYToYRow_Any_MSA;
- UYVYToUV422Row = UYVYToUV422Row_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- UYVYToYRow = UYVYToYRow_MSA;
- UYVYToUV422Row = UYVYToUV422Row_MSA;
+#if defined(HAS_MIRRORROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MirrorRow_16 = MirrorRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_MIRRORUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_Any_LSX);
+ if (IS_ALIGNED(width, 8)) {
+ MirrorRow_16 =
+ reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_LSX);
}
}
#endif
-#if defined(HAS_UYVYTOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- UYVYToYRow = UYVYToYRow_Any_MMI;
- UYVYToUV422Row = UYVYToUV422Row_Any_MMI;
+#if defined(HAS_MIRRORUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_Any_LASX);
if (IS_ALIGNED(width, 16)) {
- UYVYToYRow = UYVYToYRow_MMI;
- UYVYToUV422Row = UYVYToUV422Row_MMI;
+ MirrorRow_16 =
+ reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_LASX);
+ }
+ }
+#endif
+#if defined(HAS_MIRRORUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_RVV);
+ }
+#endif
+#if defined(HAS_MIRRORUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_Any_WASMSIMD);
+ if (IS_ALIGNED(width, 8)) {
+ MirrorRow_16 =
+ reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
+ MirrorUVRow_WASMSIMD);
}
}
#endif
+#if defined(HAS_MIRRORROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MirrorRow_16 = MirrorRow_16_HVX;
+ }
+#endif
+ // Mirror plane
for (y = 0; y < height; ++y) {
- UYVYToUV422Row(src_uyvy, dst_u, dst_v, width);
- UYVYToYRow(src_uyvy, dst_y, width);
- src_uyvy += src_stride_uyvy;
+ MirrorRow_16(src_y, dst_y, width);
+ src_y += src_stride_y;
dst_y += dst_stride_y;
- dst_u += dst_stride_u;
- dst_v += dst_stride_v;
}
- return 0;
}
-// Convert YUY2 to Y.
+// Mirror a plane of UV data.
LIBYUV_API
-int YUY2ToY(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_y,
- int dst_stride_y,
- int width,
- int height) {
+void MirrorUVPlane(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
int y;
- void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
- YUY2ToYRow_C;
- if (!src_yuy2 || !dst_y || width <= 0 || height == 0) {
- return -1;
+ void (*MirrorUVRow)(const uint8_t* src, uint8_t* dst, int width) =
+ MirrorUVRow_C;
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2;
- src_stride_yuy2 = -src_stride_yuy2;
+ src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
}
- // Coalesce rows.
- if (src_stride_yuy2 == width * 2 && dst_stride_y == width) {
- width *= height;
- height = 1;
- src_stride_yuy2 = dst_stride_y = 0;
+#if defined(HAS_MIRRORUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MirrorUVRow = MirrorUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 32)) {
+ MirrorUVRow = MirrorUVRow_NEON;
+ }
}
-#if defined(HAS_YUY2TOYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- YUY2ToYRow = YUY2ToYRow_Any_SSE2;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToYRow = YUY2ToYRow_SSE2;
+#endif
+#if defined(HAS_MIRRORUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ MirrorUVRow = MirrorUVRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ MirrorUVRow = MirrorUVRow_SSSE3;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_AVX2)
+#if defined(HAS_MIRRORUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- YUY2ToYRow = YUY2ToYRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
- YUY2ToYRow = YUY2ToYRow_AVX2;
+ MirrorUVRow = MirrorUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorUVRow = MirrorUVRow_AVX2;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- YUY2ToYRow = YUY2ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- YUY2ToYRow = YUY2ToYRow_NEON;
+#if defined(HAS_MIRRORUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MirrorUVRow = MirrorUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MIRRORUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MirrorUVRow = MirrorUVRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ MirrorUVRow = MirrorUVRow_LSX;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- YUY2ToYRow = YUY2ToYRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- YUY2ToYRow = YUY2ToYRow_MSA;
+#if defined(HAS_MIRRORUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MirrorUVRow = MirrorUVRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorUVRow = MirrorUVRow_LASX;
}
}
#endif
-#if defined(HAS_YUY2TOYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- YUY2ToYRow = YUY2ToYRow_Any_MMI;
+#if defined(HAS_MIRRORUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MirrorUVRow = MirrorUVRow_RVV;
+ }
+#endif
+#if defined(HAS_MIRRORUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MirrorUVRow = MirrorUVRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
- YUY2ToYRow = YUY2ToYRow_MMI;
+ MirrorUVRow = MirrorUVRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_MIRRORUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MirrorUVRow = MirrorUVRow_HVX;
+ }
+#endif
+ // MirrorUV plane
for (y = 0; y < height; ++y) {
- YUY2ToYRow(src_yuy2, dst_y, width);
- src_yuy2 += src_stride_yuy2;
- dst_y += dst_stride_y;
+ MirrorUVRow(src_uv, dst_uv, width);
+ src_uv += src_stride_uv;
+ dst_uv += dst_stride_uv;
}
- return 0;
}
// Mirror I400 with optional flipping
@@ -1055,13 +3165,13 @@ int I400Mirror(const uint8_t* src_y,
int dst_stride_y,
int width,
int height) {
- if (!src_y || !dst_y || width <= 0 || height == 0) {
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
@@ -1087,17 +3197,19 @@ int I420Mirror(const uint8_t* src_y,
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_y || !src_u || !src_v || !dst_y || !dst_u || !dst_v || width <= 0 ||
- height == 0) {
+
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
return -1;
}
+
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -1111,6 +3223,44 @@ int I420Mirror(const uint8_t* src_y,
return 0;
}
+// NV12 mirror.
+LIBYUV_API
+int NV12Mirror(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(halfheight - 1) * src_stride_uv;
+ src_stride_y = -src_stride_y;
+ src_stride_uv = -src_stride_uv;
+ }
+
+ if (dst_y) {
+ MirrorPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ }
+ MirrorUVPlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, halfwidth,
+ halfheight);
+ return 0;
+}
+
// ARGB mirror.
LIBYUV_API
int ARGBMirror(const uint8_t* src_argb,
@@ -1122,19 +3272,20 @@ int ARGBMirror(const uint8_t* src_argb,
int y;
void (*ARGBMirrorRow)(const uint8_t* src, uint8_t* dst, int width) =
ARGBMirrorRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
#if defined(HAS_ARGBMIRRORROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBMirrorRow = ARGBMirrorRow_Any_NEON;
- if (IS_ALIGNED(width, 4)) {
+ if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_NEON;
}
}
@@ -1147,51 +3298,169 @@ int ARGBMirror(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_ARGBMIRRORROW_AVX2)
+#if defined(HAS_ARGBMIRRORROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBMirrorRow = ARGBMirrorRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBMirrorRow = ARGBMirrorRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBMirrorRow = ARGBMirrorRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBMirrorRow = ARGBMirrorRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBMirrorRow = ARGBMirrorRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBMirrorRow = ARGBMirrorRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBMirrorRow = ARGBMirrorRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBMirrorRow = ARGBMirrorRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBMirrorRow = ARGBMirrorRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBMIRRORROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBMirrorRow = ARGBMirrorRow_HVX;
+ }
+#endif
+
+ // Mirror plane
+ for (y = 0; y < height; ++y) {
+ ARGBMirrorRow(src_argb, dst_argb, width);
+ src_argb += src_stride_argb;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// RGB24 mirror.
+LIBYUV_API
+int RGB24Mirror(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_rgb24,
+ int dst_stride_rgb24,
+ int width,
+ int height) {
+ int y;
+ void (*RGB24MirrorRow)(const uint8_t* src, uint8_t* dst, int width) =
+ RGB24MirrorRow_C;
+ if (!src_rgb24 || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_rgb24 = src_rgb24 + (ptrdiff_t)(height - 1) * src_stride_rgb24;
+ src_stride_rgb24 = -src_stride_rgb24;
+ }
+#if defined(HAS_RGB24MIRRORROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB24MirrorRow = RGB24MirrorRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24MirrorRow = RGB24MirrorRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RGB24MIRRORROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RGB24MirrorRow = RGB24MirrorRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24MirrorRow = RGB24MirrorRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RGB24MIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_AVX2;
- if (IS_ALIGNED(width, 8)) {
- ARGBMirrorRow = ARGBMirrorRow_AVX2;
+ RGB24MirrorRow = RGB24MirrorRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RGB24MirrorRow = RGB24MirrorRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBMIRRORROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBMirrorRow = ARGBMirrorRow_MSA;
- }
+#if defined(HAS_RGB24MIRRORROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ RGB24MirrorRow = RGB24MirrorRow_AVX512VBMI;
}
#endif
-#if defined(HAS_ARGBMIRRORROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBMirrorRow = ARGBMirrorRow_MMI;
+#if defined(HAS_RGB24MIRRORROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RGB24MirrorRow = RGB24MirrorRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RGB24MirrorRow = RGB24MirrorRow_WASMSIMD;
}
}
#endif
// Mirror plane
for (y = 0; y < height; ++y) {
- ARGBMirrorRow(src_argb, dst_argb, width);
- src_argb += src_stride_argb;
- dst_argb += dst_stride_argb;
+ RGB24MirrorRow(src_rgb24, dst_rgb24, width);
+ src_rgb24 += src_stride_rgb24;
+ dst_rgb24 += dst_stride_rgb24;
}
return 0;
}
-// Get a blender that optimized for the CPU and pixel count.
-// As there are 6 blenders to choose from, the caller should try to use
-// the same blend function for all pixels if possible.
+// Alpha Blend 2 ARGB images and store to destination.
LIBYUV_API
-ARGBBlendRow GetARGBBlend(void) {
+int ARGBBlend(const uint8_t* src_argb0,
+ int src_stride_argb0,
+ const uint8_t* src_argb1,
+ int src_stride_argb1,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ int y;
void (*ARGBBlendRow)(const uint8_t* src_argb, const uint8_t* src_argb1,
uint8_t* dst_argb, int width) = ARGBBlendRow_C;
-#if defined(HAS_ARGBBLENDROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBBlendRow = ARGBBlendRow_SSSE3;
- return ARGBBlendRow;
+ if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
+ dst_stride_argb = -dst_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 &&
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
+ }
+#if defined(HAS_ARGBBLENDROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBBlendRow = ARGBBlendRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBBLENDROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBBlendRow = ARGBBlendRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBBLENDROW_NEON)
@@ -1199,51 +3468,151 @@ ARGBBlendRow GetARGBBlend(void) {
ARGBBlendRow = ARGBBlendRow_NEON;
}
#endif
-#if defined(HAS_ARGBBLENDROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBBlendRow = ARGBBlendRow_MSA;
+#if defined(HAS_ARGBBLENDROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBBlendRow = ARGBBlendRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBBlendRow = ARGBBlendRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBBLENDROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBBlendRow = ARGBBlendRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBBlendRow = ARGBBlendRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBBLENDROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBBlendRow = ARGBBlendRow_RVV;
}
#endif
-#if defined(HAS_ARGBBLENDROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBBlendRow = ARGBBlendRow_MMI;
+#if defined(HAS_ARGBBLENDROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBBlendRow = ARGBBlendRow_WASMSIMD;
}
#endif
- return ARGBBlendRow;
+#if defined(HAS_ARGBBLENDROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBBlendRow = ARGBBlendRow_HVX;
+ }
+#endif
+ for (y = 0; y < height; ++y) {
+ ARGBBlendRow(src_argb0, src_argb1, dst_argb, width);
+ src_argb0 += src_stride_argb0;
+ src_argb1 += src_stride_argb1;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
}
-// Alpha Blend 2 ARGB images and store to destination.
+// Alpha Blend 2 unattenuated ARGB images and store to destination.
LIBYUV_API
-int ARGBBlend(const uint8_t* src_argb0,
- int src_stride_argb0,
- const uint8_t* src_argb1,
- int src_stride_argb1,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int ARGBAlphaBlend(const uint8_t* src_argb0,
+ int src_stride_argb0,
+ const uint8_t* src_argb1,
+ int src_stride_argb1,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
int y;
- void (*ARGBBlendRow)(const uint8_t* src_argb, const uint8_t* src_argb1,
- uint8_t* dst_argb, int width) = GetARGBBlend();
- if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) {
+ void (*ARGBAlphaBlendRow)(const uint8_t* src_argb, const uint8_t* src_argb1,
+ uint8_t* dst_argb, int width) = ARGBAlphaBlendRow_C;
+ if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 &&
- dst_stride_argb == width * 4) {
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
}
-
+#if defined(HAS_ARGBALPHABLENDROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBALPHABLENDROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAlphaBlendRow = ARGBAlphaBlendRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
- ARGBBlendRow(src_argb0, src_argb1, dst_argb, width);
+ ARGBAlphaBlendRow(src_argb0, src_argb1, dst_argb, width);
src_argb0 += src_stride_argb0;
src_argb1 += src_stride_argb1;
dst_argb += dst_stride_argb;
@@ -1267,19 +3636,21 @@ int BlendPlane(const uint8_t* src_y0,
void (*BlendPlaneRow)(const uint8_t* src0, const uint8_t* src1,
const uint8_t* alpha, uint8_t* dst, int width) =
BlendPlaneRow_C;
- if (!src_y0 || !src_y1 || !alpha || !dst_y || width <= 0 || height == 0) {
+ if (!src_y0 || !src_y1 || !alpha || !dst_y || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
// Coalesce rows for Y plane.
if (src_stride_y0 == width && src_stride_y1 == width &&
- alpha_stride == width && dst_stride_y == width) {
+ alpha_stride == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y0 = src_stride_y1 = alpha_stride = dst_stride_y = 0;
@@ -1301,14 +3672,37 @@ int BlendPlane(const uint8_t* src_y0,
}
}
#endif
-#if defined(HAS_BLENDPLANEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- BlendPlaneRow = BlendPlaneRow_Any_MMI;
+#if defined(HAS_BLENDPLANEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ BlendPlaneRow = BlendPlaneRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ BlendPlaneRow = BlendPlaneRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ BlendPlaneRow = BlendPlaneRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ BlendPlaneRow = BlendPlaneRow_RVV;
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ BlendPlaneRow = BlendPlaneRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
- BlendPlaneRow = BlendPlaneRow_MMI;
+ BlendPlaneRow = BlendPlaneRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_BLENDPLANEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ BlendPlaneRow = BlendPlaneRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
BlendPlaneRow(src_y0, src_y1, alpha, dst_y, width);
@@ -1353,15 +3747,17 @@ int I420Blend(const uint8_t* src_y0,
BlendPlaneRow_C;
void (*ScaleRowDown2)(const uint8_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, int dst_width) = ScaleRowDown2Box_C;
+
if (!src_y0 || !src_u0 || !src_v0 || !src_y1 || !src_u1 || !src_v1 ||
- !alpha || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ !alpha || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
@@ -1385,13 +3781,36 @@ int I420Blend(const uint8_t* src_y0,
}
}
#endif
-#if defined(HAS_BLENDPLANEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- BlendPlaneRow = BlendPlaneRow_Any_MMI;
+#if defined(HAS_BLENDPLANEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ BlendPlaneRow = BlendPlaneRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ BlendPlaneRow = BlendPlaneRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ BlendPlaneRow = BlendPlaneRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ BlendPlaneRow = BlendPlaneRow_RVV;
+ }
+#endif
+#if defined(HAS_BLENDPLANEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ BlendPlaneRow = BlendPlaneRow_Any_WASMSIMD;
if (IS_ALIGNED(halfwidth, 8)) {
- BlendPlaneRow = BlendPlaneRow_MMI;
+ BlendPlaneRow = BlendPlaneRow_WASMSIMD;
}
}
+#endif
+#if defined(HAS_BLENDPLANEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ BlendPlaneRow = BlendPlaneRow_HVX;
+ }
#endif
if (!IS_ALIGNED(width, 2)) {
ScaleRowDown2 = ScaleRowDown2Box_Odd_C;
@@ -1429,20 +3848,41 @@ int I420Blend(const uint8_t* src_y0,
}
}
#endif
-#if defined(HAS_SCALEROWDOWN2_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ScaleRowDown2 = ScaleRowDown2Box_Odd_MMI;
+#if defined(HAS_SCALEROWDOWN2_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ScaleRowDown2 = ScaleRowDown2Box_Odd_AVX512BW;
if (IS_ALIGNED(width, 2)) {
- ScaleRowDown2 = ScaleRowDown2Box_Any_MMI;
- if (IS_ALIGNED(halfwidth, 8)) {
- ScaleRowDown2 = ScaleRowDown2Box_MMI;
+ ScaleRowDown2 = ScaleRowDown2Box_AVX512BW;
+ }
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN2_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowDown2 = ScaleRowDown2Box_RVV;
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ScaleRowDown2 = ScaleRowDown2Box_Odd_WASMSIMD;
+ if (IS_ALIGNED(width, 2)) {
+ ScaleRowDown2 = ScaleRowDown2Box_Any_WASMSIMD;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ ScaleRowDown2 = ScaleRowDown2Box_WASMSIMD;
}
}
}
#endif
+#if defined(HAS_SCALEROWDOWN2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleRowDown2 =
+ IS_ALIGNED(width, 2) ? ScaleRowDown2Box_HVX : ScaleRowDown2Box_Odd_HVX;
+ }
+#endif
// Row buffer for intermediate alpha pixels.
align_buffer_64(halfalpha, halfwidth);
+ if (!halfalpha)
+ return 1;
for (y = 0; y < height; y += 2) {
// last row of odd height image use 1 row of alpha instead of 2.
if (y == (height - 1)) {
@@ -1450,7 +3890,7 @@ int I420Blend(const uint8_t* src_y0,
}
// Subsample 2 rows of UV to half width and half height.
ScaleRowDown2(alpha, alpha_stride, halfalpha, halfwidth);
- alpha += alpha_stride * 2;
+ alpha += (ptrdiff_t)alpha_stride * 2;
BlendPlaneRow(src_u0, src_u1, halfalpha, dst_u, halfwidth);
BlendPlaneRow(src_v0, src_v1, halfalpha, dst_v, halfwidth);
src_u0 += src_stride_u0;
@@ -1477,18 +3917,19 @@ int ARGBMultiply(const uint8_t* src_argb0,
int y;
void (*ARGBMultiplyRow)(const uint8_t* src0, const uint8_t* src1,
uint8_t* dst, int width) = ARGBMultiplyRow_C;
- if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 &&
- dst_stride_argb == width * 4) {
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
@@ -1509,6 +3950,11 @@ int ARGBMultiply(const uint8_t* src_argb0,
}
}
#endif
+#if defined(HAS_ARGBMULTIPLYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_AVX512BW;
+ }
+#endif
#if defined(HAS_ARGBMULTIPLYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBMultiplyRow = ARGBMultiplyRow_Any_NEON;
@@ -1517,22 +3963,37 @@ int ARGBMultiply(const uint8_t* src_argb0,
}
}
#endif
-#if defined(HAS_ARGBMULTIPLYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBMultiplyRow = ARGBMultiplyRow_Any_MSA;
+#if defined(HAS_ARGBMULTIPLYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_SME;
+ }
+#endif
+#if defined(HAS_ARGBMULTIPLYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_Any_LSX;
if (IS_ALIGNED(width, 4)) {
- ARGBMultiplyRow = ARGBMultiplyRow_MSA;
+ ARGBMultiplyRow = ARGBMultiplyRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBMULTIPLYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBMultiplyRow = ARGBMultiplyRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBMultiplyRow = ARGBMultiplyRow_MMI;
+#if defined(HAS_ARGBMULTIPLYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_LASX;
}
}
#endif
+#if defined(HAS_ARGBMULTIPLYROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBMULTIPLYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_HVX;
+ }
+#endif
// Multiply plane
for (y = 0; y < height; ++y) {
@@ -1557,28 +4018,29 @@ int ARGBAdd(const uint8_t* src_argb0,
int y;
void (*ARGBAddRow)(const uint8_t* src0, const uint8_t* src1, uint8_t* dst,
int width) = ARGBAddRow_C;
- if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 &&
- dst_stride_argb == width * 4) {
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
}
-#if defined(HAS_ARGBADDROW_SSE2) && (defined(_MSC_VER) && !defined(__clang__))
+#if defined(HAS_ARGBADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBAddRow = ARGBAddRow_SSE2;
}
#endif
-#if defined(HAS_ARGBADDROW_SSE2) && !(defined(_MSC_VER) && !defined(__clang__))
+#if defined(HAS_ARGBADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBAddRow = ARGBAddRow_Any_SSE2;
if (IS_ALIGNED(width, 4)) {
@@ -1602,22 +4064,27 @@ int ARGBAdd(const uint8_t* src_argb0,
}
}
#endif
-#if defined(HAS_ARGBADDROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBAddRow = ARGBAddRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBAddRow = ARGBAddRow_MSA;
+#if defined(HAS_ARGBADDROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAddRow = ARGBAddRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAddRow = ARGBAddRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBADDROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBAddRow = ARGBAddRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBAddRow = ARGBAddRow_MMI;
+#if defined(HAS_ARGBADDROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAddRow = ARGBAddRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAddRow = ARGBAddRow_LASX;
}
}
#endif
+#if defined(HAS_ARGBADDROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAddRow = ARGBAddRow_HVX;
+ }
+#endif
// Add plane
for (y = 0; y < height; ++y) {
@@ -1642,18 +4109,19 @@ int ARGBSubtract(const uint8_t* src_argb0,
int y;
void (*ARGBSubtractRow)(const uint8_t* src0, const uint8_t* src1,
uint8_t* dst, int width) = ARGBSubtractRow_C;
- if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 &&
- dst_stride_argb == width * 4) {
+ dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
@@ -1682,22 +4150,27 @@ int ARGBSubtract(const uint8_t* src_argb0,
}
}
#endif
-#if defined(HAS_ARGBSUBTRACTROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBSubtractRow = ARGBSubtractRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBSubtractRow = ARGBSubtractRow_MSA;
+#if defined(HAS_ARGBSUBTRACTROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBSubtractRow = ARGBSubtractRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBSubtractRow = ARGBSubtractRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBSUBTRACTROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBSubtractRow = ARGBSubtractRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBSubtractRow = ARGBSubtractRow_MMI;
+#if defined(HAS_ARGBSUBTRACTROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBSubtractRow = ARGBSubtractRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBSubtractRow = ARGBSubtractRow_LASX;
}
}
#endif
+#if defined(HAS_ARGBSUBTRACTROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBSubtractRow = ARGBSubtractRow_HVX;
+ }
+#endif
// Subtract plane
for (y = 0; y < height; ++y) {
@@ -1708,174 +4181,113 @@ int ARGBSubtract(const uint8_t* src_argb0,
}
return 0;
}
-// Convert I422 to RGBA with matrix
-static int I422ToRGBAMatrix(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- const struct YuvConstants* yuvconstants,
- int width,
- int height) {
- int y;
- void (*I422ToRGBARow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) =
- I422ToRGBARow_C;
- if (!src_y || !src_u || !src_v || !dst_rgba || width <= 0 || height == 0) {
+
+// Bitwise OR ARGB image with a specified ARGB value.
+LIBYUV_API
+int ARGBOr(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height,
+ uint32_t value) {
+ int y;
+ void (*ARGBOrRow)(const uint8_t* src_argb, uint8_t* dst_argb, uint32_t value,
+ int width) = ARGBOrRow_C;
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst_rgba = dst_rgba + (height - 1) * dst_stride_rgba;
- dst_stride_rgba = -dst_stride_rgba;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
}
-#if defined(HAS_I422TORGBAROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_SSSE3;
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_argb = 0;
+ }
+#if defined(HAS_ARGBORROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ARGBOrRow = ARGBOrRow_Any_SSE2;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBOrRow = ARGBOrRow_SSE2;
}
}
#endif
-#if defined(HAS_I422TORGBAROW_AVX2)
+#if defined(HAS_ARGBORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToRGBARow = I422ToRGBARow_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- I422ToRGBARow = I422ToRGBARow_AVX2;
+ ARGBOrRow = ARGBOrRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBOrRow = ARGBOrRow_AVX2;
}
}
#endif
-#if defined(HAS_I422TORGBAROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToRGBARow = I422ToRGBARow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_NEON;
- }
+#if defined(HAS_ARGBORROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBOrRow = ARGBOrRow_AVX512BW;
}
#endif
-#if defined(HAS_I422TORGBAROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToRGBARow = I422ToRGBARow_Any_MSA;
+#if defined(HAS_ARGBORROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBOrRow = ARGBOrRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
- I422ToRGBARow = I422ToRGBARow_MSA;
+ ARGBOrRow = ARGBOrRow_NEON;
}
}
#endif
-
- for (y = 0; y < height; ++y) {
- I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
- dst_rgba += dst_stride_rgba;
- src_y += src_stride_y;
- src_u += src_stride_u;
- src_v += src_stride_v;
+#if defined(HAS_ARGBORROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBOrRow = ARGBOrRow_SVE2;
}
- return 0;
-}
-
-// Convert I422 to RGBA.
-LIBYUV_API
-int I422ToRGBA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_rgba,
- int dst_stride_rgba,
- int width,
- int height) {
- return I422ToRGBAMatrix(src_y, src_stride_y, src_u, src_stride_u, src_v,
- src_stride_v, dst_rgba, dst_stride_rgba,
- &kYuvI601Constants, width, height);
-}
-
-// Convert I422 to BGRA.
-LIBYUV_API
-int I422ToBGRA(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint8_t* dst_bgra,
- int dst_stride_bgra,
- int width,
- int height) {
- return I422ToRGBAMatrix(src_y, src_stride_y, src_v,
- src_stride_v, // Swap U and V
- src_u, src_stride_u, dst_bgra, dst_stride_bgra,
- &kYvuI601Constants, // Use Yvu matrix
- width, height);
-}
-
-// Convert NV12 to RGB565.
-LIBYUV_API
-int NV12ToRGB565(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_uv,
- int src_stride_uv,
- uint8_t* dst_rgb565,
- int dst_stride_rgb565,
- int width,
- int height) {
- int y;
- void (*NV12ToRGB565Row)(
- const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants, int width) = NV12ToRGB565Row_C;
- if (!src_y || !src_uv || !dst_rgb565 || width <= 0 || height == 0) {
- return -1;
+#endif
+#if defined(HAS_ARGBORROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBOrRow = ARGBOrRow_SME;
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- dst_rgb565 = dst_rgb565 + (height - 1) * dst_stride_rgb565;
- dst_stride_rgb565 = -dst_stride_rgb565;
+#endif
+#if defined(HAS_ARGBORROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBOrRow = ARGBOrRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBOrRow = ARGBOrRow_LSX;
+ }
}
-#if defined(HAS_NV12TORGB565ROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- NV12ToRGB565Row = NV12ToRGB565Row_Any_SSSE3;
+#endif
+#if defined(HAS_ARGBORROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBOrRow = ARGBOrRow_Any_LASX;
if (IS_ALIGNED(width, 8)) {
- NV12ToRGB565Row = NV12ToRGB565Row_SSSE3;
+ ARGBOrRow = ARGBOrRow_LASX;
}
}
#endif
-#if defined(HAS_NV12TORGB565ROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- NV12ToRGB565Row = NV12ToRGB565Row_Any_AVX2;
- if (IS_ALIGNED(width, 16)) {
- NV12ToRGB565Row = NV12ToRGB565Row_AVX2;
- }
+#if defined(HAS_ARGBORROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBOrRow = ARGBOrRow_RVV;
}
#endif
-#if defined(HAS_NV12TORGB565ROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- NV12ToRGB565Row = NV12ToRGB565Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- NV12ToRGB565Row = NV12ToRGB565Row_NEON;
+#if defined(HAS_ARGBORROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBOrRow = ARGBOrRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBOrRow = ARGBOrRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_NV12TORGB565ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- NV12ToRGB565Row = NV12ToRGB565Row_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- NV12ToRGB565Row = NV12ToRGB565Row_MSA;
- }
+#if defined(HAS_ARGBORROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBOrRow = ARGBOrRow_HVX;
}
#endif
-
for (y = 0; y < height; ++y) {
- NV12ToRGB565Row(src_y, src_uv, dst_rgb565, &kYuvI601Constants, width);
- dst_rgb565 += dst_stride_rgb565;
- src_y += src_stride_y;
- if (y & 1) {
- src_uv += src_stride_uv;
- }
+ ARGBOrRow(src_argb, dst_argb, value, width);
+ src_argb += src_stride_argb;
+ dst_argb += dst_stride_argb;
}
return 0;
}
@@ -1891,17 +4303,19 @@ int RAWToRGB24(const uint8_t* src_raw,
int y;
void (*RAWToRGB24Row)(const uint8_t* src_rgb, uint8_t* dst_rgb24, int width) =
RAWToRGB24Row_C;
- if (!src_raw || !dst_rgb24 || width <= 0 || height == 0) {
+ if (!src_raw || !dst_rgb24 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_raw = src_raw + (height - 1) * src_stride_raw;
+ src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw;
src_stride_raw = -src_stride_raw;
}
// Coalesce rows.
- if (src_stride_raw == width * 3 && dst_stride_rgb24 == width * 3) {
+ if (src_stride_raw == width * 3 && dst_stride_rgb24 == width * 3 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_raw = dst_stride_rgb24 = 0;
@@ -1914,27 +4328,63 @@ int RAWToRGB24(const uint8_t* src_raw,
}
}
#endif
+#if defined(HAS_RAWTORGB24ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToRGB24Row = RAWToRGB24Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToRGB24Row = RAWToRGB24Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RAWTORGB24ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ RAWToRGB24Row = RAWToRGB24Row_AVX512BW;
+ }
+#endif
+#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI;
+ }
+#endif
#if defined(HAS_RAWTORGB24ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RAWToRGB24Row = RAWToRGB24Row_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
+ if (IS_ALIGNED(width, 16)) {
RAWToRGB24Row = RAWToRGB24Row_NEON;
}
}
#endif
-#if defined(HAS_RAWTORGB24ROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- RAWToRGB24Row = RAWToRGB24Row_Any_MSA;
+#if defined(HAS_RAWTORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToRGB24Row = RAWToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_RAWTORGB24ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RAWToRGB24Row = RAWToRGB24Row_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- RAWToRGB24Row = RAWToRGB24Row_MSA;
+ RAWToRGB24Row = RAWToRGB24Row_LSX;
}
}
#endif
-#if defined(HAS_RAWTORGB24ROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- RAWToRGB24Row = RAWToRGB24Row_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- RAWToRGB24Row = RAWToRGB24Row_MMI;
+#if defined(HAS_RAWTORGB24ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RAWToRGB24Row = RAWToRGB24Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToRGB24Row = RAWToRGB24Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTORGB24ROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RAWToRGB24Row = RAWToRGB24Row_RVV;
+ }
+#endif
+#if defined(HAS_RAWTORGB24ROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ RAWToRGB24Row = RAWToRGB24Row_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToRGB24Row = RAWToRGB24Row_WASMSIMD;
}
}
#endif
@@ -1947,6 +4397,7 @@ int RAWToRGB24(const uint8_t* src_raw,
return 0;
}
+// TODO(fbarchard): Consider uint8_t value
LIBYUV_API
void SetPlane(uint8_t* dst_y,
int dst_stride_y,
@@ -1954,14 +4405,18 @@ void SetPlane(uint8_t* dst_y,
int height,
uint32_t value) {
int y;
- void (*SetRow)(uint8_t * dst, uint8_t value, int width) = SetRow_C;
+ void (*SetRow)(uint8_t* dst, uint8_t value, int width) = SetRow_C;
+
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
if (height < 0) {
height = -height;
- dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_y = dst_y + (ptrdiff_t)(height - 1) * dst_stride_y;
dst_stride_y = -dst_stride_y;
}
// Coalesce rows.
- if (dst_stride_y == width) {
+ if (dst_stride_y == width && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_y = 0;
@@ -1974,11 +4429,11 @@ void SetPlane(uint8_t* dst_y,
}
}
#endif
-#if defined(HAS_SETROW_X86)
- if (TestCpuFlag(kCpuHasX86)) {
- SetRow = SetRow_Any_X86;
- if (IS_ALIGNED(width, 4)) {
- SetRow = SetRow_X86;
+#if defined(HAS_SETROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SetRow = SetRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ SetRow = SetRow_AVX2;
}
}
#endif
@@ -1987,15 +4442,31 @@ void SetPlane(uint8_t* dst_y,
SetRow = SetRow_ERMS;
}
#endif
-#if defined(HAS_SETROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 16)) {
- SetRow = SetRow_MSA;
+#if defined(HAS_SETROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SetRow = SetRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ SetRow = SetRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SETROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SetRow = SetRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ SetRow = SetRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_SETROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SetRow = SetRow_HVX;
}
#endif
// Set plane
for (y = 0; y < height; ++y) {
- SetRow(dst_y, value, width);
+ SetRow(dst_y, (uint8_t)value, width);
dst_y += dst_stride_y;
}
}
@@ -2020,9 +4491,10 @@ int I420Rect(uint8_t* dst_y,
uint8_t* start_y = dst_y + y * dst_stride_y + x;
uint8_t* start_u = dst_u + (y / 2) * dst_stride_u + (x / 2);
uint8_t* start_v = dst_v + (y / 2) * dst_stride_v + (x / 2);
- if (!dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || x < 0 ||
- y < 0 || value_y < 0 || value_y > 255 || value_u < 0 || value_u > 255 ||
- value_v < 0 || value_v > 255) {
+
+ if (!dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN || x < 0 || y < 0 || value_y < 0 || value_y > 255 ||
+ value_u < 0 || value_u > 255 || value_v < 0 || value_v > 255) {
return -1;
}
@@ -2042,19 +4514,20 @@ int ARGBRect(uint8_t* dst_argb,
int height,
uint32_t value) {
int y;
- void (*ARGBSetRow)(uint8_t * dst_argb, uint32_t value, int width) =
+ void (*ARGBSetRow)(uint8_t* dst_argb, uint32_t value, int width) =
ARGBSetRow_C;
- if (!dst_argb || width <= 0 || height == 0 || dst_x < 0 || dst_y < 0) {
+ if (!dst_argb || width <= 0 || height == 0 || height == INT_MIN ||
+ dst_x < 0 || dst_y < 0) {
return -1;
}
if (height < 0) {
height = -height;
- dst_argb = dst_argb + (height - 1) * dst_stride_argb;
+ dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
dst_argb += dst_y * dst_stride_argb + dst_x * 4;
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
@@ -2068,19 +4541,40 @@ int ARGBRect(uint8_t* dst_argb,
}
}
#endif
-#if defined(HAS_ARGBSETROW_X86)
- if (TestCpuFlag(kCpuHasX86)) {
- ARGBSetRow = ARGBSetRow_X86;
+#if defined(HAS_ARGBSETROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBSetRow = ARGBSetRow_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBSetRow = ARGBSetRow_AVX2;
+ }
}
#endif
-#if defined(HAS_ARGBSETROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBSetRow = ARGBSetRow_Any_MSA;
+#if defined(HAS_ARGBSETROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBSetRow = ARGBSetRow_Any_LSX;
if (IS_ALIGNED(width, 4)) {
- ARGBSetRow = ARGBSetRow_MSA;
+ ARGBSetRow = ARGBSetRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSETROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBSetRow = ARGBSetRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBSetRow = ARGBSetRow_LASX;
}
}
#endif
+#if defined(HAS_ARGBSETROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBSetRow = ARGBSetRow_HVX;
+ }
+#endif
+#if defined(HAS_ARGBSETROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBSetRow = ARGBSetRow_RVV;
+ }
+#endif
// Set plane
for (y = 0; y < height; ++y) {
@@ -2113,16 +4607,18 @@ int ARGBAttenuate(const uint8_t* src_argb,
int y;
void (*ARGBAttenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
int width) = ARGBAttenuateRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
@@ -2143,6 +4639,11 @@ int ARGBAttenuate(const uint8_t* src_argb,
}
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_ARGBATTENUATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON;
@@ -2151,22 +4652,32 @@ int ARGBAttenuate(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_ARGBATTENUATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_MSA;
- if (IS_ALIGNED(width, 8)) {
- ARGBAttenuateRow = ARGBAttenuateRow_MSA;
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
}
}
#endif
-#if defined(HAS_ARGBATTENUATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBAttenuateRow = ARGBAttenuateRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBAttenuateRow = ARGBAttenuateRow_MMI;
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
}
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
ARGBAttenuateRow(src_argb, dst_argb, width);
@@ -2187,16 +4698,18 @@ int ARGBUnattenuate(const uint8_t* src_argb,
int y;
void (*ARGBUnattenuateRow)(const uint8_t* src_argb, uint8_t* dst_argb,
int width) = ARGBUnattenuateRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
@@ -2217,7 +4730,14 @@ int ARGBUnattenuate(const uint8_t* src_argb,
}
}
#endif
- // TODO(fbarchard): Neon version.
+#if defined(HAS_ARGBUNATTENUATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBUnattenuateRow = ARGBUnattenuateRow_Any_NEON;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBUnattenuateRow = ARGBUnattenuateRow_NEON;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
ARGBUnattenuateRow(src_argb, dst_argb, width);
@@ -2238,23 +4758,35 @@ int ARGBGrayTo(const uint8_t* src_argb,
int y;
void (*ARGBGrayRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
ARGBGrayRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
-#if defined(HAS_ARGBGRAYROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
- ARGBGrayRow = ARGBGrayRow_SSSE3;
+#if defined(HAS_ARGBGRAYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBGrayRow = ARGBGrayRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBGrayRow = ARGBGrayRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBGrayRow = ARGBGrayRow_AVX512VNNI;
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON)
@@ -2262,14 +4794,19 @@ int ARGBGrayTo(const uint8_t* src_argb,
ARGBGrayRow = ARGBGrayRow_NEON;
}
#endif
-#if defined(HAS_ARGBGRAYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
- ARGBGrayRow = ARGBGrayRow_MSA;
+#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_LSX;
}
#endif
-#if defined(HAS_ARGBGRAYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 2)) {
- ARGBGrayRow = ARGBGrayRow_MMI;
+#if defined(HAS_ARGBGRAYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
+ ARGBGrayRow = ARGBGrayRow_LASX;
}
#endif
@@ -2297,14 +4834,24 @@ int ARGBGray(uint8_t* dst_argb,
return -1;
}
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
}
-#if defined(HAS_ARGBGRAYROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
- ARGBGrayRow = ARGBGrayRow_SSSE3;
+#if defined(HAS_ARGBGRAYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBGrayRow = ARGBGrayRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBGrayRow = ARGBGrayRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBGrayRow = ARGBGrayRow_AVX512VNNI;
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON)
@@ -2312,14 +4859,19 @@ int ARGBGray(uint8_t* dst_argb,
ARGBGrayRow = ARGBGrayRow_NEON;
}
#endif
-#if defined(HAS_ARGBGRAYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
- ARGBGrayRow = ARGBGrayRow_MSA;
+#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
+ }
+#endif
+#if defined(HAS_ARGBGRAYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_LSX;
}
#endif
-#if defined(HAS_ARGBGRAYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 2)) {
- ARGBGrayRow = ARGBGrayRow_MMI;
+#if defined(HAS_ARGBGRAYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
+ ARGBGrayRow = ARGBGrayRow_LASX;
}
#endif
@@ -2339,20 +4891,25 @@ int ARGBSepia(uint8_t* dst_argb,
int width,
int height) {
int y;
- void (*ARGBSepiaRow)(uint8_t * dst_argb, int width) = ARGBSepiaRow_C;
+ void (*ARGBSepiaRow)(uint8_t* dst_argb, int width) = ARGBSepiaRow_C;
uint8_t* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4;
if (!dst_argb || width <= 0 || height <= 0 || dst_x < 0 || dst_y < 0) {
return -1;
}
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
}
-#if defined(HAS_ARGBSEPIAROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
- ARGBSepiaRow = ARGBSepiaRow_SSSE3;
+#if defined(HAS_ARGBSEPIAROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBSepiaRow = ARGBSepiaRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBSEPIAROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBSepiaRow = ARGBSepiaRow_AVX512VNNI;
}
#endif
#if defined(HAS_ARGBSEPIAROW_NEON)
@@ -2360,14 +4917,19 @@ int ARGBSepia(uint8_t* dst_argb,
ARGBSepiaRow = ARGBSepiaRow_NEON;
}
#endif
-#if defined(HAS_ARGBSEPIAROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
- ARGBSepiaRow = ARGBSepiaRow_MSA;
+#if defined(HAS_ARGBSEPIAROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd;
+ }
+#endif
+#if defined(HAS_ARGBSEPIAROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
+ ARGBSepiaRow = ARGBSepiaRow_LSX;
}
#endif
-#if defined(HAS_ARGBSEPIAROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 2)) {
- ARGBSepiaRow = ARGBSepiaRow_MMI;
+#if defined(HAS_ARGBSEPIAROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
+ ARGBSepiaRow = ARGBSepiaRow_LASX;
}
#endif
@@ -2392,23 +4954,30 @@ int ARGBColorMatrix(const uint8_t* src_argb,
void (*ARGBColorMatrixRow)(const uint8_t* src_argb, uint8_t* dst_argb,
const int8_t* matrix_argb, int width) =
ARGBColorMatrixRow_C;
- if (!src_argb || !dst_argb || !matrix_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || !matrix_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
-#if defined(HAS_ARGBCOLORMATRIXROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
- ARGBColorMatrixRow = ARGBColorMatrixRow_SSSE3;
+#if defined(HAS_ARGBCOLORMATRIXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBCOLORMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_AVX512VNNI;
}
#endif
#if defined(HAS_ARGBCOLORMATRIXROW_NEON)
@@ -2416,14 +4985,19 @@ int ARGBColorMatrix(const uint8_t* src_argb,
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON;
}
#endif
-#if defined(HAS_ARGBCOLORMATRIXROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
- ARGBColorMatrixRow = ARGBColorMatrixRow_MSA;
+#if defined(HAS_ARGBCOLORMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM) && IS_ALIGNED(width, 8)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM;
+ }
+#endif
+#if defined(HAS_ARGBCOLORMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_LSX;
}
#endif
-#if defined(HAS_ARGBCOLORMATRIXROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 2)) {
- ARGBColorMatrixRow = ARGBColorMatrixRow_MMI;
+#if defined(HAS_ARGBCOLORMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_LASX;
}
#endif
for (y = 0; y < height; ++y) {
@@ -2482,7 +5056,7 @@ int ARGBColorTable(uint8_t* dst_argb,
int width,
int height) {
int y;
- void (*ARGBColorTableRow)(uint8_t * dst_argb, const uint8_t* table_argb,
+ void (*ARGBColorTableRow)(uint8_t* dst_argb, const uint8_t* table_argb,
int width) = ARGBColorTableRow_C;
uint8_t* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4;
if (!dst_argb || !table_argb || width <= 0 || height <= 0 || dst_x < 0 ||
@@ -2490,14 +5064,19 @@ int ARGBColorTable(uint8_t* dst_argb,
return -1;
}
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
}
-#if defined(HAS_ARGBCOLORTABLEROW_X86)
- if (TestCpuFlag(kCpuHasX86)) {
- ARGBColorTableRow = ARGBColorTableRow_X86;
+#if defined(HAS_ARGBCOLORTABLEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBColorTableRow = ARGBColorTableRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBCOLORTABLEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBColorTableRow = ARGBColorTableRow_NEON;
}
#endif
for (y = 0; y < height; ++y) {
@@ -2518,7 +5097,7 @@ int RGBColorTable(uint8_t* dst_argb,
int width,
int height) {
int y;
- void (*RGBColorTableRow)(uint8_t * dst_argb, const uint8_t* table_argb,
+ void (*RGBColorTableRow)(uint8_t* dst_argb, const uint8_t* table_argb,
int width) = RGBColorTableRow_C;
uint8_t* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4;
if (!dst_argb || !table_argb || width <= 0 || height <= 0 || dst_x < 0 ||
@@ -2526,14 +5105,19 @@ int RGBColorTable(uint8_t* dst_argb,
return -1;
}
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
}
-#if defined(HAS_RGBCOLORTABLEROW_X86)
- if (TestCpuFlag(kCpuHasX86)) {
- RGBColorTableRow = RGBColorTableRow_X86;
+#if defined(HAS_RGBCOLORTABLEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RGBColorTableRow = RGBColorTableRow_AVX2;
+ }
+#endif
+#if defined(HAS_RGBCOLORTABLEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGBColorTableRow = RGBColorTableRow_NEON;
}
#endif
for (y = 0; y < height; ++y) {
@@ -2563,7 +5147,7 @@ int ARGBQuantize(uint8_t* dst_argb,
int width,
int height) {
int y;
- void (*ARGBQuantizeRow)(uint8_t * dst_argb, int scale, int interval_size,
+ void (*ARGBQuantizeRow)(uint8_t* dst_argb, int scale, int interval_size,
int interval_offset, int width) = ARGBQuantizeRow_C;
uint8_t* dst = dst_argb + dst_y * dst_stride_argb + dst_x * 4;
if (!dst_argb || width <= 0 || height <= 0 || dst_x < 0 || dst_y < 0 ||
@@ -2571,14 +5155,14 @@ int ARGBQuantize(uint8_t* dst_argb,
return -1;
}
// Coalesce rows.
- if (dst_stride_argb == width * 4) {
+ if (dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
dst_stride_argb = 0;
}
-#if defined(HAS_ARGBQUANTIZEROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) {
- ARGBQuantizeRow = ARGBQuantizeRow_SSE2;
+#if defined(HAS_ARGBQUANTIZEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBQuantizeRow = ARGBQuantizeRow_AVX2;
}
#endif
#if defined(HAS_ARGBQUANTIZEROW_NEON)
@@ -2586,9 +5170,14 @@ int ARGBQuantize(uint8_t* dst_argb,
ARGBQuantizeRow = ARGBQuantizeRow_NEON;
}
#endif
-#if defined(HAS_ARGBQUANTIZEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
- ARGBQuantizeRow = ARGBQuantizeRow_MSA;
+#if defined(HAS_ARGBQUANTIZEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
+ ARGBQuantizeRow = ARGBQuantizeRow_LSX;
+ }
+#endif
+#if defined(HAS_ARGBQUANTIZEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
+ ARGBQuantizeRow = ARGBQuantizeRow_LASX;
}
#endif
for (y = 0; y < height; ++y) {
@@ -2615,14 +5204,14 @@ int ARGBComputeCumulativeSum(const uint8_t* src_argb,
if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) {
return -1;
}
-#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2;
+#if defined(HAS_COMPUTECUMULATIVESUMROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2;
}
#endif
-#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ComputeCumulativeSumRow = ComputeCumulativeSumRow_MMI;
+#if defined(HAS_COMPUTECUMULATIVESUMROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ComputeCumulativeSumRow = ComputeCumulativeSumRow_NEON;
}
#endif
@@ -2661,12 +5250,13 @@ int ARGBBlur(const uint8_t* src_argb,
int32_t* max_cumsum_bot_row;
int32_t* cumsum_top_row;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
if (radius > height) {
@@ -2675,18 +5265,19 @@ int ARGBBlur(const uint8_t* src_argb,
if (radius > (width / 2 - 1)) {
radius = width / 2 - 1;
}
- if (radius <= 0) {
+ if (radius <= 0 || height <= 1) {
return -1;
}
-#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2;
- CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2;
+#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2;
+ CumulativeSumToAverageRow = CumulativeSumToAverageRow_AVX2;
}
#endif
-#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ComputeCumulativeSumRow = ComputeCumulativeSumRow_MMI;
+#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ComputeCumulativeSumRow = ComputeCumulativeSumRow_NEON;
+ CumulativeSumToAverageRow = CumulativeSumToAverageRow_NEON;
}
#endif
// Compute enough CumulativeSum for first row to be blurred. After this
@@ -2766,23 +5357,25 @@ int ARGBShade(const uint8_t* src_argb,
int y;
void (*ARGBShadeRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width,
uint32_t value) = ARGBShadeRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0 || value == 0u) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN || value == 0u) {
return -1;
}
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
-#if defined(HAS_ARGBSHADEROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) {
- ARGBShadeRow = ARGBShadeRow_SSE2;
+#if defined(HAS_ARGBSHADEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBShadeRow = ARGBShadeRow_AVX2;
}
#endif
#if defined(HAS_ARGBSHADEROW_NEON)
@@ -2790,14 +5383,14 @@ int ARGBShade(const uint8_t* src_argb,
ARGBShadeRow = ARGBShadeRow_NEON;
}
#endif
-#if defined(HAS_ARGBSHADEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 4)) {
- ARGBShadeRow = ARGBShadeRow_MSA;
+#if defined(HAS_ARGBSHADEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
+ ARGBShadeRow = ARGBShadeRow_LSX;
}
#endif
-#if defined(HAS_ARGBSHADEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 2)) {
- ARGBShadeRow = ARGBShadeRow_MMI;
+#if defined(HAS_ARGBSHADEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
+ ARGBShadeRow = ARGBShadeRow_LASX;
}
#endif
@@ -2821,32 +5414,26 @@ int InterpolatePlane(const uint8_t* src0,
int height,
int interpolation) {
int y;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- if (!src0 || !src1 || !dst || width <= 0 || height == 0) {
+ if (!src0 || !src1 || !dst || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- dst = dst + (height - 1) * dst_stride;
+ dst = dst + (ptrdiff_t)(height - 1) * dst_stride;
dst_stride = -dst_stride;
}
// Coalesce rows.
- if (src_stride0 == width && src_stride1 == width && dst_stride == width) {
+ if (src_stride0 == width && src_stride1 == width && dst_stride == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride0 = src_stride1 = dst_stride = 0;
}
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -2855,6 +5442,11 @@ int InterpolatePlane(const uint8_t* src0,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -2863,22 +5455,50 @@ int InterpolatePlane(const uint8_t* src0,
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+ InterpolateRow = InterpolateRow_LASX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- InterpolateRow = InterpolateRow_MMI;
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
for (y = 0; y < height; ++y) {
InterpolateRow(dst, src0, src1 - src0, width, interpolation);
@@ -2889,6 +5509,87 @@ int InterpolatePlane(const uint8_t* src0,
return 0;
}
+// Interpolate 2 planes by specified amount (0 to 255).
+LIBYUV_API
+int InterpolatePlane_16(const uint16_t* src0,
+ int src_stride0,
+ const uint16_t* src1,
+ int src_stride1,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height,
+ int interpolation) {
+ int y;
+ void (*InterpolateRow_16)(uint16_t* dst_ptr, const uint16_t* src_ptr,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_16_C;
+ if (!src0 || !src1 || !dst || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst = dst + (ptrdiff_t)(height - 1) * dst_stride;
+ dst_stride = -dst_stride;
+ }
+ // Coalesce rows.
+ if (src_stride0 == width && src_stride1 == width && dst_stride == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride0 = src_stride1 = dst_stride = 0;
+ }
+#if defined(HAS_INTERPOLATEROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow_16 = InterpolateRow_16_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ InterpolateRow_16 = InterpolateRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow_16 = InterpolateRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow_16 = InterpolateRow_16_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ InterpolateRow_16 = InterpolateRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow_16 = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ InterpolateRow_16 = InterpolateRow_16_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow_16 = InterpolateRow_16_HVX;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ InterpolateRow_16(dst, src0, src1 - src0, width, interpolation);
+ src0 += src_stride0;
+ src1 += src_stride1;
+ dst += dst_stride;
+ }
+ return 0;
+}
+
// Interpolate 2 ARGB images by specified amount (0 to 255).
LIBYUV_API
int ARGBInterpolate(const uint8_t* src_argb0,
@@ -2930,10 +5631,13 @@ int I420Interpolate(const uint8_t* src0_y,
int interpolation) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
+
if (!src0_y || !src0_u || !src0_v || !src1_y || !src1_u || !src1_v ||
- !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+
InterpolatePlane(src0_y, src0_stride_y, src1_y, src1_stride_y, dst_y,
dst_stride_y, width, height, interpolation);
InterpolatePlane(src0_u, src0_stride_u, src1_u, src1_stride_u, dst_u,
@@ -2945,30 +5649,32 @@ int I420Interpolate(const uint8_t* src0_y,
// Shuffle ARGB channel order. e.g. BGRA to ARGB.
LIBYUV_API
-int ARGBShuffle(const uint8_t* src_bgra,
- int src_stride_bgra,
+int ARGBShuffle(const uint8_t* src_argb,
+ int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
const uint8_t* shuffler,
int width,
int height) {
int y;
- void (*ARGBShuffleRow)(const uint8_t* src_bgra, uint8_t* dst_argb,
+ void (*ARGBShuffleRow)(const uint8_t* src_argb, uint8_t* dst_argb,
const uint8_t* shuffler, int width) = ARGBShuffleRow_C;
- if (!src_bgra || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_bgra = src_bgra + (height - 1) * src_stride_bgra;
- src_stride_bgra = -src_stride_bgra;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_bgra == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
- src_stride_bgra = dst_stride_argb = 0;
+ src_stride_argb = dst_stride_argb = 0;
}
#if defined(HAS_ARGBSHUFFLEROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
@@ -2986,6 +5692,11 @@ int ARGBShuffle(const uint8_t* src_bgra,
}
}
#endif
+#if defined(HAS_ARGBSHUFFLEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBShuffleRow = ARGBShuffleRow_AVX512BW;
+ }
+#endif
#if defined(HAS_ARGBSHUFFLEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBShuffleRow = ARGBShuffleRow_Any_NEON;
@@ -2994,27 +5705,208 @@ int ARGBShuffle(const uint8_t* src_bgra,
}
}
#endif
-#if defined(HAS_ARGBSHUFFLEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBShuffleRow = ARGBShuffleRow_Any_MSA;
+#if defined(HAS_ARGBSHUFFLEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBShuffleRow = ARGBShuffleRow_Any_LSX;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBShuffleRow = ARGBShuffleRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBShuffleRow = ARGBShuffleRow_Any_LASX;
if (IS_ALIGNED(width, 8)) {
- ARGBShuffleRow = ARGBShuffleRow_MSA;
+ ARGBShuffleRow = ARGBShuffleRow_LASX;
}
}
#endif
-#if defined(HAS_ARGBSHUFFLEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBShuffleRow = ARGBShuffleRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBShuffleRow = ARGBShuffleRow_MMI;
+#if defined(HAS_ARGBSHUFFLEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBShuffleRow = ARGBShuffleRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 4)) {
+ ARGBShuffleRow = ARGBShuffleRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBShuffleRow = ARGBShuffleRow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBShuffleRow(src_argb, dst_argb, shuffler, width);
+ src_argb += src_stride_argb;
+ dst_argb += dst_stride_argb;
+ }
+ return 0;
+}
+
+// Shuffle AR64 channel order. e.g. AR64 to AB64.
+LIBYUV_API
+int AR64Shuffle(const uint16_t* src_ar64,
+ int src_stride_ar64,
+ uint16_t* dst_ar64,
+ int dst_stride_ar64,
+ const uint8_t* shuffler,
+ int width,
+ int height) {
+ int y;
+ void (*AR64ShuffleRow)(const uint8_t* src_ar64, uint8_t* dst_ar64,
+ const uint8_t* shuffler, int width) = AR64ShuffleRow_C;
+ if (!src_ar64 || !dst_ar64 || width <= 0 || height == 0 ||
+ height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_ar64 = src_ar64 + (ptrdiff_t)(height - 1) * src_stride_ar64;
+ src_stride_ar64 = -src_stride_ar64;
+ }
+ // Coalesce rows.
+ if (src_stride_ar64 == width * 4 && dst_stride_ar64 == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
+ width *= height;
+ height = 1;
+ src_stride_ar64 = dst_stride_ar64 = 0;
+ }
+ // Assembly versions can be reused if it's implemented with shuffle.
+#if defined(HAS_ARGBSHUFFLEROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ AR64ShuffleRow = ARGBShuffleRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ AR64ShuffleRow = ARGBShuffleRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ AR64ShuffleRow = ARGBShuffleRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ AR64ShuffleRow = ARGBShuffleRow_AVX2;
}
}
#endif
+#if defined(HAS_ARGBSHUFFLEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ AR64ShuffleRow = ARGBShuffleRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ AR64ShuffleRow = ARGBShuffleRow_Any_NEON;
+ if (IS_ALIGNED(width, 4)) {
+ AR64ShuffleRow = ARGBShuffleRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ AR64ShuffleRow = ARGBShuffleRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 4)) {
+ AR64ShuffleRow = ARGBShuffleRow_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_ARGBSHUFFLEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ AR64ShuffleRow = ARGBShuffleRow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ AR64ShuffleRow((uint8_t*)(src_ar64), (uint8_t*)(dst_ar64), shuffler,
+ width * 2);
+ src_ar64 += src_stride_ar64;
+ dst_ar64 += dst_stride_ar64;
+ }
+ return 0;
+}
+
+// Gauss blur a float plane using Gaussian 5x5 filter with
+// coefficients of 1, 4, 6, 4, 1.
+// Each destination pixel is a blur of the 5x5
+// pixels from the source.
+// Source edges are clamped.
+// Edge is 2 pixels on each side, and interior is multiple of 4.
+LIBYUV_API
+int GaussPlane_F32(const float* src,
+ int src_stride,
+ float* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int y;
+ void (*GaussCol_F32)(const float* src0, const float* src1, const float* src2,
+ const float* src3, const float* src4, float* dst,
+ int width) = GaussCol_F32_C;
+ void (*GaussRow_F32)(const float* src, float* dst, int width) =
+ GaussRow_F32_C;
+ if (!src || !dst || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src = src + (ptrdiff_t)(height - 1) * src_stride;
+ src_stride = -src_stride;
+ }
+
+#if defined(HAS_GAUSSCOL_F32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ GaussCol_F32 = GaussCol_F32_AVX2;
+ }
+#endif
+#if defined(HAS_GAUSSCOL_F32_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
+ GaussCol_F32 = GaussCol_F32_NEON;
+ }
+#endif
+#if defined(HAS_GAUSSROW_F32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ GaussRow_F32 = GaussRow_F32_AVX2;
+ }
+#endif
+#if defined(HAS_GAUSSROW_F32_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
+ GaussRow_F32 = GaussRow_F32_NEON;
+ }
+#endif
+ {
+ // 2 pixels on each side, but aligned out to 16 bytes.
+ align_buffer_64(rowbuf, (4 + width + 4) * 4);
+ if (!rowbuf)
+ return 1;
+ memset(rowbuf, 0, 16);
+ memset(rowbuf + (4 + width) * 4, 0, 16);
+ float* row = (float*)(rowbuf + 16);
+ const float* src0 = src;
+ const float* src1 = src;
+ const float* src2 = src;
+ const float* src3 = src2 + ((height > 1) ? src_stride : 0);
+ const float* src4 = src3 + ((height > 2) ? src_stride : 0);
+
+ for (y = 0; y < height; ++y) {
+ GaussCol_F32(src0, src1, src2, src3, src4, row, width);
+
+ // Extrude edge by 2 floats
+ row[-2] = row[-1] = row[0];
+ row[width + 1] = row[width] = row[width - 1];
+
+ GaussRow_F32(row - 2, dst, width);
- for (y = 0; y < height; ++y) {
- ARGBShuffleRow(src_bgra, dst_argb, shuffler, width);
- src_bgra += src_stride_bgra;
- dst_argb += dst_stride_argb;
+ src0 = src1;
+ src1 = src2;
+ src2 = src3;
+ src3 = src4;
+ if ((y + 2) < (height - 1)) {
+ src4 += src_stride;
+ }
+ dst += dst_stride;
+ }
+ free_aligned_buffer_64(rowbuf);
}
return 0;
}
@@ -3031,121 +5923,150 @@ static int ARGBSobelize(const uint8_t* src_argb,
uint8_t* dst,
int width)) {
int y;
- void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_g, int width) =
- ARGBToYJRow_C;
+ void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_g, int width,
+ const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
void (*SobelYRow)(const uint8_t* src_y0, const uint8_t* src_y1,
uint8_t* dst_sobely, int width) = SobelYRow_C;
void (*SobelXRow)(const uint8_t* src_y0, const uint8_t* src_y1,
const uint8_t* src_y2, uint8_t* dst_sobely, int width) =
SobelXRow_C;
const int kEdge = 16; // Extra pixels at start of row for extrude/align.
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
-#if defined(HAS_ARGBTOYJROW_SSSE3)
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_SSSE3;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_AVX2)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- ARGBToYJRow = ARGBToYJRow_AVX2;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_NEON)
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBToYJRow = ARGBToYJRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_NEON;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBToYJRow = ARGBToYJRow_Any_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
- ARGBToYJRow = ARGBToYJRow_MSA;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
-#if defined(HAS_ARGBTOYJROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBToYJRow = ARGBToYJRow_Any_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
+ }
+#endif
+#if defined(HAS_ARGBTOYMATRIXROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- ARGBToYJRow = ARGBToYJRow_MMI;
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
-
-#if defined(HAS_SOBELYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- SobelYRow = SobelYRow_SSE2;
+#if defined(HAS_ARGBTOYMATRIXROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
+ }
}
#endif
-#if defined(HAS_SOBELYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- SobelYRow = SobelYRow_NEON;
+#if defined(HAS_ARGBTOYMATRIXROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
}
#endif
-#if defined(HAS_SOBELYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SobelYRow = SobelYRow_MSA;
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD;
+ }
}
#endif
-#if defined(HAS_SOBELYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SobelYRow = SobelYRow_MMI;
+#if defined(HAS_ARGBTOYMATRIXROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
}
#endif
-#if defined(HAS_SOBELXROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- SobelXRow = SobelXRow_SSE2;
+
+#if defined(HAS_SOBELYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SobelYRow = SobelYRow_AVX2;
}
#endif
-#if defined(HAS_SOBELXROW_NEON)
+#if defined(HAS_SOBELYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- SobelXRow = SobelXRow_NEON;
+ SobelYRow = SobelYRow_NEON;
}
#endif
-#if defined(HAS_SOBELXROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SobelXRow = SobelXRow_MSA;
+#if defined(HAS_SOBELXROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SobelXRow = SobelXRow_AVX2;
}
#endif
-#if defined(HAS_SOBELXROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SobelXRow = SobelXRow_MMI;
+#if defined(HAS_SOBELXROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ SobelXRow = SobelXRow_NEON;
}
#endif
{
// 3 rows with edges before/after.
- const int kRowSize = (width + kEdge + 31) & ~31;
- align_buffer_64(rows, kRowSize * 2 + (kEdge + kRowSize * 3 + kEdge));
+ const int row_size = (width + kEdge + 31) & ~31;
+ align_buffer_64(rows, row_size * 2 + (kEdge + row_size * 3 + kEdge));
uint8_t* row_sobelx = rows;
- uint8_t* row_sobely = rows + kRowSize;
- uint8_t* row_y = rows + kRowSize * 2;
+ uint8_t* row_sobely = rows + row_size;
+ uint8_t* row_y = rows + row_size * 2;
// Convert first row.
uint8_t* row_y0 = row_y + kEdge;
- uint8_t* row_y1 = row_y0 + kRowSize;
- uint8_t* row_y2 = row_y1 + kRowSize;
- ARGBToYJRow(src_argb, row_y0, width);
+ uint8_t* row_y1 = row_y0 + row_size;
+ uint8_t* row_y2 = row_y1 + row_size;
+ if (!rows)
+ return 1;
+ ARGBToYMatrixRow(src_argb, row_y0, width, &kArgbJPEGConstants);
row_y0[-1] = row_y0[0];
memset(row_y0 + width, row_y0[width - 1], 16); // Extrude 16 for valgrind.
- ARGBToYJRow(src_argb, row_y1, width);
+ ARGBToYMatrixRow(src_argb, row_y1, width, &kArgbJPEGConstants);
row_y1[-1] = row_y1[0];
memset(row_y1 + width, row_y1[width - 1], 16);
memset(row_y2 + width, 0, 16);
@@ -3155,7 +6076,7 @@ static int ARGBSobelize(const uint8_t* src_argb,
if (y < (height - 1)) {
src_argb += src_stride_argb;
}
- ARGBToYJRow(src_argb, row_y2, width);
+ ARGBToYMatrixRow(src_argb, row_y2, width, &kArgbJPEGConstants);
row_y2[-1] = row_y2[0];
row_y2[width] = row_y2[width - 1];
@@ -3188,11 +6109,11 @@ int ARGBSobel(const uint8_t* src_argb,
int height) {
void (*SobelRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
uint8_t* dst_argb, int width) = SobelRow_C;
-#if defined(HAS_SOBELROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- SobelRow = SobelRow_Any_SSE2;
+#if defined(HAS_SOBELROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SobelRow = SobelRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- SobelRow = SobelRow_SSE2;
+ SobelRow = SobelRow_AVX2;
}
}
#endif
@@ -3204,19 +6125,19 @@ int ARGBSobel(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_SOBELROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SobelRow = SobelRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- SobelRow = SobelRow_MSA;
+#if defined(HAS_SOBELROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SobelRow = SobelRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ SobelRow = SobelRow_LSX;
}
}
#endif
-#if defined(HAS_SOBELROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SobelRow = SobelRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SobelRow = SobelRow_MMI;
+#if defined(HAS_SOBELROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SobelRow = SobelRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ SobelRow = SobelRow_LASX;
}
}
#endif
@@ -3234,11 +6155,11 @@ int ARGBSobelToPlane(const uint8_t* src_argb,
int height) {
void (*SobelToPlaneRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
uint8_t* dst_, int width) = SobelToPlaneRow_C;
-#if defined(HAS_SOBELTOPLANEROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- SobelToPlaneRow = SobelToPlaneRow_Any_SSE2;
- if (IS_ALIGNED(width, 16)) {
- SobelToPlaneRow = SobelToPlaneRow_SSE2;
+#if defined(HAS_SOBELTOPLANEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SobelToPlaneRow = SobelToPlaneRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ SobelToPlaneRow = SobelToPlaneRow_AVX2;
}
}
#endif
@@ -3250,19 +6171,19 @@ int ARGBSobelToPlane(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_SOBELTOPLANEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SobelToPlaneRow = SobelToPlaneRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- SobelToPlaneRow = SobelToPlaneRow_MSA;
+#if defined(HAS_SOBELTOPLANEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SobelToPlaneRow = SobelToPlaneRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ SobelToPlaneRow = SobelToPlaneRow_LSX;
}
}
#endif
-#if defined(HAS_SOBELTOPLANEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SobelToPlaneRow = SobelToPlaneRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SobelToPlaneRow = SobelToPlaneRow_MMI;
+#if defined(HAS_SOBELTOPLANEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SobelToPlaneRow = SobelToPlaneRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ SobelToPlaneRow = SobelToPlaneRow_LASX;
}
}
#endif
@@ -3281,11 +6202,11 @@ int ARGBSobelXY(const uint8_t* src_argb,
int height) {
void (*SobelXYRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
uint8_t* dst_argb, int width) = SobelXYRow_C;
-#if defined(HAS_SOBELXYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- SobelXYRow = SobelXYRow_Any_SSE2;
+#if defined(HAS_SOBELXYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SobelXYRow = SobelXYRow_Any_AVX2;
if (IS_ALIGNED(width, 16)) {
- SobelXYRow = SobelXYRow_SSE2;
+ SobelXYRow = SobelXYRow_AVX2;
}
}
#endif
@@ -3297,19 +6218,19 @@ int ARGBSobelXY(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_SOBELXYROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SobelXYRow = SobelXYRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- SobelXYRow = SobelXYRow_MSA;
+#if defined(HAS_SOBELXYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SobelXYRow = SobelXYRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ SobelXYRow = SobelXYRow_LSX;
}
}
#endif
-#if defined(HAS_SOBELXYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SobelXYRow = SobelXYRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SobelXYRow = SobelXYRow_MMI;
+#if defined(HAS_SOBELXYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SobelXYRow = SobelXYRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ SobelXYRow = SobelXYRow_LASX;
}
}
#endif
@@ -3329,30 +6250,45 @@ int ARGBPolynomial(const uint8_t* src_argb,
int y;
void (*ARGBPolynomialRow)(const uint8_t* src_argb, uint8_t* dst_argb,
const float* poly, int width) = ARGBPolynomialRow_C;
- if (!src_argb || !dst_argb || !poly || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || !poly || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
#if defined(HAS_ARGBPOLYNOMIALROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 2)) {
- ARGBPolynomialRow = ARGBPolynomialRow_SSE2;
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_Any_SSE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_SSE2;
+ }
}
#endif
#if defined(HAS_ARGBPOLYNOMIALROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3) &&
- IS_ALIGNED(width, 2)) {
- ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
+ if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_Any_AVX2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBPOLYNOMIALROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_Any_NEON;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBPolynomialRow = ARGBPolynomialRow_NEON;
+ }
}
#endif
@@ -3377,7 +6313,7 @@ int HalfFloatPlane(const uint16_t* src_y,
int y;
void (*HalfFloatRow)(const uint16_t* src, uint16_t* dst, float scale,
int width) = HalfFloatRow_C;
- if (!src_y || !dst_y || width <= 0 || height == 0) {
+ if (!src_y || !dst_y || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
src_stride_y >>= 1;
@@ -3385,23 +6321,16 @@ int HalfFloatPlane(const uint16_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
// Coalesce rows.
- if (src_stride_y == width && dst_stride_y == width) {
+ if (src_stride_y == width && dst_stride_y == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_y = 0;
}
-#if defined(HAS_HALFFLOATROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- HalfFloatRow = HalfFloatRow_Any_SSE2;
- if (IS_ALIGNED(width, 8)) {
- HalfFloatRow = HalfFloatRow_SSE2;
- }
- }
-#endif
#if defined(HAS_HALFFLOATROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
HalfFloatRow = HalfFloatRow_Any_AVX2;
@@ -3419,20 +6348,45 @@ int HalfFloatPlane(const uint16_t* src_y,
}
}
#endif
+#if defined(HAS_HALFFLOATROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HalfFloatRow = HalfFloatRow_AVX512BW;
+ }
+#endif
#if defined(HAS_HALFFLOATROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- HalfFloatRow =
- (scale == 1.0f) ? HalfFloat1Row_Any_NEON : HalfFloatRow_Any_NEON;
+ if (TestCpuFlag(kCpuHasNEON)
+#if defined(__arm__)
+ // When scale is 1/65535 the scale * 2^-112 used to convert is a denormal.
+ // But when Neon vmul is asked to multiply a normal float by that
+ // denormal scale, even though the result would have been normal, it
+ // flushes to zero. The scalar version of vmul supports denormals.
+ && scale >= 1.0f / 4096.0f
+#endif
+ ) {
+ HalfFloatRow = HalfFloatRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ HalfFloatRow = HalfFloatRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_HALFFLOATROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SVE2 : HalfFloatRow_SVE2;
+ }
+#endif
+#if defined(HAS_HALFFLOATROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ HalfFloatRow = HalfFloatRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
- HalfFloatRow = (scale == 1.0f) ? HalfFloat1Row_NEON : HalfFloatRow_NEON;
+ HalfFloatRow = HalfFloatRow_LSX;
}
}
#endif
-#if defined(HAS_HALFFLOATROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- HalfFloatRow = HalfFloatRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- HalfFloatRow = HalfFloatRow_MSA;
+#if defined(HAS_HALFFLOATROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ HalfFloatRow = HalfFloatRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ HalfFloatRow = HalfFloatRow_LASX;
}
}
#endif
@@ -3453,6 +6407,14 @@ int ByteToFloat(const uint8_t* src_y, float* dst_y, float scale, int width) {
if (!src_y || !dst_y || width <= 0) {
return -1;
}
+#if defined(HAS_BYTETOFLOATROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ByteToFloatRow = ByteToFloatRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ ByteToFloatRow = ByteToFloatRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_BYTETOFLOATROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ByteToFloatRow = ByteToFloatRow_Any_NEON;
@@ -3479,24 +6441,31 @@ int ARGBLumaColorTable(const uint8_t* src_argb,
void (*ARGBLumaColorTableRow)(
const uint8_t* src_argb, uint8_t* dst_argb, int width,
const uint8_t* luma, const uint32_t lumacoeff) = ARGBLumaColorTableRow_C;
- if (!src_argb || !dst_argb || !luma || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || !luma || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
-#if defined(HAS_ARGBLUMACOLORTABLEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 4)) {
- ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3;
+#if defined(HAS_ARGBLUMACOLORTABLEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBLumaColorTableRow = ARGBLumaColorTableRow_AVX2;
+ }
+#endif
+#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) {
+ ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON;
}
#endif
@@ -3519,21 +6488,31 @@ int ARGBCopyAlpha(const uint8_t* src_argb,
int y;
void (*ARGBCopyAlphaRow)(const uint8_t* src_argb, uint8_t* dst_argb,
int width) = ARGBCopyAlphaRow_C;
- if (!src_argb || !dst_argb || width <= 0 || height == 0) {
+ if (!src_argb || !dst_argb || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_argb == width * 4) {
+ if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_argb = 0;
}
+#if defined(HAS_ARGBCOPYALPHAROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBCopyAlphaRow = ARGBCopyAlphaRow_NEON;
+ }
+ }
+#endif
#if defined(HAS_ARGBCOPYALPHAROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_SSE2;
@@ -3550,12 +6529,9 @@ int ARGBCopyAlpha(const uint8_t* src_argb,
}
}
#endif
-#if defined(HAS_ARGBCOPYALPHAROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBCopyAlphaRow = ARGBCopyAlphaRow_MMI;
- }
+#if defined(HAS_ARGBCOPYALPHAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBCopyAlphaRow = ARGBCopyAlphaRow_HVX;
}
#endif
@@ -3575,17 +6551,18 @@ int ARGBExtractAlpha(const uint8_t* src_argb,
int dst_stride_a,
int width,
int height) {
- if (!src_argb || !dst_a || width <= 0 || height == 0) {
+ if (!src_argb || !dst_a || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb += (height - 1) * src_stride_argb;
+ src_argb += (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
// Coalesce rows.
- if (src_stride_argb == width * 4 && dst_stride_a == width) {
+ if (src_stride_argb == width * 4 && dst_stride_a == width &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_argb = dst_stride_a = 0;
@@ -3604,22 +6581,44 @@ int ARGBExtractAlpha(const uint8_t* src_argb,
: ARGBExtractAlphaRow_Any_AVX2;
}
#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW;
+ }
+#endif
#if defined(HAS_ARGBEXTRACTALPHAROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON
: ARGBExtractAlphaRow_Any_NEON;
}
#endif
-#if defined(HAS_ARGBEXTRACTALPHAROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_MSA
- : ARGBExtractAlphaRow_Any_MSA;
+#if defined(HAS_ARGBEXTRACTALPHAROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_LSX
+ : ARGBExtractAlphaRow_Any_LSX;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LASX
+ : ARGBExtractAlphaRow_Any_LASX;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBExtractAlphaRow = IS_ALIGNED(width, 16)
+ ? ARGBExtractAlphaRow_WASMSIMD
+ : ARGBExtractAlphaRow_Any_WASMSIMD;
+ }
+#endif
+#if defined(HAS_ARGBEXTRACTALPHAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV;
}
#endif
-#if defined(HAS_ARGBEXTRACTALPHAROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_MMI
- : ARGBExtractAlphaRow_Any_MMI;
+#if defined(HAS_ARGBEXTRACTALPHAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBExtractAlphaRow = ARGBExtractAlphaRow_HVX;
}
#endif
@@ -3642,17 +6641,18 @@ int ARGBCopyYToAlpha(const uint8_t* src_y,
int y;
void (*ARGBCopyYToAlphaRow)(const uint8_t* src_y, uint8_t* dst_argb,
int width) = ARGBCopyYToAlphaRow_C;
- if (!src_y || !dst_argb || width <= 0 || height == 0) {
+ if (!src_y || !dst_argb || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
// Coalesce rows.
- if (src_stride_y == width && dst_stride_argb == width * 4) {
+ if (src_stride_y == width && dst_stride_argb == width * 4 &&
+ (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = dst_stride_argb = 0;
@@ -3673,12 +6673,32 @@ int ARGBCopyYToAlpha(const uint8_t* src_y,
}
}
#endif
-#if defined(HAS_ARGBCOPYYTOALPHAROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_MMI;
- }
+#if defined(HAS_ARGBCOPYYTOALPHAROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_ARGBCOPYYTOALPHAROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16) ? ARGBCopyYToAlphaRow_NEON
+ : ARGBCopyYToAlphaRow_Any_NEON;
+ }
+#endif
+#if defined(HAS_ARGBCOPYYTOALPHAROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16)
+ ? ARGBCopyYToAlphaRow_WASMSIMD
+ : ARGBCopyYToAlphaRow_Any_WASMSIMD;
+ }
+#endif
+#if defined(HAS_ARGBCOPYYTOALPHAROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_RVV;
+ }
+#endif
+#if defined(HAS_ARGBCOPYYTOALPHAROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_HVX;
}
#endif
@@ -3690,9 +6710,6 @@ int ARGBCopyYToAlpha(const uint8_t* src_y,
return 0;
}
-// TODO(fbarchard): Consider if width is even Y channel can be split
-// directly. A SplitUVRow_Odd function could copy the remaining chroma.
-
LIBYUV_API
int YUY2ToNV12(const uint8_t* src_yuy2,
int src_stride_yuy2,
@@ -3703,124 +6720,113 @@ int YUY2ToNV12(const uint8_t* src_yuy2,
int width,
int height) {
int y;
- int halfwidth = (width + 1) >> 1;
- void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
- int width) = SplitUVRow_C;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
- ptrdiff_t src_stride, int dst_width,
- int source_y_fraction) = InterpolateRow_C;
- if (!src_yuy2 || !dst_y || !dst_uv || width <= 0 || height == 0) {
+ void (*YUY2ToYRow)(const uint8_t* src_yuy2, uint8_t* dst_y, int width) =
+ YUY2ToYRow_C;
+ void (*YUY2ToNVUVRow)(const uint8_t* src_yuy2, int stride_yuy2,
+ uint8_t* dst_uv, int width) = YUY2ToNVUVRow_C;
+ if (!src_yuy2 || !dst_y || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_yuy2 = src_yuy2 + (height - 1) * src_stride_yuy2;
+ src_yuy2 = src_yuy2 + (ptrdiff_t)(height - 1) * src_stride_yuy2;
src_stride_yuy2 = -src_stride_yuy2;
}
-#if defined(HAS_SPLITUVROW_SSE2)
+#if defined(HAS_YUY2TOYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
- SplitUVRow = SplitUVRow_Any_SSE2;
+ YUY2ToYRow = YUY2ToYRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
- SplitUVRow = SplitUVRow_SSE2;
+ YUY2ToYRow = YUY2ToYRow_SSE2;
}
}
#endif
-#if defined(HAS_SPLITUVROW_AVX2)
+#if defined(HAS_YUY2TOYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- SplitUVRow = SplitUVRow_Any_AVX2;
+ YUY2ToYRow = YUY2ToYRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- SplitUVRow = SplitUVRow_AVX2;
+ YUY2ToYRow = YUY2ToYRow_AVX2;
}
}
#endif
-#if defined(HAS_SPLITUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- SplitUVRow = SplitUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
- SplitUVRow = SplitUVRow_NEON;
- }
+#if defined(HAS_YUY2TOYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ YUY2ToYRow = YUY2ToYRow_AVX512BW;
}
#endif
-#if defined(HAS_SPLITUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SplitUVRow = SplitUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- SplitUVRow = SplitUVRow_MSA;
+#if defined(HAS_YUY2TOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToYRow = YUY2ToYRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToYRow = YUY2ToYRow_NEON;
}
}
#endif
-#if defined(HAS_SPLITUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SplitUVRow = SplitUVRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SplitUVRow = SplitUVRow_MMI;
- }
+#if defined(HAS_YUY2TOYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToYRow = YUY2ToYRow_SVE2;
}
#endif
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
+#if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
+ YUY2ToYRow = YUY2ToYRow_LSX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_AVX2;
+#if defined(HAS_YUY2TOYROW_LASX) && defined(HAS_YUY2TOUV422ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ YUY2ToYRow = YUY2ToYRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_AVX2;
+ YUY2ToYRow = YUY2ToYRow_LASX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_NEON;
+#if defined(HAS_YUY2TOYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ YUY2ToYRow = YUY2ToYRow_HVX;
+ }
+#endif
+
+#if defined(HAS_YUY2TONVUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ YUY2ToNVUVRow = YUY2ToNVUVRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_NEON;
+ YUY2ToNVUVRow = YUY2ToNVUVRow_SSE2;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
+#if defined(HAS_YUY2TONVUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ YUY2ToNVUVRow = YUY2ToNVUVRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+ YUY2ToNVUVRow = YUY2ToNVUVRow_AVX2;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- InterpolateRow = InterpolateRow_MMI;
+#if defined(HAS_YUY2TONVUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ YUY2ToNVUVRow = YUY2ToNVUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ YUY2ToNVUVRow = YUY2ToNVUVRow_NEON;
}
}
#endif
- {
- int awidth = halfwidth * 2;
- // row of y and 2 rows of uv
- align_buffer_64(rows, awidth * 3);
-
- for (y = 0; y < height - 1; y += 2) {
- // Split Y from UV.
- SplitUVRow(src_yuy2, rows, rows + awidth, awidth);
- memcpy(dst_y, rows, width);
- SplitUVRow(src_yuy2 + src_stride_yuy2, rows, rows + awidth * 2, awidth);
- memcpy(dst_y + dst_stride_y, rows, width);
- InterpolateRow(dst_uv, rows + awidth, awidth, awidth, 128);
- src_yuy2 += src_stride_yuy2 * 2;
- dst_y += dst_stride_y * 2;
- dst_uv += dst_stride_uv;
- }
- if (height & 1) {
- // Split Y from UV.
- SplitUVRow(src_yuy2, rows, dst_uv, awidth);
- memcpy(dst_y, rows, width);
- }
- free_aligned_buffer_64(rows);
+ for (y = 0; y < height - 1; y += 2) {
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ YUY2ToYRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, width);
+ YUY2ToNVUVRow(src_yuy2, src_stride_yuy2, dst_uv, width);
+ src_yuy2 += src_stride_yuy2 * 2;
+ dst_y += dst_stride_y * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ YUY2ToYRow(src_yuy2, dst_y, width);
+ YUY2ToNVUVRow(src_yuy2, 0, dst_uv, width);
}
return 0;
}
@@ -3836,24 +6842,30 @@ int UYVYToNV12(const uint8_t* src_uyvy,
int height) {
int y;
int halfwidth = (width + 1) >> 1;
+ // UYVY rows are processed as awidth (even) samples. Select kernels on the
+ // width actually passed to them.
+ int awidth = halfwidth * 2;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- if (!src_uyvy || !dst_y || !dst_uv || width <= 0 || height == 0) {
+
+ if (!src_uyvy || !dst_y || !dst_uv || width <= 0 || height == 0 ||
+ height == INT_MIN) {
return -1;
}
+
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_uyvy = src_uyvy + (height - 1) * src_stride_uyvy;
+ src_uyvy = src_uyvy + (ptrdiff_t)(height - 1) * src_stride_uyvy;
src_stride_uyvy = -src_stride_uyvy;
}
#if defined(HAS_SPLITUVROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
SplitUVRow = SplitUVRow_Any_SSE2;
- if (IS_ALIGNED(width, 16)) {
+ if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_SSE2;
}
}
@@ -3861,80 +6873,127 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
SplitUVRow = SplitUVRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
+ if (IS_ALIGNED(awidth, 32)) {
SplitUVRow = SplitUVRow_AVX2;
}
}
#endif
+#if defined(HAS_SPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ SplitUVRow = SplitUVRow_AVX512BW;
+ }
+#endif
#if defined(HAS_SPLITUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SplitUVRow = SplitUVRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
+ if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_NEON;
}
}
#endif
-#if defined(HAS_SPLITUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- SplitUVRow = SplitUVRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- SplitUVRow = SplitUVRow_MSA;
- }
+#if defined(HAS_SPLITUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ SplitUVRow = SplitUVRow_SVE2;
}
#endif
-#if defined(HAS_SPLITUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- SplitUVRow = SplitUVRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- SplitUVRow = SplitUVRow_MMI;
+#if defined(HAS_SPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ SplitUVRow = SplitUVRow_Any_LSX;
+ if (IS_ALIGNED(awidth, 16)) {
+ SplitUVRow = SplitUVRow_LSX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
+#if defined(HAS_SPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ SplitUVRow = SplitUVRow_Any_LASX;
+ if (IS_ALIGNED(awidth, 32)) {
+ SplitUVRow = SplitUVRow_LASX;
}
}
#endif
+#if defined(HAS_SPLITUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SplitUVRow = SplitUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SPLITUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SplitUVRow = SplitUVRow_HVX;
+ }
+#endif
+
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
- if (IS_ALIGNED(width, 32)) {
+ if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
+ if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
- if (IS_ALIGNED(width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(awidth, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- InterpolateRow = InterpolateRow_MMI;
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(awidth, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(awidth, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
{
- int awidth = halfwidth * 2;
// row of y and 2 rows of uv
align_buffer_64(rows, awidth * 3);
+ if (!rows)
+ return 1;
for (y = 0; y < height - 1; y += 2) {
// Split Y from UV.
@@ -3957,6 +7016,79 @@ int UYVYToNV12(const uint8_t* src_uyvy,
return 0;
}
+// width and height are src size allowing odd size handling.
+LIBYUV_API
+void HalfMergeUVPlane(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int y;
+ void (*HalfMergeUVRow)(const uint8_t* src_u, int src_stride_u,
+ const uint8_t* src_v, int src_stride_v,
+ uint8_t* dst_uv, int width) = HalfMergeUVRow_C;
+
+ if (width <= 0 || height == 0 || height == INT_MIN) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+#if defined(HAS_HALFMERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 16)) {
+ HalfMergeUVRow = HalfMergeUVRow_NEON;
+ }
+#endif
+#if defined(HAS_HALFMERGEUVROW_SVE2)
+ // The average of two values must be retained for the final odd element if
+ // present. Existing code just handles this by falling back to the C
+ // implementation, so mirror that for SVE2 as well.
+ if (TestCpuFlag(kCpuHasSVE2) && IS_ALIGNED(width, 2)) {
+ HalfMergeUVRow = HalfMergeUVRow_SVE2;
+ }
+#endif
+#if defined(HAS_HALFMERGEUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ HalfMergeUVRow = HalfMergeUVRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ HalfMergeUVRow = HalfMergeUVRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_HALFMERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ HalfMergeUVRow = HalfMergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ HalfMergeUVRow = HalfMergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_HALFMERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ HalfMergeUVRow = HalfMergeUVRow_AVX512BW;
+ }
+#endif
+
+ for (y = 0; y < height - 1; y += 2) {
+ // Merge a row of U and V into a row of UV.
+ HalfMergeUVRow(src_u, src_stride_u, src_v, src_stride_v, dst_uv, width);
+ src_u += src_stride_u * 2;
+ src_v += src_stride_v * 2;
+ dst_uv += dst_stride_uv;
+ }
+ if (height & 1) {
+ HalfMergeUVRow(src_u, 0, src_v, 0, dst_uv, width);
+ }
+}
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/rotate.cc b/libs/libyuv/source/rotate.cc
index d414186a5a..db72843a25 100644
--- a/libs/libyuv/source/rotate.cc
+++ b/libs/libyuv/source/rotate.cc
@@ -10,6 +10,9 @@
#include "libyuv/rotate.h"
+#include <assert.h>
+#include <limits.h>
+
#include "libyuv/convert.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
@@ -21,6 +24,21 @@ namespace libyuv {
extern "C" {
#endif
+static void TransposeWxH(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ int j;
+ for (j = 0; j < height; ++j) {
+ dst[i * dst_stride + j] = src[j * src_stride + i];
+ }
+ }
+}
+
LIBYUV_API
void TransposePlane(const uint8_t* src,
int src_stride,
@@ -29,57 +47,123 @@ void TransposePlane(const uint8_t* src,
int width,
int height) {
int i = height;
-#if defined(HAS_TRANSPOSEWX16_MSA)
+#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
+ void (*TransposeWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width, int height) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \
+ defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_WASMSIMD)
+ void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width) = NULL;
+#elif defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx16_C;
-#else
+#endif
+#if !defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx8_C;
#endif
+
#if defined(HAS_TRANSPOSEWX8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- TransposeWx8 = TransposeWx8_NEON;
+ TransposeWx8 = TransposeWx8_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx8 = TransposeWx8_NEON;
+ }
}
#endif
-#if defined(HAS_TRANSPOSEWX8_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- TransposeWx8 = TransposeWx8_Any_SSSE3;
- if (IS_ALIGNED(width, 8)) {
- TransposeWx8 = TransposeWx8_SSSE3;
+#if defined(HAS_TRANSPOSEWX16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx16 = TransposeWx16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx16 = TransposeWx16_NEON;
}
}
#endif
-#if defined(HAS_TRANSPOSEWX8_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- TransposeWx8 = TransposeWx8_MMI;
+#if defined(HAS_TRANSPOSEWXH_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeWxH_fn = TransposeWxH_SME;
}
#endif
-#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- TransposeWx8 = TransposeWx8_Fast_Any_SSSE3;
+#if defined(HAS_TRANSPOSEWXH_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ TransposeWxH_fn = TransposeWxH_HVX;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx8 = TransposeWx8_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx8 = TransposeWx8_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx16 = TransposeWx16_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx16 = TransposeWx16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ TransposeWx16 = TransposeWx16_AVX512BW;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ TransposeWx8 = TransposeWx8_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx8 = TransposeWx8_LSX;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ TransposeWx16 = TransposeWx16_Any_LASX;
if (IS_ALIGNED(width, 16)) {
- TransposeWx8 = TransposeWx8_Fast_SSSE3;
+ TransposeWx16 = TransposeWx16_LASX;
}
}
#endif
-#if defined(HAS_TRANSPOSEWX16_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- TransposeWx16 = TransposeWx16_Any_MSA;
+#if defined(HAS_TRANSPOSEWX8_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeWx8 = TransposeWx8_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
- TransposeWx16 = TransposeWx16_MSA;
+ TransposeWx8 = TransposeWx8_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeWx16 = TransposeWx16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx16 = TransposeWx16_WASMSIMD;
}
}
#endif
-#if defined(HAS_TRANSPOSEWX16_MSA)
+#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
+ if (TransposeWxH_fn) {
+ TransposeWxH_fn(src, src_stride, dst, dst_stride, width, height);
+ return;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \
+ defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_NEON) || \
+ defined(HAS_TRANSPOSEWX16_WASMSIMD)
// Work across the source in 16x16 tiles
- while (i >= 16) {
- TransposeWx16(src, src_stride, dst, dst_stride, width);
- src += 16 * src_stride; // Go down 16 rows.
- dst += 16; // Move over 16 columns.
- i -= 16;
+ if (TransposeWx16) {
+ while (i >= 16) {
+ TransposeWx16(src, src_stride, dst, dst_stride, width);
+ src += 16 * src_stride; // Go down 16 rows.
+ dst += 16; // Move over 16 columns.
+ i -= 16;
+ }
}
-#else
+#endif
+#if !defined(HAS_TRANSPOSEWX16_NEON)
// Work across the source in 8x8 tiles
while (i >= 8) {
TransposeWx8(src, src_stride, dst, dst_stride, width);
@@ -90,7 +174,7 @@ void TransposePlane(const uint8_t* src,
#endif
if (i > 0) {
- TransposeWxH_C(src, src_stride, dst, dst_stride, width, i);
+ TransposeWxH(src, src_stride, dst, dst_stride, width, i);
}
}
@@ -104,7 +188,7 @@ void RotatePlane90(const uint8_t* src,
// Rotate by 90 is a transpose with the source read
// from bottom to top. So set the source pointer to the end
// of the buffer and flip the sign of the source stride.
- src += src_stride * (height - 1);
+ src += (ptrdiff_t)src_stride * (height - 1);
src_stride = -src_stride;
TransposePlane(src, src_stride, dst, dst_stride, width, height);
}
@@ -119,7 +203,7 @@ void RotatePlane270(const uint8_t* src,
// Rotate by 270 is a transpose with the destination written
// from bottom to top. So set the destination pointer to the end
// of the buffer and flip the sign of the destination stride.
- dst += dst_stride * (width - 1);
+ dst += (ptrdiff_t)dst_stride * (width - 1);
dst_stride = -dst_stride;
TransposePlane(src, src_stride, dst, dst_stride, width, height);
}
@@ -131,10 +215,13 @@ void RotatePlane180(const uint8_t* src,
int dst_stride,
int width,
int height) {
- // Swap first and last row and mirror the content. Uses a temporary row.
+ // Swap top and bottom row and mirror the content. Uses a temporary row.
align_buffer_64(row, width);
- const uint8_t* src_bot = src + src_stride * (height - 1);
- uint8_t* dst_bot = dst + dst_stride * (height - 1);
+ assert(row);
+ if (!row)
+ return;
+ const uint8_t* src_bot = src + (ptrdiff_t)src_stride * (height - 1);
+ uint8_t* dst_bot = dst + (ptrdiff_t)dst_stride * (height - 1);
int half_height = (height + 1) >> 1;
int y;
void (*MirrorRow)(const uint8_t* src, uint8_t* dst, int width) = MirrorRow_C;
@@ -142,7 +229,7 @@ void RotatePlane180(const uint8_t* src,
#if defined(HAS_MIRRORROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
MirrorRow = MirrorRow_Any_NEON;
- if (IS_ALIGNED(width, 16)) {
+ if (IS_ALIGNED(width, 32)) {
MirrorRow = MirrorRow_NEON;
}
}
@@ -163,22 +250,40 @@ void RotatePlane180(const uint8_t* src,
}
}
#endif
-#if defined(HAS_MIRRORROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- MirrorRow = MirrorRow_Any_MSA;
- if (IS_ALIGNED(width, 64)) {
- MirrorRow = MirrorRow_MSA;
+#if defined(HAS_MIRRORROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MirrorRow = MirrorRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MIRRORROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MirrorRow = MirrorRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorRow = MirrorRow_LSX;
}
}
#endif
-#if defined(HAS_MIRRORROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- MirrorRow = MirrorRow_Any_MMI;
- if (IS_ALIGNED(width, 8)) {
- MirrorRow = MirrorRow_MMI;
+#if defined(HAS_MIRRORROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ MirrorRow = MirrorRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ MirrorRow = MirrorRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_MIRRORROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ MirrorRow = MirrorRow_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorRow = MirrorRow_WASMSIMD;
}
}
#endif
+#if defined(HAS_MIRRORROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MirrorRow = MirrorRow_HVX;
+ }
+#endif
#if defined(HAS_COPYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
@@ -189,6 +294,11 @@ void RotatePlane180(const uint8_t* src,
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
+#if defined(HAS_COPYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ CopyRow = CopyRow_AVX512BW;
+ }
+#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
@@ -199,85 +309,188 @@ void RotatePlane180(const uint8_t* src,
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
-#if defined(HAS_COPYROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- CopyRow = IS_ALIGNED(width, 8) ? CopyRow_MMI : CopyRow_Any_MMI;
+#if defined(HAS_COPYROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ CopyRow = CopyRow_SVE2;
+ }
+#endif
+#if defined(HAS_COPYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ CopyRow = CopyRow_SME;
+ }
+#endif
+#if defined(HAS_COPYROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ CopyRow = CopyRow_RVV;
+ }
+#endif
+#if defined(HAS_COPYROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ CopyRow = CopyRow_HVX;
+ }
+#endif
+#if defined(HAS_COPYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ CopyRow = IS_ALIGNED(width, 16) ? CopyRow_LSX : CopyRow_Any_LSX;
+ }
+#endif
+#if defined(HAS_COPYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LASX : CopyRow_Any_LASX;
}
#endif
// Odd height will harmlessly mirror the middle row twice.
for (y = 0; y < half_height; ++y) {
- MirrorRow(src, row, width); // Mirror first row into a buffer
+ CopyRow(src, row, width); // Copy top row into buffer
+ MirrorRow(src_bot, dst, width); // Mirror bottom row into top row
+ MirrorRow(row, dst_bot, width); // Mirror buffer into bottom row
src += src_stride;
- MirrorRow(src_bot, dst, width); // Mirror last row into first row
dst += dst_stride;
- CopyRow(row, dst_bot, width); // Copy first mirrored row into last
src_bot -= src_stride;
dst_bot -= dst_stride;
}
free_aligned_buffer_64(row);
}
+static void TransposeUVWxH(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ int i;
+ for (i = 0; i < width * 2; i += 2) {
+ int j;
+ for (j = 0; j < height; ++j) {
+ dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)];
+ dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1];
+ }
+ }
+}
+
LIBYUV_API
-void TransposeUV(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
+void SplitTransposeUV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
int i = height;
-#if defined(HAS_TRANSPOSEUVWX16_MSA)
+#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
+ void (*TransposeUVWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst_a,
+ int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
+ int width, int height) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \
+ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \
+ defined(HAS_TRANSPOSEUVWX16_LASX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
- int width) = TransposeUVWx16_C;
-#else
+ int width) = NULL;
+#endif
void (*TransposeUVWx8)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width) = TransposeUVWx8_C;
+
+#if defined(HAS_TRANSPOSEUVWX8_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ TransposeUVWx8 = TransposeUVWx8_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx8 = TransposeUVWx8_LSX;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWX16_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ TransposeUVWx16 = TransposeUVWx16_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx16 = TransposeUVWx16_LASX;
+ }
+ }
#endif
#if defined(HAS_TRANSPOSEUVWX8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- TransposeUVWx8 = TransposeUVWx8_NEON;
+ TransposeUVWx8 = TransposeUVWx8_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx8 = TransposeUVWx8_NEON;
+ }
}
#endif
-#if defined(HAS_TRANSPOSEUVWX8_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- TransposeUVWx8 = TransposeUVWx8_Any_SSE2;
+#if defined(HAS_TRANSPOSEUVWXH_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeUVWxH_fn = TransposeUVWxH_SME;
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWXH_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ TransposeUVWxH_fn = TransposeUVWxH_HVX;
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWX8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeUVWx8 = TransposeUVWx8_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx8 = TransposeUVWx8_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWX16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeUVWx16 = TransposeUVWx16_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
- TransposeUVWx8 = TransposeUVWx8_SSE2;
+ TransposeUVWx16 = TransposeUVWx16_AVX2;
}
}
#endif
-#if defined(HAS_TRANSPOSEUVWX8_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- TransposeUVWx8 = TransposeUVWx8_Any_MMI;
- if (IS_ALIGNED(width, 4)) {
- TransposeUVWx8 = TransposeUVWx8_MMI;
+#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ TransposeUVWx16 = TransposeUVWx16_AVX512BW;
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeUVWx8 = TransposeUVWx8_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx8 = TransposeUVWx8_WASMSIMD;
}
}
#endif
-#if defined(HAS_TRANSPOSEUVWX16_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- TransposeUVWx16 = TransposeUVWx16_Any_MSA;
+#if defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeUVWx16 = TransposeUVWx16_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
- TransposeUVWx16 = TransposeUVWx16_MSA;
+ TransposeUVWx16 = TransposeUVWx16_WASMSIMD;
}
}
#endif
-#if defined(HAS_TRANSPOSEUVWX16_MSA)
- // Work through the source in 8x8 tiles.
- while (i >= 16) {
- TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
- width);
- src += 16 * src_stride; // Go down 16 rows.
- dst_a += 16; // Move over 8 columns.
- dst_b += 16; // Move over 8 columns.
- i -= 16;
- }
-#else
+#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
+ if (TransposeUVWxH_fn) {
+ TransposeUVWxH_fn(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width, i);
+ return;
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \
+ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \
+ defined(HAS_TRANSPOSEUVWX16_LASX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
+ // Work through the source in 16x16 tiles.
+ if (TransposeUVWx16) {
+ while (i >= 16) {
+ TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width);
+ src += 16 * src_stride; // Go down 16 rows.
+ dst_a += 16; // Move over 16 columns.
+ dst_b += 16; // Move over 16 columns.
+ i -= 16;
+ }
+ }
+#endif
// Work through the source in 8x8 tiles.
while (i >= 8) {
TransposeUVWx8(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
@@ -287,79 +500,104 @@ void TransposeUV(const uint8_t* src,
dst_b += 8; // Move over 8 columns.
i -= 8;
}
-#endif
if (i > 0) {
- TransposeUVWxH_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
- width, i);
+ TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width, i);
}
}
LIBYUV_API
-void RotateUV90(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
- src += src_stride * (height - 1);
+void SplitRotateUV90(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ src += (ptrdiff_t)src_stride * (height - 1);
src_stride = -src_stride;
- TransposeUV(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, width,
- height);
+ SplitTransposeUV(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width, height);
}
LIBYUV_API
-void RotateUV270(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
+void SplitRotateUV270(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
dst_a += dst_stride_a * (width - 1);
dst_b += dst_stride_b * (width - 1);
dst_stride_a = -dst_stride_a;
dst_stride_b = -dst_stride_b;
- TransposeUV(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, width,
- height);
+ SplitTransposeUV(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width, height);
}
// Rotate 180 is a horizontal and vertical flip.
LIBYUV_API
-void RotateUV180(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
+void SplitRotateUV180(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
int i;
- void (*MirrorUVRow)(const uint8_t* src, uint8_t* dst_u, uint8_t* dst_v,
- int width) = MirrorUVRow_C;
-#if defined(HAS_MIRRORUVROW_NEON)
- if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
- MirrorUVRow = MirrorUVRow_NEON;
+ void (*MirrorSplitUVRow)(const uint8_t* src, uint8_t* dst_u, uint8_t* dst_v,
+ int width) = MirrorSplitUVRow_C;
+#if defined(HAS_MIRRORSPLITUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 16)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_NEON;
+ }
+#endif
+#if defined(HAS_MIRRORSPLITUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 8)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_MIRRORSPLITUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_Any_AVX2;
+ if (IS_ALIGNED(width, 16)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MIRRORSPLITUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_MIRRORSPLITUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 16)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_LSX;
}
#endif
-#if defined(HAS_MIRRORUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16)) {
- MirrorUVRow = MirrorUVRow_SSSE3;
+#if defined(HAS_MIRRORSPLITUVROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 32)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_LASX;
}
#endif
-#if defined(HAS_MIRRORUVROW_MSA)
- if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 32)) {
- MirrorUVRow = MirrorUVRow_MSA;
+#if defined(HAS_MIRRORSPLITUVROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(width, 16)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_WASMSIMD;
}
#endif
-#if defined(HAS_MIRRORUVROW_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(width, 8)) {
- MirrorUVRow = MirrorUVRow_MMI;
+#if defined(HAS_MIRRORSPLITUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ MirrorSplitUVRow = MirrorSplitUVRow_HVX;
}
#endif
@@ -367,13 +605,60 @@ void RotateUV180(const uint8_t* src,
dst_b += dst_stride_b * (height - 1);
for (i = 0; i < height; ++i) {
- MirrorUVRow(src, dst_a, dst_b, width);
+ MirrorSplitUVRow(src, dst_a, dst_b, width);
src += src_stride;
dst_a -= dst_stride_a;
dst_b -= dst_stride_b;
}
}
+// Rotate UV and split into planar.
+// width and height expected to be half size for NV12
+LIBYUV_API
+int SplitRotateUV(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ if (!src_uv || width <= 0 || height == 0 || height == INT_MIN || !dst_u ||
+ !dst_v) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ SplitUVPlane(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, width, height);
+ return 0;
+ case kRotate90:
+ SplitRotateUV90(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, width, height);
+ return 0;
+ case kRotate270:
+ SplitRotateUV270(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, width, height);
+ return 0;
+ case kRotate180:
+ SplitRotateUV180(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, width, height);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
LIBYUV_API
int RotatePlane(const uint8_t* src,
int src_stride,
@@ -382,14 +667,14 @@ int RotatePlane(const uint8_t* src,
int width,
int height,
enum RotationMode mode) {
- if (!src || width <= 0 || height == 0 || !dst) {
+ if (!src || width <= 0 || height == 0 || height == INT_MIN || !dst) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src = src + (height - 1) * src_stride;
+ src = src + (ptrdiff_t)(height - 1) * src_stride;
src_stride = -src_stride;
}
@@ -413,6 +698,235 @@ int RotatePlane(const uint8_t* src,
return -1;
}
+static void TransposeWxH_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ int j;
+ for (j = 0; j < height; ++j) {
+ dst[i * dst_stride + j] = src[j * src_stride + i];
+ }
+ }
+}
+
+static void TransposePlane_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int i = height;
+#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
+ defined(HAS_TRANSPOSEWXH_16_HVX)
+ void (*TransposeWxH_16_fn)(const uint16_t* src, int src_stride, uint16_t* dst,
+ int dst_stride, int width, int height) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+ void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst,
+ int dst_stride, int width) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
+ void (*TransposeWx16_16)(const uint16_t* src, int src_stride, uint16_t* dst,
+ int dst_stride, int width) = NULL;
+#endif
+ void (*TransposeWx8_16)(const uint16_t* src, int src_stride, uint16_t* dst,
+ int dst_stride, int width) = TransposeWx8_16_C;
+
+#if defined(HAS_TRANSPOSEWX8_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx8_16 = TransposeWx8_16_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx8_16 = TransposeWx8_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_16_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ TransposeWx8_16 = TransposeWx8_16_SVE2;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeWxH_16_fn = TransposeWxH_16_SME;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_16_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ TransposeWxH_16_fn = TransposeWxH_16_RVV;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ TransposeWxH_16_fn = TransposeWxH_16_HVX;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx8_16 = TransposeWx8_16_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx8_16 = TransposeWx8_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx16_16 = TransposeWx16_16_Any_AVX2;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx16_16 = TransposeWx16_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ TransposeWx32_16 = TransposeWx32_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_16_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ TransposeWx8_16 = TransposeWx8_16_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx8_16 = TransposeWx8_16_LSX;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_16_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ TransposeWx16_16 = TransposeWx16_16_Any_LASX;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx16_16 = TransposeWx16_16_LASX;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeWx8_16 = TransposeWx8_16_Any_WASMSIMD;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeWx8_16 = TransposeWx8_16_WASMSIMD;
+ }
+ }
+#endif
+
+#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
+ defined(HAS_TRANSPOSEWXH_16_HVX)
+ if (TransposeWxH_16_fn) {
+ TransposeWxH_16_fn(src, src_stride, dst, dst_stride, width, height);
+ return;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+ if (TransposeWx32_16) {
+ while (i >= 32) {
+ TransposeWx32_16(src, src_stride, dst, dst_stride, width);
+ src += 32 * (ptrdiff_t)src_stride;
+ dst += 32;
+ i -= 32;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
+ if (TransposeWx16_16) {
+ while (i >= 16) {
+ TransposeWx16_16(src, src_stride, dst, dst_stride, width);
+ src += 16 * (ptrdiff_t)src_stride;
+ dst += 16;
+ i -= 16;
+ }
+ }
+#endif
+ // Work across the source in 8x8 tiles
+ while (i >= 8) {
+ TransposeWx8_16(src, src_stride, dst, dst_stride, width);
+ src += 8 * (ptrdiff_t)src_stride; // Go down 8 rows.
+ dst += 8; // Move over 8 columns.
+ i -= 8;
+ }
+
+ if (i > 0) {
+ TransposeWxH_16(src, src_stride, dst, dst_stride, width, i);
+ }
+}
+
+static void RotatePlane90_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ // Rotate by 90 is a transpose with the source read
+ // from bottom to top. So set the source pointer to the end
+ // of the buffer and flip the sign of the source stride.
+ src += (ptrdiff_t)src_stride * (height - 1);
+ src_stride = -src_stride;
+ TransposePlane_16(src, src_stride, dst, dst_stride, width, height);
+}
+
+static void RotatePlane270_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ // Rotate by 270 is a transpose with the destination written
+ // from bottom to top. So set the destination pointer to the end
+ // of the buffer and flip the sign of the destination stride.
+ dst += (ptrdiff_t)dst_stride * (width - 1);
+ dst_stride = -dst_stride;
+ TransposePlane_16(src, src_stride, dst, dst_stride, width, height);
+}
+
+static void RotatePlane180_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ // Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
+ MirrorPlane_16(src, src_stride, dst, dst_stride, width, -height);
+}
+
+LIBYUV_API
+int RotatePlane_16(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ if (!src || width <= 0 || height == 0 || height == INT_MIN || !dst) {
+ return -1;
+ }
+
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src = src + (ptrdiff_t)(height - 1) * src_stride;
+ src_stride = -src_stride;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ // copy frame
+ CopyPlane_16(src, src_stride, dst, dst_stride, width, height);
+ return 0;
+ case kRotate90:
+ RotatePlane90_16(src, src_stride, dst, dst_stride, width, height);
+ return 0;
+ case kRotate270:
+ RotatePlane270_16(src, src_stride, dst, dst_stride, width, height);
+ return 0;
+ case kRotate180:
+ RotatePlane180_16(src, src_stride, dst, dst_stride, width, height);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
LIBYUV_API
int I420Rotate(const uint8_t* src_y,
int src_stride_y,
@@ -431,8 +945,8 @@ int I420Rotate(const uint8_t* src_y,
enum RotationMode mode) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_y || !src_u || !src_v || width <= 0 || height == 0 || !dst_y ||
- !dst_u || !dst_v) {
+ if ((!src_y && dst_y) || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v) {
return -1;
}
@@ -440,9 +954,9 @@ int I420Rotate(const uint8_t* src_y,
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (halfheight - 1) * src_stride_u;
- src_v = src_v + (halfheight - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@@ -481,6 +995,107 @@ int I420Rotate(const uint8_t* src_y,
return -1;
}
+// I422 has half width x full height UV planes, so rotate by 90 and 270
+// require scaling to maintain 422 subsampling.
+LIBYUV_API
+int I422Rotate(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ int r;
+ if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ // Copy frame
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, height);
+ CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, height);
+ return 0;
+
+ // Note on temporary Y plane for UV.
+ // Rotation of UV first fits within the Y destination plane rows.
+ // Y plane is width x height
+ // Y plane rotated is height x width
+ // UV plane is (width / 2) x height
+ // UV plane rotated is height x (width / 2)
+ // UV plane rotated+scaled is (height / 2) x width.
+ // UV plane rotated is a temporary that fits within the Y plane rotated.
+
+ case kRotate90:
+ RotatePlane90(src_u, src_stride_u, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane(dst_y, dst_stride_y, height, halfwidth, dst_u,
+ dst_stride_u, halfheight, width, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane90(src_v, src_stride_v, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane(dst_y, dst_stride_y, height, halfwidth, dst_v,
+ dst_stride_v, halfheight, width, kFilterLinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane90(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ return 0;
+ case kRotate270:
+ RotatePlane270(src_u, src_stride_u, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane(dst_y, dst_stride_y, height, halfwidth, dst_u,
+ dst_stride_u, halfheight, width, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane270(src_v, src_stride_v, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane(dst_y, dst_stride_y, height, halfwidth, dst_v,
+ dst_stride_v, halfheight, width, kFilterLinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane270(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ return 0;
+ case kRotate180:
+ RotatePlane180(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ RotatePlane180(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth,
+ height);
+ RotatePlane180(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth,
+ height);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
LIBYUV_API
int I444Rotate(const uint8_t* src_y,
int src_stride_y,
@@ -496,41 +1111,41 @@ int I444Rotate(const uint8_t* src_y,
int dst_stride_v,
int width,
int height,
- enum libyuv::RotationMode mode) {
- if (!src_y || !src_u || !src_v || width <= 0 || height == 0 || !dst_y ||
- !dst_u || !dst_v) {
+ enum RotationMode mode) {
+ if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_u = src_u + (height - 1) * src_stride_u;
- src_v = src_v + (height - 1) * src_stride_v;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
switch (mode) {
- case libyuv::kRotate0:
+ case kRotate0:
// copy frame
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
CopyPlane(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
CopyPlane(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
return 0;
- case libyuv::kRotate90:
+ case kRotate90:
RotatePlane90(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
RotatePlane90(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
RotatePlane90(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
return 0;
- case libyuv::kRotate270:
+ case kRotate270:
RotatePlane270(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
RotatePlane270(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
RotatePlane270(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
return 0;
- case libyuv::kRotate180:
+ case kRotate180:
RotatePlane180(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
RotatePlane180(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
RotatePlane180(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
@@ -557,8 +1172,8 @@ int NV12ToI420Rotate(const uint8_t* src_y,
enum RotationMode mode) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_y || !src_uv || width <= 0 || height == 0 || !dst_y || !dst_u ||
- !dst_v) {
+ if (!src_y || !src_uv || width <= 0 || height == 0 || height == INT_MIN ||
+ !dst_y || !dst_u || !dst_v) {
return -1;
}
@@ -566,8 +1181,8 @@ int NV12ToI420Rotate(const uint8_t* src_y,
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
- src_y = src_y + (height - 1) * src_stride_y;
- src_uv = src_uv + (halfheight - 1) * src_stride_uv;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(halfheight - 1) * src_stride_uv;
src_stride_y = -src_stride_y;
src_stride_uv = -src_stride_uv;
}
@@ -580,18 +1195,347 @@ int NV12ToI420Rotate(const uint8_t* src_y,
width, height);
case kRotate90:
RotatePlane90(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
- RotateUV90(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
- dst_stride_v, halfwidth, halfheight);
+ SplitRotateUV90(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, halfwidth, halfheight);
return 0;
case kRotate270:
RotatePlane270(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
- RotateUV270(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
- dst_stride_v, halfwidth, halfheight);
+ SplitRotateUV270(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, halfwidth, halfheight);
return 0;
case kRotate180:
RotatePlane180(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
- RotateUV180(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
- dst_stride_v, halfwidth, halfheight);
+ SplitRotateUV180(src_uv, src_stride_uv, dst_u, dst_stride_u, dst_v,
+ dst_stride_v, halfwidth, halfheight);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
+static void SplitPixels(const uint8_t* src_u,
+ int src_pixel_stride_uv,
+ uint8_t* dst_u,
+ int width) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ *dst_u = *src_u;
+ ++dst_u;
+ src_u += src_pixel_stride_uv;
+ }
+}
+
+// Convert Android420 to I420 with Rotate
+LIBYUV_API
+int Android420ToI420Rotate(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_pixel_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode rotation) {
+ int y;
+ const ptrdiff_t vu_off = src_v - src_u;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ if (dst_y) {
+ RotatePlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height,
+ rotation);
+ }
+
+ // Copy UV planes - I420
+ if (src_pixel_stride_uv == 1) {
+ RotatePlane(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, halfheight,
+ rotation);
+ RotatePlane(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, halfheight,
+ rotation);
+ return 0;
+ }
+ // Split UV planes - NV21
+ if (src_pixel_stride_uv == 2 && vu_off == -1 &&
+ src_stride_u == src_stride_v) {
+ SplitRotateUV(src_v, src_stride_v, dst_v, dst_stride_v, dst_u, dst_stride_u,
+ halfwidth, halfheight, rotation);
+ return 0;
+ }
+ // Split UV planes - NV12
+ if (src_pixel_stride_uv == 2 && vu_off == 1 && src_stride_u == src_stride_v) {
+ SplitRotateUV(src_u, src_stride_u, dst_u, dst_stride_u, dst_v, dst_stride_v,
+ halfwidth, halfheight, rotation);
+ return 0;
+ }
+
+ if (rotation == 0) {
+ for (y = 0; y < halfheight; ++y) {
+ SplitPixels(src_u, src_pixel_stride_uv, dst_u, halfwidth);
+ SplitPixels(src_v, src_pixel_stride_uv, dst_v, halfwidth);
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ return 0;
+ }
+ // unsupported type and/or rotation.
+ return -1;
+}
+
+LIBYUV_API
+int I010Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_stride_y < 0) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ // copy frame
+ return I010Copy(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+ dst_v, dst_stride_v, width, height);
+ case kRotate90:
+ RotatePlane90_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ RotatePlane90_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth,
+ halfheight);
+ RotatePlane90_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth,
+ halfheight);
+ return 0;
+ case kRotate270:
+ RotatePlane270_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ RotatePlane270_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth,
+ halfheight);
+ RotatePlane270_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth,
+ halfheight);
+ return 0;
+ case kRotate180:
+ RotatePlane180_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ RotatePlane180_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth,
+ halfheight);
+ RotatePlane180_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth,
+ halfheight);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
+// I210 has half width x full height UV planes, so rotate by 90 and 270
+// require scaling to maintain 422 subsampling.
+LIBYUV_API
+int I210Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ int r;
+ if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ // Copy frame
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ CopyPlane_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth, height);
+ CopyPlane_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth, height);
+ return 0;
+
+ // Note on temporary Y plane for UV.
+ // Rotation of UV first fits within the Y destination plane rows.
+ // Y plane is width x height
+ // Y plane rotated is height x width
+ // UV plane is (width / 2) x height
+ // UV plane rotated is height x (width / 2)
+ // UV plane rotated+scaled is (height / 2) x width.
+ // UV plane rotated is a temporary that fits within the Y plane rotated.
+
+ case kRotate90:
+ RotatePlane90_16(src_u, src_stride_u, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane_16(dst_y, dst_stride_y, height, halfwidth, dst_u,
+ dst_stride_u, halfheight, width, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane90_16(src_v, src_stride_v, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane_16(dst_y, dst_stride_y, height, halfwidth, dst_v,
+ dst_stride_v, halfheight, width, kFilterLinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane90_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ return 0;
+ case kRotate270:
+ RotatePlane270_16(src_u, src_stride_u, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane_16(dst_y, dst_stride_y, height, halfwidth, dst_u,
+ dst_stride_u, halfheight, width, kFilterBilinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane270_16(src_v, src_stride_v, dst_y, dst_stride_y, halfwidth,
+ height);
+ r = ScalePlane_16(dst_y, dst_stride_y, height, halfwidth, dst_v,
+ dst_stride_v, halfheight, width, kFilterLinear);
+ if (r != 0) {
+ return r;
+ }
+ RotatePlane270_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ return 0;
+ case kRotate180:
+ RotatePlane180_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ RotatePlane180_16(src_u, src_stride_u, dst_u, dst_stride_u, halfwidth,
+ height);
+ RotatePlane180_16(src_v, src_stride_v, dst_v, dst_stride_v, halfwidth,
+ height);
+ return 0;
+ default:
+ break;
+ }
+ return -1;
+}
+
+LIBYUV_API
+int I410Rotate(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ enum RotationMode mode) {
+ if (!src_y || !src_u || !src_v || width <= 0 || height == 0 ||
+ height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_stride_y < 0) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
+ src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
+ src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ switch (mode) {
+ case kRotate0:
+ // copy frame
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ CopyPlane_16(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
+ CopyPlane_16(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
+ return 0;
+ case kRotate90:
+ RotatePlane90_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+ RotatePlane90_16(src_u, src_stride_u, dst_u, dst_stride_u, width, height);
+ RotatePlane90_16(src_v, src_stride_v, dst_v, dst_stride_v, width, height);
+ return 0;
+ case kRotate270:
+ RotatePlane270_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ RotatePlane270_16(src_u, src_stride_u, dst_u, dst_stride_u, width,
+ height);
+ RotatePlane270_16(src_v, src_stride_v, dst_v, dst_stride_v, width,
+ height);
+ return 0;
+ case kRotate180:
+ RotatePlane180_16(src_y, src_stride_y, dst_y, dst_stride_y, width,
+ height);
+ RotatePlane180_16(src_u, src_stride_u, dst_u, dst_stride_u, width,
+ height);
+ RotatePlane180_16(src_v, src_stride_v, dst_v, dst_stride_v, width,
+ height);
return 0;
default:
break;
diff --git a/libs/libyuv/source/rotate_any.cc b/libs/libyuv/source/rotate_any.cc
index b3baf084d0..bc203655a9 100644
--- a/libs/libyuv/source/rotate_any.cc
+++ b/libs/libyuv/source/rotate_any.cc
@@ -11,42 +11,50 @@
#include "libyuv/rotate.h"
#include "libyuv/rotate_row.h"
-#include "libyuv/basic_types.h"
-
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#define TANY(NAMEANY, TPOS_SIMD, MASK) \
- void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
- int dst_stride, int width) { \
- int r = width & MASK; \
- int n = width - r; \
- if (n > 0) { \
- TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
- } \
- TransposeWx8_C(src + n, src_stride, dst + n * dst_stride, dst_stride, r); \
+#define TANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
+ void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
+ int dst_stride, int width) { \
+ int r = width & MASK; \
+ int n = width - r; \
+ if (n > 0) { \
+ TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
+ } \
+ ptrdiff_t np = n; \
+ TPOS_C(src + np, src_stride, dst + np * dst_stride, dst_stride, r); \
}
#ifdef HAS_TRANSPOSEWX8_NEON
-TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, 7)
+TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
+#endif
+#ifdef HAS_TRANSPOSEWX16_NEON
+TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
+#endif
+#ifdef HAS_TRANSPOSEWX8_AVX2
+TANY(TransposeWx8_Any_AVX2, TransposeWx8_AVX2, TransposeWx8_C, 15)
+#endif
+#ifdef HAS_TRANSPOSEWX16_AVX2
+TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15)
#endif
-#ifdef HAS_TRANSPOSEWX8_SSSE3
-TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, 7)
+#ifdef HAS_TRANSPOSEWX8_LSX
+TANY(TransposeWx8_Any_LSX, TransposeWx8_LSX, TransposeWx8_C, 15)
#endif
-#ifdef HAS_TRANSPOSEWX8_MMI
-TANY(TransposeWx8_Any_MMI, TransposeWx8_MMI, 7)
+#ifdef HAS_TRANSPOSEWX16_LASX
+TANY(TransposeWx16_Any_LASX, TransposeWx16_LASX, TransposeWx16_C, 15)
#endif
-#ifdef HAS_TRANSPOSEWX8_FAST_SSSE3
-TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, 15)
+#ifdef HAS_TRANSPOSEWX8_WASMSIMD
+TANY(TransposeWx8_Any_WASMSIMD, TransposeWx8_WASMSIMD, TransposeWx8_C, 15)
#endif
-#ifdef HAS_TRANSPOSEWX16_MSA
-TANY(TransposeWx16_Any_MSA, TransposeWx16_MSA, 15)
+#ifdef HAS_TRANSPOSEWX16_WASMSIMD
+TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15)
#endif
#undef TANY
-#define TUVANY(NAMEANY, TPOS_SIMD, MASK) \
+#define TUVANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst_a, \
int dst_stride_a, uint8_t* dst_b, int dst_stride_b, \
int width) { \
@@ -55,24 +63,126 @@ TANY(TransposeWx16_Any_MSA, TransposeWx16_MSA, 15)
if (n > 0) { \
TPOS_SIMD(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, n); \
} \
- TransposeUVWx8_C(src + n * 2, src_stride, dst_a + n * dst_stride_a, \
- dst_stride_a, dst_b + n * dst_stride_b, dst_stride_b, r); \
+ ptrdiff_t np = n; \
+ TPOS_C(src + np * 2, src_stride, dst_a + np * dst_stride_a, dst_stride_a, \
+ dst_b + np * dst_stride_b, dst_stride_b, r); \
}
#ifdef HAS_TRANSPOSEUVWX8_NEON
-TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, 7)
+TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7)
#endif
-#ifdef HAS_TRANSPOSEUVWX8_SSE2
-TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, 7)
+#ifdef HAS_TRANSPOSEUVWX8_AVX2
+TUVANY(TransposeUVWx8_Any_AVX2, TransposeUVWx8_AVX2, TransposeUVWx8_C, 7)
#endif
-#ifdef HAS_TRANSPOSEUVWX8_MMI
-TUVANY(TransposeUVWx8_Any_MMI, TransposeUVWx8_MMI, 7)
+#ifdef HAS_TRANSPOSEUVWX16_AVX2
+TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7)
#endif
-#ifdef HAS_TRANSPOSEUVWX16_MSA
-TUVANY(TransposeUVWx16_Any_MSA, TransposeUVWx16_MSA, 7)
+#ifdef HAS_TRANSPOSEUVWX8_LSX
+TUVANY(TransposeUVWx8_Any_LSX, TransposeUVWx8_LSX, TransposeUVWx8_C, 7)
+#endif
+#ifdef HAS_TRANSPOSEUVWX16_LASX
+TUVANY(TransposeUVWx16_Any_LASX, TransposeUVWx16_LASX, TransposeUVWx16_C, 7)
+#endif
+#ifdef HAS_TRANSPOSEUVWX8_WASMSIMD
+TUVANY(TransposeUVWx8_Any_WASMSIMD,
+ TransposeUVWx8_WASMSIMD,
+ TransposeUVWx8_C,
+ 7)
+#endif
+#ifdef HAS_TRANSPOSEUVWX16_WASMSIMD
+TUVANY(TransposeUVWx16_Any_WASMSIMD,
+ TransposeUVWx16_WASMSIMD,
+ TransposeUVWx16_C,
+ 7)
#endif
#undef TUVANY
+#define TANY32(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
+ void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
+ int dst_stride, int width) { \
+ int r = width & MASK; \
+ int n = width - r; \
+ if (n > 0) { \
+ TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
+ } \
+ if (r > 0) { \
+ TPOS_C(src + (ptrdiff_t)n * src_stride, src_stride, \
+ dst + (ptrdiff_t)n * 4, dst_stride, 4, r); \
+ } \
+ }
+
+#ifdef HAS_TRANSPOSEWX4_32_AVX2
+TANY32(TransposeWx4_32_Any_AVX2, TransposeWx4_32_AVX2, TransposeWxH_32_C, 3)
+#endif
+#ifdef HAS_TRANSPOSEWX4_32_LSX
+TANY32(TransposeWx4_32_Any_LSX, TransposeWx4_32_LSX, TransposeWxH_32_C, 3)
+#endif
+#ifdef HAS_TRANSPOSEWX4_32_LASX
+TANY32(TransposeWx4_32_Any_LASX, TransposeWx4_32_LASX, TransposeWxH_32_C, 7)
+#endif
+#ifdef HAS_TRANSPOSEWX4_32_WASMSIMD
+TANY32(TransposeWx4_32_Any_WASMSIMD,
+ TransposeWx4_32_WASMSIMD,
+ TransposeWxH_32_C,
+ 3)
+#endif
+#undef TANY32
+
+#define TANY32_SRC(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \
+ void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
+ int dst_stride, int width) { \
+ int r = width & MASK; \
+ int n = width - r; \
+ if (n > 0) { \
+ TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
+ } \
+ ptrdiff_t np = n; \
+ TransposeWxH_32_C(src + np * 4, src_stride, dst + np * dst_stride, \
+ dst_stride, r, HEIGHT); \
+ }
+
+#ifdef HAS_TRANSPOSEWX8_32_AVX2
+TANY32_SRC(TransposeWx8_32_Any_AVX2, TransposeWx8_32_AVX2, 3, 8)
+#endif
+#undef TANY32_SRC
+
+#define TANY16(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \
+ void NAMEANY(const uint16_t* src, int src_stride, uint16_t* dst, \
+ int dst_stride, int width) { \
+ int r = width & MASK; \
+ int n = width - r; \
+ if (n > 0) { \
+ TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
+ } \
+ ptrdiff_t np = n; \
+ TransposeWx8_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \
+ r); \
+ if ((HEIGHT) == 16) { \
+ TransposeWx8_16_C(src + np + 8 * src_stride, src_stride, \
+ dst + np * dst_stride + 8, dst_stride, r); \
+ } \
+ }
+
+#ifdef HAS_TRANSPOSEWX8_16_NEON
+TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8)
+#endif
+#ifdef HAS_TRANSPOSEWX8_16_AVX2
+TANY16(TransposeWx8_16_Any_AVX2, TransposeWx8_16_AVX2, 7, 8)
+#endif
+#ifdef HAS_TRANSPOSEWX16_16_AVX2
+TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16)
+#endif
+#ifdef HAS_TRANSPOSEWX8_16_LSX
+TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8)
+#endif
+#ifdef HAS_TRANSPOSEWX16_16_LASX
+TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16)
+#endif
+#ifdef HAS_TRANSPOSEWX8_16_WASMSIMD
+TANY16(TransposeWx8_16_Any_WASMSIMD, TransposeWx8_16_WASMSIMD, 7, 8)
+#endif
+#undef TANY16
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/rotate_argb.cc b/libs/libyuv/source/rotate_argb.cc
index a93fd55f92..1b0bf7417d 100644
--- a/libs/libyuv/source/rotate_argb.cc
+++ b/libs/libyuv/source/rotate_argb.cc
@@ -8,188 +8,217 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/rotate.h"
+#include "libyuv/rotate_argb.h"
+
+#include <limits.h>
#include "libyuv/convert.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
+#include "libyuv/rotate.h"
+#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
-#include "libyuv/scale_row.h" /* for ScaleARGBRowDownEven_ */
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-static void ARGBTranspose(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- int i;
- int src_pixel_step = src_stride_argb >> 2;
- void (*ScaleARGBRowDownEven)(
- const uint8_t* src_argb, ptrdiff_t src_stride_argb, int src_step,
- uint8_t* dst_argb, int dst_width) = ScaleARGBRowDownEven_C;
-#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_SSE2;
- if (IS_ALIGNED(height, 4)) { // Width of dest.
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_SSE2;
- }
- }
+static int ARGBTranspose(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
+ defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
+ defined(HAS_TRANSPOSEWXH_32_AVX2)
+ void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width, int height) = NULL;
#endif
-#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_NEON;
+#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+ void (*TransposeWx16_32)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEWX8_32_AVX2)
+ void (*TransposeWx8_32)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width) = NULL;
+#endif
+ void (*TransposeWx4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width) = NULL;
+ // Check stride is a multiple of 4.
+ if (src_stride_argb & 3) {
+ return -1;
+ }
+#if defined(HAS_TRANSPOSEWX4_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx4_32 = TransposeWx4_32_Any_AVX2;
if (IS_ALIGNED(height, 4)) { // Width of dest.
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_NEON;
+ TransposeWx4_32 = TransposeWx4_32_AVX2;
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWNEVEN_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_MSA;
- if (IS_ALIGNED(height, 4)) { // Width of dest.
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_MSA;
+#if defined(HAS_TRANSPOSEWX8_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx8_32 = TransposeWx8_32_Any_AVX2;
+ if (IS_ALIGNED(width, 4)) {
+ TransposeWx8_32 = TransposeWx8_32_AVX2;
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWNEVEN_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_MMI;
- if (IS_ALIGNED(height, 4)) { // Width of dest.
- ScaleARGBRowDownEven = ScaleARGBRowDownEven_MMI;
- }
+#if defined(HAS_TRANSPOSEWXH_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWxH_32 = TransposeWxH_32_AVX2;
}
#endif
-
- for (i = 0; i < width; ++i) { // column of source to row of dest.
- ScaleARGBRowDownEven(src_argb, 0, src_pixel_step, dst_argb, height);
- dst_argb += dst_stride_argb;
- src_argb += 4;
+#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ TransposeWx16_32 = TransposeWx16_32_AVX512BW;
}
-}
-
-void ARGBRotate90(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- // Rotate by 90 is a ARGBTranspose with the source read
- // from bottom to top. So set the source pointer to the end
- // of the buffer and flip the sign of the source stride.
- src_argb += src_stride_argb * (height - 1);
- src_stride_argb = -src_stride_argb;
- ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
- height);
-}
-
-void ARGBRotate270(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- // Rotate by 270 is a ARGBTranspose with the destination written
- // from bottom to top. So set the destination pointer to the end
- // of the buffer and flip the sign of the destination stride.
- dst_argb += dst_stride_argb * (width - 1);
- dst_stride_argb = -dst_stride_argb;
- ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
- height);
-}
-
-void ARGBRotate180(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
- // Swap first and last row and mirror the content. Uses a temporary row.
- align_buffer_64(row, width * 4);
- const uint8_t* src_bot = src_argb + src_stride_argb * (height - 1);
- uint8_t* dst_bot = dst_argb + dst_stride_argb * (height - 1);
- int half_height = (height + 1) >> 1;
- int y;
- void (*ARGBMirrorRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
- ARGBMirrorRow_C;
- void (*CopyRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
- CopyRow_C;
-#if defined(HAS_ARGBMIRRORROW_NEON)
+#endif
+#if defined(HAS_TRANSPOSEWXH_32_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_NEON;
- if (IS_ALIGNED(width, 4)) {
- ARGBMirrorRow = ARGBMirrorRow_NEON;
- }
+ TransposeWxH_32 = TransposeWxH_32_NEON;
}
#endif
-#if defined(HAS_ARGBMIRRORROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_SSE2;
- if (IS_ALIGNED(width, 4)) {
- ARGBMirrorRow = ARGBMirrorRow_SSE2;
- }
+#if defined(HAS_TRANSPOSEWX4_32_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ TransposeWx4_32 = TransposeWx4_32_SVE2;
+#if defined(HAS_TRANSPOSEWXH_32_NEON)
+ TransposeWxH_32 = NULL;
+#endif
}
#endif
-#if defined(HAS_ARGBMIRRORROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_AVX2;
- if (IS_ALIGNED(width, 8)) {
- ARGBMirrorRow = ARGBMirrorRow_AVX2;
- }
+#if defined(HAS_TRANSPOSEWXH_32_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeWxH_32 = TransposeWxH_32_SME;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_32_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ TransposeWxH_32 = TransposeWxH_32_RVV;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_32_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ TransposeWxH_32 = TransposeWxH_32_HVX;
}
#endif
-#if defined(HAS_ARGBMIRRORROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_MSA;
- if (IS_ALIGNED(width, 16)) {
- ARGBMirrorRow = ARGBMirrorRow_MSA;
+#if defined(HAS_TRANSPOSEWX4_32_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ TransposeWx4_32 = TransposeWx4_32_Any_LSX;
+ if (IS_ALIGNED(height, 4)) { // Width of dest.
+ TransposeWx4_32 = TransposeWx4_32_LSX;
}
}
#endif
-#if defined(HAS_ARGBMIRRORROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ARGBMirrorRow = ARGBMirrorRow_Any_MMI;
- if (IS_ALIGNED(width, 2)) {
- ARGBMirrorRow = ARGBMirrorRow_MMI;
+#if defined(HAS_TRANSPOSEWX4_32_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ TransposeWx4_32 = TransposeWx4_32_Any_LASX;
+ if (IS_ALIGNED(height, 8)) { // Width of dest.
+ TransposeWx4_32 = TransposeWx4_32_LASX;
}
}
#endif
-#if defined(HAS_COPYROW_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
+#if defined(HAS_TRANSPOSEWX4_32_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ TransposeWx4_32 = TransposeWx4_32_Any_WASMSIMD;
+ if (IS_ALIGNED(height, 4)) { // Width of dest.
+ TransposeWx4_32 = TransposeWx4_32_WASMSIMD;
+ }
}
#endif
-#if defined(HAS_COPYROW_AVX)
- if (TestCpuFlag(kCpuHasAVX)) {
- CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
+
+#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+ if (TransposeWx16_32) {
+ while (height >= 16) {
+ TransposeWx16_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width);
+ src_argb += 16 * (ptrdiff_t)src_stride_argb;
+ dst_argb += 16 * 4;
+ height -= 16;
+ }
}
#endif
-#if defined(HAS_COPYROW_ERMS)
- if (TestCpuFlag(kCpuHasERMS)) {
- CopyRow = CopyRow_ERMS;
+#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
+ defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
+ defined(HAS_TRANSPOSEWXH_32_AVX2)
+ if (TransposeWxH_32) {
+ if (height > 0) {
+ TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
+ }
+ return 0;
}
#endif
-#if defined(HAS_COPYROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
+#if defined(HAS_TRANSPOSEWX8_32_AVX2)
+ if (TransposeWx8_32) {
+ while (height >= 8) {
+ TransposeWx8_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width);
+ src_argb += 8 * (ptrdiff_t)src_stride_argb;
+ dst_argb += 8 * 4;
+ height -= 8;
+ }
}
#endif
+ if (height > 0) {
+ if (TransposeWx4_32) {
+ while (width >= 4) {
+ TransposeWx4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ height);
+ src_argb += 4 * 4;
+ dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
+ width -= 4;
+ }
+ }
+ if (width > 0) {
+ TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
+ }
+ }
+ return 0;
+}
+
+static int ARGBRotate90(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ // Rotate by 90 is a ARGBTranspose with the source read
+ // from bottom to top. So set the source pointer to the end
+ // of the buffer and flip the sign of the source stride.
+ src_argb += (ptrdiff_t)src_stride_argb * (height - 1);
+ src_stride_argb = -src_stride_argb;
+ return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
+}
+
+static int ARGBRotate270(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ // Rotate by 270 is a ARGBTranspose with the destination written
+ // from bottom to top. So set the destination pointer to the end
+ // of the buffer and flip the sign of the destination stride.
+ dst_argb += (ptrdiff_t)dst_stride_argb * (width - 1);
+ dst_stride_argb = -dst_stride_argb;
+ return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
+}
- // Odd height will harmlessly mirror the middle row twice.
- for (y = 0; y < half_height; ++y) {
- ARGBMirrorRow(src_argb, row, width); // Mirror first row into a buffer
- ARGBMirrorRow(src_bot, dst_argb, width); // Mirror last row into first row
- CopyRow(row, dst_bot, width * 4); // Copy first mirrored row into last
- src_argb += src_stride_argb;
- dst_argb += dst_stride_argb;
- src_bot -= src_stride_argb;
- dst_bot -= dst_stride_argb;
- }
- free_aligned_buffer_64(row);
+static int ARGBRotate180(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ // Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
+ return ARGBMirror(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
+ -height);
}
LIBYUV_API
@@ -200,14 +229,15 @@ int ARGBRotate(const uint8_t* src_argb,
int width,
int height,
enum RotationMode mode) {
- if (!src_argb || width <= 0 || height == 0 || !dst_argb) {
+ if (!src_argb || width <= 0 || height == 0 || height == INT_MIN ||
+ !dst_argb) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
- src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
@@ -217,17 +247,14 @@ int ARGBRotate(const uint8_t* src_argb,
return ARGBCopy(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate90:
- ARGBRotate90(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
- height);
- return 0;
+ return ARGBRotate90(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
case kRotate270:
- ARGBRotate270(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
- height);
- return 0;
+ return ARGBRotate270(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
case kRotate180:
- ARGBRotate180(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
- height);
- return 0;
+ return ARGBRotate180(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
+ width, height);
default:
break;
}
diff --git a/libs/libyuv/source/rotate_common.cc b/libs/libyuv/source/rotate_common.cc
index ff212adebc..e4aab102f5 100644
--- a/libs/libyuv/source/rotate_common.cc
+++ b/libs/libyuv/source/rotate_common.cc
@@ -36,6 +36,16 @@ void TransposeWx8_C(const uint8_t* src,
}
}
+void TransposeWx16_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_C(src, src_stride, dst, dst_stride, width);
+ TransposeWx8_C((src + 8 * src_stride), src_stride, (dst + 8), dst_stride,
+ width);
+}
+
void TransposeUVWx8_C(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@@ -67,35 +77,99 @@ void TransposeUVWx8_C(const uint8_t* src,
}
}
-void TransposeWxH_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width,
- int height) {
+void TransposeUVWx16_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeUVWx8_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width);
+ TransposeUVWx8_C((src + 8 * src_stride), src_stride, (dst_a + 8),
+ dst_stride_a, (dst_b + 8), dst_stride_b, width);
+}
+
+void TransposeWx8_16_C(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
int i;
for (i = 0; i < width; ++i) {
- int j;
- for (j = 0; j < height; ++j) {
- dst[i * dst_stride + j] = src[j * src_stride + i];
- }
+ dst[0] = src[0 * src_stride];
+ dst[1] = src[1 * src_stride];
+ dst[2] = src[2 * src_stride];
+ dst[3] = src[3 * src_stride];
+ dst[4] = src[4 * src_stride];
+ dst[5] = src[5 * src_stride];
+ dst[6] = src[6 * src_stride];
+ dst[7] = src[7 * src_stride];
+ ++src;
+ dst += dst_stride;
}
}
-void TransposeUVWxH_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width,
- int height) {
+static inline void Copy4(uint8_t* dst, const uint8_t* src) {
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
+ dst[3] = src[3];
+}
+
+// Transpose 32 bit values (ARGB)
+void TransposeWx4_32_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const uint8_t* src1 = src + src_stride;
+ const uint8_t* src2 = src1 + src_stride;
+ const uint8_t* src3 = src2 + src_stride;
+ uint8_t* dst1 = dst + dst_stride;
+ uint8_t* dst2 = dst1 + dst_stride;
+ uint8_t* dst3 = dst2 + dst_stride;
+ int i;
+ for (i = 0; i < width; i += 4) {
+ Copy4(dst + 0, src + 0);
+ Copy4(dst + 4, src1 + 0);
+ Copy4(dst + 8, src2 + 0);
+ Copy4(dst + 12, src3 + 0);
+ Copy4(dst1 + 0, src + 4);
+ Copy4(dst1 + 4, src1 + 4);
+ Copy4(dst1 + 8, src2 + 4);
+ Copy4(dst1 + 12, src3 + 4);
+ Copy4(dst2 + 0, src + 8);
+ Copy4(dst2 + 4, src1 + 8);
+ Copy4(dst2 + 8, src2 + 8);
+ Copy4(dst2 + 12, src3 + 8);
+ Copy4(dst3 + 0, src + 12);
+ Copy4(dst3 + 4, src1 + 12);
+ Copy4(dst3 + 8, src2 + 12);
+ Copy4(dst3 + 12, src3 + 12);
+ src += (ptrdiff_t)src_stride * 4; // advance 4 rows
+ src1 += (ptrdiff_t)src_stride * 4;
+ src2 += (ptrdiff_t)src_stride * 4;
+ src3 += (ptrdiff_t)src_stride * 4;
+ dst += 4 * 4; // advance 4 columns
+ dst1 += 4 * 4;
+ dst2 += 4 * 4;
+ dst3 += 4 * 4;
+ }
+}
+
+void TransposeWxH_32_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
int i;
- for (i = 0; i < width * 2; i += 2) {
+ for (i = 0; i < width; ++i) {
int j;
for (j = 0; j < height; ++j) {
- dst_a[j + ((i >> 1) * dst_stride_a)] = src[i + (j * src_stride)];
- dst_b[j + ((i >> 1) * dst_stride_b)] = src[i + (j * src_stride) + 1];
+ Copy4(dst + (ptrdiff_t)i * dst_stride + j * 4,
+ src + (ptrdiff_t)j * src_stride + i * 4);
}
}
}
diff --git a/libs/libyuv/source/rotate_gcc.cc b/libs/libyuv/source/rotate_gcc.cc
index 04e19e29ee..734638023f 100644
--- a/libs/libyuv/source/rotate_gcc.cc
+++ b/libs/libyuv/source/rotate_gcc.cc
@@ -11,362 +11,1702 @@
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
+#include <string.h>
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+// This module is for GCC and clang x86 and x64.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (!defined(_MSC_VER) || defined(__clang__))
+
+#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
+// Transpose 16x8 bytes using 2 lanes of AVX2. Source row r is in lane r / 4
+// of ymm(r % 4). Byte and word unpacks produce 4-row column chunks in each
+// lane; vperm2i128 + dword unpack join rows 0..3 and 4..7. Each 128-bit lane
+// holds 8 bytes for dst_a (low 64 bits) and 8 bytes for dst_b (high 64 bits).
+// Width is a multiple of 16 bytes.
+static void TransposeWx8_Byte_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t d_a = dst_stride_a;
+ const ptrdiff_t d_b = dst_stride_b;
+ uintptr_t temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
+ "lea (%[src],%[src_stride],2),%[temp] \n"
+ "lea 0x10(%[src]),%[src] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n"
+
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n"
+ "vpunpcklbw %%ymm3,%%ymm2,%%ymm6 \n"
+ "vpunpckhbw %%ymm3,%%ymm2,%%ymm7 \n"
+
+ "vpunpcklwd %%ymm6,%%ymm4,%%ymm0 \n"
+ "vpunpckhwd %%ymm6,%%ymm4,%%ymm1 \n"
+ "vpunpcklwd %%ymm7,%%ymm5,%%ymm2 \n"
+ "vpunpckhwd %%ymm7,%%ymm5,%%ymm3 \n"
+
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm4 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm5 \n"
+ "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm6 \n"
+ "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm7 \n"
+
+ "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n"
+ "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n"
+ "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n"
+ "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n"
+
+ "vmovq %%xmm0,(%[dst_a]) \n"
+ "vmovhpd %%xmm0,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vmovq %%xmm1,(%[dst_a]) \n"
+ "vmovhpd %%xmm1,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vextracti128 $1,%%ymm0,%%xmm4 \n"
+ "vextracti128 $1,%%ymm1,%%xmm5 \n"
+ "vmovq %%xmm4,(%[dst_a]) \n"
+ "vmovhpd %%xmm4,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vmovq %%xmm5,(%[dst_a]) \n"
+ "vmovhpd %%xmm5,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vmovq %%xmm2,(%[dst_a]) \n"
+ "vmovhpd %%xmm2,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vmovq %%xmm3,(%[dst_a]) \n"
+ "vmovhpd %%xmm3,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vextracti128 $1,%%ymm2,%%xmm6 \n"
+ "vextracti128 $1,%%ymm3,%%xmm7 \n"
+ "vmovq %%xmm6,(%[dst_a]) \n"
+ "vmovhpd %%xmm6,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "vmovq %%xmm7,(%[dst_a]) \n"
+ "vmovhpd %%xmm7,(%[dst_b]) \n"
+ "add %[dst_stride_a],%[dst_a] \n"
+ "add %[dst_stride_b],%[dst_b] \n"
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
+ [width] "+rm"(byte_width), [temp] "=&r"(temp)
+ : [src_stride] "r"((ptrdiff_t)src_stride),
+ [dst_stride_a] "rm"(d_a),
+ [dst_stride_b] "rm"(d_b)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEWX8_AVX2)
+void TransposeWx8_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
+ dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEUVWX8_AVX2)
+void TransposeUVWx8_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+// Dword permutes to gather 16 rows of a column from 4 lanes.
+static const uint32_t kPermdTranspose_AVX512BW[32] = {
+ 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
+ 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
+
+// Transpose 16x16 bytes held in 4 lanes of 4 zmm registers.
+// Source row r is in lane r / 4 of zmm(r % 4). Outputs zmm0..zmm3 are 16
+// destination rows, lane n of zmm0/zmm2 for dst_a and lane n of zmm1/zmm3
+// for dst_b.
+#define TRANSPOSE16X16_AVX512BW \
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm4 \n" \
+ "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" \
+ "vpunpcklbw %%zmm3,%%zmm2,%%zmm0 \n" \
+ "vpunpckhbw %%zmm3,%%zmm2,%%zmm1 \n" \
+ "vpunpcklwd %%zmm0,%%zmm4,%%zmm2 \n" \
+ "vpunpckhwd %%zmm0,%%zmm4,%%zmm3 \n" \
+ "vpunpcklwd %%zmm1,%%zmm5,%%zmm4 \n" \
+ "vpunpckhwd %%zmm1,%%zmm5,%%zmm5 \n" \
+ "vmovdqa64 %%zmm2,%%zmm0 \n" \
+ "vpermt2d %%zmm4,%%zmm6,%%zmm0 \n" \
+ "vmovdqa64 %%zmm2,%%zmm1 \n" \
+ "vpermt2d %%zmm4,%%zmm7,%%zmm1 \n" \
+ "vmovdqa64 %%zmm3,%%zmm2 \n" \
+ "vpermt2d %%zmm5,%%zmm6,%%zmm2 \n" \
+ "vpermt2d %%zmm5,%%zmm7,%%zmm3 \n"
+
+// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
+// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
+// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
+// TODO(fbarchard): Port to rotate_win.cc using intrinsics.
+// TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes.
+// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
+// which is within 10% of AVX512BW on most CPUs.
+LIBYUV_TARGET_AVX512BW
+static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ uintptr_t temp;
+ asm volatile(
+ "vmovdqu32 (%[shuf]),%%zmm6 \n"
+ "vmovdqu32 0x40(%[shuf]),%%zmm7 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ // Main loop: 16 source rows x 16 bytes.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
+ "lea (%[src],%[src_stride],2),%[temp] \n"
+ "lea 0x10(%[src]),%[src] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
+
+ TRANSPOSE16X16_AVX512BW
+
+ "vmovdqu %%xmm0,(%[dst_a]) \n"
+ "vmovdqu %%xmm1,(%[dst_b]) \n"
+ "vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vmovdqu %%xmm2,(%[dst_a]) \n"
+ "vmovdqu %%xmm3,(%[dst_b]) \n"
+ "vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
+ "vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
+ "vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
+ "vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
+ "vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti32x4 $3,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ // Remainder: 1 to 15 bytes, masked loads.
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "je 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k[width],%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu8 (%[src]),%%xmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm1%{%%k1%}%{z%} \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm2%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm3%{%%k1%}%{z%} \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm4,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm4,%%zmm1,%%zmm1 \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm4,%%zmm2,%%zmm2 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm4,%%zmm3,%%zmm3 \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm4,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm4,%%zmm1,%%zmm1 \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm4,%%zmm2,%%zmm2 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm4,%%zmm3,%%zmm3 \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm4,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm4,%%zmm1,%%zmm1 \n"
+ "lea (%[src],%[src_stride],2),%[src] \n"
+ "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm4,%%zmm2,%%zmm2 \n"
+ "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm4,%%zmm3,%%zmm3 \n"
+
+ TRANSPOSE16X16_AVX512BW
+
+ // Store 1 destination row per remaining byte.
+ "vmovdqu %%xmm0,(%[dst_a]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vmovdqu %%xmm1,(%[dst_b]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vmovdqu %%xmm2,(%[dst_a]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vmovdqu %%xmm3,(%[dst_b]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
+ "sub $0x1,%[width] \n"
+ "je 99f \n"
+ "vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
+ [width] "+r"(byte_width), [temp] "=&r"(temp)
+ : [src_stride] "r"((ptrdiff_t)src_stride),
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a),
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b),
+ [shuf] "r"(kPermdTranspose_AVX512BW)
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#undef TRANSPOSE16X16_AVX512BW
+#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
+ // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEWX16_AVX512BW)
+void TransposeWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
+ dst + (width > 1 ? dst_stride : 0),
+ dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+void TransposeUVWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+
+#if defined(__x86_64__)
+#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
+// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
+// of ymm(r % 8). 3 unpack stages leave each lane with 2 columns x 8 rows, and
+// vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an
+// even column for dst_a and lane 1 an odd column for dst_b.
+// Width is a multiple of 16 bytes.
+static void TransposeWx16_Byte_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ uintptr_t temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
+ "lea (%[src],%[src_stride],2),%[temp] \n"
+ "lea 0x10(%[src]),%[src] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm4 \n"
+ "vmovdqu (%[temp],%[src_stride]),%%xmm5 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm6 \n"
+ "vmovdqu (%[temp],%[src_stride]),%%xmm7 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm5,%%ymm5 \n"
+ "lea (%[temp],%[src_stride],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n"
+ "vinserti128 $1,(%[temp],%[src_stride]),%%ymm7,%%ymm7 \n"
+
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm8 \n"
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm9 \n"
+ "vpunpcklbw %%ymm3,%%ymm2,%%ymm10 \n"
+ "vpunpckhbw %%ymm3,%%ymm2,%%ymm11 \n"
+ "vpunpcklbw %%ymm5,%%ymm4,%%ymm12 \n"
+ "vpunpckhbw %%ymm5,%%ymm4,%%ymm13 \n"
+ "vpunpcklbw %%ymm7,%%ymm6,%%ymm14 \n"
+ "vpunpckhbw %%ymm7,%%ymm6,%%ymm15 \n"
+
+ "vpunpcklwd %%ymm10,%%ymm8,%%ymm0 \n"
+ "vpunpckhwd %%ymm10,%%ymm8,%%ymm1 \n"
+ "vpunpcklwd %%ymm11,%%ymm9,%%ymm2 \n"
+ "vpunpckhwd %%ymm11,%%ymm9,%%ymm3 \n"
+ "vpunpcklwd %%ymm14,%%ymm12,%%ymm4 \n"
+ "vpunpckhwd %%ymm14,%%ymm12,%%ymm5 \n"
+ "vpunpcklwd %%ymm15,%%ymm13,%%ymm6 \n"
+ "vpunpckhwd %%ymm15,%%ymm13,%%ymm7 \n"
-// Transpose 8x8. 32 or 64 bit, but not NaCL for 64 bit.
-#if defined(HAS_TRANSPOSEWX8_SSSE3)
-void TransposeWx8_SSSE3(const uint8_t* src,
+ "vpunpckldq %%ymm4,%%ymm0,%%ymm8 \n"
+ "vpunpckhdq %%ymm4,%%ymm0,%%ymm9 \n"
+ "vpunpckldq %%ymm5,%%ymm1,%%ymm10 \n"
+ "vpunpckhdq %%ymm5,%%ymm1,%%ymm11 \n"
+ "vpunpckldq %%ymm6,%%ymm2,%%ymm12 \n"
+ "vpunpckhdq %%ymm6,%%ymm2,%%ymm13 \n"
+ "vpunpckldq %%ymm7,%%ymm3,%%ymm14 \n"
+ "vpunpckhdq %%ymm7,%%ymm3,%%ymm15 \n"
+
+ "vpermq $0xd8,%%ymm8,%%ymm8 \n"
+ "vpermq $0xd8,%%ymm9,%%ymm9 \n"
+ "vpermq $0xd8,%%ymm10,%%ymm10 \n"
+ "vpermq $0xd8,%%ymm11,%%ymm11 \n"
+ "vpermq $0xd8,%%ymm12,%%ymm12 \n"
+ "vpermq $0xd8,%%ymm13,%%ymm13 \n"
+ "vpermq $0xd8,%%ymm14,%%ymm14 \n"
+ "vpermq $0xd8,%%ymm15,%%ymm15 \n"
+
+ "vmovdqu %%xmm8,(%[dst_a]) \n"
+ "vextracti128 $1,%%ymm8,(%[dst_b]) \n"
+ "vmovdqu %%xmm9,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti128 $1,%%ymm9,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vmovdqu %%xmm10,(%[dst_a]) \n"
+ "vextracti128 $1,%%ymm10,(%[dst_b]) \n"
+ "vmovdqu %%xmm11,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti128 $1,%%ymm11,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vmovdqu %%xmm12,(%[dst_a]) \n"
+ "vextracti128 $1,%%ymm12,(%[dst_b]) \n"
+ "vmovdqu %%xmm13,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti128 $1,%%ymm13,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "vmovdqu %%xmm14,(%[dst_a]) \n"
+ "vextracti128 $1,%%ymm14,(%[dst_b]) \n"
+ "vmovdqu %%xmm15,(%[dst_a],%[dst_stride_a]) \n"
+ "vextracti128 $1,%%ymm15,(%[dst_b],%[dst_stride_b]) \n"
+ "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
+ "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
+ [width] "+r"(byte_width), [temp] "=&r"(temp)
+ : [src_stride] "r"((ptrdiff_t)src_stride),
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a),
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
+ "xmm15");
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_AVX2)
+void TransposeWx16_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
+ TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2,
+ dst + dst_stride, dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEUVWX16_AVX2)
+void TransposeUVWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
+#else // !defined(__x86_64__)
+#if defined(HAS_TRANSPOSEWX16_AVX2)
+void TransposeWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_AVX2(src, src_stride, dst, dst_stride, width);
+ TransposeWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8,
+ dst_stride, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEUVWX16_AVX2)
+void TransposeUVWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeUVWx8_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width);
+ TransposeUVWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst_a + 8,
+ dst_stride_a, dst_b + 8, dst_stride_b, width);
+}
+#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
+#endif // defined(__x86_64__)
+
+#if defined(HAS_TRANSPOSEWX16_16_AVX2)
+#if defined(__x86_64__)
+void TransposeWx16_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* src_b = (const uint8_t*)src;
+ uint8_t* dst_b = (uint8_t*)dst;
+ const uint8_t* temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm4 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm6 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm5,%%ymm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm7,%%ymm7 \n"
+ "add $0x10,%[src] \n"
+
+ // Stage 1: 16-bit unpack -> ymm8..ymm15
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm8 \n"
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm9 \n"
+ "vpunpcklwd %%ymm3,%%ymm2,%%ymm10 \n"
+ "vpunpckhwd %%ymm3,%%ymm2,%%ymm11 \n"
+ "vpunpcklwd %%ymm5,%%ymm4,%%ymm12 \n"
+ "vpunpckhwd %%ymm5,%%ymm4,%%ymm13 \n"
+ "vpunpcklwd %%ymm7,%%ymm6,%%ymm14 \n"
+ "vpunpckhwd %%ymm7,%%ymm6,%%ymm15 \n"
+
+ // Stage 2: 32-bit unpack -> ymm0..ymm7
+ "vpunpckldq %%ymm10,%%ymm8,%%ymm0 \n"
+ "vpunpckhdq %%ymm10,%%ymm8,%%ymm1 \n"
+ "vpunpckldq %%ymm11,%%ymm9,%%ymm2 \n"
+ "vpunpckhdq %%ymm11,%%ymm9,%%ymm3 \n"
+ "vpunpckldq %%ymm14,%%ymm12,%%ymm4 \n"
+ "vpunpckhdq %%ymm14,%%ymm12,%%ymm5 \n"
+ "vpunpckldq %%ymm15,%%ymm13,%%ymm6 \n"
+ "vpunpckhdq %%ymm15,%%ymm13,%%ymm7 \n"
+
+ // Stage 3: 64-bit unpack -> ymm8..ymm15
+ "vpunpcklqdq %%ymm4,%%ymm0,%%ymm8 \n"
+ "vpunpckhqdq %%ymm4,%%ymm0,%%ymm9 \n"
+ "vpunpcklqdq %%ymm5,%%ymm1,%%ymm10 \n"
+ "vpunpckhqdq %%ymm5,%%ymm1,%%ymm11 \n"
+ "vpunpcklqdq %%ymm6,%%ymm2,%%ymm12 \n"
+ "vpunpckhqdq %%ymm6,%%ymm2,%%ymm13 \n"
+ "vpunpcklqdq %%ymm7,%%ymm3,%%ymm14 \n"
+ "vpunpckhqdq %%ymm7,%%ymm3,%%ymm15 \n"
+
+ "vmovdqu %%ymm8,(%[dst]) \n"
+ "vmovdqu %%ymm9,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%ymm10,(%[dst]) \n"
+ "vmovdqu %%ymm11,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%ymm12,(%[dst]) \n"
+ "vmovdqu %%ymm13,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%ymm14,(%[dst]) \n"
+ "vmovdqu %%ymm15,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x8,%[w] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_b), // %[src]
+ [dst] "+r"(dst_b), // %[dst]
+ [w] "+r"(width), // %[w]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
+ "xmm15");
+}
+#endif // defined(__x86_64__)
+#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX8_16_AVX2)
+void TransposeWx8_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* src_b = (const uint8_t*)src;
+ uint8_t* dst_b = (uint8_t*)dst;
+ const uint8_t* temp;
asm volatile(
- // Read in the data from the source pointer.
- // First round of bit swap.
- LABELALIGN
"1: \n"
- "movq (%0),%%xmm0 \n"
- "movq (%0,%3),%%xmm1 \n"
- "lea (%0,%3,2),%0 \n"
- "punpcklbw %%xmm1,%%xmm0 \n"
- "movq (%0),%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "palignr $0x8,%%xmm1,%%xmm1 \n"
- "movq (%0,%3),%%xmm3 \n"
- "lea (%0,%3,2),%0 \n"
- "punpcklbw %%xmm3,%%xmm2 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "movq (%0),%%xmm4 \n"
- "palignr $0x8,%%xmm3,%%xmm3 \n"
- "movq (%0,%3),%%xmm5 \n"
- "lea (%0,%3,2),%0 \n"
- "punpcklbw %%xmm5,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "movq (%0),%%xmm6 \n"
- "palignr $0x8,%%xmm5,%%xmm5 \n"
- "movq (%0,%3),%%xmm7 \n"
- "lea (%0,%3,2),%0 \n"
- "punpcklbw %%xmm7,%%xmm6 \n"
- "neg %3 \n"
- "movdqa %%xmm6,%%xmm7 \n"
- "lea 0x8(%0,%3,8),%0 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- "neg %3 \n"
- // Second round of bit swap.
- "punpcklwd %%xmm2,%%xmm0 \n"
- "punpcklwd %%xmm3,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "movdqa %%xmm1,%%xmm3 \n"
- "palignr $0x8,%%xmm2,%%xmm2 \n"
- "palignr $0x8,%%xmm3,%%xmm3 \n"
- "punpcklwd %%xmm6,%%xmm4 \n"
- "punpcklwd %%xmm7,%%xmm5 \n"
- "movdqa %%xmm4,%%xmm6 \n"
- "movdqa %%xmm5,%%xmm7 \n"
- "palignr $0x8,%%xmm6,%%xmm6 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- // Third round of bit swap.
- // Write to the destination pointer.
- "punpckldq %%xmm4,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "movdqa %%xmm0,%%xmm4 \n"
- "palignr $0x8,%%xmm4,%%xmm4 \n"
- "movq %%xmm4,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm6,%%xmm2 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "movq %%xmm2,(%1) \n"
- "palignr $0x8,%%xmm6,%%xmm6 \n"
- "punpckldq %%xmm5,%%xmm1 \n"
- "movq %%xmm6,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "movdqa %%xmm1,%%xmm5 \n"
- "movq %%xmm1,(%1) \n"
- "palignr $0x8,%%xmm5,%%xmm5 \n"
- "movq %%xmm5,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm7,%%xmm3 \n"
- "movq %%xmm3,(%1) \n"
- "movdqa %%xmm3,%%xmm7 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- "sub $0x8,%2 \n"
- "movq %%xmm7,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "r"((intptr_t)(dst_stride)) // %4
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n"
+ "add $0x10,%[src] \n"
+
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm5 \n"
+ "vpunpcklwd %%ymm3,%%ymm2,%%ymm6 \n"
+ "vpunpckhwd %%ymm3,%%ymm2,%%ymm7 \n"
+
+ "vpunpckldq %%ymm6,%%ymm4,%%ymm0 \n"
+ "vpunpckhdq %%ymm6,%%ymm4,%%ymm1 \n"
+ "vpunpckldq %%ymm7,%%ymm5,%%ymm2 \n"
+ "vpunpckhdq %%ymm7,%%ymm5,%%ymm3 \n"
+
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm2,%%ymm2 \n"
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "vextracti128 $1,%%ymm0,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%xmm1,(%[dst]) \n"
+ "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%xmm2,(%[dst]) \n"
+ "vextracti128 $1,%%ymm2,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%xmm3,(%[dst]) \n"
+ "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "subl $0x8,%[w] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_b),
+ [dst] "+r"(dst_b),
+#if defined(__i386__)
+ [w] "+m"(width),
+#else
+ [w] "+rm"(width),
+#endif
+ [temp] "=&r"(temp)
+ : [s] "r"(s),
+ [d] "r"(d)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
-
-// Transpose 16x8. 64 bit
-#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
-void TransposeWx8_Fast_SSSE3(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
+#endif // defined(HAS_TRANSPOSEWX8_16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__)
+void TransposeWx16_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_16_AVX2(src, src_stride, dst, dst_stride, width);
+ TransposeWx8_16_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8,
+ dst_stride, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__)
+
+#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+// Transpose 32 rows of 16 bit plane (arbitrary width) using 8x32 tiles in
+// zmm0..zmm15 with k1-masked right tail.
+LIBYUV_TARGET_AVX512BW
+void TransposeWx32_16_AVX512BW(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* src_b = (const uint8_t*)src;
+ uint8_t* dst_b = (uint8_t*)dst;
+ const int n = width & ~7;
+ if (n > 0) {
+ int cnt = n;
+ const uint8_t* temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm4 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm6 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm4,%%zmm4 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm6,%%zmm6 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm4,%%zmm4 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm6,%%zmm6 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm4,%%zmm4 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm6,%%zmm6 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
+ "add $0x10,%[src] \n"
+
+ // Stage 1: 16-bit unpack -> zmm8..zmm15
+ "vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n"
+ "vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n"
+ "vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n"
+ "vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n"
+ "vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n"
+ "vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n"
+ "vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n"
+ "vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n"
+
+ // Stage 2: 32-bit unpack -> zmm0..zmm7
+ "vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n"
+ "vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n"
+ "vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n"
+ "vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n"
+ "vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n"
+ "vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n"
+ "vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n"
+ "vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n"
+
+ // Stage 3: 64-bit unpack -> zmm8..zmm15
+ "vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n"
+ "vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n"
+ "vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n"
+ "vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n"
+ "vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n"
+ "vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n"
+ "vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n"
+
+ "vmovdqu16 %%zmm8,(%[dst]) \n"
+ "vmovdqu16 %%zmm9,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm10,(%[dst]) \n"
+ "vmovdqu16 %%zmm11,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm12,(%[dst]) \n"
+ "vmovdqu16 %%zmm13,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm14,(%[dst]) \n"
+ "vmovdqu16 %%zmm15,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x8,%[cnt] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_b), // %[src]
+ [dst] "+r"(dst_b), // %[dst]
+ [cnt] "+r"(cnt), // %[cnt]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",
+ "xmm14", "xmm15");
+ width -= n;
+ }
+ if (width > 0) {
+ const uint32_t col_mask = (1u << width) - 1;
+ const uint8_t* temp;
+ asm volatile(
+ "kmovw %[col_mask],%%k1 \n"
+ "vmovdqu16 (%[src]),%%xmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[src],%[s]),%%xmm1%{%%k1%}%{z%} \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm2%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm3%{%%k1%}%{z%} \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm4%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm5%{%%k1%}%{z%} \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm6%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm7%{%%k1%}%{z%} \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm2,%%zmm2 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm4,%%zmm4 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm16,%%zmm7,%%zmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm2,%%zmm2 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm4,%%zmm4 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm16,%%zmm7,%%zmm7 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm2,%%zmm2 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm4,%%zmm4 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm5,%%zmm5 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu16 (%[temp]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[temp],%[s]),%%xmm16%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm16,%%zmm7,%%zmm7 \n"
+
+ // Stage 1: 16-bit unpack -> zmm8..zmm15
+ "vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n"
+ "vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n"
+ "vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n"
+ "vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n"
+ "vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n"
+ "vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n"
+ "vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n"
+ "vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n"
+
+ // Stage 2: 32-bit unpack -> zmm0..zmm7
+ "vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n"
+ "vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n"
+ "vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n"
+ "vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n"
+ "vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n"
+ "vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n"
+ "vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n"
+ "vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n"
+
+ // Stage 3: 64-bit unpack -> zmm8..zmm15
+ "vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n"
+ "vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n"
+ "vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n"
+ "vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n"
+ "vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n"
+ "vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n"
+ "vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n"
+
+ "vmovdqu16 %%zmm8,(%[dst]) \n"
+ "cmp $1,%[width] \n"
+ "jle 2f \n"
+ "vmovdqu16 %%zmm9,(%[dst],%[d]) \n"
+ "cmp $2,%[width] \n"
+ "jle 2f \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm10,(%[dst]) \n"
+ "cmp $3,%[width] \n"
+ "jle 2f \n"
+ "vmovdqu16 %%zmm11,(%[dst],%[d]) \n"
+ "cmp $4,%[width] \n"
+ "jle 2f \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm12,(%[dst]) \n"
+ "cmp $5,%[width] \n"
+ "jle 2f \n"
+ "vmovdqu16 %%zmm13,(%[dst],%[d]) \n"
+ "cmp $6,%[width] \n"
+ "jle 2f \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu16 %%zmm14,(%[dst]) \n"
+ "2: \n"
+ : [dst] "+r"(dst_b), // %[dst]
+ [temp] "=&r"(temp) // %[temp]
+ : [src] "r"(src_b), // %[src]
+ [s] "r"(s), // %[s]
+ [d] "r"(d), // %[d]
+ [col_mask] "r"(col_mask), // %[col_mask]
+ [width] "r"(width) // %[width]
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",
+ "xmm14", "xmm15", "xmm16");
+ }
+ asm volatile("vzeroupper \n");
+}
+#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEWX4_32_AVX2)
+// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per
+// AVX2 iteration (with a 4-row VEX 128-bit tail) and write 4 destination rows.
+void TransposeWx4_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ uintptr_t temp;
asm volatile(
- // Read in the data from the source pointer.
- // First round of bit swap.
- LABELALIGN
+ "subl $0x8,%[width] \n"
+ "jl 2f \n"
+
+ // Main loop: 8 source rows of 4 pixels -> 4 dst rows of 8 pixels (32 B).
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu (%0,%3),%%xmm1 \n"
- "lea (%0,%3,2),%0 \n"
- "movdqa %%xmm0,%%xmm8 \n"
- "punpcklbw %%xmm1,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm8 \n"
- "movdqu (%0),%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm8,%%xmm9 \n"
- "palignr $0x8,%%xmm1,%%xmm1 \n"
- "palignr $0x8,%%xmm9,%%xmm9 \n"
- "movdqu (%0,%3),%%xmm3 \n"
- "lea (%0,%3,2),%0 \n"
- "movdqa %%xmm2,%%xmm10 \n"
- "punpcklbw %%xmm3,%%xmm2 \n"
- "punpckhbw %%xmm3,%%xmm10 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "movdqa %%xmm10,%%xmm11 \n"
- "movdqu (%0),%%xmm4 \n"
- "palignr $0x8,%%xmm3,%%xmm3 \n"
- "palignr $0x8,%%xmm11,%%xmm11 \n"
- "movdqu (%0,%3),%%xmm5 \n"
- "lea (%0,%3,2),%0 \n"
- "movdqa %%xmm4,%%xmm12 \n"
- "punpcklbw %%xmm5,%%xmm4 \n"
- "punpckhbw %%xmm5,%%xmm12 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "movdqa %%xmm12,%%xmm13 \n"
- "movdqu (%0),%%xmm6 \n"
- "palignr $0x8,%%xmm5,%%xmm5 \n"
- "palignr $0x8,%%xmm13,%%xmm13 \n"
- "movdqu (%0,%3),%%xmm7 \n"
- "lea (%0,%3,2),%0 \n"
- "movdqa %%xmm6,%%xmm14 \n"
- "punpcklbw %%xmm7,%%xmm6 \n"
- "punpckhbw %%xmm7,%%xmm14 \n"
- "neg %3 \n"
- "movdqa %%xmm6,%%xmm7 \n"
- "movdqa %%xmm14,%%xmm15 \n"
- "lea 0x10(%0,%3,8),%0 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- "palignr $0x8,%%xmm15,%%xmm15 \n"
- "neg %3 \n"
- // Second round of bit swap.
- "punpcklwd %%xmm2,%%xmm0 \n"
- "punpcklwd %%xmm3,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "movdqa %%xmm1,%%xmm3 \n"
- "palignr $0x8,%%xmm2,%%xmm2 \n"
- "palignr $0x8,%%xmm3,%%xmm3 \n"
- "punpcklwd %%xmm6,%%xmm4 \n"
- "punpcklwd %%xmm7,%%xmm5 \n"
- "movdqa %%xmm4,%%xmm6 \n"
- "movdqa %%xmm5,%%xmm7 \n"
- "palignr $0x8,%%xmm6,%%xmm6 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- "punpcklwd %%xmm10,%%xmm8 \n"
- "punpcklwd %%xmm11,%%xmm9 \n"
- "movdqa %%xmm8,%%xmm10 \n"
- "movdqa %%xmm9,%%xmm11 \n"
- "palignr $0x8,%%xmm10,%%xmm10 \n"
- "palignr $0x8,%%xmm11,%%xmm11 \n"
- "punpcklwd %%xmm14,%%xmm12 \n"
- "punpcklwd %%xmm15,%%xmm13 \n"
- "movdqa %%xmm12,%%xmm14 \n"
- "movdqa %%xmm13,%%xmm15 \n"
- "palignr $0x8,%%xmm14,%%xmm14 \n"
- "palignr $0x8,%%xmm15,%%xmm15 \n"
- // Third round of bit swap.
- // Write to the destination pointer.
- "punpckldq %%xmm4,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "movdqa %%xmm0,%%xmm4 \n"
- "palignr $0x8,%%xmm4,%%xmm4 \n"
- "movq %%xmm4,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm6,%%xmm2 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "movq %%xmm2,(%1) \n"
- "palignr $0x8,%%xmm6,%%xmm6 \n"
- "punpckldq %%xmm5,%%xmm1 \n"
- "movq %%xmm6,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "movdqa %%xmm1,%%xmm5 \n"
- "movq %%xmm1,(%1) \n"
- "palignr $0x8,%%xmm5,%%xmm5 \n"
- "movq %%xmm5,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm7,%%xmm3 \n"
- "movq %%xmm3,(%1) \n"
- "movdqa %%xmm3,%%xmm7 \n"
- "palignr $0x8,%%xmm7,%%xmm7 \n"
- "movq %%xmm7,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm12,%%xmm8 \n"
- "movq %%xmm8,(%1) \n"
- "movdqa %%xmm8,%%xmm12 \n"
- "palignr $0x8,%%xmm12,%%xmm12 \n"
- "movq %%xmm12,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm14,%%xmm10 \n"
- "movdqa %%xmm10,%%xmm14 \n"
- "movq %%xmm10,(%1) \n"
- "palignr $0x8,%%xmm14,%%xmm14 \n"
- "punpckldq %%xmm13,%%xmm9 \n"
- "movq %%xmm14,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "movdqa %%xmm9,%%xmm13 \n"
- "movq %%xmm9,(%1) \n"
- "palignr $0x8,%%xmm13,%%xmm13 \n"
- "movq %%xmm13,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "punpckldq %%xmm15,%%xmm11 \n"
- "movq %%xmm11,(%1) \n"
- "movdqa %%xmm11,%%xmm15 \n"
- "palignr $0x8,%%xmm15,%%xmm15 \n"
- "sub $0x10,%2 \n"
- "movq %%xmm15,(%1,%4) \n"
- "lea (%1,%4,2),%1 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "r"((intptr_t)(dst_stride)) // %4
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n"
+ "lea (%[temp],%[s],2),%[src] \n"
+
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n"
+ "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n"
+
+ "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n"
+ "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n"
+ "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n"
+ "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n"
+
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "vmovdqu %%ymm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[temp] \n"
+ "vmovdqu %%ymm2,(%[temp]) \n"
+ "vmovdqu %%ymm3,(%[temp],%[d]) \n"
+ "add $0x20,%[dst] \n"
+ "subl $0x8,%[width] \n"
+ "jge 1b \n"
+
+ // Tail: 4 source rows of 4 pixels -> 4 dst rows of 4 pixels (16 B).
+ "2: \n"
+ "addl $0x8,%[width] \n"
+ "je 99f \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
+ "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
+ "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
+
+ "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
+ "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
+ "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
+ "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
+
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "vmovdqu %%xmm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[temp] \n"
+ "vmovdqu %%xmm2,(%[temp]) \n"
+ "vmovdqu %%xmm3,(%[temp],%[d]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+rm"(width), // %[width]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
- "xmm15");
+ "xmm7");
}
-#endif // defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
+#endif // defined(HAS_TRANSPOSEWX4_32_AVX2)
-// Transpose UV 8x8. 64 bit.
-#if defined(HAS_TRANSPOSEUVWX8_SSE2)
-void TransposeUVWx8_SSE2(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width) {
+#if defined(HAS_TRANSPOSEWX8_32_AVX2)
+// Transpose 8 rows of 32 bit plane (width multiple of 4) using 4x8 YMM vertical
+// tiles in ymm0..ymm7.
+void TransposeWx8_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const uint8_t* temp;
asm volatile(
- // Read in the data from the source pointer.
- // First round of bit swap.
- LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu (%0,%4),%%xmm1 \n"
- "lea (%0,%4,2),%0 \n"
- "movdqa %%xmm0,%%xmm8 \n"
- "punpcklbw %%xmm1,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm8 \n"
- "movdqa %%xmm8,%%xmm1 \n"
- "movdqu (%0),%%xmm2 \n"
- "movdqu (%0,%4),%%xmm3 \n"
- "lea (%0,%4,2),%0 \n"
- "movdqa %%xmm2,%%xmm8 \n"
- "punpcklbw %%xmm3,%%xmm2 \n"
- "punpckhbw %%xmm3,%%xmm8 \n"
- "movdqa %%xmm8,%%xmm3 \n"
- "movdqu (%0),%%xmm4 \n"
- "movdqu (%0,%4),%%xmm5 \n"
- "lea (%0,%4,2),%0 \n"
- "movdqa %%xmm4,%%xmm8 \n"
- "punpcklbw %%xmm5,%%xmm4 \n"
- "punpckhbw %%xmm5,%%xmm8 \n"
- "movdqa %%xmm8,%%xmm5 \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu (%0,%4),%%xmm7 \n"
- "lea (%0,%4,2),%0 \n"
- "movdqa %%xmm6,%%xmm8 \n"
- "punpcklbw %%xmm7,%%xmm6 \n"
- "neg %4 \n"
- "lea 0x10(%0,%4,8),%0 \n"
- "punpckhbw %%xmm7,%%xmm8 \n"
- "movdqa %%xmm8,%%xmm7 \n"
- "neg %4 \n"
- // Second round of bit swap.
- "movdqa %%xmm0,%%xmm8 \n"
- "movdqa %%xmm1,%%xmm9 \n"
- "punpckhwd %%xmm2,%%xmm8 \n"
- "punpckhwd %%xmm3,%%xmm9 \n"
- "punpcklwd %%xmm2,%%xmm0 \n"
- "punpcklwd %%xmm3,%%xmm1 \n"
- "movdqa %%xmm8,%%xmm2 \n"
- "movdqa %%xmm9,%%xmm3 \n"
- "movdqa %%xmm4,%%xmm8 \n"
- "movdqa %%xmm5,%%xmm9 \n"
- "punpckhwd %%xmm6,%%xmm8 \n"
- "punpckhwd %%xmm7,%%xmm9 \n"
- "punpcklwd %%xmm6,%%xmm4 \n"
- "punpcklwd %%xmm7,%%xmm5 \n"
- "movdqa %%xmm8,%%xmm6 \n"
- "movdqa %%xmm9,%%xmm7 \n"
- // Third round of bit swap.
- // Write to the destination pointer.
- "movdqa %%xmm0,%%xmm8 \n"
- "punpckldq %%xmm4,%%xmm0 \n"
- "movlpd %%xmm0,(%1) \n" // Write back U channel
- "movhpd %%xmm0,(%2) \n" // Write back V channel
- "punpckhdq %%xmm4,%%xmm8 \n"
- "movlpd %%xmm8,(%1,%5) \n"
- "lea (%1,%5,2),%1 \n"
- "movhpd %%xmm8,(%2,%6) \n"
- "lea (%2,%6,2),%2 \n"
- "movdqa %%xmm2,%%xmm8 \n"
- "punpckldq %%xmm6,%%xmm2 \n"
- "movlpd %%xmm2,(%1) \n"
- "movhpd %%xmm2,(%2) \n"
- "punpckhdq %%xmm6,%%xmm8 \n"
- "movlpd %%xmm8,(%1,%5) \n"
- "lea (%1,%5,2),%1 \n"
- "movhpd %%xmm8,(%2,%6) \n"
- "lea (%2,%6,2),%2 \n"
- "movdqa %%xmm1,%%xmm8 \n"
- "punpckldq %%xmm5,%%xmm1 \n"
- "movlpd %%xmm1,(%1) \n"
- "movhpd %%xmm1,(%2) \n"
- "punpckhdq %%xmm5,%%xmm8 \n"
- "movlpd %%xmm8,(%1,%5) \n"
- "lea (%1,%5,2),%1 \n"
- "movhpd %%xmm8,(%2,%6) \n"
- "lea (%2,%6,2),%2 \n"
- "movdqa %%xmm3,%%xmm8 \n"
- "punpckldq %%xmm7,%%xmm3 \n"
- "movlpd %%xmm3,(%1) \n"
- "movhpd %%xmm3,(%2) \n"
- "punpckhdq %%xmm7,%%xmm8 \n"
- "sub $0x8,%3 \n"
- "movlpd %%xmm8,(%1,%5) \n"
- "lea (%1,%5,2),%1 \n"
- "movhpd %%xmm8,(%2,%6) \n"
- "lea (%2,%6,2),%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst_a), // %1
- "+r"(dst_b), // %2
- "+r"(width) // %3
- : "r"((intptr_t)(src_stride)), // %4
- "r"((intptr_t)(dst_stride_a)), // %5
- "r"((intptr_t)(dst_stride_b)) // %6
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
+ "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n"
+ "add $0x10,%[src] \n"
+
+ // Stage 1: 32-bit unpack across both 128-bit lanes
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n"
+ "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n"
+
+ // Stage 2: 64-bit unpack across both 128-bit lanes
+ "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n"
+ "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n"
+ "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n"
+ "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n"
+
+ // Store 4 full 256-bit (32 B) rows = 8 32-bit pixels each
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "vmovdqu %%ymm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%ymm2,(%[dst]) \n"
+ "vmovdqu %%ymm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "subl $0x4,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+rm"(width), // %[width]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // defined(HAS_TRANSPOSEWX8_32_AVX2)
+
+#if defined(HAS_TRANSPOSEWXH_32_AVX2)
+// Transpose rem_w (1..3) columns x 8 rows of 32-bit pixels using AVX2.
+static void TransposeRemWx8_32_AVX2(const uint8_t* src,
+ ptrdiff_t s,
+ uint8_t* dst,
+ ptrdiff_t d,
+ int rem_w) {
+ const uint8_t* temp;
+ asm volatile(
+ "cmpl $2,%[rem_w] \n"
+ "jl 1f \n"
+ "je 2f \n"
+
+ // rem_w == 3: load 12 bytes per row from 8 rows.
+ "vmovq (%[src]),%%xmm0 \n"
+ "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n"
+ "vmovq (%[src],%[s]),%%xmm1 \n"
+ "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm2 \n"
+ "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm3 \n"
+ "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm4 \n"
+ "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n"
+ "vmovq (%[temp],%[s]),%%xmm5 \n"
+ "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm4 \n"
+ "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm5 \n"
+ "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n"
+ "jmp 3f \n"
+
+ // rem_w == 2: load 8 bytes per row from 8 rows.
+ "2: \n"
+ "vmovq (%[src]),%%xmm0 \n"
+ "vmovq (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n"
+ "vmovq (%[temp],%[s]),%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n"
+ "jmp 3f \n"
+
+ // rem_w == 1: load 4 bytes per row from 8 rows.
+ "1: \n"
+ "vmovd (%[src]),%%xmm0 \n"
+ "vmovd (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovd (%[temp]),%%xmm2 \n"
+ "vmovd (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovd (%[temp]),%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n"
+ "vmovd (%[temp],%[s]),%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovd (%[temp]),%%xmm4 \n"
+ "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n"
+ "vmovd (%[temp],%[s]),%%xmm5 \n"
+ "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n"
+
+ "3: \n"
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
+ "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n"
+ "vmovdqu %%ymm6,(%[dst]) \n"
+ "cmpl $1,%[rem_w] \n"
+ "jle 4f \n"
+ "vpunpckhqdq %%ymm5,%%ymm4,%%ymm7 \n"
+ "vmovdqu %%ymm7,(%[dst],%[d]) \n"
+ "cmpl $2,%[rem_w] \n"
+ "jle 4f \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckhdq %%ymm3,%%ymm2,%%ymm5 \n"
+ "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n"
+ "vmovdqu %%ymm6,(%[dst],%[d],2) \n"
+ "4: \n"
+ "vzeroupper \n"
+ : [temp] "=&r"(temp)
+ : [src] "r"(src),
+ [s] "r"(s),
+ [dst] "r"(dst),
+ [d] "r"(d),
+ [rem_w] "rm"(rem_w)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7", "xmm8", "xmm9");
+ "xmm7");
+}
+
+// Transpose rem_w (1..3) columns x 4 rows of 32-bit pixels using VEX 128-bit.
+static void TransposeRemWx4_32_AVX2(const uint8_t* src,
+ ptrdiff_t s,
+ uint8_t* dst,
+ ptrdiff_t d,
+ int rem_w) {
+ const uint8_t* temp;
+ asm volatile(
+ "cmpl $2,%[rem_w] \n"
+ "jl 1f \n"
+ "je 2f \n"
+
+ // rem_w == 3: load 12 bytes per row from 4 rows.
+ "vmovq (%[src]),%%xmm0 \n"
+ "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n"
+ "vmovq (%[src],%[s]),%%xmm1 \n"
+ "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm2 \n"
+ "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm3 \n"
+ "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n"
+ "jmp 3f \n"
+
+ // rem_w == 2: load 8 bytes per row from 4 rows.
+ "2: \n"
+ "vmovq (%[src]),%%xmm0 \n"
+ "vmovq (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovq (%[temp]),%%xmm2 \n"
+ "vmovq (%[temp],%[s]),%%xmm3 \n"
+ "jmp 3f \n"
+
+ // rem_w == 1: load 4 bytes per row from 4 rows.
+ "1: \n"
+ "vmovd (%[src]),%%xmm0 \n"
+ "vmovd (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovd (%[temp]),%%xmm2 \n"
+ "vmovd (%[temp],%[s]),%%xmm3 \n"
+
+ "3: \n"
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
+ "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n"
+ "vmovdqu %%xmm6,(%[dst]) \n"
+ "cmpl $1,%[rem_w] \n"
+ "jle 4f \n"
+ "vpunpckhqdq %%xmm5,%%xmm4,%%xmm7 \n"
+ "vmovdqu %%xmm7,(%[dst],%[d]) \n"
+ "cmpl $2,%[rem_w] \n"
+ "jle 4f \n"
+ "vpunpckhdq %%xmm1,%%xmm0,%%xmm4 \n"
+ "vpunpckhdq %%xmm3,%%xmm2,%%xmm5 \n"
+ "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n"
+ "vmovdqu %%xmm6,(%[dst],%[d],2) \n"
+ "4: \n"
+ : [temp] "=&r"(temp)
+ : [src] "r"(src),
+ [s] "r"(s),
+ [dst] "r"(dst),
+ [d] "r"(d),
+ [rem_w] "rm"(rem_w)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+
+// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3}
+// tiles plus exact remainder handling).
+void TransposeWxH_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const int w4 = width & ~3;
+ const int rem_w = width & 3;
+
+ // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail.
+ while (height >= 8) {
+ if (w4 > 0) {
+ TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4);
+ }
+ if (rem_w > 0) {
+ TransposeRemWx8_32_AVX2(src + (ptrdiff_t)w4 * 4, s,
+ dst + (ptrdiff_t)w4 * d, d, rem_w);
+ }
+ src += 8 * s;
+ dst += 32;
+ height -= 8;
+ }
+
+ // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail.
+ if (height >= 4) {
+ const int w8 = width & ~7;
+ if (w8 > 0) {
+ const uint8_t* src_p = src;
+ uint8_t* dst_p = dst;
+ int cnt = w8;
+ const uint8_t* temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "vmovdqu (%[src],%[s]),%%ymm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%ymm2 \n"
+ "vmovdqu (%[temp],%[s]),%%ymm3 \n"
+ "add $0x20,%[src] \n"
+
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
+ "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n"
+ "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n"
+
+ "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n"
+ "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n"
+ "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n"
+ "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n"
+
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "vmovdqu %%xmm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu %%xmm2,(%[dst]) \n"
+ "vmovdqu %%xmm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vextracti128 $1,%%ymm0,(%[dst]) \n"
+ "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vextracti128 $1,%%ymm2,(%[dst]) \n"
+ "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "subl $0x8,%[cnt] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_p),
+ [dst] "+r"(dst_p),
+ [cnt] "+rm"(cnt),
+ [temp] "=&r"(temp)
+ : [s] "r"(s),
+ [d] "r"(d)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ }
+ if (width & 4) {
+ const uint8_t* src_p = src + (ptrdiff_t)w8 * 4;
+ uint8_t* dst_p = dst + (ptrdiff_t)w8 * d;
+ uintptr_t temp;
+ asm volatile(
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
+ "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
+ "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
+
+ "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
+ "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
+ "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
+ "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
+
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "vmovdqu %%xmm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[temp] \n"
+ "vmovdqu %%xmm2,(%[temp]) \n"
+ "vmovdqu %%xmm3,(%[temp],%[d]) \n"
+ : [temp] "=&r"(temp)
+ : [src] "r"(src_p),
+ [dst] "r"(dst_p),
+ [s] "r"(s),
+ [d] "r"(d)
+ : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+ }
+ if (rem_w > 0) {
+ TransposeRemWx4_32_AVX2(src + (ptrdiff_t)w4 * 4, s,
+ dst + (ptrdiff_t)w4 * d, d, rem_w);
+ }
+ src += 4 * s;
+ dst += 16;
+ height -= 4;
+ }
+
+ // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner.
+ if (height > 0) {
+ if (w4 > 0) {
+ const uint8_t* src_p = src;
+ uint8_t* dst_p = dst;
+ int cnt = w4;
+ if (height == 1) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "add $0x10,%[src] \n"
+ "vmovd %%xmm0,(%[dst]) \n"
+ "vpextrd $1,%%xmm0,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vpextrd $2,%%xmm0,(%[dst]) \n"
+ "vpextrd $3,%%xmm0,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x4,%[cnt] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_p),
+ [dst] "+r"(dst_p),
+ [cnt] "+r"(cnt)
+ : [d] "r"(d)
+ : "memory", "cc", "xmm0");
+ } else if (height == 2) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "add $0x10,%[src] \n"
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm2 \n"
+ "vpunpckhdq %%xmm1,%%xmm0,%%xmm3 \n"
+ "vmovq %%xmm2,(%[dst]) \n"
+ "vmovhpd %%xmm2,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovq %%xmm3,(%[dst]) \n"
+ "vmovhpd %%xmm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x4,%[cnt] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_p),
+ [dst] "+r"(dst_p),
+ [cnt] "+r"(cnt)
+ : [s] "r"(s),
+ [d] "r"(d)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ } else {
+ asm volatile(
+ "vpxor %%xmm3,%%xmm3,%%xmm3 \n"
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "vmovdqu (%[src],%[s],2),%%xmm2 \n"
+ "add $0x10,%[src] \n"
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
+ "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
+ "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
+ "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
+ "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
+ "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
+ "vmovq %%xmm0,(%[dst]) \n"
+ "vpextrd $2,%%xmm0,8(%[dst]) \n"
+ "vmovq %%xmm1,(%[dst],%[d]) \n"
+ "vpextrd $2,%%xmm1,8(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovq %%xmm2,(%[dst]) \n"
+ "vpextrd $2,%%xmm2,8(%[dst]) \n"
+ "vpunpckhqdq %%xmm7,%%xmm6,%%xmm4 \n"
+ "vmovq %%xmm4,(%[dst],%[d]) \n"
+ "vpextrd $2,%%xmm4,8(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x4,%[cnt] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_p),
+ [dst] "+r"(dst_p),
+ [cnt] "+r"(cnt)
+ : [s] "r"(s),
+ [d] "r"(d)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ }
+ }
+ for (int x = w4; x < width; ++x) {
+ uint8_t* dst_row = dst + (ptrdiff_t)x * d;
+ for (int y = 0; y < height; ++y) {
+ memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4);
+ }
+ }
+ }
}
-#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
-#endif // defined(__x86_64__) || defined(__i386__)
+#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+// Transpose 16 rows of 32 bit plane (arbitrary width) using 4x16 ZMM vertical
+// tiles in zmm0..zmm7 with k1-masked right tail.
+LIBYUV_TARGET_AVX512BW
+void TransposeWx16_32_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const int n = width & ~3;
+ if (n > 0) {
+ int cnt = n;
+ const uint8_t* temp;
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vmovdqu (%[src],%[s]),%%xmm1 \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm2 \n"
+ "vmovdqu (%[temp],%[s]),%%xmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
+ "vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
+ "add $0x10,%[src] \n"
+
+ // Stage 1: 32-bit unpack across all 4 128-bit lanes
+ "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
+ "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
+ "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
+ "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
+
+ // Stage 2: 64-bit unpack across all 4 128-bit lanes
+ "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
+ "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
+ "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
+ "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
+
+ // Store 4 full 512-bit (64 B) rows = 16 32-bit pixels each
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "vmovdqu32 %%zmm1,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu32 %%zmm2,(%[dst]) \n"
+ "vmovdqu32 %%zmm3,(%[dst],%[d]) \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "sub $0x4,%[cnt] \n"
+ "jg 1b \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [cnt] "+r"(cnt), // %[cnt]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ width -= n;
+ }
+ if (width > 0) {
+ const uint32_t col_mask = (1u << width) - 1;
+ const uint8_t* temp;
+ asm volatile(
+ "kmovw %[col_mask],%%k1 \n"
+ "vmovdqu32 (%[src]),%%xmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 (%[src],%[s]),%%xmm1%{%%k1%}%{z%} \n"
+ "lea (%[src],%[s],2),%[temp] \n"
+ "vmovdqu32 (%[temp]),%%xmm2%{%%k1%}%{z%} \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm3%{%%k1%}%{z%} \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm8,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm8,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm8,%%zmm2,%%zmm2 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $1,%%xmm8,%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm8,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm8,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm8,%%zmm2,%%zmm2 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $2,%%xmm8,%%zmm3,%%zmm3 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm8,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm8,%%zmm1,%%zmm1 \n"
+ "lea (%[temp],%[s],2),%[temp] \n"
+ "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm8,%%zmm2,%%zmm2 \n"
+ "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
+ "vinserti32x4 $3,%%xmm8,%%zmm3,%%zmm3 \n"
+
+ "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
+ "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
+ "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
+ "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
+
+ "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
+ "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
+ "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
+ "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
+
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "cmp $1,%[width] \n"
+ "jle 2f \n"
+ "vmovdqu32 %%zmm1,(%[dst],%[d]) \n"
+ "cmp $2,%[width] \n"
+ "jle 2f \n"
+ "lea (%[dst],%[d],2),%[dst] \n"
+ "vmovdqu32 %%zmm2,(%[dst]) \n"
+ "2: \n"
+ : [dst] "+r"(dst), // %[dst]
+ [temp] "=&r"(temp) // %[temp]
+ : [src] "r"(src), // %[src]
+ [s] "r"(s), // %[s]
+ [d] "r"(d), // %[d]
+ [col_mask] "r"(col_mask), // %[col_mask]
+ [width] "r"(width) // %[width]
+ : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7", "xmm8");
+ }
+ asm volatile("vzeroupper \n");
+}
+#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+
+#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
+ // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN)
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/source/rotate_hvx.cc b/libs/libyuv/source/rotate_hvx.cc
new file mode 100644
index 0000000000..b3a0606f91
--- /dev/null
+++ b/libs/libyuv/source/rotate_hvx.cc
@@ -0,0 +1,388 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#include <hexagon_protos.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <string.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+static inline HVX_Vector LoadAligned_HVX(const void* ptr) {
+ HVX_Vector v;
+ memcpy(&v, (const HVX_Vector*)ptr, sizeof(v));
+ return v;
+}
+
+static inline HVX_Vector LoadU_HVX(const void* ptr) {
+ HVX_UVector v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+}
+
+static inline void StoreU_HVX(void* ptr, HVX_Vector v) {
+ memcpy(ptr, &v, sizeof(HVX_UVector));
+}
+
+static inline HVX_Vector LoadPartial_HVX(const void* ptr, int n_bytes) {
+ if (n_bytes >= 128) {
+ return LoadU_HVX(ptr);
+ }
+ const uint8_t* p = (const uint8_t*)ptr;
+ const uintptr_t addr = (uintptr_t)p;
+ const uintptr_t offset = addr & 127u;
+ HVX_Vector v0 = LoadAligned_HVX(p);
+ HVX_Vector v1 =
+ (offset + (uintptr_t)n_bytes > 128u) ? LoadAligned_HVX(p + 128) : v0;
+ return Q6_V_valign_VVR(v1, v0, addr);
+}
+
+static inline void StorePartial_HVX(void* ptr, HVX_Vector v, int n_bytes) {
+ if (n_bytes >= 128) {
+ StoreU_HVX(ptr, v);
+ return;
+ }
+ uint8_t* p = (uint8_t*)ptr;
+ const uintptr_t addr = (uintptr_t)p;
+ const uintptr_t offset = addr & 127u;
+ HVX_Vector value = Q6_V_vlalign_VVR(v, v, addr);
+ HVX_VectorPred ql_not = Q6_Q_vsetq_R(addr);
+ HVX_VectorPred qr = Q6_Q_vsetq2_R(offset + (uintptr_t)n_bytes);
+ if (offset + (uintptr_t)n_bytes > 128u) {
+ Q6_vmem_QRIV(qr, p + 128, value);
+ qr = Q6_Q_vcmp_eq_VbVb(value, value);
+ }
+ ql_not = Q6_Q_or_QQn(ql_not, qr);
+ Q6_vmem_QnRIV(ql_not, p, value);
+}
+
+static inline void Interleave4_HVX(HVX_Vector* x0,
+ HVX_Vector* x1,
+ HVX_Vector* x2,
+ HVX_Vector* x3,
+ int stride) {
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(*x1, *x0, -stride);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(*x3, *x2, -stride);
+ HVX_VectorPair q01 =
+ Q6_W_vshuff_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -2 * stride);
+ HVX_VectorPair q23 =
+ Q6_W_vshuff_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -2 * stride);
+ *x0 = Q6_V_lo_W(q01);
+ *x1 = Q6_V_hi_W(q01);
+ *x2 = Q6_V_lo_W(q23);
+ *x3 = Q6_V_hi_W(q23);
+}
+
+static inline void Interleave32_HVX(HVX_Vector* x,
+ HVX_Vector* out,
+ int stride) {
+ int g, q, j;
+ for (g = 0; g < 32; g += 4) {
+ Interleave4_HVX(&x[g], &x[g + 1], &x[g + 2], &x[g + 3], stride);
+ }
+ for (q = 0; q < 4; ++q) {
+ Interleave4_HVX(&x[q], &x[4 + q], &x[8 + q], &x[12 + q], 4 * stride);
+ Interleave4_HVX(&x[16 + q], &x[20 + q], &x[24 + q], &x[28 + q], 4 * stride);
+ for (j = 0; j < 4; ++j) {
+ HVX_VectorPair p =
+ Q6_W_vshuff_VVR(x[16 + 4 * j + q], x[4 * j + q], -16 * stride);
+ out[8 * q + 2 * j] = Q6_V_lo_W(p);
+ out[8 * q + 2 * j + 1] = Q6_V_hi_W(p);
+ }
+ }
+}
+
+#if defined(HAS_TRANSPOSEWXH_HVX) || defined(HAS_TRANSPOSEUVWXH_HVX)
+static void TransposeWxH_Byte_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width,
+ int height) {
+ const ptrdiff_t s_stride = src_stride;
+ const ptrdiff_t da_stride = dst_stride_a;
+ const ptrdiff_t db_stride = dst_stride_b;
+ HVX_Vector buf[4][32];
+ HVX_Vector v[32];
+ const HVX_Vector vzero = Q6_V_vzero();
+
+ do {
+ const int block_h = height > 128 ? 128 : height;
+ const uint8_t* src2 = src;
+ uint8_t* dst2_a = dst_a;
+ uint8_t* dst2_b = dst_b;
+ int bw = byte_width;
+
+ do {
+ const int block_w = bw > 128 ? 128 : bw;
+ int s, r, k;
+ for (s = 0; s < 4; ++s) {
+ if (s * 32 < block_h) {
+ int strip_h = block_h - s * 32;
+ const uint8_t* src_row = src2 + (ptrdiff_t)(s * 32) * s_stride;
+ if (strip_h > 32) {
+ strip_h = 32;
+ }
+ if (block_w == 128) {
+ for (r = 0; r < strip_h; ++r) {
+ v[r] = LoadU_HVX(src_row);
+ src_row += s_stride;
+ }
+ } else {
+ for (r = 0; r < strip_h; ++r) {
+ v[r] = LoadPartial_HVX(src_row, block_w);
+ src_row += s_stride;
+ }
+ }
+ for (; r < 32; ++r) {
+ v[r] = vzero;
+ }
+ Interleave32_HVX(v, buf[s], 1);
+ }
+ }
+
+ if (block_h == 128) {
+ for (k = 0; k < 32 && k * 4 < block_w; ++k) {
+ HVX_Vector b0 = buf[0][k];
+ HVX_Vector b1 = buf[1][k];
+ HVX_Vector b2 = buf[2][k];
+ HVX_Vector b3 = buf[3][k];
+ Interleave4_HVX(&b0, &b1, &b2, &b3, 32);
+ StoreU_HVX(dst2_a, b0);
+ if (k * 4 + 1 < block_w) {
+ StoreU_HVX(dst2_b, b1);
+ }
+ if (k * 4 + 2 < block_w) {
+ StoreU_HVX(dst2_a + da_stride, b2);
+ }
+ if (k * 4 + 3 < block_w) {
+ StoreU_HVX(dst2_b + db_stride, b3);
+ }
+ dst2_a += 2 * da_stride;
+ dst2_b += 2 * db_stride;
+ }
+ } else {
+ for (k = 0; k < 32 && k * 4 < block_w; ++k) {
+ HVX_Vector b0 = buf[0][k];
+ HVX_Vector b1 = (block_h > 32) ? buf[1][k] : vzero;
+ HVX_Vector b2 = (block_h > 64) ? buf[2][k] : vzero;
+ HVX_Vector b3 = (block_h > 96) ? buf[3][k] : vzero;
+ Interleave4_HVX(&b0, &b1, &b2, &b3, 32);
+ StorePartial_HVX(dst2_a, b0, block_h);
+ if (k * 4 + 1 < block_w) {
+ StorePartial_HVX(dst2_b, b1, block_h);
+ }
+ if (k * 4 + 2 < block_w) {
+ StorePartial_HVX(dst2_a + da_stride, b2, block_h);
+ }
+ if (k * 4 + 3 < block_w) {
+ StorePartial_HVX(dst2_b + db_stride, b3, block_h);
+ }
+ dst2_a += 2 * da_stride;
+ dst2_b += 2 * db_stride;
+ }
+ }
+
+ src2 += 128;
+ bw -= 128;
+ } while (bw > 0);
+
+ src += 128 * s_stride;
+ dst_a += 128;
+ dst_b += 128;
+ height -= 128;
+ } while (height > 0);
+}
+#endif // defined(HAS_TRANSPOSEWXH_HVX) || defined(HAS_TRANSPOSEUVWXH_HVX)
+
+#ifdef HAS_TRANSPOSEWXH_HVX
+void TransposeWxH_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ TransposeWxH_Byte_HVX(src, src_stride, dst, dst_stride * 2,
+ dst + (width > 1 ? dst_stride : 0), dst_stride * 2,
+ width, height);
+}
+#endif // HAS_TRANSPOSEWXH_HVX
+
+#ifdef HAS_TRANSPOSEUVWXH_HVX
+void TransposeUVWxH_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ TransposeWxH_Byte_HVX(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2, height);
+}
+#endif // HAS_TRANSPOSEUVWXH_HVX
+
+#ifdef HAS_TRANSPOSEWXH_16_HVX
+void TransposeWxH_16_HVX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t s_stride = src_stride;
+ const ptrdiff_t d_stride = dst_stride;
+ HVX_Vector buf[2][32];
+ HVX_Vector v[32];
+ const HVX_Vector vzero = Q6_V_vzero();
+
+ do {
+ const int block_h = height > 64 ? 64 : height;
+ const uint16_t* src2 = src;
+ uint16_t* dst2 = dst;
+ int w = width;
+
+ do {
+ const int block_w = w > 64 ? 64 : w;
+ int s, r, k;
+ for (s = 0; s < 2; ++s) {
+ if (s * 32 < block_h) {
+ int strip_h = block_h - s * 32;
+ const uint16_t* src_row = src2 + (ptrdiff_t)(s * 32) * s_stride;
+ if (strip_h > 32) {
+ strip_h = 32;
+ }
+ if (block_w == 64) {
+ for (r = 0; r < strip_h; ++r) {
+ v[r] = LoadU_HVX(src_row);
+ src_row += s_stride;
+ }
+ } else {
+ for (r = 0; r < strip_h; ++r) {
+ v[r] = LoadPartial_HVX(src_row, block_w * 2);
+ src_row += s_stride;
+ }
+ }
+ for (; r < 32; ++r) {
+ v[r] = vzero;
+ }
+ Interleave32_HVX(v, buf[s], 2);
+ }
+ }
+
+ if (block_h == 64) {
+ for (k = 0; k < 32 && k * 2 < block_w; ++k) {
+ HVX_VectorPair c01 = Q6_W_vshuff_VVR(buf[1][k], buf[0][k], -64);
+ StoreU_HVX(dst2, Q6_V_lo_W(c01));
+ if (k * 2 + 1 < block_w) {
+ StoreU_HVX(dst2 + d_stride, Q6_V_hi_W(c01));
+ }
+ dst2 += 2 * d_stride;
+ }
+ } else {
+ for (k = 0; k < 32 && k * 2 < block_w; ++k) {
+ HVX_Vector b0 = buf[0][k];
+ HVX_Vector b1 = (block_h > 32) ? buf[1][k] : vzero;
+ HVX_VectorPair c01 = Q6_W_vshuff_VVR(b1, b0, -64);
+ StorePartial_HVX(dst2, Q6_V_lo_W(c01), block_h * 2);
+ if (k * 2 + 1 < block_w) {
+ StorePartial_HVX(dst2 + d_stride, Q6_V_hi_W(c01), block_h * 2);
+ }
+ dst2 += 2 * d_stride;
+ }
+ }
+
+ src2 += 64;
+ w -= 64;
+ } while (w > 0);
+
+ src += 64 * s_stride;
+ dst += 64;
+ height -= 64;
+ } while (height > 0);
+}
+#endif // HAS_TRANSPOSEWXH_16_HVX
+
+#ifdef HAS_TRANSPOSEWXH_32_HVX
+void TransposeWxH_32_HVX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t s_stride = src_stride;
+ const ptrdiff_t d_stride = dst_stride;
+ HVX_Vector v[32];
+ HVX_Vector out[32];
+ const HVX_Vector vzero = Q6_V_vzero();
+
+ do {
+ const int block_h = height > 32 ? 32 : height;
+ const uint8_t* src2 = src;
+ uint8_t* dst2 = dst;
+ int w = width;
+
+ do {
+ const int block_w = w > 32 ? 32 : w;
+ const uint8_t* src_row = src2;
+ int r, c;
+ if (block_w == 32) {
+ for (r = 0; r < block_h; ++r) {
+ v[r] = LoadU_HVX(src_row);
+ src_row += s_stride;
+ }
+ } else {
+ for (r = 0; r < block_h; ++r) {
+ v[r] = LoadPartial_HVX(src_row, block_w * 4);
+ src_row += s_stride;
+ }
+ }
+ for (; r < 32; ++r) {
+ v[r] = vzero;
+ }
+ Interleave32_HVX(v, out, 4);
+
+ if (block_h == 32) {
+ for (c = 0; c < block_w; ++c) {
+ StoreU_HVX(dst2, out[c]);
+ dst2 += d_stride;
+ }
+ } else {
+ for (c = 0; c < block_w; ++c) {
+ StorePartial_HVX(dst2, out[c], block_h * 4);
+ dst2 += d_stride;
+ }
+ }
+
+ src2 += 128;
+ w -= 32;
+ } while (w > 0);
+
+ src += 32 * s_stride;
+ dst += 128;
+ height -= 32;
+ } while (height > 0);
+}
+#endif // HAS_TRANSPOSEWXH_32_HVX
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
diff --git a/libs/libyuv/source/rotate_lasx.cc b/libs/libyuv/source/rotate_lasx.cc
new file mode 100644
index 0000000000..8a50cbe2a0
--- /dev/null
+++ b/libs/libyuv/source/rotate_lasx.cc
@@ -0,0 +1,506 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+
+#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+void TransposeWx16_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr8, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr9, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr10, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr11, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr12, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr13, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr14, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr15, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "xvpermi.q $xr0, $xr8, 0x02 \n"
+ "xvpermi.q $xr1, $xr9, 0x02 \n"
+ "xvpermi.q $xr2, $xr10, 0x02 \n"
+ "xvpermi.q $xr3, $xr11, 0x02 \n"
+ "xvpermi.q $xr4, $xr12, 0x02 \n"
+ "xvpermi.q $xr5, $xr13, 0x02 \n"
+ "xvpermi.q $xr6, $xr14, 0x02 \n"
+ "xvpermi.q $xr7, $xr15, 0x02 \n"
+
+ "xvilvl.b $xr8, $xr1, $xr0 \n"
+ "xvilvh.b $xr9, $xr1, $xr0 \n"
+ "xvilvl.b $xr10, $xr3, $xr2 \n"
+ "xvilvh.b $xr11, $xr3, $xr2 \n"
+ "xvilvl.b $xr12, $xr5, $xr4 \n"
+ "xvilvh.b $xr13, $xr5, $xr4 \n"
+ "xvilvl.b $xr14, $xr7, $xr6 \n"
+ "xvilvh.b $xr15, $xr7, $xr6 \n"
+
+ "xvilvl.h $xr0, $xr10, $xr8 \n"
+ "xvilvh.h $xr1, $xr10, $xr8 \n"
+ "xvilvl.h $xr2, $xr11, $xr9 \n"
+ "xvilvh.h $xr3, $xr11, $xr9 \n"
+ "xvilvl.h $xr4, $xr14, $xr12 \n"
+ "xvilvh.h $xr5, $xr14, $xr12 \n"
+ "xvilvl.h $xr6, $xr15, $xr13 \n"
+ "xvilvh.h $xr7, $xr15, $xr13 \n"
+
+ "xvilvl.w $xr8, $xr4, $xr0 \n"
+ "xvilvh.w $xr9, $xr4, $xr0 \n"
+ "xvilvl.w $xr10, $xr5, $xr1 \n"
+ "xvilvh.w $xr11, $xr5, $xr1 \n"
+ "xvilvl.w $xr12, $xr6, $xr2 \n"
+ "xvilvh.w $xr13, $xr6, $xr2 \n"
+ "xvilvl.w $xr14, $xr7, $xr3 \n"
+ "xvilvh.w $xr15, $xr7, $xr3 \n"
+
+ "xvpermi.d $xr0, $xr8, 0x08 \n"
+ "xvpermi.d $xr1, $xr8, 0x0d \n"
+ "xvpermi.d $xr2, $xr9, 0x08 \n"
+ "xvpermi.d $xr3, $xr9, 0x0d \n"
+ "xvpermi.d $xr4, $xr10, 0x08 \n"
+ "xvpermi.d $xr5, $xr10, 0x0d \n"
+ "xvpermi.d $xr6, $xr11, 0x08 \n"
+ "xvpermi.d $xr7, $xr11, 0x0d \n"
+ "xvpermi.d $xr8, $xr12, 0x08 \n"
+ "xvpermi.d $xr9, $xr12, 0x0d \n"
+ "xvpermi.d $xr10, $xr13, 0x08 \n"
+ "xvpermi.d $xr11, $xr13, 0x0d \n"
+ "xvpermi.d $xr12, $xr14, 0x08 \n"
+ "xvpermi.d $xr13, $xr14, 0x0d \n"
+ "xvpermi.d $xr14, $xr15, 0x08 \n"
+ "xvpermi.d $xr15, $xr15, 0x0d \n"
+
+ "vst $vr0, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr1, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr2, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr3, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr4, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr5, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr6, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr7, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr8, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr9, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr10, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr11, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr12, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr13, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr14, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr15, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15");
+}
+
+void TransposeUVWx16_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t da = dst_stride_a;
+ const ptrdiff_t db = dst_stride_b;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr8, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr9, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr10, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr11, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr12, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr13, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr14, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr15, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "xvpermi.q $xr0, $xr8, 0x02 \n"
+ "xvpermi.q $xr1, $xr9, 0x02 \n"
+ "xvpermi.q $xr2, $xr10, 0x02 \n"
+ "xvpermi.q $xr3, $xr11, 0x02 \n"
+ "xvpermi.q $xr4, $xr12, 0x02 \n"
+ "xvpermi.q $xr5, $xr13, 0x02 \n"
+ "xvpermi.q $xr6, $xr14, 0x02 \n"
+ "xvpermi.q $xr7, $xr15, 0x02 \n"
+
+ "xvilvl.b $xr8, $xr1, $xr0 \n"
+ "xvilvh.b $xr9, $xr1, $xr0 \n"
+ "xvilvl.b $xr10, $xr3, $xr2 \n"
+ "xvilvh.b $xr11, $xr3, $xr2 \n"
+ "xvilvl.b $xr12, $xr5, $xr4 \n"
+ "xvilvh.b $xr13, $xr5, $xr4 \n"
+ "xvilvl.b $xr14, $xr7, $xr6 \n"
+ "xvilvh.b $xr15, $xr7, $xr6 \n"
+
+ "xvilvl.h $xr0, $xr10, $xr8 \n"
+ "xvilvh.h $xr1, $xr10, $xr8 \n"
+ "xvilvl.h $xr2, $xr11, $xr9 \n"
+ "xvilvh.h $xr3, $xr11, $xr9 \n"
+ "xvilvl.h $xr4, $xr14, $xr12 \n"
+ "xvilvh.h $xr5, $xr14, $xr12 \n"
+ "xvilvl.h $xr6, $xr15, $xr13 \n"
+ "xvilvh.h $xr7, $xr15, $xr13 \n"
+
+ "xvilvl.w $xr8, $xr4, $xr0 \n"
+ "xvilvh.w $xr9, $xr4, $xr0 \n"
+ "xvilvl.w $xr10, $xr5, $xr1 \n"
+ "xvilvh.w $xr11, $xr5, $xr1 \n"
+ "xvilvl.w $xr12, $xr6, $xr2 \n"
+ "xvilvh.w $xr13, $xr6, $xr2 \n"
+ "xvilvl.w $xr14, $xr7, $xr3 \n"
+ "xvilvh.w $xr15, $xr7, $xr3 \n"
+
+ "xvpermi.d $xr0, $xr8, 0x08 \n"
+ "xvpermi.d $xr1, $xr8, 0x0d \n"
+ "xvpermi.d $xr2, $xr9, 0x08 \n"
+ "xvpermi.d $xr3, $xr9, 0x0d \n"
+ "xvpermi.d $xr4, $xr10, 0x08 \n"
+ "xvpermi.d $xr5, $xr10, 0x0d \n"
+ "xvpermi.d $xr6, $xr11, 0x08 \n"
+ "xvpermi.d $xr7, $xr11, 0x0d \n"
+ "xvpermi.d $xr8, $xr12, 0x08 \n"
+ "xvpermi.d $xr9, $xr12, 0x0d \n"
+ "xvpermi.d $xr10, $xr13, 0x08 \n"
+ "xvpermi.d $xr11, $xr13, 0x0d \n"
+ "xvpermi.d $xr12, $xr14, 0x08 \n"
+ "xvpermi.d $xr13, $xr14, 0x0d \n"
+ "xvpermi.d $xr14, $xr15, 0x08 \n"
+ "xvpermi.d $xr15, $xr15, 0x0d \n"
+
+ "vst $vr0, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr1, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr2, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr3, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr4, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr5, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr6, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr7, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr8, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr9, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr10, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr11, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr12, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr13, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vst $vr14, %[dst_a], 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vst $vr15, %[dst_b], 0 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [da] "r"(da), // %[da]
+ [db] "r"(db) // %[db]
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15");
+}
+
+void TransposeWx4_32_LASX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ ptrdiff_t s = src_stride;
+ ptrdiff_t s2 = s * 2;
+ ptrdiff_t s3 = s + s2;
+ ptrdiff_t s4 = s * 4;
+ uint8_t* dst0 = dst;
+ uint8_t* dst1 = dst0 + dst_stride;
+ uint8_t* dst2 = dst1 + dst_stride;
+ uint8_t* dst3 = dst2 + dst_stride;
+ const uint8_t* src4;
+
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vldx $vr1, %[src], %[s] \n"
+ "vldx $vr2, %[src], %[s2] \n"
+ "vldx $vr3, %[src], %[s3] \n"
+ "add.d %[src4], %[src], %[s4] \n"
+ "vld $vr4, %[src4], 0 \n"
+ "vldx $vr5, %[src4], %[s] \n"
+ "vldx $vr6, %[src4], %[s2] \n"
+ "vldx $vr7, %[src4], %[s3] \n"
+ "add.d %[src], %[src4], %[s4] \n"
+
+ "xvpermi.q $xr0, $xr4, 0x02 \n"
+ "xvpermi.q $xr1, $xr5, 0x02 \n"
+ "xvpermi.q $xr2, $xr6, 0x02 \n"
+ "xvpermi.q $xr3, $xr7, 0x02 \n"
+
+ "xvilvl.w $xr4, $xr1, $xr0 \n"
+ "xvilvh.w $xr5, $xr1, $xr0 \n"
+ "xvilvl.w $xr6, $xr3, $xr2 \n"
+ "xvilvh.w $xr7, $xr3, $xr2 \n"
+
+ "xvilvl.d $xr0, $xr6, $xr4 \n"
+ "xvilvh.d $xr1, $xr6, $xr4 \n"
+ "xvilvl.d $xr2, $xr7, $xr5 \n"
+ "xvilvh.d $xr3, $xr7, $xr5 \n"
+
+ "xvst $xr0, %[dst0], 0 \n"
+ "xvst $xr1, %[dst1], 0 \n"
+ "xvst $xr2, %[dst2], 0 \n"
+ "xvst $xr3, %[dst3], 0 \n"
+ "addi.d %[dst0], %[dst0], 32 \n"
+ "addi.d %[dst1], %[dst1], 32 \n"
+ "addi.d %[dst2], %[dst2], 32 \n"
+ "addi.d %[dst3], %[dst3], 32 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [src4] "=&r"(src4), // %[src4]
+ [dst0] "+r"(dst0), // %[dst0]
+ [dst1] "+r"(dst1), // %[dst1]
+ [dst2] "+r"(dst2), // %[dst2]
+ [dst3] "+r"(dst3), // %[dst3]
+ [w] "+r"(width) // %[w]
+ : [s] "r"(s), // %[s]
+ [s2] "r"(s2), // %[s2]
+ [s3] "r"(s3), // %[s3]
+ [s4] "r"(s4) // %[s4]
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7");
+}
+
+void TransposeWx16_16_LASX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* src_b = (const uint8_t*)src;
+ uint8_t* dst_b = (uint8_t*)dst;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+
+ "vld $vr8, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr9, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr10, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr11, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr12, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr13, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr14, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr15, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "xvpermi.q $xr0, $xr8, 0x02 \n"
+ "xvpermi.q $xr1, $xr9, 0x02 \n"
+ "xvpermi.q $xr2, $xr10, 0x02 \n"
+ "xvpermi.q $xr3, $xr11, 0x02 \n"
+ "xvpermi.q $xr4, $xr12, 0x02 \n"
+ "xvpermi.q $xr5, $xr13, 0x02 \n"
+ "xvpermi.q $xr6, $xr14, 0x02 \n"
+ "xvpermi.q $xr7, $xr15, 0x02 \n"
+
+ "xvilvl.h $xr8, $xr1, $xr0 \n"
+ "xvilvh.h $xr9, $xr1, $xr0 \n"
+ "xvilvl.h $xr10, $xr3, $xr2 \n"
+ "xvilvh.h $xr11, $xr3, $xr2 \n"
+ "xvilvl.h $xr12, $xr5, $xr4 \n"
+ "xvilvh.h $xr13, $xr5, $xr4 \n"
+ "xvilvl.h $xr14, $xr7, $xr6 \n"
+ "xvilvh.h $xr15, $xr7, $xr6 \n"
+
+ "xvilvl.w $xr0, $xr10, $xr8 \n"
+ "xvilvh.w $xr1, $xr10, $xr8 \n"
+ "xvilvl.w $xr2, $xr11, $xr9 \n"
+ "xvilvh.w $xr3, $xr11, $xr9 \n"
+ "xvilvl.w $xr4, $xr14, $xr12 \n"
+ "xvilvh.w $xr5, $xr14, $xr12 \n"
+ "xvilvl.w $xr6, $xr15, $xr13 \n"
+ "xvilvh.w $xr7, $xr15, $xr13 \n"
+
+ "xvilvl.d $xr8, $xr4, $xr0 \n"
+ "xvilvh.d $xr9, $xr4, $xr0 \n"
+ "xvilvl.d $xr10, $xr5, $xr1 \n"
+ "xvilvh.d $xr11, $xr5, $xr1 \n"
+ "xvilvl.d $xr12, $xr6, $xr2 \n"
+ "xvilvh.d $xr13, $xr6, $xr2 \n"
+ "xvilvl.d $xr14, $xr7, $xr3 \n"
+ "xvilvh.d $xr15, $xr7, $xr3 \n"
+
+ "xvst $xr8, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr9, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr10, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr11, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr12, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr13, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr14, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "xvst $xr15, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_b), // %[src]
+ [dst] "+r"(dst_b), // %[dst]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15");
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
diff --git a/libs/libyuv/source/rotate_lsx.cc b/libs/libyuv/source/rotate_lsx.cc
new file mode 100644
index 0000000000..531145bf18
--- /dev/null
+++ b/libs/libyuv/source/rotate_lsx.cc
@@ -0,0 +1,383 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+void TransposeWx8_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "vilvl.b $vr8, $vr1, $vr0 \n"
+ "vilvh.b $vr9, $vr1, $vr0 \n"
+ "vilvl.b $vr10, $vr3, $vr2 \n"
+ "vilvh.b $vr11, $vr3, $vr2 \n"
+ "vilvl.b $vr12, $vr5, $vr4 \n"
+ "vilvh.b $vr13, $vr5, $vr4 \n"
+ "vilvl.b $vr14, $vr7, $vr6 \n"
+ "vilvh.b $vr15, $vr7, $vr6 \n"
+
+ "vilvl.h $vr0, $vr10, $vr8 \n"
+ "vilvh.h $vr1, $vr10, $vr8 \n"
+ "vilvl.h $vr2, $vr11, $vr9 \n"
+ "vilvh.h $vr3, $vr11, $vr9 \n"
+ "vilvl.h $vr4, $vr14, $vr12 \n"
+ "vilvh.h $vr5, $vr14, $vr12 \n"
+ "vilvl.h $vr6, $vr15, $vr13 \n"
+ "vilvh.h $vr7, $vr15, $vr13 \n"
+
+ "vilvl.w $vr8, $vr4, $vr0 \n"
+ "vilvh.w $vr9, $vr4, $vr0 \n"
+ "vilvl.w $vr10, $vr5, $vr1 \n"
+ "vilvh.w $vr11, $vr5, $vr1 \n"
+ "vilvl.w $vr12, $vr6, $vr2 \n"
+ "vilvh.w $vr13, $vr6, $vr2 \n"
+ "vilvl.w $vr14, $vr7, $vr3 \n"
+ "vilvh.w $vr15, $vr7, $vr3 \n"
+
+ "vstelm.d $vr8, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr8, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr9, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr9, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr10, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr10, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr11, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr11, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr12, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr12, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr13, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr13, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr14, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr14, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr15, %[dst], 0, 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vstelm.d $vr15, %[dst], 0, 1 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15");
+}
+
+void TransposeUVWx8_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t da = dst_stride_a;
+ const ptrdiff_t db = dst_stride_b;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "vilvl.b $vr8, $vr1, $vr0 \n"
+ "vilvh.b $vr9, $vr1, $vr0 \n"
+ "vilvl.b $vr10, $vr3, $vr2 \n"
+ "vilvh.b $vr11, $vr3, $vr2 \n"
+ "vilvl.b $vr12, $vr5, $vr4 \n"
+ "vilvh.b $vr13, $vr5, $vr4 \n"
+ "vilvl.b $vr14, $vr7, $vr6 \n"
+ "vilvh.b $vr15, $vr7, $vr6 \n"
+
+ "vilvl.h $vr0, $vr10, $vr8 \n"
+ "vilvh.h $vr1, $vr10, $vr8 \n"
+ "vilvl.h $vr2, $vr11, $vr9 \n"
+ "vilvh.h $vr3, $vr11, $vr9 \n"
+ "vilvl.h $vr4, $vr14, $vr12 \n"
+ "vilvh.h $vr5, $vr14, $vr12 \n"
+ "vilvl.h $vr6, $vr15, $vr13 \n"
+ "vilvh.h $vr7, $vr15, $vr13 \n"
+
+ "vilvl.w $vr8, $vr4, $vr0 \n"
+ "vilvh.w $vr9, $vr4, $vr0 \n"
+ "vilvl.w $vr10, $vr5, $vr1 \n"
+ "vilvh.w $vr11, $vr5, $vr1 \n"
+ "vilvl.w $vr12, $vr6, $vr2 \n"
+ "vilvh.w $vr13, $vr6, $vr2 \n"
+ "vilvl.w $vr14, $vr7, $vr3 \n"
+ "vilvh.w $vr15, $vr7, $vr3 \n"
+
+ "vstelm.d $vr8, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr8, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr9, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr9, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr10, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr10, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr11, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr11, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr12, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr12, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr13, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr13, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr14, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr14, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "vstelm.d $vr15, %[dst_a], 0, 0 \n"
+ "add.d %[dst_a], %[dst_a], %[da] \n"
+ "vstelm.d $vr15, %[dst_b], 0, 1 \n"
+ "add.d %[dst_b], %[dst_b], %[db] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [da] "r"(da), // %[da]
+ [db] "r"(db) // %[db]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15");
+}
+
+void TransposeWx4_32_LSX(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t s2 = s * 2;
+ const ptrdiff_t s3 = s + s2;
+ const ptrdiff_t s4 = s * 4;
+ uint8_t* dst0 = dst;
+ uint8_t* dst1 = dst0 + dst_stride;
+ uint8_t* dst2 = dst1 + dst_stride;
+ uint8_t* dst3 = dst2 + dst_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vldx $vr1, %[src], %[s] \n"
+ "vldx $vr2, %[src], %[s2] \n"
+ "vldx $vr3, %[src], %[s3] \n"
+ "add.d %[src], %[src], %[s4] \n"
+
+ "vilvl.w $vr4, $vr1, $vr0 \n"
+ "vilvh.w $vr5, $vr1, $vr0 \n"
+ "vilvl.w $vr6, $vr3, $vr2 \n"
+ "vilvh.w $vr7, $vr3, $vr2 \n"
+
+ "vilvl.d $vr0, $vr6, $vr4 \n"
+ "vilvh.d $vr1, $vr6, $vr4 \n"
+ "vilvl.d $vr2, $vr7, $vr5 \n"
+ "vilvh.d $vr3, $vr7, $vr5 \n"
+
+ "vst $vr0, %[dst0], 0 \n"
+ "vst $vr1, %[dst1], 0 \n"
+ "vst $vr2, %[dst2], 0 \n"
+ "vst $vr3, %[dst3], 0 \n"
+ "addi.d %[dst0], %[dst0], 16 \n"
+ "addi.d %[dst1], %[dst1], 16 \n"
+ "addi.d %[dst2], %[dst2], 16 \n"
+ "addi.d %[dst3], %[dst3], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src), // %[src]
+ [dst0] "+r"(dst0), // %[dst0]
+ [dst1] "+r"(dst1), // %[dst1]
+ [dst2] "+r"(dst2), // %[dst2]
+ [dst3] "+r"(dst3), // %[dst3]
+ [w] "+r"(width) // %[w]
+ : [s] "r"(s), // %[s]
+ [s2] "r"(s2), // %[s2]
+ [s3] "r"(s3), // %[s3]
+ [s4] "r"(s4) // %[s4]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void TransposeWx8_16_LSX(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* src_b = (const uint8_t*)src;
+ uint8_t* dst_b = (uint8_t*)dst;
+ const uint8_t* t;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "move %[t], %[src] \n"
+ "vld $vr0, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr1, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr2, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr3, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr4, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr5, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr6, %[t], 0 \n"
+ "add.d %[t], %[t], %[s] \n"
+ "vld $vr7, %[t], 0 \n"
+ "addi.d %[src], %[src], 16 \n"
+
+ "vilvl.h $vr8, $vr1, $vr0 \n"
+ "vilvh.h $vr9, $vr1, $vr0 \n"
+ "vilvl.h $vr10, $vr3, $vr2 \n"
+ "vilvh.h $vr11, $vr3, $vr2 \n"
+ "vilvl.h $vr12, $vr5, $vr4 \n"
+ "vilvh.h $vr13, $vr5, $vr4 \n"
+ "vilvl.h $vr14, $vr7, $vr6 \n"
+ "vilvh.h $vr15, $vr7, $vr6 \n"
+
+ "vilvl.w $vr0, $vr10, $vr8 \n"
+ "vilvh.w $vr1, $vr10, $vr8 \n"
+ "vilvl.w $vr2, $vr11, $vr9 \n"
+ "vilvh.w $vr3, $vr11, $vr9 \n"
+ "vilvl.w $vr4, $vr14, $vr12 \n"
+ "vilvh.w $vr5, $vr14, $vr12 \n"
+ "vilvl.w $vr6, $vr15, $vr13 \n"
+ "vilvh.w $vr7, $vr15, $vr13 \n"
+
+ "vilvl.d $vr8, $vr4, $vr0 \n"
+ "vilvh.d $vr9, $vr4, $vr0 \n"
+ "vilvl.d $vr10, $vr5, $vr1 \n"
+ "vilvh.d $vr11, $vr5, $vr1 \n"
+ "vilvl.d $vr12, $vr6, $vr2 \n"
+ "vilvh.d $vr13, $vr6, $vr2 \n"
+ "vilvl.d $vr14, $vr7, $vr3 \n"
+ "vilvh.d $vr15, $vr7, $vr3 \n"
+
+ "vst $vr8, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr9, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr10, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr11, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr12, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr13, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr14, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "vst $vr15, %[dst], 0 \n"
+ "add.d %[dst], %[dst], %[d] \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_b), // %[src]
+ [dst] "+r"(dst_b), // %[dst]
+ [w] "+r"(width), // %[w]
+ [t] "=&r"(t) // %[t]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15");
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
diff --git a/libs/libyuv/source/rotate_mmi.cc b/libs/libyuv/source/rotate_mmi.cc
deleted file mode 100644
index f8de60834d..0000000000
--- a/libs/libyuv/source/rotate_mmi.cc
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Copyright 2011 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "libyuv/rotate_row.h"
-#include "libyuv/row.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-// This module is for Mips MMI.
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-void TransposeWx8_MMI(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6;
- uint64_t tmp7, tmp8, tmp9, tmp10, tmp11, tmp12, tmp13;
- uint8_t* src_tmp = nullptr;
-
- __asm__ volatile(
- "1: \n\t"
- "ldc1 %[tmp12], 0x00(%[src]) \n\t"
- "dadd %[src_tmp], %[src], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp0 = (00 10 01 11 02 12 03 13) */
- "punpcklbh %[tmp0], %[tmp12], %[tmp13] \n\t"
- /* tmp1 = (04 14 05 15 06 16 07 17) */
- "punpckhbh %[tmp1], %[tmp12], %[tmp13] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp2 = (20 30 21 31 22 32 23 33) */
- "punpcklbh %[tmp2], %[tmp12], %[tmp13] \n\t"
- /* tmp3 = (24 34 25 35 26 36 27 37) */
- "punpckhbh %[tmp3], %[tmp12], %[tmp13] \n\t"
-
- /* tmp4 = (00 10 20 30 01 11 21 31) */
- "punpcklhw %[tmp4], %[tmp0], %[tmp2] \n\t"
- /* tmp5 = (02 12 22 32 03 13 23 33) */
- "punpckhhw %[tmp5], %[tmp0], %[tmp2] \n\t"
- /* tmp6 = (04 14 24 34 05 15 25 35) */
- "punpcklhw %[tmp6], %[tmp1], %[tmp3] \n\t"
- /* tmp7 = (06 16 26 36 07 17 27 37) */
- "punpckhhw %[tmp7], %[tmp1], %[tmp3] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp0 = (40 50 41 51 42 52 43 53) */
- "punpcklbh %[tmp0], %[tmp12], %[tmp13] \n\t"
- /* tmp1 = (44 54 45 55 46 56 47 57) */
- "punpckhbh %[tmp1], %[tmp12], %[tmp13] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp2 = (60 70 61 71 62 72 63 73) */
- "punpcklbh %[tmp2], %[tmp12], %[tmp13] \n\t"
- /* tmp3 = (64 74 65 75 66 76 67 77) */
- "punpckhbh %[tmp3], %[tmp12], %[tmp13] \n\t"
-
- /* tmp8 = (40 50 60 70 41 51 61 71) */
- "punpcklhw %[tmp8], %[tmp0], %[tmp2] \n\t"
- /* tmp9 = (42 52 62 72 43 53 63 73) */
- "punpckhhw %[tmp9], %[tmp0], %[tmp2] \n\t"
- /* tmp10 = (44 54 64 74 45 55 65 75) */
- "punpcklhw %[tmp10], %[tmp1], %[tmp3] \n\t"
- /* tmp11 = (46 56 66 76 47 57 67 77) */
- "punpckhhw %[tmp11], %[tmp1], %[tmp3] \n\t"
-
- /* tmp0 = (00 10 20 30 40 50 60 70) */
- "punpcklwd %[tmp0], %[tmp4], %[tmp8] \n\t"
- /* tmp1 = (01 11 21 31 41 51 61 71) */
- "punpckhwd %[tmp1], %[tmp4], %[tmp8] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst]) \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst]) \n\t"
-
- /* tmp0 = (02 12 22 32 42 52 62 72) */
- "punpcklwd %[tmp0], %[tmp5], %[tmp9] \n\t"
- /* tmp1 = (03 13 23 33 43 53 63 73) */
- "punpckhwd %[tmp1], %[tmp5], %[tmp9] \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst]) \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst]) \n\t"
-
- /* tmp0 = (04 14 24 34 44 54 64 74) */
- "punpcklwd %[tmp0], %[tmp6], %[tmp10] \n\t"
- /* tmp1 = (05 15 25 35 45 55 65 75) */
- "punpckhwd %[tmp1], %[tmp6], %[tmp10] \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst]) \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst]) \n\t"
-
- /* tmp0 = (06 16 26 36 46 56 66 76) */
- "punpcklwd %[tmp0], %[tmp7], %[tmp11] \n\t"
- /* tmp1 = (07 17 27 37 47 57 67 77) */
- "punpckhwd %[tmp1], %[tmp7], %[tmp11] \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst]) \n\t"
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst]) \n\t"
-
- "dadd %[dst], %[dst], %[dst_stride] \n\t"
- "daddi %[src], %[src], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
-
- : [tmp0] "=&f"(tmp0), [tmp1] "=&f"(tmp1), [tmp2] "=&f"(tmp2),
- [tmp3] "=&f"(tmp3), [tmp4] "=&f"(tmp4), [tmp5] "=&f"(tmp5),
- [tmp6] "=&f"(tmp6), [tmp7] "=&f"(tmp7), [tmp8] "=&f"(tmp8),
- [tmp9] "=&f"(tmp9), [tmp10] "=&f"(tmp10), [tmp11] "=&f"(tmp11),
- [tmp12] "=&f"(tmp12), [tmp13] "=&f"(tmp13), [dst] "+&r"(dst),
- [src_tmp] "+&r"(src_tmp)
- : [src] "r"(src), [width] "r"(width), [src_stride] "r"(src_stride),
- [dst_stride] "r"(dst_stride)
- : "memory");
-}
-
-void TransposeUVWx8_MMI(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width) {
- uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6;
- uint64_t tmp7, tmp8, tmp9, tmp10, tmp11, tmp12, tmp13;
- uint8_t* src_tmp = nullptr;
-
- __asm__ volatile(
- "1: \n\t"
- /* tmp12 = (u00 v00 u01 v01 u02 v02 u03 v03) */
- "ldc1 %[tmp12], 0x00(%[src]) \n\t"
- "dadd %[src_tmp], %[src], %[src_stride] \n\t"
- /* tmp13 = (u10 v10 u11 v11 u12 v12 u13 v13) */
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp0 = (u00 u10 v00 v10 u01 u11 v01 v11) */
- "punpcklbh %[tmp0], %[tmp12], %[tmp13] \n\t"
- /* tmp1 = (u02 u12 v02 v12 u03 u13 v03 v13) */
- "punpckhbh %[tmp1], %[tmp12], %[tmp13] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- /* tmp12 = (u20 v20 u21 v21 u22 v22 u23 v23) */
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- /* tmp13 = (u30 v30 u31 v31 u32 v32 u33 v33) */
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp2 = (u20 u30 v20 v30 u21 u31 v21 v31) */
- "punpcklbh %[tmp2], %[tmp12], %[tmp13] \n\t"
- /* tmp3 = (u22 u32 v22 v32 u23 u33 v23 v33) */
- "punpckhbh %[tmp3], %[tmp12], %[tmp13] \n\t"
-
- /* tmp4 = (u00 u10 u20 u30 v00 v10 v20 v30) */
- "punpcklhw %[tmp4], %[tmp0], %[tmp2] \n\t"
- /* tmp5 = (u01 u11 u21 u31 v01 v11 v21 v31) */
- "punpckhhw %[tmp5], %[tmp0], %[tmp2] \n\t"
- /* tmp6 = (u02 u12 u22 u32 v02 v12 v22 v32) */
- "punpcklhw %[tmp6], %[tmp1], %[tmp3] \n\t"
- /* tmp7 = (u03 u13 u23 u33 v03 v13 v23 v33) */
- "punpckhhw %[tmp7], %[tmp1], %[tmp3] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- /* tmp12 = (u40 v40 u41 v41 u42 v42 u43 v43) */
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- /* tmp13 = (u50 v50 u51 v51 u52 v52 u53 v53) */
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp0 = (u40 u50 v40 v50 u41 u51 v41 v51) */
- "punpcklbh %[tmp0], %[tmp12], %[tmp13] \n\t"
- /* tmp1 = (u42 u52 v42 v52 u43 u53 v43 v53) */
- "punpckhbh %[tmp1], %[tmp12], %[tmp13] \n\t"
-
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- /* tmp12 = (u60 v60 u61 v61 u62 v62 u63 v63) */
- "ldc1 %[tmp12], 0x00(%[src_tmp]) \n\t"
- /* tmp13 = (u70 v70 u71 v71 u72 v72 u73 v73) */
- "dadd %[src_tmp], %[src_tmp], %[src_stride] \n\t"
- "ldc1 %[tmp13], 0x00(%[src_tmp]) \n\t"
-
- /* tmp2 = (u60 u70 v60 v70 u61 u71 v61 v71) */
- "punpcklbh %[tmp2], %[tmp12], %[tmp13] \n\t"
- /* tmp3 = (u62 u72 v62 v72 u63 u73 v63 v73) */
- "punpckhbh %[tmp3], %[tmp12], %[tmp13] \n\t"
-
- /* tmp8 = (u40 u50 u60 u70 v40 v50 v60 v70) */
- "punpcklhw %[tmp8], %[tmp0], %[tmp2] \n\t"
- /* tmp9 = (u41 u51 u61 u71 v41 v51 v61 v71) */
- "punpckhhw %[tmp9], %[tmp0], %[tmp2] \n\t"
- /* tmp10 = (u42 u52 u62 u72 v42 v52 v62 v72) */
- "punpcklhw %[tmp10], %[tmp1], %[tmp3] \n\t"
- /* tmp11 = (u43 u53 u63 u73 v43 v53 v63 v73) */
- "punpckhhw %[tmp11], %[tmp1], %[tmp3] \n\t"
-
- /* tmp0 = (u00 u10 u20 u30 u40 u50 u60 u70) */
- "punpcklwd %[tmp0], %[tmp4], %[tmp8] \n\t"
- /* tmp1 = (v00 v10 v20 v30 v40 v50 v60 v70) */
- "punpckhwd %[tmp1], %[tmp4], %[tmp8] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst_a]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst_a]) \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst_b]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst_b]) \n\t"
-
- /* tmp0 = (u01 u11 u21 u31 u41 u51 u61 u71) */
- "punpcklwd %[tmp0], %[tmp5], %[tmp9] \n\t"
- /* tmp1 = (v01 v11 v21 v31 v41 v51 v61 v71) */
- "punpckhwd %[tmp1], %[tmp5], %[tmp9] \n\t"
- "dadd %[dst_a], %[dst_a], %[dst_stride_a] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst_a]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst_a]) \n\t"
- "dadd %[dst_b], %[dst_b], %[dst_stride_b] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst_b]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst_b]) \n\t"
-
- /* tmp0 = (u02 u12 u22 u32 u42 u52 u62 u72) */
- "punpcklwd %[tmp0], %[tmp6], %[tmp10] \n\t"
- /* tmp1 = (v02 v12 v22 v32 v42 v52 v62 v72) */
- "punpckhwd %[tmp1], %[tmp6], %[tmp10] \n\t"
- "dadd %[dst_a], %[dst_a], %[dst_stride_a] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst_a]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst_a]) \n\t"
- "dadd %[dst_b], %[dst_b], %[dst_stride_b] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst_b]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst_b]) \n\t"
-
- /* tmp0 = (u03 u13 u23 u33 u43 u53 u63 u73) */
- "punpcklwd %[tmp0], %[tmp7], %[tmp11] \n\t"
- /* tmp1 = (v03 v13 v23 v33 v43 v53 v63 v73) */
- "punpckhwd %[tmp1], %[tmp7], %[tmp11] \n\t"
- "dadd %[dst_a], %[dst_a], %[dst_stride_a] \n\t"
- "gssdlc1 %[tmp0], 0x07(%[dst_a]) \n\t"
- "gssdrc1 %[tmp0], 0x00(%[dst_a]) \n\t"
- "dadd %[dst_b], %[dst_b], %[dst_stride_b] \n\t"
- "gssdlc1 %[tmp1], 0x07(%[dst_b]) \n\t"
- "gssdrc1 %[tmp1], 0x00(%[dst_b]) \n\t"
-
- "dadd %[dst_a], %[dst_a], %[dst_stride_a] \n\t"
- "dadd %[dst_b], %[dst_b], %[dst_stride_b] \n\t"
- "daddiu %[src], %[src], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
-
- : [tmp0] "=&f"(tmp0), [tmp1] "=&f"(tmp1), [tmp2] "=&f"(tmp2),
- [tmp3] "=&f"(tmp3), [tmp4] "=&f"(tmp4), [tmp5] "=&f"(tmp5),
- [tmp6] "=&f"(tmp6), [tmp7] "=&f"(tmp7), [tmp8] "=&f"(tmp8),
- [tmp9] "=&f"(tmp9), [tmp10] "=&f"(tmp10), [tmp11] "=&f"(tmp11),
- [tmp12] "=&f"(tmp12), [tmp13] "=&f"(tmp13), [dst_a] "+&r"(dst_a),
- [dst_b] "+&r"(dst_b), [src_tmp] "+&r"(src_tmp)
- : [src] "r"(src), [width] "r"(width), [dst_stride_a] "r"(dst_stride_a),
- [dst_stride_b] "r"(dst_stride_b), [src_stride] "r"(src_stride)
- : "memory");
-}
-
-#endif // !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
diff --git a/libs/libyuv/source/rotate_msa.cc b/libs/libyuv/source/rotate_msa.cc
deleted file mode 100644
index 99bdca65b3..0000000000
--- a/libs/libyuv/source/rotate_msa.cc
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Copyright 2016 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "libyuv/rotate_row.h"
-
-// This module is for GCC MSA
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#include "libyuv/macros_msa.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-#define ILVRL_B(in0, in1, in2, in3, out0, out1, out2, out3) \
- { \
- out0 = (v16u8)__msa_ilvr_b((v16i8)in1, (v16i8)in0); \
- out1 = (v16u8)__msa_ilvl_b((v16i8)in1, (v16i8)in0); \
- out2 = (v16u8)__msa_ilvr_b((v16i8)in3, (v16i8)in2); \
- out3 = (v16u8)__msa_ilvl_b((v16i8)in3, (v16i8)in2); \
- }
-
-#define ILVRL_H(in0, in1, in2, in3, out0, out1, out2, out3) \
- { \
- out0 = (v16u8)__msa_ilvr_h((v8i16)in1, (v8i16)in0); \
- out1 = (v16u8)__msa_ilvl_h((v8i16)in1, (v8i16)in0); \
- out2 = (v16u8)__msa_ilvr_h((v8i16)in3, (v8i16)in2); \
- out3 = (v16u8)__msa_ilvl_h((v8i16)in3, (v8i16)in2); \
- }
-
-#define ILVRL_W(in0, in1, in2, in3, out0, out1, out2, out3) \
- { \
- out0 = (v16u8)__msa_ilvr_w((v4i32)in1, (v4i32)in0); \
- out1 = (v16u8)__msa_ilvl_w((v4i32)in1, (v4i32)in0); \
- out2 = (v16u8)__msa_ilvr_w((v4i32)in3, (v4i32)in2); \
- out3 = (v16u8)__msa_ilvl_w((v4i32)in3, (v4i32)in2); \
- }
-
-#define ILVRL_D(in0, in1, in2, in3, out0, out1, out2, out3) \
- { \
- out0 = (v16u8)__msa_ilvr_d((v2i64)in1, (v2i64)in0); \
- out1 = (v16u8)__msa_ilvl_d((v2i64)in1, (v2i64)in0); \
- out2 = (v16u8)__msa_ilvr_d((v2i64)in3, (v2i64)in2); \
- out3 = (v16u8)__msa_ilvl_d((v2i64)in3, (v2i64)in2); \
- }
-
-void TransposeWx16_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- TransposeWx8_C(src, src_stride, dst, dst_stride, width);
- TransposeWx8_C((src + 8 * src_stride), src_stride, (dst + 8), dst_stride,
- width);
-}
-
-void TransposeUVWx16_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width) {
- TransposeUVWx8_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
- width);
- TransposeUVWx8_C((src + 8 * src_stride), src_stride, (dst_a + 8),
- dst_stride_a, (dst_b + 8), dst_stride_b, width);
-}
-
-void TransposeWx16_MSA(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- int x;
- const uint8_t* s;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3;
- v16u8 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
- v16u8 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
-
- for (x = 0; x < width; x += 16) {
- s = src;
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg0, reg1, reg2, reg3);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg4, reg5, reg6, reg7);
- ILVRL_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
- ILVRL_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg0, reg1, reg2, reg3);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg4, reg5, reg6, reg7);
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg4, (v4i32)reg0);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg4, (v4i32)reg0);
- ILVRL_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
- ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
- dst += dst_stride * 4;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg5, (v4i32)reg1);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg5, (v4i32)reg1);
- ILVRL_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
- ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
- dst += dst_stride * 4;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg6, (v4i32)reg2);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg6, (v4i32)reg2);
- ILVRL_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
- ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
- dst += dst_stride * 4;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg7, (v4i32)reg3);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg7, (v4i32)reg3);
- ILVRL_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
- ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
- src += 16;
- dst += dst_stride * 4;
- }
-}
-
-void TransposeUVWx16_MSA(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width) {
- int x;
- const uint8_t* s;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3;
- v16u8 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
- v16u8 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
-
- for (x = 0; x < width; x += 8) {
- s = src;
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg0, reg1, reg2, reg3);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg4, reg5, reg6, reg7);
- ILVRL_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
- ILVRL_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg0, reg1, reg2, reg3);
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- s += src_stride;
- ILVRL_B(src0, src1, src2, src3, vec0, vec1, vec2, vec3);
- ILVRL_H(vec0, vec2, vec1, vec3, reg4, reg5, reg6, reg7);
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg4, (v4i32)reg0);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg4, (v4i32)reg0);
- ILVRL_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
- ST_UB2(dst0, dst2, dst_a, dst_stride_a);
- ST_UB2(dst1, dst3, dst_b, dst_stride_b);
- dst_a += dst_stride_a * 2;
- dst_b += dst_stride_b * 2;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg5, (v4i32)reg1);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg5, (v4i32)reg1);
- ILVRL_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
- ST_UB2(dst0, dst2, dst_a, dst_stride_a);
- ST_UB2(dst1, dst3, dst_b, dst_stride_b);
- dst_a += dst_stride_a * 2;
- dst_b += dst_stride_b * 2;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg6, (v4i32)reg2);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg6, (v4i32)reg2);
- ILVRL_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
- ST_UB2(dst0, dst2, dst_a, dst_stride_a);
- ST_UB2(dst1, dst3, dst_b, dst_stride_b);
- dst_a += dst_stride_a * 2;
- dst_b += dst_stride_b * 2;
- res8 = (v16u8)__msa_ilvr_w((v4i32)reg7, (v4i32)reg3);
- res9 = (v16u8)__msa_ilvl_w((v4i32)reg7, (v4i32)reg3);
- ILVRL_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
- ST_UB2(dst0, dst2, dst_a, dst_stride_a);
- ST_UB2(dst1, dst3, dst_b, dst_stride_b);
- src += 16;
- dst_a += dst_stride_a * 2;
- dst_b += dst_stride_b * 2;
- }
-}
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
-
-#endif // !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
diff --git a/libs/libyuv/source/rotate_neon.cc b/libs/libyuv/source/rotate_neon.cc
index fdc0dd476c..292930ef8f 100644
--- a/libs/libyuv/source/rotate_neon.cc
+++ b/libs/libyuv/source/rotate_neon.cc
@@ -11,187 +11,87 @@
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
-#include "libyuv/basic_types.h"
-
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
-static const uvec8 kVTbl4x4Transpose = {0, 4, 8, 12, 1, 5, 9, 13,
- 2, 6, 10, 14, 3, 7, 11, 15};
-
void TransposeWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
- const uint8_t* src_temp;
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %5, #8 \n"
-
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "vld1.8 {d0}, [%0], %2 \n"
- "vld1.8 {d1}, [%0], %2 \n"
- "vld1.8 {d2}, [%0], %2 \n"
- "vld1.8 {d3}, [%0], %2 \n"
- "vld1.8 {d4}, [%0], %2 \n"
- "vld1.8 {d5}, [%0], %2 \n"
- "vld1.8 {d6}, [%0], %2 \n"
- "vld1.8 {d7}, [%0] \n"
-
- "vtrn.8 d1, d0 \n"
- "vtrn.8 d3, d2 \n"
- "vtrn.8 d5, d4 \n"
- "vtrn.8 d7, d6 \n"
-
- "vtrn.16 d1, d3 \n"
- "vtrn.16 d0, d2 \n"
- "vtrn.16 d5, d7 \n"
- "vtrn.16 d4, d6 \n"
-
- "vtrn.32 d1, d5 \n"
- "vtrn.32 d0, d4 \n"
- "vtrn.32 d3, d7 \n"
- "vtrn.32 d2, d6 \n"
-
- "vrev16.8 q0, q0 \n"
- "vrev16.8 q1, q1 \n"
- "vrev16.8 q2, q2 \n"
- "vrev16.8 q3, q3 \n"
-
- "mov %0, %3 \n"
-
- "vst1.8 {d1}, [%0], %4 \n"
- "vst1.8 {d0}, [%0], %4 \n"
- "vst1.8 {d3}, [%0], %4 \n"
- "vst1.8 {d2}, [%0], %4 \n"
- "vst1.8 {d5}, [%0], %4 \n"
- "vst1.8 {d4}, [%0], %4 \n"
- "vst1.8 {d7}, [%0], %4 \n"
- "vst1.8 {d6}, [%0] \n"
-
- "add %1, #8 \n" // src += 8
- "add %3, %3, %4, lsl #3 \n" // dst += 8 * dst_stride
- "subs %5, #8 \n" // w -= 8
- "bge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %5, #8 \n"
- "beq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %5, #2 \n"
- "blt 3f \n"
-
- "cmp %5, #4 \n"
- "blt 2f \n"
-
- // 4x8 block
- "mov %0, %1 \n"
- "vld1.32 {d0[0]}, [%0], %2 \n"
- "vld1.32 {d0[1]}, [%0], %2 \n"
- "vld1.32 {d1[0]}, [%0], %2 \n"
- "vld1.32 {d1[1]}, [%0], %2 \n"
- "vld1.32 {d2[0]}, [%0], %2 \n"
- "vld1.32 {d2[1]}, [%0], %2 \n"
- "vld1.32 {d3[0]}, [%0], %2 \n"
- "vld1.32 {d3[1]}, [%0] \n"
-
- "mov %0, %3 \n"
-
- "vld1.8 {q3}, [%6] \n"
-
- "vtbl.8 d4, {d0, d1}, d6 \n"
- "vtbl.8 d5, {d0, d1}, d7 \n"
- "vtbl.8 d0, {d2, d3}, d6 \n"
- "vtbl.8 d1, {d2, d3}, d7 \n"
-
- // TODO(frkoenig): Rework shuffle above to
- // write out with 4 instead of 8 writes.
- "vst1.32 {d4[0]}, [%0], %4 \n"
- "vst1.32 {d4[1]}, [%0], %4 \n"
- "vst1.32 {d5[0]}, [%0], %4 \n"
- "vst1.32 {d5[1]}, [%0] \n"
-
- "add %0, %3, #4 \n"
- "vst1.32 {d0[0]}, [%0], %4 \n"
- "vst1.32 {d0[1]}, [%0], %4 \n"
- "vst1.32 {d1[0]}, [%0], %4 \n"
- "vst1.32 {d1[1]}, [%0] \n"
-
- "add %1, #4 \n" // src += 4
- "add %3, %3, %4, lsl #2 \n" // dst += 4 * dst_stride
- "subs %5, #4 \n" // w -= 4
- "beq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %5, #2 \n"
- "blt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "vld1.16 {d0[0]}, [%0], %2 \n"
- "vld1.16 {d1[0]}, [%0], %2 \n"
- "vld1.16 {d0[1]}, [%0], %2 \n"
- "vld1.16 {d1[1]}, [%0], %2 \n"
- "vld1.16 {d0[2]}, [%0], %2 \n"
- "vld1.16 {d1[2]}, [%0], %2 \n"
- "vld1.16 {d0[3]}, [%0], %2 \n"
- "vld1.16 {d1[3]}, [%0] \n"
-
- "vtrn.8 d0, d1 \n"
-
- "mov %0, %3 \n"
-
- "vst1.64 {d0}, [%0], %4 \n"
- "vst1.64 {d1}, [%0] \n"
-
- "add %1, #2 \n" // src += 2
- "add %3, %3, %4, lsl #1 \n" // dst += 2 * dst_stride
- "subs %5, #2 \n" // w -= 2
- "beq 4f \n"
-
- // 1x8 block
- "3: \n"
- "vld1.8 {d0[0]}, [%1], %2 \n"
- "vld1.8 {d0[1]}, [%1], %2 \n"
- "vld1.8 {d0[2]}, [%1], %2 \n"
- "vld1.8 {d0[3]}, [%1], %2 \n"
- "vld1.8 {d0[4]}, [%1], %2 \n"
- "vld1.8 {d0[5]}, [%1], %2 \n"
- "vld1.8 {d0[6]}, [%1], %2 \n"
- "vld1.8 {d0[7]}, [%1] \n"
-
- "vst1.64 {d0}, [%3] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(src_stride), // %2
- "+r"(dst), // %3
- "+r"(dst_stride), // %4
- "+r"(width) // %5
- : "r"(&kVTbl4x4Transpose) // %6
+ "sub %[width], #8 \n"
+
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "vld1.8 {d0}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d1}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d2}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d3}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d4}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d5}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d6}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d7}, [%[temp]] \n"
+ "add %[src], #8 \n"
+
+ "vtrn.8 d1, d0 \n"
+ "vtrn.8 d3, d2 \n"
+ "vtrn.8 d5, d4 \n"
+ "vtrn.8 d7, d6 \n"
+ "subs %[width], #8 \n"
+
+ "vtrn.16 d1, d3 \n"
+ "vtrn.16 d0, d2 \n"
+ "vtrn.16 d5, d7 \n"
+ "vtrn.16 d4, d6 \n"
+
+ "vtrn.32 d1, d5 \n"
+ "vtrn.32 d0, d4 \n"
+ "vtrn.32 d3, d7 \n"
+ "vtrn.32 d2, d6 \n"
+
+ "vrev16.8 q0, q0 \n"
+ "vrev16.8 q1, q1 \n"
+ "vrev16.8 q2, q2 \n"
+ "vrev16.8 q3, q3 \n"
+
+ "mov %[temp], %[dst] \n"
+ "vst1.8 {d1}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d0}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d3}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d2}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d5}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d4}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d7}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d6}, [%[temp]] \n"
+ "add %[dst], %[dst], %[dst_stride], lsl #3 \n"
+
+ "bge 1b \n"
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [dst_stride] "r"(dst_stride) // %[dst_stride]
: "memory", "cc", "q0", "q1", "q2", "q3");
}
-static const uvec8 kVTbl4x4TransposeDi = {0, 8, 1, 9, 2, 10, 3, 11,
- 4, 12, 5, 13, 6, 14, 7, 15};
-
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@@ -199,215 +99,384 @@ void TransposeUVWx8_NEON(const uint8_t* src,
uint8_t* dst_b,
int dst_stride_b,
int width) {
- const uint8_t* src_temp;
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %7, #8 \n"
+ "sub %[width], #8 \n"
+
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "vld2.8 {d0, d1}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d2, d3}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d4, d5}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d6, d7}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d16, d17}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d18, d19}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d20, d21}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d22, d23}, [%[temp]] \n"
+ "add %[src], #8*2 \n"
+
+ "vtrn.8 q1, q0 \n"
+ "vtrn.8 q3, q2 \n"
+ "vtrn.8 q9, q8 \n"
+ "vtrn.8 q11, q10 \n"
+ "subs %[width], #8 \n"
+
+ "vtrn.16 q1, q3 \n"
+ "vtrn.16 q0, q2 \n"
+ "vtrn.16 q9, q11 \n"
+ "vtrn.16 q8, q10 \n"
+
+ "vtrn.32 q1, q9 \n"
+ "vtrn.32 q0, q8 \n"
+ "vtrn.32 q3, q11 \n"
+ "vtrn.32 q2, q10 \n"
+
+ "vrev16.8 q0, q0 \n"
+ "vrev16.8 q1, q1 \n"
+ "vrev16.8 q2, q2 \n"
+ "vrev16.8 q3, q3 \n"
+ "vrev16.8 q8, q8 \n"
+ "vrev16.8 q9, q9 \n"
+ "vrev16.8 q10, q10 \n"
+ "vrev16.8 q11, q11 \n"
+
+ "mov %[temp], %[dst_a] \n"
+ "vst1.8 {d2}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d0}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d6}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d4}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d18}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d16}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d22}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d20}, [%[temp]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
+
+ "mov %[temp], %[dst_b] \n"
+ "vst1.8 {d3}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d1}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d7}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d5}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d19}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d17}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d23}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d21}, [%[temp]] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
+
+ "bge 1b \n"
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [dst_stride_a] "r"(dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"(dst_stride_b) // %[dst_stride_b]
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
+}
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "vld2.8 {d0, d1}, [%0], %2 \n"
- "vld2.8 {d2, d3}, [%0], %2 \n"
- "vld2.8 {d4, d5}, [%0], %2 \n"
- "vld2.8 {d6, d7}, [%0], %2 \n"
- "vld2.8 {d16, d17}, [%0], %2 \n"
- "vld2.8 {d18, d19}, [%0], %2 \n"
- "vld2.8 {d20, d21}, [%0], %2 \n"
- "vld2.8 {d22, d23}, [%0] \n"
-
- "vtrn.8 q1, q0 \n"
- "vtrn.8 q3, q2 \n"
- "vtrn.8 q9, q8 \n"
- "vtrn.8 q11, q10 \n"
-
- "vtrn.16 q1, q3 \n"
- "vtrn.16 q0, q2 \n"
- "vtrn.16 q9, q11 \n"
- "vtrn.16 q8, q10 \n"
-
- "vtrn.32 q1, q9 \n"
- "vtrn.32 q0, q8 \n"
- "vtrn.32 q3, q11 \n"
- "vtrn.32 q2, q10 \n"
-
- "vrev16.8 q0, q0 \n"
- "vrev16.8 q1, q1 \n"
- "vrev16.8 q2, q2 \n"
- "vrev16.8 q3, q3 \n"
- "vrev16.8 q8, q8 \n"
- "vrev16.8 q9, q9 \n"
- "vrev16.8 q10, q10 \n"
- "vrev16.8 q11, q11 \n"
-
- "mov %0, %3 \n"
-
- "vst1.8 {d2}, [%0], %4 \n"
- "vst1.8 {d0}, [%0], %4 \n"
- "vst1.8 {d6}, [%0], %4 \n"
- "vst1.8 {d4}, [%0], %4 \n"
- "vst1.8 {d18}, [%0], %4 \n"
- "vst1.8 {d16}, [%0], %4 \n"
- "vst1.8 {d22}, [%0], %4 \n"
- "vst1.8 {d20}, [%0] \n"
-
- "mov %0, %5 \n"
-
- "vst1.8 {d3}, [%0], %6 \n"
- "vst1.8 {d1}, [%0], %6 \n"
- "vst1.8 {d7}, [%0], %6 \n"
- "vst1.8 {d5}, [%0], %6 \n"
- "vst1.8 {d19}, [%0], %6 \n"
- "vst1.8 {d17}, [%0], %6 \n"
- "vst1.8 {d23}, [%0], %6 \n"
- "vst1.8 {d21}, [%0] \n"
-
- "add %1, #8*2 \n" // src += 8*2
- "add %3, %3, %4, lsl #3 \n" // dst_a += 8 * dst_stride_a
- "add %5, %5, %6, lsl #3 \n" // dst_b += 8 * dst_stride_b
- "subs %7, #8 \n" // w -= 8
- "bge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %7, #8 \n"
- "beq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %7, #2 \n"
- "blt 3f \n"
-
- "cmp %7, #4 \n"
+// Transpose 32 bit values (ARGB) in Wx4 tiles using vld4.32 with lane insertion
+// and lane stores for 1..3 remainder rows.
+void TransposeWx4_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ const ptrdiff_t d = dst_stride;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ uint8_t* dst3 = dst2 + d;
+ asm volatile(
+ "subs %8, %8, #4 \n"
"blt 2f \n"
- // TODO(frkoenig): Clean this up
- // 4x8 block
- "mov %0, %1 \n"
- "vld1.64 {d0}, [%0], %2 \n"
- "vld1.64 {d1}, [%0], %2 \n"
- "vld1.64 {d2}, [%0], %2 \n"
- "vld1.64 {d3}, [%0], %2 \n"
- "vld1.64 {d4}, [%0], %2 \n"
- "vld1.64 {d5}, [%0], %2 \n"
- "vld1.64 {d6}, [%0], %2 \n"
- "vld1.64 {d7}, [%0] \n"
-
- "vld1.8 {q15}, [%8] \n"
-
- "vtrn.8 q0, q1 \n"
- "vtrn.8 q2, q3 \n"
-
- "vtbl.8 d16, {d0, d1}, d30 \n"
- "vtbl.8 d17, {d0, d1}, d31 \n"
- "vtbl.8 d18, {d2, d3}, d30 \n"
- "vtbl.8 d19, {d2, d3}, d31 \n"
- "vtbl.8 d20, {d4, d5}, d30 \n"
- "vtbl.8 d21, {d4, d5}, d31 \n"
- "vtbl.8 d22, {d6, d7}, d30 \n"
- "vtbl.8 d23, {d6, d7}, d31 \n"
-
- "mov %0, %3 \n"
-
- "vst1.32 {d16[0]}, [%0], %4 \n"
- "vst1.32 {d16[1]}, [%0], %4 \n"
- "vst1.32 {d17[0]}, [%0], %4 \n"
- "vst1.32 {d17[1]}, [%0], %4 \n"
-
- "add %0, %3, #4 \n"
- "vst1.32 {d20[0]}, [%0], %4 \n"
- "vst1.32 {d20[1]}, [%0], %4 \n"
- "vst1.32 {d21[0]}, [%0], %4 \n"
- "vst1.32 {d21[1]}, [%0] \n"
-
- "mov %0, %5 \n"
-
- "vst1.32 {d18[0]}, [%0], %6 \n"
- "vst1.32 {d18[1]}, [%0], %6 \n"
- "vst1.32 {d19[0]}, [%0], %6 \n"
- "vst1.32 {d19[1]}, [%0], %6 \n"
-
- "add %0, %5, #4 \n"
- "vst1.32 {d22[0]}, [%0], %6 \n"
- "vst1.32 {d22[1]}, [%0], %6 \n"
- "vst1.32 {d23[0]}, [%0], %6 \n"
- "vst1.32 {d23[1]}, [%0] \n"
-
- "add %1, #4*2 \n" // src += 4 * 2
- "add %3, %3, %4, lsl #2 \n" // dst_a += 4 *
- // dst_stride_a
- "add %5, %5, %6, lsl #2 \n" // dst_b += 4 *
- // dst_stride_b
- "subs %7, #4 \n" // w -= 4
- "beq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %7, #2 \n"
- "blt 3f \n"
-
- // 2x8 block
+ // Main loop transpose 4x4. Read a column, write a row.
+ "1: \n"
+ "vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n"
+ "vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n"
+ "vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n"
+ "vld4.32 {d1[1], d3[1], d5[1], d7[1]}, [%3], %9 \n"
+ "subs %8, %8, #4 \n"
+ "vst1.8 {q0}, [%4]! \n"
+ "vst1.8 {q1}, [%5]! \n"
+ "vst1.8 {q2}, [%6]! \n"
+ "vst1.8 {q3}, [%7]! \n"
+ "bge 1b \n"
+
"2: \n"
- "mov %0, %1 \n"
- "vld2.16 {d0[0], d2[0]}, [%0], %2 \n"
- "vld2.16 {d1[0], d3[0]}, [%0], %2 \n"
- "vld2.16 {d0[1], d2[1]}, [%0], %2 \n"
- "vld2.16 {d1[1], d3[1]}, [%0], %2 \n"
- "vld2.16 {d0[2], d2[2]}, [%0], %2 \n"
- "vld2.16 {d1[2], d3[2]}, [%0], %2 \n"
- "vld2.16 {d0[3], d2[3]}, [%0], %2 \n"
- "vld2.16 {d1[3], d3[3]}, [%0] \n"
-
- "vtrn.8 d0, d1 \n"
- "vtrn.8 d2, d3 \n"
-
- "mov %0, %3 \n"
-
- "vst1.64 {d0}, [%0], %4 \n"
- "vst1.64 {d2}, [%0] \n"
-
- "mov %0, %5 \n"
-
- "vst1.64 {d1}, [%0], %6 \n"
- "vst1.64 {d3}, [%0] \n"
-
- "add %1, #2*2 \n" // src += 2 * 2
- "add %3, %3, %4, lsl #1 \n" // dst_a += 2 *
- // dst_stride_a
- "add %5, %5, %6, lsl #1 \n" // dst_b += 2 *
- // dst_stride_b
- "subs %7, #2 \n" // w -= 2
- "beq 4f \n"
-
- // 1x8 block
+ "adds %8, %8, #4 \n"
+ "beq 5f \n"
+
+ // Remainder of 1, 2, or 3 source rows.
+ "vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0] \n"
+ "cmp %8, #2 \n"
+ "blt 4f \n"
+ "vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1] \n"
+ "bgt 3f \n"
+ "vst1.32 {d0}, [%4] \n"
+ "vst1.32 {d2}, [%5] \n"
+ "vst1.32 {d4}, [%6] \n"
+ "vst1.32 {d6}, [%7] \n"
+ "b 5f \n"
+
"3: \n"
- "vld2.8 {d0[0], d1[0]}, [%1], %2 \n"
- "vld2.8 {d0[1], d1[1]}, [%1], %2 \n"
- "vld2.8 {d0[2], d1[2]}, [%1], %2 \n"
- "vld2.8 {d0[3], d1[3]}, [%1], %2 \n"
- "vld2.8 {d0[4], d1[4]}, [%1], %2 \n"
- "vld2.8 {d0[5], d1[5]}, [%1], %2 \n"
- "vld2.8 {d0[6], d1[6]}, [%1], %2 \n"
- "vld2.8 {d0[7], d1[7]}, [%1] \n"
-
- "vst1.64 {d0}, [%3] \n"
- "vst1.64 {d1}, [%5] \n"
+ "vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2] \n"
+ "vst1.32 {d0}, [%4]! \n"
+ "vst1.32 {d2}, [%5]! \n"
+ "vst1.32 {d4}, [%6]! \n"
+ "vst1.32 {d6}, [%7]! \n"
+ "vst1.32 {d1[0]}, [%4] \n"
+ "vst1.32 {d3[0]}, [%5] \n"
+ "vst1.32 {d5[0]}, [%6] \n"
+ "vst1.32 {d7[0]}, [%7] \n"
+ "b 5f \n"
"4: \n"
+ "vst1.32 {d0[0]}, [%4] \n"
+ "vst1.32 {d2[0]}, [%5] \n"
+ "vst1.32 {d4[0]}, [%6] \n"
+ "vst1.32 {d6[0]}, [%7] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(dst2), // %6
+ "+r"(dst3), // %7
+ "+r"(width) // %8
+ : "r"(s * 4) // %9
+ : "memory", "cc", "q0", "q1", "q2", "q3");
+}
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(src_stride), // %2
- "+r"(dst_a), // %3
- "+r"(dst_stride_a), // %4
- "+r"(dst_b), // %5
- "+r"(dst_stride_b), // %6
- "+r"(width) // %7
- : "r"(&kVTbl4x4TransposeDi) // %8
+void TransposeWxH_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t d = dst_stride;
+ while (width >= 4) {
+ TransposeWx4_32_NEON(src, src_stride, dst, dst_stride, height);
+ src += 16;
+ dst += 4 * d;
+ width -= 4;
+ }
+ if (width == 3) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ asm volatile(
+ "subs %7, %7, #4 \n"
+ "blt 2f \n"
+ "1: \n"
+ "vld3.32 {d0[0], d2[0], d4[0]}, [%0], %8 \n"
+ "vld3.32 {d0[1], d2[1], d4[1]}, [%1], %8 \n"
+ "vld3.32 {d1[0], d3[0], d5[0]}, [%2], %8 \n"
+ "vld3.32 {d1[1], d3[1], d5[1]}, [%3], %8 \n"
+ "subs %7, %7, #4 \n"
+ "vst1.8 {q0}, [%4]! \n"
+ "vst1.8 {q1}, [%5]! \n"
+ "vst1.8 {q2}, [%6]! \n"
+ "bge 1b \n"
+ "2: \n"
+ "adds %7, %7, #4 \n"
+ "beq 5f \n"
+ "vld3.32 {d0[0], d2[0], d4[0]}, [%0] \n"
+ "cmp %7, #2 \n"
+ "blt 4f \n"
+ "vld3.32 {d0[1], d2[1], d4[1]}, [%1] \n"
+ "bgt 3f \n"
+ "vst1.32 {d0}, [%4] \n"
+ "vst1.32 {d2}, [%5] \n"
+ "vst1.32 {d4}, [%6] \n"
+ "b 5f \n"
+ "3: \n"
+ "vld3.32 {d1[0], d3[0], d5[0]}, [%2] \n"
+ "vst1.32 {d0}, [%4]! \n"
+ "vst1.32 {d2}, [%5]! \n"
+ "vst1.32 {d4}, [%6]! \n"
+ "vst1.32 {d1[0]}, [%4] \n"
+ "vst1.32 {d3[0]}, [%5] \n"
+ "vst1.32 {d5[0]}, [%6] \n"
+ "b 5f \n"
+ "4: \n"
+ "vst1.32 {d0[0]}, [%4] \n"
+ "vst1.32 {d2[0]}, [%5] \n"
+ "vst1.32 {d4[0]}, [%6] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(dst2), // %6
+ "+r"(height) // %7
+ : "r"(s * 4) // %8
+ : "memory", "cc", "q0", "q1", "q2");
+ } else if (width == 2) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ uint8_t* dst1 = dst + d;
+ asm volatile(
+ "subs %6, %6, #4 \n"
+ "blt 2f \n"
+ "1: \n"
+ "vld2.32 {d0[0], d2[0]}, [%0], %7 \n"
+ "vld2.32 {d0[1], d2[1]}, [%1], %7 \n"
+ "vld2.32 {d1[0], d3[0]}, [%2], %7 \n"
+ "vld2.32 {d1[1], d3[1]}, [%3], %7 \n"
+ "subs %6, %6, #4 \n"
+ "vst1.8 {q0}, [%4]! \n"
+ "vst1.8 {q1}, [%5]! \n"
+ "bge 1b \n"
+ "2: \n"
+ "adds %6, %6, #4 \n"
+ "beq 5f \n"
+ "vld2.32 {d0[0], d2[0]}, [%0] \n"
+ "cmp %6, #2 \n"
+ "blt 4f \n"
+ "vld2.32 {d0[1], d2[1]}, [%1] \n"
+ "bgt 3f \n"
+ "vst1.32 {d0}, [%4] \n"
+ "vst1.32 {d2}, [%5] \n"
+ "b 5f \n"
+ "3: \n"
+ "vld2.32 {d1[0], d3[0]}, [%2] \n"
+ "vst1.32 {d0}, [%4]! \n"
+ "vst1.32 {d2}, [%5]! \n"
+ "vst1.32 {d1[0]}, [%4] \n"
+ "vst1.32 {d3[0]}, [%5] \n"
+ "b 5f \n"
+ "4: \n"
+ "vst1.32 {d0[0]}, [%4] \n"
+ "vst1.32 {d2[0]}, [%5] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(height) // %6
+ : "r"(s * 4) // %7
+ : "memory", "cc", "q0", "q1");
+ } else if (width == 1) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ asm volatile(
+ "subs %5, %5, #4 \n"
+ "blt 2f \n"
+ "1: \n"
+ "vld1.32 {d0[0]}, [%0], %6 \n"
+ "vld1.32 {d0[1]}, [%1], %6 \n"
+ "vld1.32 {d1[0]}, [%2], %6 \n"
+ "vld1.32 {d1[1]}, [%3], %6 \n"
+ "subs %5, %5, #4 \n"
+ "vst1.8 {q0}, [%4]! \n"
+ "bge 1b \n"
+ "2: \n"
+ "adds %5, %5, #4 \n"
+ "beq 5f \n"
+ "vld1.32 {d0[0]}, [%0] \n"
+ "cmp %5, #2 \n"
+ "blt 4f \n"
+ "vld1.32 {d0[1]}, [%1] \n"
+ "bgt 3f \n"
+ "vst1.32 {d0}, [%4] \n"
+ "b 5f \n"
+ "3: \n"
+ "vld1.32 {d1[0]}, [%2] \n"
+ "vst1.32 {d0}, [%4]! \n"
+ "vst1.32 {d1[0]}, [%4] \n"
+ "b 5f \n"
+ "4: \n"
+ "vst1.32 {d0[0]}, [%4] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(height) // %5
+ : "r"(s * 4) // %6
+ : "memory", "cc", "q0");
+ }
+}
+
+void TransposeWx8_16_NEON(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint8_t* temp;
+ asm volatile(
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "vld1.8 {q0}, [%[temp]], %[s] \n"
+ "vld1.8 {q1}, [%[temp]], %[s] \n"
+ "vld1.8 {q2}, [%[temp]], %[s] \n"
+ "vld1.8 {q3}, [%[temp]], %[s] \n"
+ "vld1.8 {q8}, [%[temp]], %[s] \n"
+ "vld1.8 {q9}, [%[temp]], %[s] \n"
+ "vld1.8 {q10}, [%[temp]], %[s] \n"
+ "vld1.8 {q11}, [%[temp]] \n"
+ "add %[src], %[src], #16 \n"
+
+ // Stage 1: 16-bit transpose
+ "vtrn.16 q0, q1 \n"
+ "vtrn.16 q2, q3 \n"
+ "vtrn.16 q8, q9 \n"
+ "vtrn.16 q10, q11 \n"
+
+ // Stage 2: 32-bit transpose
+ "vtrn.32 q0, q2 \n"
+ "vtrn.32 q1, q3 \n"
+ "vtrn.32 q8, q10 \n"
+ "vtrn.32 q9, q11 \n"
+
+ // Stage 3: 64-bit swap
+ "vswp d1, d16 \n"
+ "vswp d3, d18 \n"
+ "vswp d5, d20 \n"
+ "vswp d7, d22 \n"
+
+ "subs %[w], %[w], #8 \n"
+ "vst1.8 {q0}, [%[dst]], %[d] \n"
+ "vst1.8 {q1}, [%[dst]], %[d] \n"
+ "vst1.8 {q2}, [%[dst]], %[d] \n"
+ "vst1.8 {q3}, [%[dst]], %[d] \n"
+ "vst1.8 {q8}, [%[dst]], %[d] \n"
+ "vst1.8 {q9}, [%[dst]], %[d] \n"
+ "vst1.8 {q10}, [%[dst]], %[d] \n"
+ "vst1.8 {q11}, [%[dst]], %[d] \n"
+ "bgt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
+
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/rotate_neon64.cc b/libs/libyuv/source/rotate_neon64.cc
index f469baacf6..5844c0d28e 100644
--- a/libs/libyuv/source/rotate_neon64.cc
+++ b/libs/libyuv/source/rotate_neon64.cc
@@ -11,413 +11,542 @@
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
-#include "libyuv/basic_types.h"
-
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
-static const uvec8 kVTbl4x4Transpose = {0, 4, 8, 12, 1, 5, 9, 13,
- 2, 6, 10, 14, 3, 7, 11, 15};
-
-void TransposeWx8_NEON(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
+void TransposeWx16_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
const uint8_t* src_temp;
+ asm volatile(
+ "1: \n"
+ "mov %[src_temp], %[src] \n"
+
+ "ld1 {v16.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v17.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v18.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v19.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v20.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v21.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v22.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v23.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v24.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v25.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v26.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v27.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v28.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v29.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v30.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v31.16b}, [%[src_temp]], %[src_stride] \n"
+
+ "add %[src], %[src], #16 \n"
+
+ // Transpose bytes within each 2x2 block.
+ "trn1 v0.16b, v16.16b, v17.16b \n"
+ "trn2 v1.16b, v16.16b, v17.16b \n"
+ "trn1 v2.16b, v18.16b, v19.16b \n"
+ "trn2 v3.16b, v18.16b, v19.16b \n"
+ "trn1 v4.16b, v20.16b, v21.16b \n"
+ "trn2 v5.16b, v20.16b, v21.16b \n"
+ "trn1 v6.16b, v22.16b, v23.16b \n"
+ "trn2 v7.16b, v22.16b, v23.16b \n"
+ "trn1 v8.16b, v24.16b, v25.16b \n"
+ "trn2 v9.16b, v24.16b, v25.16b \n"
+ "trn1 v10.16b, v26.16b, v27.16b \n"
+ "trn2 v11.16b, v26.16b, v27.16b \n"
+ "trn1 v12.16b, v28.16b, v29.16b \n"
+ "trn2 v13.16b, v28.16b, v29.16b \n"
+ "trn1 v14.16b, v30.16b, v31.16b \n"
+ "trn2 v15.16b, v30.16b, v31.16b \n"
+
+ // Transpose 2x2-byte blocks within each 4x4 block.
+ "trn1 v16.8h, v0.8h, v2.8h \n"
+ "trn1 v17.8h, v1.8h, v3.8h \n"
+ "trn2 v18.8h, v0.8h, v2.8h \n"
+ "trn2 v19.8h, v1.8h, v3.8h \n"
+ "trn1 v20.8h, v4.8h, v6.8h \n"
+ "trn1 v21.8h, v5.8h, v7.8h \n"
+ "trn2 v22.8h, v4.8h, v6.8h \n"
+ "trn2 v23.8h, v5.8h, v7.8h \n"
+ "trn1 v24.8h, v8.8h, v10.8h \n"
+ "trn1 v25.8h, v9.8h, v11.8h \n"
+ "trn2 v26.8h, v8.8h, v10.8h \n"
+ "trn2 v27.8h, v9.8h, v11.8h \n"
+ "trn1 v28.8h, v12.8h, v14.8h \n"
+ "trn1 v29.8h, v13.8h, v15.8h \n"
+ "trn2 v30.8h, v12.8h, v14.8h \n"
+ "trn2 v31.8h, v13.8h, v15.8h \n"
+
+ "subs %w[width], %w[width], #16 \n"
+
+ // Transpose 4x4-byte blocks within each 8x8 block.
+ "trn1 v0.4s, v16.4s, v20.4s \n"
+ "trn1 v2.4s, v17.4s, v21.4s \n"
+ "trn1 v4.4s, v18.4s, v22.4s \n"
+ "trn1 v6.4s, v19.4s, v23.4s \n"
+ "trn2 v8.4s, v16.4s, v20.4s \n"
+ "trn2 v10.4s, v17.4s, v21.4s \n"
+ "trn2 v12.4s, v18.4s, v22.4s \n"
+ "trn2 v14.4s, v19.4s, v23.4s \n"
+ "trn1 v1.4s, v24.4s, v28.4s \n"
+ "trn1 v3.4s, v25.4s, v29.4s \n"
+ "trn1 v5.4s, v26.4s, v30.4s \n"
+ "trn1 v7.4s, v27.4s, v31.4s \n"
+ "trn2 v9.4s, v24.4s, v28.4s \n"
+ "trn2 v11.4s, v25.4s, v29.4s \n"
+ "trn2 v13.4s, v26.4s, v30.4s \n"
+ "trn2 v15.4s, v27.4s, v31.4s \n"
+
+ // Transpose 8x8-byte blocks and store.
+ "st2 {v0.d, v1.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v2.d, v3.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v4.d, v5.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v6.d, v7.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v8.d, v9.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v10.d, v11.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v12.d, v13.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v14.d, v15.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v0.d, v1.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v2.d, v3.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v4.d, v5.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v6.d, v7.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v8.d, v9.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v10.d, v11.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v12.d, v13.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v14.d, v15.d}[1], [%[dst]], %[dst_stride] \n"
+
+ "b.gt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [src_temp] "=&r"(src_temp), // %[src_temp]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride) // %[dst_stride]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
+ "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",
+ "v29", "v30", "v31");
+}
+
+void TransposeUVWx8_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %w3, %w3, #8 \n"
+ "sub %w[width], %w[width], #8 \n"
+
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "ld1 {v0.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v1.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v2.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v3.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v4.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v5.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v6.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v7.16b}, [%[temp]] \n"
+ "add %[src], %[src], #16 \n"
+
+ "trn1 v16.16b, v0.16b, v1.16b \n"
+ "trn2 v17.16b, v0.16b, v1.16b \n"
+ "trn1 v18.16b, v2.16b, v3.16b \n"
+ "trn2 v19.16b, v2.16b, v3.16b \n"
+ "trn1 v20.16b, v4.16b, v5.16b \n"
+ "trn2 v21.16b, v4.16b, v5.16b \n"
+ "trn1 v22.16b, v6.16b, v7.16b \n"
+ "trn2 v23.16b, v6.16b, v7.16b \n"
+
+ "subs %w[width], %w[width], #8 \n"
+
+ "trn1 v0.8h, v16.8h, v18.8h \n"
+ "trn2 v1.8h, v16.8h, v18.8h \n"
+ "trn1 v2.8h, v20.8h, v22.8h \n"
+ "trn2 v3.8h, v20.8h, v22.8h \n"
+ "trn1 v4.8h, v17.8h, v19.8h \n"
+ "trn2 v5.8h, v17.8h, v19.8h \n"
+ "trn1 v6.8h, v21.8h, v23.8h \n"
+ "trn2 v7.8h, v21.8h, v23.8h \n"
+
+ "trn1 v16.4s, v0.4s, v2.4s \n"
+ "trn2 v17.4s, v0.4s, v2.4s \n"
+ "trn1 v18.4s, v1.4s, v3.4s \n"
+ "trn2 v19.4s, v1.4s, v3.4s \n"
+ "trn1 v20.4s, v4.4s, v6.4s \n"
+ "trn2 v21.4s, v4.4s, v6.4s \n"
+ "trn1 v22.4s, v5.4s, v7.4s \n"
+ "trn2 v23.4s, v5.4s, v7.4s \n"
+
+ "mov %[temp], %[dst_a] \n"
+ "st1 {v16.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v18.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v17.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v19.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v16.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v18.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v17.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v19.d}[1], [%[temp]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
+
+ "mov %[temp], %[dst_b] \n"
+ "st1 {v20.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v22.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v21.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v23.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v20.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v22.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v21.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v23.d}[1], [%[temp]] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "ld1 {v0.8b}, [%0], %5 \n"
- "ld1 {v1.8b}, [%0], %5 \n"
- "ld1 {v2.8b}, [%0], %5 \n"
- "ld1 {v3.8b}, [%0], %5 \n"
- "ld1 {v4.8b}, [%0], %5 \n"
- "ld1 {v5.8b}, [%0], %5 \n"
- "ld1 {v6.8b}, [%0], %5 \n"
- "ld1 {v7.8b}, [%0] \n"
-
- "trn2 v16.8b, v0.8b, v1.8b \n"
- "trn1 v17.8b, v0.8b, v1.8b \n"
- "trn2 v18.8b, v2.8b, v3.8b \n"
- "trn1 v19.8b, v2.8b, v3.8b \n"
- "trn2 v20.8b, v4.8b, v5.8b \n"
- "trn1 v21.8b, v4.8b, v5.8b \n"
- "trn2 v22.8b, v6.8b, v7.8b \n"
- "trn1 v23.8b, v6.8b, v7.8b \n"
-
- "trn2 v3.4h, v17.4h, v19.4h \n"
- "trn1 v1.4h, v17.4h, v19.4h \n"
- "trn2 v2.4h, v16.4h, v18.4h \n"
- "trn1 v0.4h, v16.4h, v18.4h \n"
- "trn2 v7.4h, v21.4h, v23.4h \n"
- "trn1 v5.4h, v21.4h, v23.4h \n"
- "trn2 v6.4h, v20.4h, v22.4h \n"
- "trn1 v4.4h, v20.4h, v22.4h \n"
-
- "trn2 v21.2s, v1.2s, v5.2s \n"
- "trn1 v17.2s, v1.2s, v5.2s \n"
- "trn2 v20.2s, v0.2s, v4.2s \n"
- "trn1 v16.2s, v0.2s, v4.2s \n"
- "trn2 v23.2s, v3.2s, v7.2s \n"
- "trn1 v19.2s, v3.2s, v7.2s \n"
- "trn2 v22.2s, v2.2s, v6.2s \n"
- "trn1 v18.2s, v2.2s, v6.2s \n"
-
- "mov %0, %2 \n"
-
- "st1 {v17.8b}, [%0], %6 \n"
- "st1 {v16.8b}, [%0], %6 \n"
- "st1 {v19.8b}, [%0], %6 \n"
- "st1 {v18.8b}, [%0], %6 \n"
- "st1 {v21.8b}, [%0], %6 \n"
- "st1 {v20.8b}, [%0], %6 \n"
- "st1 {v23.8b}, [%0], %6 \n"
- "st1 {v22.8b}, [%0] \n"
-
- "add %1, %1, #8 \n" // src += 8
- "add %2, %2, %6, lsl #3 \n" // dst += 8 * dst_stride
- "subs %w3, %w3, #8 \n" // w -= 8
"b.ge 1b \n"
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b) // %[dst_stride_b]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
+}
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %w3, %w3, #8 \n"
- "b.eq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %w3, #2 \n"
- "b.lt 3f \n"
-
- "cmp %w3, #4 \n"
+// Transpose 32 bit values (ARGB) in Wx4 tiles using ld4 with lane insertion
+// and lane stores for 1..3 remainder rows.
+void TransposeWx4_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ const ptrdiff_t d = dst_stride;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ uint8_t* dst3 = dst2 + d;
+ asm volatile(
+ "subs %w8, %w8, #4 \n"
"b.lt 2f \n"
- // 4x8 block
- "mov %0, %1 \n"
- "ld1 {v0.s}[0], [%0], %5 \n"
- "ld1 {v0.s}[1], [%0], %5 \n"
- "ld1 {v0.s}[2], [%0], %5 \n"
- "ld1 {v0.s}[3], [%0], %5 \n"
- "ld1 {v1.s}[0], [%0], %5 \n"
- "ld1 {v1.s}[1], [%0], %5 \n"
- "ld1 {v1.s}[2], [%0], %5 \n"
- "ld1 {v1.s}[3], [%0] \n"
-
- "mov %0, %2 \n"
-
- "ld1 {v2.16b}, [%4] \n"
-
- "tbl v3.16b, {v0.16b}, v2.16b \n"
- "tbl v0.16b, {v1.16b}, v2.16b \n"
-
- // TODO(frkoenig): Rework shuffle above to
- // write out with 4 instead of 8 writes.
- "st1 {v3.s}[0], [%0], %6 \n"
- "st1 {v3.s}[1], [%0], %6 \n"
- "st1 {v3.s}[2], [%0], %6 \n"
- "st1 {v3.s}[3], [%0] \n"
-
- "add %0, %2, #4 \n"
- "st1 {v0.s}[0], [%0], %6 \n"
- "st1 {v0.s}[1], [%0], %6 \n"
- "st1 {v0.s}[2], [%0], %6 \n"
- "st1 {v0.s}[3], [%0] \n"
-
- "add %1, %1, #4 \n" // src += 4
- "add %2, %2, %6, lsl #2 \n" // dst += 4 * dst_stride
- "subs %w3, %w3, #4 \n" // w -= 4
- "b.eq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %w3, #2 \n"
- "b.lt 3f \n"
-
- // 2x8 block
+ // Main loop transpose 4x4. Read a column, write a row.
+ "1: \n"
+ "ldp s0, s1, [%0] \n"
+ "ldp s2, s3, [%0, #8] \n"
+ "add %0, %0, %9 \n"
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1], %9 \n"
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2], %9 \n"
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[3], [%3], %9 \n"
+ "subs %w8, %w8, #4 \n"
+ "st1 {v0.4s}, [%4], 16 \n"
+ "st1 {v1.4s}, [%5], 16 \n"
+ "st1 {v2.4s}, [%6], 16 \n"
+ "st1 {v3.4s}, [%7], 16 \n"
+ "b.ge 1b \n"
+
"2: \n"
- "mov %0, %1 \n"
- "ld1 {v0.h}[0], [%0], %5 \n"
- "ld1 {v1.h}[0], [%0], %5 \n"
- "ld1 {v0.h}[1], [%0], %5 \n"
- "ld1 {v1.h}[1], [%0], %5 \n"
- "ld1 {v0.h}[2], [%0], %5 \n"
- "ld1 {v1.h}[2], [%0], %5 \n"
- "ld1 {v0.h}[3], [%0], %5 \n"
- "ld1 {v1.h}[3], [%0] \n"
-
- "trn2 v2.8b, v0.8b, v1.8b \n"
- "trn1 v3.8b, v0.8b, v1.8b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v3.8b}, [%0], %6 \n"
- "st1 {v2.8b}, [%0] \n"
-
- "add %1, %1, #2 \n" // src += 2
- "add %2, %2, %6, lsl #1 \n" // dst += 2 * dst_stride
- "subs %w3, %w3, #2 \n" // w -= 2
- "b.eq 4f \n"
-
- // 1x8 block
- "3: \n"
- "ld1 {v0.b}[0], [%1], %5 \n"
- "ld1 {v0.b}[1], [%1], %5 \n"
- "ld1 {v0.b}[2], [%1], %5 \n"
- "ld1 {v0.b}[3], [%1], %5 \n"
- "ld1 {v0.b}[4], [%1], %5 \n"
- "ld1 {v0.b}[5], [%1], %5 \n"
- "ld1 {v0.b}[6], [%1], %5 \n"
- "ld1 {v0.b}[7], [%1] \n"
+ "adds %w8, %w8, #4 \n"
+ "b.eq 5f \n"
+
+ // Remainder of 1, 2, or 3 source rows.
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0] \n"
+ "cmp %w8, #2 \n"
+ "b.lt 4f \n"
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1] \n"
+ "b.gt 3f \n"
+ "st1 {v0.d}[0], [%4] \n"
+ "st1 {v1.d}[0], [%5] \n"
+ "st1 {v2.d}[0], [%6] \n"
+ "st1 {v3.d}[0], [%7] \n"
+ "b 5f \n"
- "st1 {v0.8b}, [%2] \n"
+ "3: \n"
+ "ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2] \n"
+ "st1 {v0.d}[0], [%4], 8 \n"
+ "st1 {v1.d}[0], [%5], 8 \n"
+ "st1 {v2.d}[0], [%6], 8 \n"
+ "st1 {v3.d}[0], [%7], 8 \n"
+ "st1 {v0.s}[2], [%4] \n"
+ "st1 {v1.s}[2], [%5] \n"
+ "st1 {v2.s}[2], [%6] \n"
+ "st1 {v3.s}[2], [%7] \n"
+ "b 5f \n"
"4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(dst), // %2
- "+r"(width) // %3
- : "r"(&kVTbl4x4Transpose), // %4
- "r"(static_cast<ptrdiff_t>(src_stride)), // %5
- "r"(static_cast<ptrdiff_t>(dst_stride)) // %6
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
- "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+ "st1 {v0.s}[0], [%4] \n"
+ "st1 {v1.s}[0], [%5] \n"
+ "st1 {v2.s}[0], [%6] \n"
+ "st1 {v3.s}[0], [%7] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(dst2), // %6
+ "+r"(dst3), // %7
+ "+r"(width) // %8
+ : "r"(s * 4) // %9
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
-static const uint8_t kVTbl4x4TransposeDi[32] = {
- 0, 16, 32, 48, 2, 18, 34, 50, 4, 20, 36, 52, 6, 22, 38, 54,
- 1, 17, 33, 49, 3, 19, 35, 51, 5, 21, 37, 53, 7, 23, 39, 55};
+void TransposeWxH_32_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t d = dst_stride;
+ while (width >= 4) {
+ TransposeWx4_32_NEON(src, src_stride, dst, dst_stride, height);
+ src += 16;
+ dst += 4 * d;
+ width -= 4;
+ }
+ if (width == 3) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ asm volatile(
+ "subs %w7, %w7, #4 \n"
+ "b.lt 2f \n"
+ "1: \n"
+ "ld3 {v0.s, v1.s, v2.s}[0], [%0], %8 \n"
+ "ld3 {v0.s, v1.s, v2.s}[1], [%1], %8 \n"
+ "ld3 {v0.s, v1.s, v2.s}[2], [%2], %8 \n"
+ "ld3 {v0.s, v1.s, v2.s}[3], [%3], %8 \n"
+ "subs %w7, %w7, #4 \n"
+ "st1 {v0.4s}, [%4], 16 \n"
+ "st1 {v1.4s}, [%5], 16 \n"
+ "st1 {v2.4s}, [%6], 16 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w7, %w7, #4 \n"
+ "b.eq 5f \n"
+ "ld3 {v0.s, v1.s, v2.s}[0], [%0] \n"
+ "cmp %w7, #2 \n"
+ "b.lt 4f \n"
+ "ld3 {v0.s, v1.s, v2.s}[1], [%1] \n"
+ "b.gt 3f \n"
+ "st1 {v0.d}[0], [%4] \n"
+ "st1 {v1.d}[0], [%5] \n"
+ "st1 {v2.d}[0], [%6] \n"
+ "b 5f \n"
+ "3: \n"
+ "ld3 {v0.s, v1.s, v2.s}[2], [%2] \n"
+ "st1 {v0.d}[0], [%4], 8 \n"
+ "st1 {v1.d}[0], [%5], 8 \n"
+ "st1 {v2.d}[0], [%6], 8 \n"
+ "st1 {v0.s}[2], [%4] \n"
+ "st1 {v1.s}[2], [%5] \n"
+ "st1 {v2.s}[2], [%6] \n"
+ "b 5f \n"
+ "4: \n"
+ "st1 {v0.s}[0], [%4] \n"
+ "st1 {v1.s}[0], [%5] \n"
+ "st1 {v2.s}[0], [%6] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(dst2), // %6
+ "+r"(height) // %7
+ : "r"(s * 4) // %8
+ : "memory", "cc", "v0", "v1", "v2");
+ } else if (width == 2) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ uint8_t* dst1 = dst + d;
+ asm volatile(
+ "subs %w6, %w6, #4 \n"
+ "b.lt 2f \n"
+ "1: \n"
+ "ld2 {v0.s, v1.s}[0], [%0], %7 \n"
+ "ld2 {v0.s, v1.s}[1], [%1], %7 \n"
+ "ld2 {v0.s, v1.s}[2], [%2], %7 \n"
+ "ld2 {v0.s, v1.s}[3], [%3], %7 \n"
+ "subs %w6, %w6, #4 \n"
+ "st1 {v0.4s}, [%4], 16 \n"
+ "st1 {v1.4s}, [%5], 16 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w6, %w6, #4 \n"
+ "b.eq 5f \n"
+ "ld2 {v0.s, v1.s}[0], [%0] \n"
+ "cmp %w6, #2 \n"
+ "b.lt 4f \n"
+ "ld2 {v0.s, v1.s}[1], [%1] \n"
+ "b.gt 3f \n"
+ "st1 {v0.d}[0], [%4] \n"
+ "st1 {v1.d}[0], [%5] \n"
+ "b 5f \n"
+ "3: \n"
+ "ld2 {v0.s, v1.s}[2], [%2] \n"
+ "st1 {v0.d}[0], [%4], 8 \n"
+ "st1 {v1.d}[0], [%5], 8 \n"
+ "st1 {v0.s}[2], [%4] \n"
+ "st1 {v1.s}[2], [%5] \n"
+ "b 5f \n"
+ "4: \n"
+ "st1 {v0.s}[0], [%4] \n"
+ "st1 {v1.s}[0], [%5] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(dst1), // %5
+ "+r"(height) // %6
+ : "r"(s * 4) // %7
+ : "memory", "cc", "v0", "v1");
+ } else if (width == 1) {
+ const ptrdiff_t s = src_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ asm volatile(
+ "subs %w5, %w5, #4 \n"
+ "b.lt 2f \n"
+ "1: \n"
+ "ld1 {v0.s}[0], [%0], %6 \n"
+ "ld1 {v0.s}[1], [%1], %6 \n"
+ "ld1 {v0.s}[2], [%2], %6 \n"
+ "ld1 {v0.s}[3], [%3], %6 \n"
+ "subs %w5, %w5, #4 \n"
+ "st1 {v0.4s}, [%4], 16 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w5, %w5, #4 \n"
+ "b.eq 5f \n"
+ "ld1 {v0.s}[0], [%0] \n"
+ "cmp %w5, #2 \n"
+ "b.lt 4f \n"
+ "ld1 {v0.s}[1], [%1] \n"
+ "b.gt 3f \n"
+ "st1 {v0.d}[0], [%4] \n"
+ "b 5f \n"
+ "3: \n"
+ "ld1 {v0.s}[2], [%2] \n"
+ "st1 {v0.d}[0], [%4], 8 \n"
+ "st1 {v0.s}[2], [%4] \n"
+ "b 5f \n"
+ "4: \n"
+ "st1 {v0.s}[0], [%4] \n"
+ "5: \n"
+ : "+r"(src), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(dst), // %4
+ "+r"(height) // %5
+ : "r"(s * 4) // %6
+ : "memory", "cc", "v0");
+ }
+}
-void TransposeUVWx8_NEON(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int width) {
- const uint8_t* src_temp;
+void TransposeWx8_16_NEON(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint16_t* temp;
asm volatile(
- // loops are on blocks of 8. loop will stop when
- // counter gets to or below 0. starting the counter
- // at w-8 allow for this
- "sub %w4, %w4, #8 \n"
-
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "ld1 {v0.16b}, [%0], %5 \n"
- "ld1 {v1.16b}, [%0], %5 \n"
- "ld1 {v2.16b}, [%0], %5 \n"
- "ld1 {v3.16b}, [%0], %5 \n"
- "ld1 {v4.16b}, [%0], %5 \n"
- "ld1 {v5.16b}, [%0], %5 \n"
- "ld1 {v6.16b}, [%0], %5 \n"
- "ld1 {v7.16b}, [%0] \n"
-
- "trn1 v16.16b, v0.16b, v1.16b \n"
- "trn2 v17.16b, v0.16b, v1.16b \n"
- "trn1 v18.16b, v2.16b, v3.16b \n"
- "trn2 v19.16b, v2.16b, v3.16b \n"
- "trn1 v20.16b, v4.16b, v5.16b \n"
- "trn2 v21.16b, v4.16b, v5.16b \n"
- "trn1 v22.16b, v6.16b, v7.16b \n"
- "trn2 v23.16b, v6.16b, v7.16b \n"
-
- "trn1 v0.8h, v16.8h, v18.8h \n"
- "trn2 v1.8h, v16.8h, v18.8h \n"
- "trn1 v2.8h, v20.8h, v22.8h \n"
- "trn2 v3.8h, v20.8h, v22.8h \n"
- "trn1 v4.8h, v17.8h, v19.8h \n"
- "trn2 v5.8h, v17.8h, v19.8h \n"
- "trn1 v6.8h, v21.8h, v23.8h \n"
- "trn2 v7.8h, v21.8h, v23.8h \n"
-
- "trn1 v16.4s, v0.4s, v2.4s \n"
- "trn2 v17.4s, v0.4s, v2.4s \n"
- "trn1 v18.4s, v1.4s, v3.4s \n"
- "trn2 v19.4s, v1.4s, v3.4s \n"
- "trn1 v20.4s, v4.4s, v6.4s \n"
- "trn2 v21.4s, v4.4s, v6.4s \n"
- "trn1 v22.4s, v5.4s, v7.4s \n"
- "trn2 v23.4s, v5.4s, v7.4s \n"
-
- "mov %0, %2 \n"
-
- "st1 {v16.d}[0], [%0], %6 \n"
- "st1 {v18.d}[0], [%0], %6 \n"
- "st1 {v17.d}[0], [%0], %6 \n"
- "st1 {v19.d}[0], [%0], %6 \n"
- "st1 {v16.d}[1], [%0], %6 \n"
- "st1 {v18.d}[1], [%0], %6 \n"
- "st1 {v17.d}[1], [%0], %6 \n"
- "st1 {v19.d}[1], [%0] \n"
-
- "mov %0, %3 \n"
-
- "st1 {v20.d}[0], [%0], %7 \n"
- "st1 {v22.d}[0], [%0], %7 \n"
- "st1 {v21.d}[0], [%0], %7 \n"
- "st1 {v23.d}[0], [%0], %7 \n"
- "st1 {v20.d}[1], [%0], %7 \n"
- "st1 {v22.d}[1], [%0], %7 \n"
- "st1 {v21.d}[1], [%0], %7 \n"
- "st1 {v23.d}[1], [%0] \n"
-
- "add %1, %1, #16 \n" // src += 8*2
- "add %2, %2, %6, lsl #3 \n" // dst_a += 8 *
- // dst_stride_a
- "add %3, %3, %7, lsl #3 \n" // dst_b += 8 *
- // dst_stride_b
- "subs %w4, %w4, #8 \n" // w -= 8
- "b.ge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %w4, %w4, #8 \n"
- "b.eq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %w4, #2 \n"
- "b.lt 3f \n"
-
- "cmp %w4, #4 \n"
- "b.lt 2f \n"
-
- // TODO(frkoenig): Clean this up
- // 4x8 block
- "mov %0, %1 \n"
- "ld1 {v0.8b}, [%0], %5 \n"
- "ld1 {v1.8b}, [%0], %5 \n"
- "ld1 {v2.8b}, [%0], %5 \n"
- "ld1 {v3.8b}, [%0], %5 \n"
- "ld1 {v4.8b}, [%0], %5 \n"
- "ld1 {v5.8b}, [%0], %5 \n"
- "ld1 {v6.8b}, [%0], %5 \n"
- "ld1 {v7.8b}, [%0] \n"
-
- "ld1 {v30.16b}, [%8], #16 \n"
- "ld1 {v31.16b}, [%8] \n"
-
- "tbl v16.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v30.16b \n"
- "tbl v17.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v31.16b \n"
- "tbl v18.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v30.16b \n"
- "tbl v19.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v31.16b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v16.s}[0], [%0], %6 \n"
- "st1 {v16.s}[1], [%0], %6 \n"
- "st1 {v16.s}[2], [%0], %6 \n"
- "st1 {v16.s}[3], [%0], %6 \n"
-
- "add %0, %2, #4 \n"
- "st1 {v18.s}[0], [%0], %6 \n"
- "st1 {v18.s}[1], [%0], %6 \n"
- "st1 {v18.s}[2], [%0], %6 \n"
- "st1 {v18.s}[3], [%0] \n"
-
- "mov %0, %3 \n"
-
- "st1 {v17.s}[0], [%0], %7 \n"
- "st1 {v17.s}[1], [%0], %7 \n"
- "st1 {v17.s}[2], [%0], %7 \n"
- "st1 {v17.s}[3], [%0], %7 \n"
-
- "add %0, %3, #4 \n"
- "st1 {v19.s}[0], [%0], %7 \n"
- "st1 {v19.s}[1], [%0], %7 \n"
- "st1 {v19.s}[2], [%0], %7 \n"
- "st1 {v19.s}[3], [%0] \n"
-
- "add %1, %1, #8 \n" // src += 4 * 2
- "add %2, %2, %6, lsl #2 \n" // dst_a += 4 *
- // dst_stride_a
- "add %3, %3, %7, lsl #2 \n" // dst_b += 4 *
- // dst_stride_b
- "subs %w4, %w4, #4 \n" // w -= 4
- "b.eq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %w4, #2 \n"
- "b.lt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "ld2 {v0.h, v1.h}[0], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[0], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[1], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[1], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[2], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[2], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[3], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[3], [%0] \n"
-
- "trn1 v4.8b, v0.8b, v2.8b \n"
- "trn2 v5.8b, v0.8b, v2.8b \n"
- "trn1 v6.8b, v1.8b, v3.8b \n"
- "trn2 v7.8b, v1.8b, v3.8b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v4.d}[0], [%0], %6 \n"
- "st1 {v6.d}[0], [%0] \n"
-
- "mov %0, %3 \n"
-
- "st1 {v5.d}[0], [%0], %7 \n"
- "st1 {v7.d}[0], [%0] \n"
-
- "add %1, %1, #4 \n" // src += 2 * 2
- "add %2, %2, %6, lsl #1 \n" // dst_a += 2 *
- // dst_stride_a
- "add %3, %3, %7, lsl #1 \n" // dst_b += 2 *
- // dst_stride_b
- "subs %w4, %w4, #2 \n" // w -= 2
- "b.eq 4f \n"
-
- // 1x8 block
- "3: \n"
- "ld2 {v0.b, v1.b}[0], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[1], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[2], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[3], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[4], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[5], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[6], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[7], [%1] \n"
-
- "st1 {v0.d}[0], [%2] \n"
- "st1 {v1.d}[0], [%3] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(dst_a), // %2
- "+r"(dst_b), // %3
- "+r"(width) // %4
- : "r"(static_cast<ptrdiff_t>(src_stride)), // %5
- "r"(static_cast<ptrdiff_t>(dst_stride_a)), // %6
- "r"(static_cast<ptrdiff_t>(dst_stride_b)), // %7
- "r"(&kVTbl4x4TransposeDi) // %8
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "ld1 {v0.8h}, [%[temp]], %[s] \n"
+ "ld1 {v1.8h}, [%[temp]], %[s] \n"
+ "ld1 {v2.8h}, [%[temp]], %[s] \n"
+ "ld1 {v3.8h}, [%[temp]], %[s] \n"
+ "ld1 {v4.8h}, [%[temp]], %[s] \n"
+ "ld1 {v5.8h}, [%[temp]], %[s] \n"
+ "ld1 {v6.8h}, [%[temp]], %[s] \n"
+ "ld1 {v7.8h}, [%[temp]] \n"
+ "add %[src], %[src], #16 \n"
+
+ // Stage 1: 16-bit transpose
+ "trn1 v16.8h, v0.8h, v1.8h \n"
+ "trn2 v17.8h, v0.8h, v1.8h \n"
+ "trn1 v18.8h, v2.8h, v3.8h \n"
+ "trn2 v19.8h, v2.8h, v3.8h \n"
+ "trn1 v20.8h, v4.8h, v5.8h \n"
+ "trn2 v21.8h, v4.8h, v5.8h \n"
+ "trn1 v22.8h, v6.8h, v7.8h \n"
+ "trn2 v23.8h, v6.8h, v7.8h \n"
+
+ // Stage 2: 32-bit transpose
+ "trn1 v0.4s, v16.4s, v18.4s \n"
+ "trn1 v1.4s, v17.4s, v19.4s \n"
+ "trn2 v2.4s, v16.4s, v18.4s \n"
+ "trn2 v3.4s, v17.4s, v19.4s \n"
+ "trn1 v4.4s, v20.4s, v22.4s \n"
+ "trn1 v5.4s, v21.4s, v23.4s \n"
+ "trn2 v6.4s, v20.4s, v22.4s \n"
+ "trn2 v7.4s, v21.4s, v23.4s \n"
+
+ // Stage 3: 64-bit transpose
+ "trn1 v16.2d, v0.2d, v4.2d \n"
+ "trn1 v17.2d, v1.2d, v5.2d \n"
+ "trn1 v18.2d, v2.2d, v6.2d \n"
+ "trn1 v19.2d, v3.2d, v7.2d \n"
+ "trn2 v20.2d, v0.2d, v4.2d \n"
+ "trn2 v21.2d, v1.2d, v5.2d \n"
+ "trn2 v22.2d, v2.2d, v6.2d \n"
+ "trn2 v23.2d, v3.2d, v7.2d \n"
+
+ "subs %w[w], %w[w], #8 \n"
+ "st1 {v16.8h}, [%[dst]], %[d] \n"
+ "st1 {v17.8h}, [%[dst]], %[d] \n"
+ "st1 {v18.8h}, [%[dst]], %[d] \n"
+ "st1 {v19.8h}, [%[dst]], %[d] \n"
+ "st1 {v20.8h}, [%[dst]], %[d] \n"
+ "st1 {v21.8h}, [%[dst]], %[d] \n"
+ "st1 {v22.8h}, [%[dst]], %[d] \n"
+ "st1 {v23.8h}, [%[dst]], %[d] \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
- "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/rotate_rvv.cc b/libs/libyuv/source/rotate_rvv.cc
new file mode 100644
index 0000000000..327f861ec5
--- /dev/null
+++ b/libs/libyuv/source/rotate_rvv.cc
@@ -0,0 +1,275 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+// This module is for RVV (RISC-V Vector extension)
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#ifndef RVV_VL_CLOBBER
+#ifdef __clang__
+#define RVV_VL_CLOBBER
+#else
+#define RVV_VL_CLOBBER "vl",
+#endif
+#endif
+
+#ifdef HAS_TRANSPOSEWX4_32_RVV
+// Transpose 32 bit values (ARGB) in Wx4 tiles, moving down W rows of source
+// (VL at a time) and across W columns of destination (4 rows).
+// Tuned for 512-bit VL (SiFive X280).
+void TransposeWx4_32_RVV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ int vl;
+ ptrdiff_t src_step;
+ ptrdiff_t dst_step;
+ uint8_t* dst1 = dst + (ptrdiff_t)dst_stride;
+ uint8_t* dst2 = dst1 + (ptrdiff_t)dst_stride;
+ uint8_t* dst3 = dst2 + (ptrdiff_t)dst_stride;
+ asm volatile(
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "slli %[dst_step], %[vl], 2 \n"
+ "mul %[src_step], %[vl], %[src_stride] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bltz %[w], 2f \n"
+ "1: \n"
+ "vlsseg4e32.v v8, (%[src]), %[src_stride] \n"
+ "add %[src], %[src], %[src_step] \n"
+ "vse32.v v8, (%[dst0]) \n"
+ "vse32.v v10, (%[dst1]) \n"
+ "vse32.v v12, (%[dst2]) \n"
+ "vse32.v v14, (%[dst3]) \n"
+ "add %[dst0], %[dst0], %[dst_step] \n"
+ "add %[dst1], %[dst1], %[dst_step] \n"
+ "add %[dst2], %[dst2], %[dst_step] \n"
+ "add %[dst3], %[dst3], %[dst_step] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bgez %[w], 1b \n"
+ "2: \n"
+ "add %[w], %[w], %[vl] \n"
+ "beqz %[w], 3f \n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vlsseg4e32.v v8, (%[src]), %[src_stride] \n"
+ "vse32.v v8, (%[dst0]) \n"
+ "vse32.v v10, (%[dst1]) \n"
+ "vse32.v v12, (%[dst2]) \n"
+ "vse32.v v14, (%[dst3]) \n"
+ "3: \n"
+ : [w] "+r"(width), // %[w]
+ [src] "+r"(src), // %[src]
+ [dst0] "+r"(dst), // %[dst0]
+ [dst1] "+r"(dst1), // %[dst1]
+ [dst2] "+r"(dst2), // %[dst2]
+ [dst3] "+r"(dst3), // %[dst3]
+ [vl] "=&r"(vl), // %[vl]
+ [src_step] "=&r"(src_step), // %[src_step]
+ [dst_step] "=&r"(dst_step) // %[dst_step]
+ : [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride]
+ : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15");
+}
+#endif // HAS_TRANSPOSEWX4_32_RVV
+
+#ifdef HAS_TRANSPOSEWXH_32_RVV
+// Transpose 32 bit plane of any width x height using RVV without scalar
+// fallback.
+void TransposeWxH_32_RVV(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ while (width >= 4) {
+ TransposeWx4_32_RVV(src, src_stride, dst, dst_stride, height);
+ src += 16;
+ dst += 4 * (ptrdiff_t)dst_stride;
+ width -= 4;
+ }
+ while (width > 0) {
+ const uint8_t* s_col = src;
+ uint8_t* d_row = dst;
+ int h = height;
+ int vl;
+ ptrdiff_t src_step;
+ ptrdiff_t dst_step;
+ asm volatile(
+ "vsetvli %[vl], %[h], e32, m8, ta, ma \n"
+ "slli %[d_step], %[vl], 2 \n"
+ "mul %[s_step], %[vl], %[src_stride] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bltz %[h], 2f \n"
+ "1: \n"
+ "vlse32.v v8, (%[s]), %[src_stride] \n"
+ "add %[s], %[s], %[s_step] \n"
+ "vse32.v v8, (%[d]) \n"
+ "add %[d], %[d], %[d_step] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bgez %[h], 1b \n"
+ "2: \n"
+ "add %[h], %[h], %[vl] \n"
+ "beqz %[h], 3f \n"
+ "vsetvli %[vl], %[h], e32, m8, ta, ma \n"
+ "vlse32.v v8, (%[s]), %[src_stride] \n"
+ "vse32.v v8, (%[d]) \n"
+ "3: \n"
+ : [h] "+r"(h), // %[h]
+ [s] "+r"(s_col), // %[s]
+ [d] "+r"(d_row), // %[d]
+ [vl] "=&r"(vl), // %[vl]
+ [s_step] "=&r"(src_step), // %[s_step]
+ [d_step] "=&r"(dst_step) // %[d_step]
+ : [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride]
+ : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15");
+ src += 4;
+ dst += (ptrdiff_t)dst_stride;
+ width -= 1;
+ }
+}
+#endif // HAS_TRANSPOSEWXH_32_RVV
+
+#ifdef HAS_TRANSPOSEWXH_16_RVV
+void TransposeWxH_16_RVV(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t s_stride = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d_stride = (ptrdiff_t)dst_stride * 2;
+ while (width >= 4) {
+ const uint16_t* s0 = src;
+ const uint16_t* s1 = src + 1;
+ const uint16_t* s2 = src + 2;
+ const uint16_t* s3 = src + 3;
+ uint8_t* d0 = (uint8_t*)dst;
+ uint8_t* d1 = d0 + d_stride;
+ uint8_t* d2 = d1 + d_stride;
+ uint8_t* d3 = d2 + d_stride;
+ int h = height;
+ int vl;
+ ptrdiff_t src_step;
+ ptrdiff_t dst_step;
+ asm volatile(
+ "vsetvli %[vl], %[h], e16, m8, ta, ma \n"
+ "slli %[d_step], %[vl], 1 \n"
+ "mul %[s_step], %[vl], %[s_stride] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bltz %[h], 2f \n"
+ "1: \n"
+ "vlse16.v v0, (%[s0]), %[s_stride] \n"
+ "vlse16.v v8, (%[s1]), %[s_stride] \n"
+ "vlse16.v v16, (%[s2]), %[s_stride] \n"
+ "vlse16.v v24, (%[s3]), %[s_stride] \n"
+ "add %[s0], %[s0], %[s_step] \n"
+ "add %[s1], %[s1], %[s_step] \n"
+ "add %[s2], %[s2], %[s_step] \n"
+ "add %[s3], %[s3], %[s_step] \n"
+ "vse16.v v0, (%[d0]) \n"
+ "vse16.v v8, (%[d1]) \n"
+ "vse16.v v16, (%[d2]) \n"
+ "vse16.v v24, (%[d3]) \n"
+ "add %[d0], %[d0], %[d_step] \n"
+ "add %[d1], %[d1], %[d_step] \n"
+ "add %[d2], %[d2], %[d_step] \n"
+ "add %[d3], %[d3], %[d_step] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bgez %[h], 1b \n"
+ "2: \n"
+ "add %[h], %[h], %[vl] \n"
+ "beqz %[h], 3f \n"
+ "vsetvli %[vl], %[h], e16, m8, ta, ma \n"
+ "vlse16.v v0, (%[s0]), %[s_stride] \n"
+ "vlse16.v v8, (%[s1]), %[s_stride] \n"
+ "vlse16.v v16, (%[s2]), %[s_stride] \n"
+ "vlse16.v v24, (%[s3]), %[s_stride] \n"
+ "vse16.v v0, (%[d0]) \n"
+ "vse16.v v8, (%[d1]) \n"
+ "vse16.v v16, (%[d2]) \n"
+ "vse16.v v24, (%[d3]) \n"
+ "3: \n"
+ : [h] "+r"(h), // %[h]
+ [s0] "+r"(s0), // %[s0]
+ [s1] "+r"(s1), // %[s1]
+ [s2] "+r"(s2), // %[s2]
+ [s3] "+r"(s3), // %[s3]
+ [d0] "+r"(d0), // %[d0]
+ [d1] "+r"(d1), // %[d1]
+ [d2] "+r"(d2), // %[d2]
+ [d3] "+r"(d3), // %[d3]
+ [vl] "=&r"(vl), // %[vl]
+ [s_step] "=&r"(src_step), // %[s_step]
+ [d_step] "=&r"(dst_step) // %[d_step]
+ : [s_stride] "r"(s_stride) // %[s_stride]
+ : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
+ "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
+ "v26", "v27", "v28", "v29", "v30", "v31");
+ src += 4;
+ dst += 4 * (ptrdiff_t)dst_stride;
+ width -= 4;
+ }
+ while (width > 0) {
+ const uint16_t* s_col = src;
+ uint16_t* d_row = dst;
+ int h = height;
+ int vl;
+ ptrdiff_t src_step;
+ ptrdiff_t dst_step;
+ asm volatile(
+ "vsetvli %[vl], %[h], e16, m8, ta, ma \n"
+ "slli %[d_step], %[vl], 1 \n"
+ "mul %[s_step], %[vl], %[s_stride] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bltz %[h], 2f \n"
+ "1: \n"
+ "vlse16.v v8, (%[s]), %[s_stride] \n"
+ "add %[s], %[s], %[s_step] \n"
+ "vse16.v v8, (%[d]) \n"
+ "add %[d], %[d], %[d_step] \n"
+ "sub %[h], %[h], %[vl] \n"
+ "bgez %[h], 1b \n"
+ "2: \n"
+ "add %[h], %[h], %[vl] \n"
+ "beqz %[h], 3f \n"
+ "vsetvli %[vl], %[h], e16, m8, ta, ma \n"
+ "vlse16.v v8, (%[s]), %[s_stride] \n"
+ "vse16.v v8, (%[d]) \n"
+ "3: \n"
+ : [h] "+r"(h), // %[h]
+ [s] "+r"(s_col), // %[s]
+ [d] "+r"(d_row), // %[d]
+ [vl] "=&r"(vl), // %[vl]
+ [s_step] "=&r"(src_step), // %[s_step]
+ [d_step] "=&r"(dst_step) // %[d_step]
+ : [s_stride] "r"(s_stride) // %[s_stride]
+ : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15");
+ src += 1;
+ dst += (ptrdiff_t)dst_stride;
+ width -= 1;
+ }
+}
+#endif // HAS_TRANSPOSEWXH_16_RVV
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
diff --git a/libs/libyuv/source/rotate_sme.cc b/libs/libyuv/source/rotate_sme.cc
new file mode 100644
index 0000000000..85d5bf6049
--- /dev/null
+++ b/libs/libyuv/source/rotate_sme.cc
@@ -0,0 +1,313 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+__arm_locally_streaming __arm_new("za") void TransposeWxH_SME(
+ const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int vl;
+ asm("cntb %x0" : "=r"(vl));
+
+ do {
+ const uint8_t* src2 = src;
+ uint8_t* dst2 = dst;
+
+ // Process up to VL elements per iteration of the inner loop.
+ int block_height = height > vl ? vl : height;
+
+ int width2 = width;
+ do {
+ const uint8_t* src3 = src2;
+
+ // Process up to VL elements per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows.
+ "whilelt p0.b, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0.
+ "1: \n"
+ "ld1b {za0h.b[w12, 0]}, p0/z, [%[src3]] \n"
+ "add %[src3], %[src3], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.b, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store W <= VL columns from ZA0.
+ "2: \n"
+ "st1b {za0v.b[w12, 0]}, p0, [%[dst2]] \n"
+ "add %[dst2], %[dst2], %[dst_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src3] "+r"(src3), // %[src3]
+ [dst2] "+r"(dst2) // %[dst2]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
+ [block_width] "r"(block_width), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "cc", "memory", "p0", "w12", "za");
+
+ src2 += vl;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += vl * src_stride;
+ dst += vl;
+ height -= vl;
+ } while (height > 0);
+}
+
+__arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME(
+ const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ int vl;
+ asm("cnth %x0" : "=r"(vl));
+
+ do {
+ const uint8_t* src2 = src;
+ uint8_t* dst2_a = dst_a;
+ uint8_t* dst2_b = dst_b;
+
+ // Process up to VL bytes per iteration of the inner loop.
+ int block_height = height > vl * 2 ? vl * 2 : height;
+
+ int width2 = width;
+ do {
+ const uint8_t* src3 = src2;
+
+ // Process up to VL 16-bit elements per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows,
+ // %[block_width] is always a multiple of two here.
+ "whilelt p0.b, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0, such that U/V components exist in
+ // alternating columns.
+ "1: \n"
+ "ld1b {za0h.b[w12, 0]}, p0/z, [%[src]] \n"
+ "add %[src], %[src], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.b, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store alternating UV data from pairs of ZA0 columns.
+ "2: \n"
+ "st1b {za0v.b[w12, 0]}, p0, [%[dst_a]] \n"
+ "st1b {za0v.b[w12, 1]}, p0, [%[dst_b]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b] \n"
+ "add w12, w12, #2 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src] "+r"(src3), // %[src]
+ [dst_a] "+r"(dst2_a), // %[dst_a]
+ [dst_b] "+r"(dst2_b) // %[dst_b]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b), // %[dst_stride_b]
+ [block_width] "r"(block_width * 2), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "cc", "memory", "p0", "w12", "za");
+
+ src2 += 2 * vl;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += 2 * vl * src_stride;
+ dst_a += 2 * vl;
+ dst_b += 2 * vl;
+ height -= 2 * vl;
+ } while (height > 0);
+}
+
+__arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME(
+ const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int vl;
+ asm("cntw %x0" : "=r"(vl));
+
+ do {
+ const uint8_t* src2 = src;
+ uint8_t* dst2 = dst;
+
+ // Process up to VL 32-bit rows per iteration of the outer loop.
+ int block_height = height > vl ? vl : height;
+
+ int width2 = width;
+ do {
+ const uint8_t* src3 = src2;
+
+ // Process up to VL 32-bit columns per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows.
+ "whilelt p0.s, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0.S.
+ "1: \n"
+ "ld1w {za0h.s[w12, 0]}, p0/z, [%[src3]] \n"
+ "add %[src3], %[src3], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.s, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store W <= VL columns from ZA0.S.
+ "2: \n"
+ "st1w {za0v.s[w12, 0]}, p0, [%[dst2]] \n"
+ "add %[dst2], %[dst2], %[dst_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src3] "+r"(src3), // %[src3]
+ [dst2] "+r"(dst2) // %[dst2]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
+ [block_width] "r"(block_width), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "cc", "memory", "p0", "w12", "za");
+
+ src2 += vl * 4;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += (ptrdiff_t)vl * src_stride;
+ dst += vl * 4;
+ height -= vl;
+ } while (height > 0);
+}
+
+__arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME(
+ const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int vl;
+ asm("cnth %x0" : "=r"(vl));
+
+ do {
+ const uint16_t* src2 = src;
+ uint16_t* dst2 = dst;
+
+ // Process up to VL 16-bit rows per iteration of the outer loop.
+ int block_height = height > vl ? vl : height;
+
+ int width2 = width;
+ do {
+ const uint16_t* src3 = src2;
+
+ // Process up to VL 16-bit columns per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows.
+ "whilelt p0.h, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0.H.
+ "1: \n"
+ "ld1h {za0h.h[w12, 0]}, p0/z, [%[src3]] \n"
+ "add %[src3], %[src3], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.h, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store W <= VL columns from ZA0.H.
+ "2: \n"
+ "st1h {za0v.h[w12, 0]}, p0, [%[dst2]] \n"
+ "add %[dst2], %[dst2], %[dst_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src3] "+r"(src3), // %[src3]
+ [dst2] "+r"(dst2) // %[dst2]
+ : [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride]
+ [block_width] "r"(block_width), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "memory", "cc", "w12", "p0", "za");
+
+ src2 += vl;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += (ptrdiff_t)vl * src_stride;
+ dst += vl;
+ height -= vl;
+ } while (height > 0);
+}
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/rotate_sve.cc b/libs/libyuv/source/rotate_sve.cc
new file mode 100644
index 0000000000..f7dc3c14dd
--- /dev/null
+++ b/libs/libyuv/source/rotate_sve.cc
@@ -0,0 +1,279 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+// Transpose 32 bit values (ARGB) in Wx4 tiles using SVE2 with predication
+// for tail widths.
+void TransposeWx4_32_SVE2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t s4 = s * 4;
+ const ptrdiff_t d = dst_stride;
+ const uint8_t* src1 = src + s;
+ const uint8_t* src2 = src1 + s;
+ const uint8_t* src3 = src2 + s;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ uint8_t* dst3 = dst2 + d;
+ asm volatile(
+ "ptrue p0.s, vl4 \n"
+ "subs %w[width], %w[width], #4 \n"
+ "b.lt 2f \n"
+
+ // Main loop: full 4x4 blocks.
+ "1: \n"
+ "ld1w {z0.s}, p0/z, [%[src0]] \n"
+ "ld1w {z1.s}, p0/z, [%[src1]] \n"
+ "ld1w {z2.s}, p0/z, [%[src2]] \n"
+ "ld1w {z3.s}, p0/z, [%[src3]] \n"
+ "add %[src0], %[src0], %[s4] \n"
+ "add %[src1], %[src1], %[s4] \n"
+ "add %[src2], %[src2], %[s4] \n"
+ "add %[src3], %[src3], %[s4] \n"
+ "trn1 z4.s, z0.s, z1.s \n"
+ "trn2 z5.s, z0.s, z1.s \n"
+ "trn1 z6.s, z2.s, z3.s \n"
+ "trn2 z7.s, z2.s, z3.s \n"
+ "trn1 z0.d, z4.d, z6.d \n" // column 0
+ "trn1 z1.d, z5.d, z7.d \n" // column 1
+ "trn2 z2.d, z4.d, z6.d \n" // column 2
+ "trn2 z3.d, z5.d, z7.d \n" // column 3
+ "subs %w[width], %w[width], #4 \n"
+ "st1w {z0.s}, p0, [%[dst0]] \n"
+ "st1w {z1.s}, p0, [%[dst1]] \n"
+ "st1w {z2.s}, p0, [%[dst2]] \n"
+ "st1w {z3.s}, p0, [%[dst3]] \n"
+ "add %[dst0], %[dst0], #16 \n"
+ "add %[dst1], %[dst1], #16 \n"
+ "add %[dst2], %[dst2], #16 \n"
+ "add %[dst3], %[dst3], #16 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], #4 \n"
+ "b.eq 3f \n"
+
+ // Tail loop: 1..3 remaining source rows / destination columns.
+ "cmp %w[width], #1 \n"
+ "csel %[src1], %[src0], %[src1], eq \n"
+ "cmp %w[width], #2 \n"
+ "csel %[src2], %[src1], %[src2], le \n"
+ "mov %[src3], %[src2] \n"
+ "ld1w {z0.s}, p0/z, [%[src0]] \n"
+ "ld1w {z1.s}, p0/z, [%[src1]] \n"
+ "ld1w {z2.s}, p0/z, [%[src2]] \n"
+ "ld1w {z3.s}, p0/z, [%[src3]] \n"
+ "trn1 z4.s, z0.s, z1.s \n"
+ "trn2 z5.s, z0.s, z1.s \n"
+ "trn1 z6.s, z2.s, z3.s \n"
+ "trn2 z7.s, z2.s, z3.s \n"
+ "trn1 z0.d, z4.d, z6.d \n" // column 0
+ "trn1 z1.d, z5.d, z7.d \n" // column 1
+ "trn2 z2.d, z4.d, z6.d \n" // column 2
+ "trn2 z3.d, z5.d, z7.d \n" // column 3
+ "whilelt p0.s, wzr, %w[width] \n"
+ "st1w {z0.s}, p0, [%[dst0]] \n"
+ "st1w {z1.s}, p0, [%[dst1]] \n"
+ "st1w {z2.s}, p0, [%[dst2]] \n"
+ "st1w {z3.s}, p0, [%[dst3]] \n"
+ "3: \n"
+ : [src0] "+r"(src), // %[src0]
+ [src1] "+r"(src1), // %[src1]
+ [src2] "+r"(src2), // %[src2]
+ [src3] "+r"(src3), // %[src3]
+ [dst0] "+r"(dst), // %[dst0]
+ [dst1] "+r"(dst1), // %[dst1]
+ [dst2] "+r"(dst2), // %[dst2]
+ [dst3] "+r"(dst3), // %[dst3]
+ [width] "+r"(width) // %[width]
+ : [s4] "r"(s4) // %[s4]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
+}
+
+void TransposeWx8_16_SVE2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
+ const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
+ const uint16_t* temp;
+ asm volatile(
+ "ptrue p0.h, vl8 \n"
+ "subs %w[w], %w[w], #8 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "ld1h {z0.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z1.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z2.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z3.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z4.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z5.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z6.h}, p0/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z7.h}, p0/z, [%[temp]] \n"
+ "add %[src], %[src], #16 \n"
+
+ "trn1 z16.h, z0.h, z1.h \n"
+ "trn2 z17.h, z0.h, z1.h \n"
+ "trn1 z18.h, z2.h, z3.h \n"
+ "trn2 z19.h, z2.h, z3.h \n"
+ "trn1 z20.h, z4.h, z5.h \n"
+ "trn2 z21.h, z4.h, z5.h \n"
+ "trn1 z22.h, z6.h, z7.h \n"
+ "trn2 z23.h, z6.h, z7.h \n"
+
+ "trn1 z0.s, z16.s, z18.s \n"
+ "trn1 z1.s, z17.s, z19.s \n"
+ "trn2 z2.s, z16.s, z18.s \n"
+ "trn2 z3.s, z17.s, z19.s \n"
+ "trn1 z4.s, z20.s, z22.s \n"
+ "trn1 z5.s, z21.s, z23.s \n"
+ "trn2 z6.s, z20.s, z22.s \n"
+ "trn2 z7.s, z21.s, z23.s \n"
+
+ "trn1 z16.d, z0.d, z4.d \n"
+ "trn1 z17.d, z1.d, z5.d \n"
+ "trn1 z18.d, z2.d, z6.d \n"
+ "trn1 z19.d, z3.d, z7.d \n"
+ "trn2 z20.d, z0.d, z4.d \n"
+ "trn2 z21.d, z1.d, z5.d \n"
+ "trn2 z22.d, z2.d, z6.d \n"
+ "trn2 z23.d, z3.d, z7.d \n"
+
+ "st1h {z16.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z17.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z18.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z19.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z20.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z21.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z22.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z23.h}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %[d] \n"
+ "subs %w[w], %w[w], #8 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[w], %w[w], #8 \n"
+ "b.eq 3f \n"
+ "whilelt p1.h, wzr, %w[w] \n"
+ "mov %[temp], %[src] \n"
+ "ld1h {z0.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z1.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z2.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z3.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z4.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z5.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z6.h}, p1/z, [%[temp]] \n"
+ "add %[temp], %[temp], %[s] \n"
+ "ld1h {z7.h}, p1/z, [%[temp]] \n"
+
+ "trn1 z16.h, z0.h, z1.h \n"
+ "trn2 z17.h, z0.h, z1.h \n"
+ "trn1 z18.h, z2.h, z3.h \n"
+ "trn2 z19.h, z2.h, z3.h \n"
+ "trn1 z20.h, z4.h, z5.h \n"
+ "trn2 z21.h, z4.h, z5.h \n"
+ "trn1 z22.h, z6.h, z7.h \n"
+ "trn2 z23.h, z6.h, z7.h \n"
+
+ "trn1 z0.s, z16.s, z18.s \n"
+ "trn1 z1.s, z17.s, z19.s \n"
+ "trn2 z2.s, z16.s, z18.s \n"
+ "trn2 z3.s, z17.s, z19.s \n"
+ "trn1 z4.s, z20.s, z22.s \n"
+ "trn1 z5.s, z21.s, z23.s \n"
+ "trn2 z6.s, z20.s, z22.s \n"
+ "trn2 z7.s, z21.s, z23.s \n"
+
+ "trn1 z16.d, z0.d, z4.d \n"
+ "trn1 z17.d, z1.d, z5.d \n"
+ "trn1 z18.d, z2.d, z6.d \n"
+ "trn1 z19.d, z3.d, z7.d \n"
+ "trn2 z20.d, z0.d, z4.d \n"
+ "trn2 z21.d, z1.d, z5.d \n"
+ "trn2 z22.d, z2.d, z6.d \n"
+ "trn2 z23.d, z3.d, z7.d \n"
+
+ "st1h {z16.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z17.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z18.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z19.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z20.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z21.h}, p0, [%[dst]] \n"
+ "subs %w[w], %w[w], #1 \n"
+ "b.eq 3f \n"
+ "add %[dst], %[dst], %[d] \n"
+ "st1h {z22.h}, p0, [%[dst]] \n"
+ "3: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [temp] "=&r"(temp) // %[temp]
+ : [s] "r"(s), // %[s]
+ [d] "r"(d) // %[d]
+ : "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6",
+ "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23");
+}
+
+#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/rotate_wasm.cc b/libs/libyuv/source/rotate_wasm.cc
new file mode 100644
index 0000000000..5a8adc95d2
--- /dev/null
+++ b/libs/libyuv/source/rotate_wasm.cc
@@ -0,0 +1,328 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#include <wasm_simd128.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Interleave the low or high halves of 2 vectors as 8, 16, 32 or 64 bit
+// elements. These map to punpck on x86 and zip on Arm.
+#define ZIPLO8(a, b) \
+ wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, \
+ 23)
+#define ZIPHI8(a, b) \
+ wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, \
+ 30, 15, 31)
+#define ZIPLO16(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11)
+#define ZIPHI16(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15)
+#define ZIPLO32(a, b) wasm_i32x4_shuffle(a, b, 0, 4, 1, 5)
+#define ZIPHI32(a, b) wasm_i32x4_shuffle(a, b, 2, 6, 3, 7)
+#define ZIPLO64(a, b) wasm_i64x2_shuffle(a, b, 0, 2)
+#define ZIPHI64(a, b) wasm_i64x2_shuffle(a, b, 1, 3)
+
+// Transpose 8 rows x 16 bytes. c[m] holds columns 2m and 2m + 1 of the 8 rows
+// in its low and high 8 bytes.
+static inline void Transpose8x16_WASMSIMD(const uint8_t* src,
+ ptrdiff_t s,
+ v128_t* c) {
+ v128_t r0 = wasm_v128_load(src);
+ v128_t r1 = wasm_v128_load(src + s);
+ v128_t r2 = wasm_v128_load(src + s * 2);
+ v128_t r3 = wasm_v128_load(src + s * 3);
+ v128_t r4 = wasm_v128_load(src + s * 4);
+ v128_t r5 = wasm_v128_load(src + s * 5);
+ v128_t r6 = wasm_v128_load(src + s * 6);
+ v128_t r7 = wasm_v128_load(src + s * 7);
+
+ // Bytes of row pairs. a0 is columns 0..7 of rows 0, 1.
+ v128_t a0 = ZIPLO8(r0, r1);
+ v128_t a1 = ZIPHI8(r0, r1);
+ v128_t a2 = ZIPLO8(r2, r3);
+ v128_t a3 = ZIPHI8(r2, r3);
+ v128_t a4 = ZIPLO8(r4, r5);
+ v128_t a5 = ZIPHI8(r4, r5);
+ v128_t a6 = ZIPLO8(r6, r7);
+ v128_t a7 = ZIPHI8(r6, r7);
+
+ // Words. b0 is columns 0..3 of rows 0..3.
+ v128_t b0 = ZIPLO16(a0, a2);
+ v128_t b1 = ZIPHI16(a0, a2);
+ v128_t b2 = ZIPLO16(a1, a3);
+ v128_t b3 = ZIPHI16(a1, a3);
+ v128_t b4 = ZIPLO16(a4, a6);
+ v128_t b5 = ZIPHI16(a4, a6);
+ v128_t b6 = ZIPLO16(a5, a7);
+ v128_t b7 = ZIPHI16(a5, a7);
+
+ // Dwords. c0 is columns 0, 1 of rows 0..7.
+ c[0] = ZIPLO32(b0, b4);
+ c[1] = ZIPHI32(b0, b4);
+ c[2] = ZIPLO32(b1, b5);
+ c[3] = ZIPHI32(b1, b5);
+ c[4] = ZIPLO32(b2, b6);
+ c[5] = ZIPHI32(b2, b6);
+ c[6] = ZIPLO32(b3, b7);
+ c[7] = ZIPHI32(b3, b7);
+}
+
+#if defined(HAS_TRANSPOSEWX8_WASMSIMD) || defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
+// Transpose 8 rows x 16 bytes. Even columns go to dst_a and odd columns go to
+// dst_b. Width is a multiple of 16 bytes.
+static void TransposeWx8_Byte_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t sa = dst_stride_a;
+ const ptrdiff_t sb = dst_stride_b;
+ while (byte_width > 0) {
+ v128_t c[8];
+ int m;
+ Transpose8x16_WASMSIMD(src, src_stride, c);
+ for (m = 0; m < 8; ++m) {
+ wasm_v128_store64_lane(dst_a + sa * m, c[m], 0);
+ wasm_v128_store64_lane(dst_b + sb * m, c[m], 1);
+ }
+ src += 16;
+ dst_a += sa * 8;
+ dst_b += sb * 8;
+ byte_width -= 16;
+ }
+}
+#endif // HAS_TRANSPOSEWX8_WASMSIMD || HAS_TRANSPOSEUVWX8_WASMSIMD
+
+#if defined(HAS_TRANSPOSEWX8_WASMSIMD)
+// Transpose 16x8. Width is a multiple of 16.
+void TransposeWx8_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2,
+ dst + dst_stride, dst_stride * 2, width);
+}
+#endif // HAS_TRANSPOSEWX8_WASMSIMD
+
+#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
+// Transpose UV 8x8. Width is a multiple of 8.
+void TransposeUVWx8_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx8_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // HAS_TRANSPOSEUVWX8_WASMSIMD
+
+#if defined(HAS_TRANSPOSEWX16_WASMSIMD) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
+// Transpose 16 rows x 16 bytes as 2 halves of 8 rows, which are joined with
+// 64 bit interleaves to store full 16 byte rows. Even columns go to dst_a and
+// odd columns go to dst_b. Width is a multiple of 16 bytes.
+static void TransposeWx16_Byte_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t sa = dst_stride_a;
+ const ptrdiff_t sb = dst_stride_b;
+ while (byte_width > 0) {
+ v128_t lo[8];
+ v128_t hi[8];
+ int m;
+ Transpose8x16_WASMSIMD(src, s, lo);
+ Transpose8x16_WASMSIMD(src + s * 8, s, hi);
+ for (m = 0; m < 8; ++m) {
+ wasm_v128_store(dst_a + sa * m, ZIPLO64(lo[m], hi[m]));
+ wasm_v128_store(dst_b + sb * m, ZIPHI64(lo[m], hi[m]));
+ }
+ src += 16;
+ dst_a += sa * 8;
+ dst_b += sb * 8;
+ byte_width -= 16;
+ }
+}
+#endif // HAS_TRANSPOSEWX16_WASMSIMD || HAS_TRANSPOSEUVWX16_WASMSIMD
+
+#if defined(HAS_TRANSPOSEWX16_WASMSIMD)
+// Transpose 16x16. Width is a multiple of 16.
+void TransposeWx16_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx16_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2,
+ dst + dst_stride, dst_stride * 2, width);
+}
+#endif // HAS_TRANSPOSEWX16_WASMSIMD
+
+#if defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
+// Transpose UV 8x16. Width is a multiple of 8.
+void TransposeUVWx16_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx16_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // HAS_TRANSPOSEUVWX16_WASMSIMD
+
+#if defined(HAS_TRANSPOSEWX4_32_WASMSIMD)
+// Transpose 32 bit values (ARGB) in Wx4 tiles, moving down 4 rows of source
+// and across 4 columns of destination. Width is a multiple of 4.
+void TransposeWx4_32_WASMSIMD(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ uint8_t* dst1 = dst + d;
+ uint8_t* dst2 = dst1 + d;
+ uint8_t* dst3 = dst2 + d;
+ while (width >= 8) {
+ v128_t r0 = wasm_v128_load(src);
+ v128_t r1 = wasm_v128_load(src + s);
+ v128_t r2 = wasm_v128_load(src + s * 2);
+ v128_t r3 = wasm_v128_load(src + s * 3);
+ src += s * 4;
+ v128_t r4 = wasm_v128_load(src);
+ v128_t r5 = wasm_v128_load(src + s);
+ v128_t r6 = wasm_v128_load(src + s * 2);
+ v128_t r7 = wasm_v128_load(src + s * 3);
+ src += s * 4;
+ v128_t a0 = ZIPLO32(r0, r1);
+ v128_t a1 = ZIPHI32(r0, r1);
+ v128_t a2 = ZIPLO32(r2, r3);
+ v128_t a3 = ZIPHI32(r2, r3);
+ v128_t b0 = ZIPLO32(r4, r5);
+ v128_t b1 = ZIPHI32(r4, r5);
+ v128_t b2 = ZIPLO32(r6, r7);
+ v128_t b3 = ZIPHI32(r6, r7);
+ wasm_v128_store(dst, ZIPLO64(a0, a2));
+ wasm_v128_store(dst1, ZIPHI64(a0, a2));
+ wasm_v128_store(dst2, ZIPLO64(a1, a3));
+ wasm_v128_store(dst3, ZIPHI64(a1, a3));
+ wasm_v128_store(dst + 16, ZIPLO64(b0, b2));
+ wasm_v128_store(dst1 + 16, ZIPHI64(b0, b2));
+ wasm_v128_store(dst2 + 16, ZIPLO64(b1, b3));
+ wasm_v128_store(dst3 + 16, ZIPHI64(b1, b3));
+ dst += 32;
+ dst1 += 32;
+ dst2 += 32;
+ dst3 += 32;
+ width -= 8;
+ }
+ while (width > 0) {
+ v128_t r0 = wasm_v128_load(src);
+ v128_t r1 = wasm_v128_load(src + s);
+ v128_t r2 = wasm_v128_load(src + s * 2);
+ v128_t r3 = wasm_v128_load(src + s * 3);
+ v128_t a0 = ZIPLO32(r0, r1);
+ v128_t a1 = ZIPHI32(r0, r1);
+ v128_t a2 = ZIPLO32(r2, r3);
+ v128_t a3 = ZIPHI32(r2, r3);
+ wasm_v128_store(dst, ZIPLO64(a0, a2));
+ wasm_v128_store(dst1, ZIPHI64(a0, a2));
+ wasm_v128_store(dst2, ZIPLO64(a1, a3));
+ wasm_v128_store(dst3, ZIPHI64(a1, a3));
+ src += s * 4;
+ dst += 16;
+ dst1 += 16;
+ dst2 += 16;
+ dst3 += 16;
+ width -= 4;
+ }
+}
+#endif // HAS_TRANSPOSEWX4_32_WASMSIMD
+
+#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
+void TransposeWx8_16_WASMSIMD(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width > 0) {
+ v128_t r0 = wasm_v128_load(src);
+ v128_t r1 = wasm_v128_load(src + s);
+ v128_t r2 = wasm_v128_load(src + s * 2);
+ v128_t r3 = wasm_v128_load(src + s * 3);
+ v128_t r4 = wasm_v128_load(src + s * 4);
+ v128_t r5 = wasm_v128_load(src + s * 5);
+ v128_t r6 = wasm_v128_load(src + s * 6);
+ v128_t r7 = wasm_v128_load(src + s * 7);
+
+ v128_t t0 = ZIPLO16(r0, r1);
+ v128_t t1 = ZIPHI16(r0, r1);
+ v128_t t2 = ZIPLO16(r2, r3);
+ v128_t t3 = ZIPHI16(r2, r3);
+ v128_t t4 = ZIPLO16(r4, r5);
+ v128_t t5 = ZIPHI16(r4, r5);
+ v128_t t6 = ZIPLO16(r6, r7);
+ v128_t t7 = ZIPHI16(r6, r7);
+
+ v128_t u0 = ZIPLO32(t0, t2);
+ v128_t u1 = ZIPHI32(t0, t2);
+ v128_t u2 = ZIPLO32(t1, t3);
+ v128_t u3 = ZIPHI32(t1, t3);
+ v128_t u4 = ZIPLO32(t4, t6);
+ v128_t u5 = ZIPHI32(t4, t6);
+ v128_t u6 = ZIPLO32(t5, t7);
+ v128_t u7 = ZIPHI32(t5, t7);
+
+ wasm_v128_store(dst, ZIPLO64(u0, u4));
+ wasm_v128_store(dst + d, ZIPHI64(u0, u4));
+ wasm_v128_store(dst + d * 2, ZIPLO64(u1, u5));
+ wasm_v128_store(dst + d * 3, ZIPHI64(u1, u5));
+ wasm_v128_store(dst + d * 4, ZIPLO64(u2, u6));
+ wasm_v128_store(dst + d * 5, ZIPHI64(u2, u6));
+ wasm_v128_store(dst + d * 6, ZIPLO64(u3, u7));
+ wasm_v128_store(dst + d * 7, ZIPHI64(u3, u7));
+ src += 8;
+ dst += d * 8;
+ width -= 8;
+ }
+}
+#endif // HAS_TRANSPOSEWX8_16_WASMSIMD
+
+#undef ZIPLO8
+#undef ZIPHI8
+#undef ZIPLO16
+#undef ZIPHI16
+#undef ZIPLO32
+#undef ZIPHI32
+#undef ZIPLO64
+#undef ZIPHI64
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
diff --git a/libs/libyuv/source/rotate_win.cc b/libs/libyuv/source/rotate_win.cc
index e887dd525c..d1d48dfea4 100644
--- a/libs/libyuv/source/rotate_win.cc
+++ b/libs/libyuv/source/rotate_win.cc
@@ -11,242 +11,1205 @@
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
+// This module is for Visual C 32/64 bit, and for clang with
+// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+
+#include <emmintrin.h>
+#include <immintrin.h> // For AVX2 and AVX512 intrinsics
+#include <string.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
+#endif
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// This module is for 32 bit Visual C x86 and clangcl
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
-
-__declspec(naked) void TransposeWx8_SSSE3(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- __asm {
- push edi
- push esi
- push ebp
- mov eax, [esp + 12 + 4] // src
- mov edi, [esp + 12 + 8] // src_stride
- mov edx, [esp + 12 + 12] // dst
- mov esi, [esp + 12 + 16] // dst_stride
- mov ecx, [esp + 12 + 20] // width
-
- // Read in the data from the source pointer.
- // First round of bit swap.
- align 4
- convertloop:
- movq xmm0, qword ptr [eax]
- lea ebp, [eax + 8]
- movq xmm1, qword ptr [eax + edi]
- lea eax, [eax + 2 * edi]
- punpcklbw xmm0, xmm1
- movq xmm2, qword ptr [eax]
- movdqa xmm1, xmm0
- palignr xmm1, xmm1, 8
- movq xmm3, qword ptr [eax + edi]
- lea eax, [eax + 2 * edi]
- punpcklbw xmm2, xmm3
- movdqa xmm3, xmm2
- movq xmm4, qword ptr [eax]
- palignr xmm3, xmm3, 8
- movq xmm5, qword ptr [eax + edi]
- punpcklbw xmm4, xmm5
- lea eax, [eax + 2 * edi]
- movdqa xmm5, xmm4
- movq xmm6, qword ptr [eax]
- palignr xmm5, xmm5, 8
- movq xmm7, qword ptr [eax + edi]
- punpcklbw xmm6, xmm7
- mov eax, ebp
- movdqa xmm7, xmm6
- palignr xmm7, xmm7, 8
- // Second round of bit swap.
- punpcklwd xmm0, xmm2
- punpcklwd xmm1, xmm3
- movdqa xmm2, xmm0
- movdqa xmm3, xmm1
- palignr xmm2, xmm2, 8
- palignr xmm3, xmm3, 8
- punpcklwd xmm4, xmm6
- punpcklwd xmm5, xmm7
- movdqa xmm6, xmm4
- movdqa xmm7, xmm5
- palignr xmm6, xmm6, 8
- palignr xmm7, xmm7, 8
- // Third round of bit swap.
- // Write to the destination pointer.
- punpckldq xmm0, xmm4
- movq qword ptr [edx], xmm0
- movdqa xmm4, xmm0
- palignr xmm4, xmm4, 8
- movq qword ptr [edx + esi], xmm4
- lea edx, [edx + 2 * esi]
- punpckldq xmm2, xmm6
- movdqa xmm6, xmm2
- palignr xmm6, xmm6, 8
- movq qword ptr [edx], xmm2
- punpckldq xmm1, xmm5
- movq qword ptr [edx + esi], xmm6
- lea edx, [edx + 2 * esi]
- movdqa xmm5, xmm1
- movq qword ptr [edx], xmm1
- palignr xmm5, xmm5, 8
- punpckldq xmm3, xmm7
- movq qword ptr [edx + esi], xmm5
- lea edx, [edx + 2 * esi]
- movq qword ptr [edx], xmm3
- movdqa xmm7, xmm3
- palignr xmm7, xmm7, 8
- sub ecx, 8
- movq qword ptr [edx + esi], xmm7
- lea edx, [edx + 2 * esi]
- jg convertloop
-
- pop ebp
- pop esi
- pop edi
- ret
- }
-}
-
-__declspec(naked) void TransposeUVWx8_SSE2(const uint8_t* src,
- int src_stride,
- uint8_t* dst_a,
- int dst_stride_a,
- uint8_t* dst_b,
- int dst_stride_b,
- int w) {
- __asm {
- push ebx
- push esi
- push edi
- push ebp
- mov eax, [esp + 16 + 4] // src
- mov edi, [esp + 16 + 8] // src_stride
- mov edx, [esp + 16 + 12] // dst_a
- mov esi, [esp + 16 + 16] // dst_stride_a
- mov ebx, [esp + 16 + 20] // dst_b
- mov ebp, [esp + 16 + 24] // dst_stride_b
- mov ecx, esp
- sub esp, 4 + 16
- and esp, ~15
- mov [esp + 16], ecx
- mov ecx, [ecx + 16 + 28] // w
-
- align 4
- // Read in the data from the source pointer.
- // First round of bit swap.
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + edi]
- lea eax, [eax + 2 * edi]
- movdqa xmm7, xmm0 // use xmm7 as temp register.
- punpcklbw xmm0, xmm1
- punpckhbw xmm7, xmm1
- movdqa xmm1, xmm7
- movdqu xmm2, [eax]
- movdqu xmm3, [eax + edi]
- lea eax, [eax + 2 * edi]
- movdqa xmm7, xmm2
- punpcklbw xmm2, xmm3
- punpckhbw xmm7, xmm3
- movdqa xmm3, xmm7
- movdqu xmm4, [eax]
- movdqu xmm5, [eax + edi]
- lea eax, [eax + 2 * edi]
- movdqa xmm7, xmm4
- punpcklbw xmm4, xmm5
- punpckhbw xmm7, xmm5
- movdqa xmm5, xmm7
- movdqu xmm6, [eax]
- movdqu xmm7, [eax + edi]
- lea eax, [eax + 2 * edi]
- movdqu [esp], xmm5 // backup xmm5
- neg edi
- movdqa xmm5, xmm6 // use xmm5 as temp register.
- punpcklbw xmm6, xmm7
- punpckhbw xmm5, xmm7
- movdqa xmm7, xmm5
- lea eax, [eax + 8 * edi + 16]
- neg edi
- // Second round of bit swap.
- movdqa xmm5, xmm0
- punpcklwd xmm0, xmm2
- punpckhwd xmm5, xmm2
- movdqa xmm2, xmm5
- movdqa xmm5, xmm1
- punpcklwd xmm1, xmm3
- punpckhwd xmm5, xmm3
- movdqa xmm3, xmm5
- movdqa xmm5, xmm4
- punpcklwd xmm4, xmm6
- punpckhwd xmm5, xmm6
- movdqa xmm6, xmm5
- movdqu xmm5, [esp] // restore xmm5
- movdqu [esp], xmm6 // backup xmm6
- movdqa xmm6, xmm5 // use xmm6 as temp register.
- punpcklwd xmm5, xmm7
- punpckhwd xmm6, xmm7
- movdqa xmm7, xmm6
-
- // Third round of bit swap.
- // Write to the destination pointer.
- movdqa xmm6, xmm0
- punpckldq xmm0, xmm4
- punpckhdq xmm6, xmm4
- movdqa xmm4, xmm6
- movdqu xmm6, [esp] // restore xmm6
- movlpd qword ptr [edx], xmm0
- movhpd qword ptr [ebx], xmm0
- movlpd qword ptr [edx + esi], xmm4
- lea edx, [edx + 2 * esi]
- movhpd qword ptr [ebx + ebp], xmm4
- lea ebx, [ebx + 2 * ebp]
- movdqa xmm0, xmm2 // use xmm0 as the temp register.
- punpckldq xmm2, xmm6
- movlpd qword ptr [edx], xmm2
- movhpd qword ptr [ebx], xmm2
- punpckhdq xmm0, xmm6
- movlpd qword ptr [edx + esi], xmm0
- lea edx, [edx + 2 * esi]
- movhpd qword ptr [ebx + ebp], xmm0
- lea ebx, [ebx + 2 * ebp]
- movdqa xmm0, xmm1 // use xmm0 as the temp register.
- punpckldq xmm1, xmm5
- movlpd qword ptr [edx], xmm1
- movhpd qword ptr [ebx], xmm1
- punpckhdq xmm0, xmm5
- movlpd qword ptr [edx + esi], xmm0
- lea edx, [edx + 2 * esi]
- movhpd qword ptr [ebx + ebp], xmm0
- lea ebx, [ebx + 2 * ebp]
- movdqa xmm0, xmm3 // use xmm0 as the temp register.
- punpckldq xmm3, xmm7
- movlpd qword ptr [edx], xmm3
- movhpd qword ptr [ebx], xmm3
- punpckhdq xmm0, xmm7
- sub ecx, 8
- movlpd qword ptr [edx + esi], xmm0
- lea edx, [edx + 2 * esi]
- movhpd qword ptr [ebx + ebp], xmm0
- lea ebx, [ebx + 2 * ebp]
- jg convertloop
-
- mov esp, [esp + 16]
- pop ebp
- pop edi
- pop esi
- pop ebx
- ret
- }
-}
-
-#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
+// Target attributes allow gcc and clang to build each function for its
+// instruction set regardless of the baseline, which may lack SSE2 for 32 bit.
+#if defined(__clang__) || defined(__GNUC__)
+#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
+#define LIBYUV_TARGET_AVX512BW \
+ __attribute__((target("avx512bw,avx512vl,avx512f")))
+#else
+#define LIBYUV_TARGET_AVX2
+#define LIBYUV_TARGET_AVX512BW
+#endif // defined(__clang__) || defined(__GNUC__)
+
+#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
+// Transpose 8 rows x 16 bytes using 2 lanes of AVX2. Source row r is in
+// lane r / 4 of register r % 4. Each 128-bit output lane holds 2 columns x 8
+// rows: low 8 bytes (even column) for dst_a and high 8 bytes (odd column) for
+// dst_b. Width is a multiple of 16 bytes.
+LIBYUV_TARGET_AVX2
+static void TransposeWx8_Byte_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t sa = dst_stride_a;
+ const ptrdiff_t sb = dst_stride_b;
+ while (byte_width > 0) {
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
+ _mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
+ _mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
+ _mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
+ _mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
+
+ __m256i a0 = _mm256_unpacklo_epi8(r0, r1);
+ __m256i a1 = _mm256_unpackhi_epi8(r0, r1);
+ __m256i a2 = _mm256_unpacklo_epi8(r2, r3);
+ __m256i a3 = _mm256_unpackhi_epi8(r2, r3);
+
+ __m256i b0 = _mm256_unpacklo_epi16(a0, a2);
+ __m256i b1 = _mm256_unpackhi_epi16(a0, a2);
+ __m256i b2 = _mm256_unpacklo_epi16(a1, a3);
+ __m256i b3 = _mm256_unpackhi_epi16(a1, a3);
+
+ __m256i t0 = _mm256_permute2x128_si256(b0, b1, 0x20);
+ __m256i t1 = _mm256_permute2x128_si256(b0, b1, 0x31);
+ __m256i t2 = _mm256_permute2x128_si256(b2, b3, 0x20);
+ __m256i t3 = _mm256_permute2x128_si256(b2, b3, 0x31);
+
+ __m256i c0 = _mm256_unpacklo_epi32(t0, t1);
+ __m256i c1 = _mm256_unpackhi_epi32(t0, t1);
+ __m256i c2 = _mm256_unpacklo_epi32(t2, t3);
+ __m256i c3 = _mm256_unpackhi_epi32(t2, t3);
+
+ __m128i lo0 = _mm256_castsi256_si128(c0);
+ __m128i lo1 = _mm256_castsi256_si128(c1);
+ __m128i hi0 = _mm256_extracti128_si256(c0, 1);
+ __m128i hi1 = _mm256_extracti128_si256(c1, 1);
+ __m128i lo2 = _mm256_castsi256_si128(c2);
+ __m128i lo3 = _mm256_castsi256_si128(c3);
+ __m128i hi2 = _mm256_extracti128_si256(c2, 1);
+ __m128i hi3 = _mm256_extracti128_si256(c3, 1);
+
+ _mm_storel_epi64((__m128i*)(dst_a), lo0);
+ _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(lo0));
+ _mm_storel_epi64((__m128i*)(dst_a + sa), lo1);
+ _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(lo1));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 2), hi0);
+ _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(hi0));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 3), hi1);
+ _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(hi1));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 4), lo2);
+ _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(lo2));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 5), lo3);
+ _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(lo3));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 6), hi2);
+ _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(hi2));
+ _mm_storel_epi64((__m128i*)(dst_a + sa * 7), hi3);
+ _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(hi3));
+
+ src += 16;
+ dst_a += sa * 8;
+ dst_b += sb * 8;
+ byte_width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEWX8_AVX2)
+// Transpose 16x8. Width is a multiple of 16.
+LIBYUV_TARGET_AVX2
+void TransposeWx8_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
+ dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEUVWX8_AVX2)
+// Transpose UV 8x8. Width is a multiple of 8.
+LIBYUV_TARGET_AVX2
+void TransposeUVWx8_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX8_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
+// Load row r into lane 0 and row r + 8 into lane 1.
+#define LOAD2ROWS_AVX2(p, s8) \
+ _mm256_inserti128_si256( \
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(p))), \
+ _mm_loadu_si128((const __m128i*)((p) + (s8))), 1)
+
+// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
+// of register r % 8. 3 unpack stages leave each lane with 2 columns x 8 rows,
+// and vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an
+// even column for dst_a and lane 1 an odd column for dst_b.
+// Width is a multiple of 16 bytes.
+LIBYUV_TARGET_AVX2
+static void TransposeWx16_Byte_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t sa = dst_stride_a;
+ const ptrdiff_t sb = dst_stride_b;
+ while (byte_width > 0) {
+ __m256i r0 = LOAD2ROWS_AVX2(src, s * 8);
+ __m256i r1 = LOAD2ROWS_AVX2(src + s, s * 8);
+ __m256i r2 = LOAD2ROWS_AVX2(src + s * 2, s * 8);
+ __m256i r3 = LOAD2ROWS_AVX2(src + s * 3, s * 8);
+ __m256i r4 = LOAD2ROWS_AVX2(src + s * 4, s * 8);
+ __m256i r5 = LOAD2ROWS_AVX2(src + s * 5, s * 8);
+ __m256i r6 = LOAD2ROWS_AVX2(src + s * 6, s * 8);
+ __m256i r7 = LOAD2ROWS_AVX2(src + s * 7, s * 8);
+
+ __m256i a0 = _mm256_unpacklo_epi8(r0, r1);
+ __m256i a1 = _mm256_unpackhi_epi8(r0, r1);
+ __m256i a2 = _mm256_unpacklo_epi8(r2, r3);
+ __m256i a3 = _mm256_unpackhi_epi8(r2, r3);
+ __m256i a4 = _mm256_unpacklo_epi8(r4, r5);
+ __m256i a5 = _mm256_unpackhi_epi8(r4, r5);
+ __m256i a6 = _mm256_unpacklo_epi8(r6, r7);
+ __m256i a7 = _mm256_unpackhi_epi8(r6, r7);
+
+ __m256i b0 = _mm256_unpacklo_epi16(a0, a2);
+ __m256i b1 = _mm256_unpackhi_epi16(a0, a2);
+ __m256i b2 = _mm256_unpacklo_epi16(a1, a3);
+ __m256i b3 = _mm256_unpackhi_epi16(a1, a3);
+ __m256i b4 = _mm256_unpacklo_epi16(a4, a6);
+ __m256i b5 = _mm256_unpackhi_epi16(a4, a6);
+ __m256i b6 = _mm256_unpacklo_epi16(a5, a7);
+ __m256i b7 = _mm256_unpackhi_epi16(a5, a7);
+
+ __m256i c0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b0, b4), 0xd8);
+ __m256i c1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b0, b4), 0xd8);
+ __m256i c2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b1, b5), 0xd8);
+ __m256i c3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b1, b5), 0xd8);
+ __m256i c4 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b2, b6), 0xd8);
+ __m256i c5 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b2, b6), 0xd8);
+ __m256i c6 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b3, b7), 0xd8);
+ __m256i c7 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b3, b7), 0xd8);
+
+ _mm_storeu_si128((__m128i*)(dst_a), _mm256_castsi256_si128(c0));
+ _mm_storeu_si128((__m128i*)(dst_b), _mm256_extracti128_si256(c0, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa), _mm256_castsi256_si128(c1));
+ _mm_storeu_si128((__m128i*)(dst_b + sb), _mm256_extracti128_si256(c1, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm256_castsi256_si128(c2));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 2),
+ _mm256_extracti128_si256(c2, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 3), _mm256_castsi256_si128(c3));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 3),
+ _mm256_extracti128_si256(c3, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 4), _mm256_castsi256_si128(c4));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 4),
+ _mm256_extracti128_si256(c4, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 5), _mm256_castsi256_si128(c5));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 5),
+ _mm256_extracti128_si256(c5, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 6), _mm256_castsi256_si128(c6));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 6),
+ _mm256_extracti128_si256(c6, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 7), _mm256_castsi256_si128(c7));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 7),
+ _mm256_extracti128_si256(c7, 1));
+
+ src += 16;
+ dst_a += sa * 8;
+ dst_b += sb * 8;
+ byte_width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#undef LOAD2ROWS_AVX2
+#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_AVX2)
+LIBYUV_TARGET_AVX2
+void TransposeWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2,
+ dst + dst_stride, dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEUVWX16_AVX2)
+LIBYUV_TARGET_AVX2
+void TransposeUVWx16_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+// Dword permutes to gather 16 rows of a column from 4 lanes.
+static const uint32_t kPermdTranspose_AVX512BW[32] = {
+ 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
+ 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
+
+// Load 16 masked bytes from rows p, p + s4, p + s4 * 2, p + s4 * 3 into
+// lanes 0..3.
+#define LOAD4ROWS_AVX512BW(p, s4, k) \
+ _mm512_inserti32x4( \
+ _mm512_inserti32x4( \
+ _mm512_inserti32x4( \
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi8(k, (p))), \
+ _mm_maskz_loadu_epi8(k, (p) + (s4)), 1), \
+ _mm_maskz_loadu_epi8(k, (p) + (s4) * 2), 2), \
+ _mm_maskz_loadu_epi8(k, (p) + (s4) * 3), 3)
+
+// Store 16 destination rows. Lane n of o0/o2 is for dst_a and lane n of
+// o1/o3 is for dst_b.
+LIBYUV_TARGET_AVX512BW
+static void Store16x16_AVX512BW(__m512i o0,
+ __m512i o1,
+ __m512i o2,
+ __m512i o3,
+ uint8_t* dst_a,
+ ptrdiff_t sa,
+ uint8_t* dst_b,
+ ptrdiff_t sb) {
+ _mm_storeu_si128((__m128i*)(dst_a), _mm512_castsi512_si128(o0));
+ _mm_storeu_si128((__m128i*)(dst_b), _mm512_castsi512_si128(o1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa), _mm512_extracti32x4_epi32(o0, 1));
+ _mm_storeu_si128((__m128i*)(dst_b + sb), _mm512_extracti32x4_epi32(o1, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm512_castsi512_si128(o2));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 2), _mm512_castsi512_si128(o3));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 3),
+ _mm512_extracti32x4_epi32(o2, 1));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 3),
+ _mm512_extracti32x4_epi32(o3, 1));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 4),
+ _mm512_extracti32x4_epi32(o0, 2));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 4),
+ _mm512_extracti32x4_epi32(o1, 2));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 5),
+ _mm512_extracti32x4_epi32(o0, 3));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 5),
+ _mm512_extracti32x4_epi32(o1, 3));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 6),
+ _mm512_extracti32x4_epi32(o2, 2));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 6),
+ _mm512_extracti32x4_epi32(o3, 2));
+ _mm_storeu_si128((__m128i*)(dst_a + sa * 7),
+ _mm512_extracti32x4_epi32(o2, 3));
+ _mm_storeu_si128((__m128i*)(dst_b + sb * 7),
+ _mm512_extracti32x4_epi32(o3, 3));
+}
+
+// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
+// Source row r is in lane r / 4 of register r % 4.
+// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
+// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
+// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
+// which is within 10% of AVX512BW on most CPUs.
+LIBYUV_TARGET_AVX512BW
+static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int byte_width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t sa = dst_stride_a;
+ const ptrdiff_t sb = dst_stride_b;
+ const __m512i perm0 = _mm512_loadu_si512(kPermdTranspose_AVX512BW);
+ const __m512i perm1 = _mm512_loadu_si512(kPermdTranspose_AVX512BW + 16);
+ while (byte_width > 0) {
+ const int n = byte_width < 16 ? byte_width : 16;
+ const __mmask16 k = (__mmask16)((1u << n) - 1);
+ __m512i z0 = LOAD4ROWS_AVX512BW(src, s * 4, k);
+ __m512i z1 = LOAD4ROWS_AVX512BW(src + s, s * 4, k);
+ __m512i z2 = LOAD4ROWS_AVX512BW(src + s * 2, s * 4, k);
+ __m512i z3 = LOAD4ROWS_AVX512BW(src + s * 3, s * 4, k);
+
+ __m512i a0 = _mm512_unpacklo_epi8(z0, z1);
+ __m512i a1 = _mm512_unpackhi_epi8(z0, z1);
+ __m512i a2 = _mm512_unpacklo_epi8(z2, z3);
+ __m512i a3 = _mm512_unpackhi_epi8(z2, z3);
+ __m512i b0 = _mm512_unpacklo_epi16(a0, a2);
+ __m512i b1 = _mm512_unpackhi_epi16(a0, a2);
+ __m512i b2 = _mm512_unpacklo_epi16(a1, a3);
+ __m512i b3 = _mm512_unpackhi_epi16(a1, a3);
+ __m512i o0 = _mm512_permutex2var_epi32(b0, perm0, b2);
+ __m512i o1 = _mm512_permutex2var_epi32(b0, perm1, b2);
+ __m512i o2 = _mm512_permutex2var_epi32(b1, perm0, b3);
+ __m512i o3 = _mm512_permutex2var_epi32(b1, perm1, b3);
+
+ if (n == 16) {
+ Store16x16_AVX512BW(o0, o1, o2, o3, dst_a, sa, dst_b, sb);
+ dst_a += sa * 8;
+ dst_b += sb * 8;
+ } else {
+ // Remainder: 1 to 15 bytes. Store 1 destination row per byte.
+ SIMD_ALIGNED(uint8_t temp[16 * 16]);
+ int i;
+ Store16x16_AVX512BW(o0, o1, o2, o3, temp, 32, temp + 16, 32);
+ for (i = 0; i < n; ++i) {
+ uint8_t* dst = (i & 1) ? dst_b + sb * (i >> 1) : dst_a + sa * (i >> 1);
+ _mm_storeu_si128((__m128i*)dst,
+ _mm_load_si128((const __m128i*)(temp + i * 16)));
+ }
+ }
+ src += 16;
+ byte_width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#undef LOAD4ROWS_AVX512BW
+#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
+ // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEWX16_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+void TransposeWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
+ dst + (width > 1 ? dst_stride : 0),
+ dst_stride * 2, width);
+}
+#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+void TransposeUVWx16_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width) {
+ TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
+ dst_stride_b, width * 2);
+}
+#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEWX8_16_AVX2)
+LIBYUV_TARGET_AVX2
+void TransposeWx8_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width > 0) {
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
+ _mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
+ _mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
+ _mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
+ _mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
+
+ __m256i t0 = _mm256_unpacklo_epi16(r0, r1);
+ __m256i t1 = _mm256_unpackhi_epi16(r0, r1);
+ __m256i t2 = _mm256_unpacklo_epi16(r2, r3);
+ __m256i t3 = _mm256_unpackhi_epi16(r2, r3);
+
+ __m256i u0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t0, t2), 0xd8);
+ __m256i u1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t0, t2), 0xd8);
+ __m256i u2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t1, t3), 0xd8);
+ __m256i u3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t1, t3), 0xd8);
+
+ _mm_storeu_si128((__m128i*)(dst), _mm256_castsi256_si128(u0));
+ _mm_storeu_si128((__m128i*)(dst + d), _mm256_extracti128_si256(u0, 1));
+ _mm_storeu_si128((__m128i*)(dst + d * 2), _mm256_castsi256_si128(u1));
+ _mm_storeu_si128((__m128i*)(dst + d * 3), _mm256_extracti128_si256(u1, 1));
+ _mm_storeu_si128((__m128i*)(dst + d * 4), _mm256_castsi256_si128(u2));
+ _mm_storeu_si128((__m128i*)(dst + d * 5), _mm256_extracti128_si256(u2, 1));
+ _mm_storeu_si128((__m128i*)(dst + d * 6), _mm256_castsi256_si128(u3));
+ _mm_storeu_si128((__m128i*)(dst + d * 7), _mm256_extracti128_si256(u3, 1));
+ src += 8;
+ dst += d * 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX8_16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_16_AVX2)
+LIBYUV_TARGET_AVX2
+void TransposeWx16_16_AVX2(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width > 0) {
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
+ _mm_loadu_si128((const __m128i*)(src + s * 8)), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
+ _mm_loadu_si128((const __m128i*)(src + s * 9)), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
+ _mm_loadu_si128((const __m128i*)(src + s * 10)), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
+ _mm_loadu_si128((const __m128i*)(src + s * 11)), 1);
+ __m256i r4 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))),
+ _mm_loadu_si128((const __m128i*)(src + s * 12)), 1);
+ __m256i r5 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))),
+ _mm_loadu_si128((const __m128i*)(src + s * 13)), 1);
+ __m256i r6 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))),
+ _mm_loadu_si128((const __m128i*)(src + s * 14)), 1);
+ __m256i r7 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))),
+ _mm_loadu_si128((const __m128i*)(src + s * 15)), 1);
+
+ __m256i t0 = _mm256_unpacklo_epi16(r0, r1);
+ __m256i t1 = _mm256_unpackhi_epi16(r0, r1);
+ __m256i t2 = _mm256_unpacklo_epi16(r2, r3);
+ __m256i t3 = _mm256_unpackhi_epi16(r2, r3);
+ __m256i t4 = _mm256_unpacklo_epi16(r4, r5);
+ __m256i t5 = _mm256_unpackhi_epi16(r4, r5);
+ __m256i t6 = _mm256_unpacklo_epi16(r6, r7);
+ __m256i t7 = _mm256_unpackhi_epi16(r6, r7);
+
+ __m256i u0 = _mm256_unpacklo_epi32(t0, t2);
+ __m256i u1 = _mm256_unpackhi_epi32(t0, t2);
+ __m256i u2 = _mm256_unpacklo_epi32(t1, t3);
+ __m256i u3 = _mm256_unpackhi_epi32(t1, t3);
+ __m256i u4 = _mm256_unpacklo_epi32(t4, t6);
+ __m256i u5 = _mm256_unpackhi_epi32(t4, t6);
+ __m256i u6 = _mm256_unpacklo_epi32(t5, t7);
+ __m256i u7 = _mm256_unpackhi_epi32(t5, t7);
+
+ _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4));
+ _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4));
+ _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5));
+ _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5));
+ _mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6));
+ _mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6));
+ _mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7));
+ _mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7));
+ src += 8;
+ dst += d * 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
+
+#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+void TransposeWx32_16_AVX512BW(const uint16_t* src,
+ int src_stride,
+ uint16_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width >= 8) {
+ __m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
+ __m512i r1 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
+ __m512i r2 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
+ __m512i r3 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
+ __m512i r4 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 4)));
+ __m512i r5 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 5)));
+ __m512i r6 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 6)));
+ __m512i r7 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 7)));
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
+ 1);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
+ 1);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
+ 1);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
+ 1);
+ r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 12)),
+ 1);
+ r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 13)),
+ 1);
+ r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 14)),
+ 1);
+ r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 15)),
+ 1);
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 16)),
+ 2);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 17)),
+ 2);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 18)),
+ 2);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 19)),
+ 2);
+ r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 20)),
+ 2);
+ r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 21)),
+ 2);
+ r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 22)),
+ 2);
+ r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 23)),
+ 2);
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 24)),
+ 3);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 25)),
+ 3);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 26)),
+ 3);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 27)),
+ 3);
+ r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 28)),
+ 3);
+ r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 29)),
+ 3);
+ r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 30)),
+ 3);
+ r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 31)),
+ 3);
+
+ __m512i t0 = _mm512_unpacklo_epi16(r0, r1);
+ __m512i t1 = _mm512_unpackhi_epi16(r0, r1);
+ __m512i t2 = _mm512_unpacklo_epi16(r2, r3);
+ __m512i t3 = _mm512_unpackhi_epi16(r2, r3);
+ __m512i t4 = _mm512_unpacklo_epi16(r4, r5);
+ __m512i t5 = _mm512_unpackhi_epi16(r4, r5);
+ __m512i t6 = _mm512_unpacklo_epi16(r6, r7);
+ __m512i t7 = _mm512_unpackhi_epi16(r6, r7);
+
+ __m512i u0 = _mm512_unpacklo_epi32(t0, t2);
+ __m512i u1 = _mm512_unpackhi_epi32(t0, t2);
+ __m512i u2 = _mm512_unpacklo_epi32(t1, t3);
+ __m512i u3 = _mm512_unpackhi_epi32(t1, t3);
+ __m512i u4 = _mm512_unpacklo_epi32(t4, t6);
+ __m512i u5 = _mm512_unpackhi_epi32(t4, t6);
+ __m512i u6 = _mm512_unpacklo_epi32(t5, t7);
+ __m512i u7 = _mm512_unpackhi_epi32(t5, t7);
+
+ _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
+ _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
+ _mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(u1, u5));
+ _mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(u1, u5));
+ _mm512_storeu_si512((__m512i*)(dst + d * 4), _mm512_unpacklo_epi64(u2, u6));
+ _mm512_storeu_si512((__m512i*)(dst + d * 5), _mm512_unpackhi_epi64(u2, u6));
+ _mm512_storeu_si512((__m512i*)(dst + d * 6), _mm512_unpacklo_epi64(u3, u7));
+ _mm512_storeu_si512((__m512i*)(dst + d * 7), _mm512_unpackhi_epi64(u3, u7));
+ src += 8;
+ dst += d * 8;
+ width -= 8;
+ }
+ if (width > 0) {
+ const __mmask8 col_mask = (1u << width) - 1;
+ __m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src));
+ __m512i r1 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s));
+ __m512i r2 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 2));
+ __m512i r3 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 3));
+ __m512i r4 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 4));
+ __m512i r5 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 5));
+ __m512i r6 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 6));
+ __m512i r7 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 7));
+
+ r0 =
+ _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 8), 1);
+ r1 =
+ _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 9), 1);
+ r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 10),
+ 1);
+ r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 11),
+ 1);
+ r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 12),
+ 1);
+ r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 13),
+ 1);
+ r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 14),
+ 1);
+ r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 15),
+ 1);
+
+ r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 16),
+ 2);
+ r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 17),
+ 2);
+ r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 18),
+ 2);
+ r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 19),
+ 2);
+ r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 20),
+ 2);
+ r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 21),
+ 2);
+ r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 22),
+ 2);
+ r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 23),
+ 2);
+
+ r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 24),
+ 3);
+ r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 25),
+ 3);
+ r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 26),
+ 3);
+ r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 27),
+ 3);
+ r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 28),
+ 3);
+ r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 29),
+ 3);
+ r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 30),
+ 3);
+ r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 31),
+ 3);
+
+ __m512i t0 = _mm512_unpacklo_epi16(r0, r1);
+ __m512i t1 = _mm512_unpackhi_epi16(r0, r1);
+ __m512i t2 = _mm512_unpacklo_epi16(r2, r3);
+ __m512i t3 = _mm512_unpackhi_epi16(r2, r3);
+ __m512i t4 = _mm512_unpacklo_epi16(r4, r5);
+ __m512i t5 = _mm512_unpackhi_epi16(r4, r5);
+ __m512i t6 = _mm512_unpacklo_epi16(r6, r7);
+ __m512i t7 = _mm512_unpackhi_epi16(r6, r7);
+
+ __m512i u0 = _mm512_unpacklo_epi32(t0, t2);
+ __m512i u1 = _mm512_unpackhi_epi32(t0, t2);
+ __m512i u2 = _mm512_unpacklo_epi32(t1, t3);
+ __m512i u3 = _mm512_unpackhi_epi32(t1, t3);
+ __m512i u4 = _mm512_unpacklo_epi32(t4, t6);
+ __m512i u5 = _mm512_unpackhi_epi32(t4, t6);
+ __m512i u6 = _mm512_unpacklo_epi32(t5, t7);
+ __m512i u7 = _mm512_unpackhi_epi32(t5, t7);
+
+ _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
+ if (width > 1) {
+ _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
+ }
+ if (width > 2) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 2),
+ _mm512_unpacklo_epi64(u1, u5));
+ }
+ if (width > 3) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 3),
+ _mm512_unpackhi_epi64(u1, u5));
+ }
+ if (width > 4) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 4),
+ _mm512_unpacklo_epi64(u2, u6));
+ }
+ if (width > 5) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 5),
+ _mm512_unpackhi_epi64(u2, u6));
+ }
+ if (width > 6) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 6),
+ _mm512_unpacklo_epi64(u3, u7));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
+
+#if defined(HAS_TRANSPOSEWX4_32_AVX2)
+// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per
+// AVX2 iteration (with a 4-row 128-bit tail) and write 4 destination rows.
+LIBYUV_TARGET_AVX2
+void TransposeWx4_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width >= 8) {
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
+ _mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
+ _mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
+ _mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
+ _mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
+
+ __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
+ __m256i t1 = _mm256_unpacklo_epi32(r2, r3);
+ __m256i t2 = _mm256_unpackhi_epi32(r0, r1);
+ __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
+
+ _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
+ _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
+ _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
+ _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
+ src += s * 8;
+ dst += 32;
+ width -= 8;
+ }
+ if (width > 0) {
+ __m128i r0 = _mm_loadu_si128((const __m128i*)(src));
+ __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
+ __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
+ __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
+ __m128i t0 = _mm_unpacklo_epi32(r0, r1);
+ __m128i t1 = _mm_unpacklo_epi32(r2, r3);
+ __m128i t2 = _mm_unpackhi_epi32(r0, r1);
+ __m128i t3 = _mm_unpackhi_epi32(r2, r3);
+ _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
+ _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
+ _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
+ _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX4_32_AVX2)
+
+#if defined(HAS_TRANSPOSEWX8_32_AVX2)
+LIBYUV_TARGET_AVX2
+void TransposeWx8_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width > 0) {
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
+ _mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
+ _mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
+ _mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
+ _mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
+
+ __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
+ __m256i t1 = _mm256_unpacklo_epi32(r2, r3);
+ __m256i t2 = _mm256_unpackhi_epi32(r0, r1);
+ __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
+
+ _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
+ _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
+ _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
+ _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
+ src += 16;
+ dst += d * 4;
+ width -= 4;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX8_32_AVX2)
+
+#if defined(HAS_TRANSPOSEWXH_32_AVX2)
+// Load n (1..3) 32-bit pixels into low lanes of __m128i with zeroed upper
+// lanes (safe at buffer boundaries and MemorySanitizer-clean).
+LIBYUV_TARGET_AVX2
+static __m128i LoadPartialRow_32_AVX2(const uint8_t* p, int n) {
+ if (n == 1) {
+ int v;
+ memcpy(&v, p, 4);
+ return _mm_cvtsi32_si128(v);
+ }
+ if (n == 2) {
+ return _mm_loadl_epi64((const __m128i*)p);
+ }
+ int v;
+ memcpy(&v, p + 8, 4);
+ return _mm_insert_epi32(_mm_loadl_epi64((const __m128i*)p), v, 2);
+}
+
+// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3}
+// tiles plus exact remainder handling).
+LIBYUV_TARGET_AVX2
+void TransposeWxH_32_AVX2(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ const int w4 = width & ~3;
+ const int rem_w = width & 3;
+
+ // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail.
+ while (height >= 8) {
+ if (w4 > 0) {
+ TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4);
+ }
+ if (rem_w > 0) {
+ const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4;
+ uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d;
+ __m256i r0 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem, rem_w)),
+ LoadPartialRow_32_AVX2(src_rem + s * 4, rem_w), 1);
+ __m256i r1 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem + s, rem_w)),
+ LoadPartialRow_32_AVX2(src_rem + s * 5, rem_w), 1);
+ __m256i r2 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(
+ LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w)),
+ LoadPartialRow_32_AVX2(src_rem + s * 6, rem_w), 1);
+ __m256i r3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(
+ LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w)),
+ LoadPartialRow_32_AVX2(src_rem + s * 7, rem_w), 1);
+
+ __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
+ __m256i t1 = _mm256_unpacklo_epi32(r2, r3);
+ _mm256_storeu_si256((__m256i*)(dst_rem), _mm256_unpacklo_epi64(t0, t1));
+ if (rem_w > 1) {
+ _mm256_storeu_si256((__m256i*)(dst_rem + d),
+ _mm256_unpackhi_epi64(t0, t1));
+ }
+ if (rem_w > 2) {
+ __m256i t2 = _mm256_unpackhi_epi32(r0, r1);
+ __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
+ _mm256_storeu_si256((__m256i*)(dst_rem + d * 2),
+ _mm256_unpacklo_epi64(t2, t3));
+ }
+ _mm256_zeroupper();
+ }
+ src += 8 * s;
+ dst += 32;
+ height -= 8;
+ }
+
+ // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail.
+ if (height >= 4) {
+ const int w8 = width & ~7;
+ if (w8 > 0) {
+ const uint8_t* src_p = src;
+ uint8_t* dst_p = dst;
+ int cnt = w8;
+ while (cnt > 0) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)(src_p));
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_p + s));
+ __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_p + s * 2));
+ __m256i r3 = _mm256_loadu_si256((const __m256i*)(src_p + s * 3));
+
+ __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
+ __m256i t1 = _mm256_unpacklo_epi32(r2, r3);
+ __m256i t2 = _mm256_unpackhi_epi32(r0, r1);
+ __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
+
+ __m256i o0 = _mm256_unpacklo_epi64(t0, t1);
+ __m256i o1 = _mm256_unpackhi_epi64(t0, t1);
+ __m256i o2 = _mm256_unpacklo_epi64(t2, t3);
+ __m256i o3 = _mm256_unpackhi_epi64(t2, t3);
+
+ _mm_storeu_si128((__m128i*)(dst_p), _mm256_castsi256_si128(o0));
+ _mm_storeu_si128((__m128i*)(dst_p + d), _mm256_castsi256_si128(o1));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm256_castsi256_si128(o2));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm256_castsi256_si128(o3));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 4),
+ _mm256_extracti128_si256(o0, 1));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 5),
+ _mm256_extracti128_si256(o1, 1));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 6),
+ _mm256_extracti128_si256(o2, 1));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 7),
+ _mm256_extracti128_si256(o3, 1));
+
+ src_p += 32;
+ dst_p += d * 8;
+ cnt -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width & 4) {
+ const uint8_t* src_p = src + (ptrdiff_t)w8 * 4;
+ uint8_t* dst_p = dst + (ptrdiff_t)w8 * d;
+ __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
+ __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
+ __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2));
+ __m128i r3 = _mm_loadu_si128((const __m128i*)(src_p + s * 3));
+ __m128i t0 = _mm_unpacklo_epi32(r0, r1);
+ __m128i t1 = _mm_unpacklo_epi32(r2, r3);
+ __m128i t2 = _mm_unpackhi_epi32(r0, r1);
+ __m128i t3 = _mm_unpackhi_epi32(r2, r3);
+ _mm_storeu_si128((__m128i*)(dst_p), _mm_unpacklo_epi64(t0, t1));
+ _mm_storeu_si128((__m128i*)(dst_p + d), _mm_unpackhi_epi64(t0, t1));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm_unpacklo_epi64(t2, t3));
+ _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm_unpackhi_epi64(t2, t3));
+ }
+ if (rem_w > 0) {
+ const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4;
+ uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d;
+ __m128i r0 = LoadPartialRow_32_AVX2(src_rem, rem_w);
+ __m128i r1 = LoadPartialRow_32_AVX2(src_rem + s, rem_w);
+ __m128i r2 = LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w);
+ __m128i r3 = LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w);
+ __m128i t0 = _mm_unpacklo_epi32(r0, r1);
+ __m128i t1 = _mm_unpacklo_epi32(r2, r3);
+ _mm_storeu_si128((__m128i*)(dst_rem), _mm_unpacklo_epi64(t0, t1));
+ if (rem_w > 1) {
+ _mm_storeu_si128((__m128i*)(dst_rem + d), _mm_unpackhi_epi64(t0, t1));
+ }
+ if (rem_w > 2) {
+ __m128i t2 = _mm_unpackhi_epi32(r0, r1);
+ __m128i t3 = _mm_unpackhi_epi32(r2, r3);
+ _mm_storeu_si128((__m128i*)(dst_rem + d * 2),
+ _mm_unpacklo_epi64(t2, t3));
+ }
+ }
+ src += 4 * s;
+ dst += 16;
+ height -= 4;
+ }
+
+ // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner.
+ if (height > 0) {
+ if (w4 > 0) {
+ const uint8_t* src_p = src;
+ uint8_t* dst_p = dst;
+ int cnt = w4;
+ if (height == 1) {
+ while (cnt > 0) {
+ __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
+ int v0 = _mm_cvtsi128_si32(r0);
+ int v1 = _mm_extract_epi32(r0, 1);
+ int v2 = _mm_extract_epi32(r0, 2);
+ int v3 = _mm_extract_epi32(r0, 3);
+ memcpy(dst_p, &v0, 4);
+ memcpy(dst_p + d, &v1, 4);
+ memcpy(dst_p + d * 2, &v2, 4);
+ memcpy(dst_p + d * 3, &v3, 4);
+ src_p += 16;
+ dst_p += d * 4;
+ cnt -= 4;
+ }
+ } else if (height == 2) {
+ while (cnt > 0) {
+ __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
+ __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
+ __m128i t0 = _mm_unpacklo_epi32(r0, r1);
+ __m128i t1 = _mm_unpackhi_epi32(r0, r1);
+ _mm_storel_epi64((__m128i*)(dst_p), t0);
+ _mm_storeh_pd((double*)(dst_p + d), _mm_castsi128_pd(t0));
+ _mm_storel_epi64((__m128i*)(dst_p + d * 2), t1);
+ _mm_storeh_pd((double*)(dst_p + d * 3), _mm_castsi128_pd(t1));
+ src_p += 16;
+ dst_p += d * 4;
+ cnt -= 4;
+ }
+ } else {
+ const __m128i zero = _mm_setzero_si128();
+ while (cnt > 0) {
+ __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
+ __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
+ __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2));
+ __m128i t0 = _mm_unpacklo_epi32(r0, r1);
+ __m128i t1 = _mm_unpacklo_epi32(r2, zero);
+ __m128i t2 = _mm_unpackhi_epi32(r0, r1);
+ __m128i t3 = _mm_unpackhi_epi32(r2, zero);
+ __m128i o0 = _mm_unpacklo_epi64(t0, t1);
+ __m128i o1 = _mm_unpackhi_epi64(t0, t1);
+ __m128i o2 = _mm_unpacklo_epi64(t2, t3);
+ __m128i o3 = _mm_unpackhi_epi64(t2, t3);
+ int v0 = _mm_extract_epi32(o0, 2);
+ int v1 = _mm_extract_epi32(o1, 2);
+ int v2 = _mm_extract_epi32(o2, 2);
+ int v3 = _mm_extract_epi32(o3, 2);
+ _mm_storel_epi64((__m128i*)(dst_p), o0);
+ memcpy(dst_p + 8, &v0, 4);
+ _mm_storel_epi64((__m128i*)(dst_p + d), o1);
+ memcpy(dst_p + d + 8, &v1, 4);
+ _mm_storel_epi64((__m128i*)(dst_p + d * 2), o2);
+ memcpy(dst_p + d * 2 + 8, &v2, 4);
+ _mm_storel_epi64((__m128i*)(dst_p + d * 3), o3);
+ memcpy(dst_p + d * 3 + 8, &v3, 4);
+ src_p += 16;
+ dst_p += d * 4;
+ cnt -= 4;
+ }
+ }
+ }
+ for (int x = w4; x < width; ++x) {
+ uint8_t* dst_row = dst + (ptrdiff_t)x * d;
+ for (int y = 0; y < height; ++y) {
+ memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4);
+ }
+ }
+ }
+}
+#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
+
+#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+void TransposeWx16_32_AVX512BW(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ const ptrdiff_t s = src_stride;
+ const ptrdiff_t d = dst_stride;
+ while (width >= 4) {
+ __m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
+ __m512i r1 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
+ __m512i r2 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
+ __m512i r3 =
+ _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)),
+ 1);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)),
+ 1);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 6)),
+ 1);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)),
+ 1);
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
+ 2);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
+ 2);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
+ 2);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
+ 2);
+
+ r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)),
+ 3);
+ r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)),
+ 3);
+ r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 14)),
+ 3);
+ r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)),
+ 3);
+
+ __m512i t0 = _mm512_unpacklo_epi32(r0, r1);
+ __m512i t1 = _mm512_unpacklo_epi32(r2, r3);
+ __m512i t2 = _mm512_unpackhi_epi32(r0, r1);
+ __m512i t3 = _mm512_unpackhi_epi32(r2, r3);
+
+ _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
+ _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
+ _mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3));
+ _mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3));
+ src += 16;
+ dst += d * 4;
+ width -= 4;
+ }
+ if (width > 0) {
+ const __mmask8 col_mask = (1u << width) - 1;
+ __m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src));
+ __m512i r1 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s));
+ __m512i r2 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 2));
+ __m512i r3 =
+ _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 3));
+
+ r0 =
+ _mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 4), 1);
+ r1 =
+ _mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 5), 1);
+ r2 =
+ _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 6), 1);
+ r3 =
+ _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 7), 1);
+
+ r0 =
+ _mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 8), 2);
+ r1 =
+ _mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 9), 2);
+ r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 10),
+ 2);
+ r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 11),
+ 2);
+
+ r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 12),
+ 3);
+ r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 13),
+ 3);
+ r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 14),
+ 3);
+ r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 15),
+ 3);
+
+ __m512i t0 = _mm512_unpacklo_epi32(r0, r1);
+ __m512i t1 = _mm512_unpacklo_epi32(r2, r3);
+ __m512i t2 = _mm512_unpackhi_epi32(r0, r1);
+ __m512i t3 = _mm512_unpackhi_epi32(r2, r3);
+
+ _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
+ if (width > 1) {
+ _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
+ }
+ if (width > 2) {
+ _mm512_storeu_si512((__m512i*)(dst + d * 2),
+ _mm512_unpacklo_epi64(t2, t3));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
+
+#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
+ // (Visual C or LIBYUV_ENABLE_ROWWIN)
diff --git a/libs/libyuv/source/row_any.cc b/libs/libyuv/source/row_any.cc
index 9fafff6020..9c6d3ccb3f 100644
--- a/libs/libyuv/source/row_any.cc
+++ b/libs/libyuv/source/row_any.cc
@@ -10,90 +10,306 @@
#include "libyuv/row.h"
+#include <assert.h>
#include <string.h> // For memset.
-#include "libyuv/basic_types.h"
-
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// memset for temp is meant to clear the source buffer (not dest) so that
-// SIMD that reads full multiple of 16 bytes will not trigger msan errors.
-// memset is not needed for production, as the garbage values are processed but
-// not used, although there may be edge cases for subsampling.
-// The size of the buffer is based on the largest read, which can be inferred
-// by the source type (e.g. ARGB) and the mask (last parameter), or by examining
-// the source code for how much the source pointers are advanced.
-
-// Subsampled source needs to be increase by 1 of not even.
+// Subsampled source needs to be increase by 1 if not even.
#define SS(width, shift) (((width) + (1 << (shift)) - 1) >> (shift))
+#ifdef LIBYUV_MSAN
+#define MSAN_MEMSET(a, b, c) memset(a, b, c)
+#else
+#define MSAN_MEMSET(a, b, c)
+#endif
+
+// Any 4 planes to 1
+#define ANY41(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
+ void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
+ const uint8_t* v_buf, const uint8_t* a_buf, uint8_t* dst_ptr, \
+ int width) { \
+ SIMD_ALIGNED(uint8_t vin[64 * 4]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 4, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, u_buf, v_buf, a_buf, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r); \
+ memcpy(vin + 64, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 128, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 192, a_buf + np, r); \
+ ANY_SIMD(vin, vin + 64, vin + 128, vin + 192, vout, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
+ }
+
+#ifdef HAS_MERGEARGBROW_SSE2
+ANY41(MergeARGBRow_Any_SSE2, MergeARGBRow_SSE2, 0, 0, 4, 7)
+#endif
+#ifdef HAS_MERGEARGBROW_AVX2
+ANY41(MergeARGBRow_Any_AVX2, MergeARGBRow_AVX2, 0, 0, 4, 15)
+#endif
+#ifdef HAS_MERGEARGBROW_NEON
+ANY41(MergeARGBRow_Any_NEON, MergeARGBRow_NEON, 0, 0, 4, 15)
+#endif
+
+// Note that odd width replication includes 444 due to implementation
+// on arm that subsamples 444 to 422 internally.
// Any 4 planes to 1 with yuvconstants
#define ANY41C(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
const uint8_t* v_buf, const uint8_t* a_buf, uint8_t* dst_ptr, \
const struct YuvConstants* yuvconstants, int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 5]); \
- memset(temp, 0, 64 * 4); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[64 * 4]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 4, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(y_buf, u_buf, v_buf, a_buf, dst_ptr, yuvconstants, n); \
} \
- memcpy(temp, y_buf + n, r); \
- memcpy(temp + 64, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- memcpy(temp + 128, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- memcpy(temp + 192, a_buf + n, r); \
- ANY_SIMD(temp, temp + 64, temp + 128, temp + 192, temp + 256, \
- yuvconstants, MASK + 1); \
- memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 256, \
- SS(r, DUVSHIFT) * BPP); \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r); \
+ memcpy(vin + 64, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 128, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 192, a_buf + np, r); \
+ if (width & 1) { \
+ vin[64 + SS(r, UVSHIFT)] = vin[64 + SS(r, UVSHIFT) - 1]; \
+ vin[128 + SS(r, UVSHIFT)] = vin[128 + SS(r, UVSHIFT) - 1]; \
+ } \
+ ANY_SIMD(vin, vin + 64, vin + 128, vin + 192, vout, yuvconstants, \
+ MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
}
+#ifdef HAS_I444ALPHATOARGBROW_SSSE3
+ANY41C(I444AlphaToARGBRow_Any_SSSE3, I444AlphaToARGBRow_SSSE3, 0, 0, 4, 7)
+#endif
+#ifdef HAS_I444ALPHATOARGBROW_AVX2
+ANY41C(I444AlphaToARGBRow_Any_AVX2, I444AlphaToARGBRow_AVX2, 0, 0, 4, 15)
+#endif
#ifdef HAS_I422ALPHATOARGBROW_SSSE3
ANY41C(I422AlphaToARGBRow_Any_SSSE3, I422AlphaToARGBRow_SSSE3, 1, 0, 4, 7)
#endif
#ifdef HAS_I422ALPHATOARGBROW_AVX2
ANY41C(I422AlphaToARGBRow_Any_AVX2, I422AlphaToARGBRow_AVX2, 1, 0, 4, 15)
#endif
+#ifdef HAS_I444ALPHATOARGBROW_NEON
+ANY41C(I444AlphaToARGBRow_Any_NEON, I444AlphaToARGBRow_NEON, 0, 0, 4, 7)
+#endif
#ifdef HAS_I422ALPHATOARGBROW_NEON
ANY41C(I422AlphaToARGBRow_Any_NEON, I422AlphaToARGBRow_NEON, 1, 0, 4, 7)
#endif
-#ifdef HAS_I422ALPHATOARGBROW_MSA
-ANY41C(I422AlphaToARGBRow_Any_MSA, I422AlphaToARGBRow_MSA, 1, 0, 4, 7)
+#ifdef HAS_I422ALPHATOARGBROW_LSX
+ANY41C(I422AlphaToARGBRow_Any_LSX, I422AlphaToARGBRow_LSX, 1, 0, 4, 7)
+#endif
+#ifdef HAS_I422ALPHATOARGBROW_LASX
+ANY41C(I422AlphaToARGBRow_Any_LASX, I422AlphaToARGBRow_LASX, 1, 0, 4, 15)
#endif
#undef ANY41C
+// Any 4 planes to 1 plane of 8 bit with yuvconstants
+#define ANY41CT(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, T, SBPP, BPP, MASK) \
+ void NAMEANY(const T* y_buf, const T* u_buf, const T* v_buf, const T* a_buf, \
+ uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, \
+ int width) { \
+ SIMD_ALIGNED(T vin[16 * 4]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 4, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, u_buf, v_buf, a_buf, dst_ptr, yuvconstants, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r * SBPP); \
+ memcpy(vin + 16, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
+ memcpy(vin + 32, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
+ memcpy(vin + 48, a_buf + np, r * SBPP); \
+ ANY_SIMD(vin, vin + 16, vin + 32, vin + 48, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
+ }
+
+#ifdef HAS_I210ALPHATOARGBROW_NEON
+ANY41CT(I210AlphaToARGBRow_Any_NEON,
+ I210AlphaToARGBRow_NEON,
+ 1,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+#ifdef HAS_I410ALPHATOARGBROW_NEON
+ANY41CT(I410AlphaToARGBRow_Any_NEON,
+ I410AlphaToARGBRow_NEON,
+ 0,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+
+#ifdef HAS_I210ALPHATOARGBROW_SSSE3
+ANY41CT(I210AlphaToARGBRow_Any_SSSE3,
+ I210AlphaToARGBRow_SSSE3,
+ 1,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+
+#ifdef HAS_I210ALPHATOARGBROW_AVX2
+ANY41CT(I210AlphaToARGBRow_Any_AVX2,
+ I210AlphaToARGBRow_AVX2,
+ 1,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 15)
+#endif
+
+#ifdef HAS_I410ALPHATOARGBROW_SSSE3
+ANY41CT(I410AlphaToARGBRow_Any_SSSE3,
+ I410AlphaToARGBRow_SSSE3,
+ 0,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+
+#ifdef HAS_I410ALPHATOARGBROW_AVX2
+ANY41CT(I410AlphaToARGBRow_Any_AVX2,
+ I410AlphaToARGBRow_AVX2,
+ 0,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 15)
+#endif
+
+#undef ANY41CT
+
+// Any 4 planes to 1 plane with parameter
+#define ANY41PT(NAMEANY, ANY_SIMD, STYPE, SBPP, DTYPE, BPP, MASK) \
+ void NAMEANY(const STYPE* r_buf, const STYPE* g_buf, const STYPE* b_buf, \
+ const STYPE* a_buf, DTYPE* dst_ptr, int depth, int width) { \
+ SIMD_ALIGNED(STYPE vin[16 * 4]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 4, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(r_buf, g_buf, b_buf, a_buf, dst_ptr, depth, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, r_buf + np, r * SBPP); \
+ memcpy(vin + 16, g_buf + np, r * SBPP); \
+ memcpy(vin + 32, b_buf + np, r * SBPP); \
+ memcpy(vin + 48, a_buf + np, r * SBPP); \
+ ANY_SIMD(vin, vin + 16, vin + 32, vin + 48, vout, depth, MASK + 1); \
+ memcpy((uint8_t*)dst_ptr + np * BPP, vout, r * BPP); \
+ }
+
+#ifdef HAS_MERGEAR64ROW_AVX2
+ANY41PT(MergeAR64Row_Any_AVX2, MergeAR64Row_AVX2, uint16_t, 2, uint16_t, 8, 15)
+#endif
+
+#ifdef HAS_MERGEAR64ROW_NEON
+ANY41PT(MergeAR64Row_Any_NEON, MergeAR64Row_NEON, uint16_t, 2, uint16_t, 8, 7)
+#endif
+
+#ifdef HAS_MERGEARGB16TO8ROW_AVX2
+ANY41PT(MergeARGB16To8Row_Any_AVX2,
+ MergeARGB16To8Row_AVX2,
+ uint16_t,
+ 2,
+ uint8_t,
+ 4,
+ 15)
+#endif
+
+#ifdef HAS_MERGEARGB16TO8ROW_NEON
+ANY41PT(MergeARGB16To8Row_Any_NEON,
+ MergeARGB16To8Row_NEON,
+ uint16_t,
+ 2,
+ uint8_t,
+ 4,
+ 7)
+#endif
+
+#undef ANY41PT
+
// Any 3 planes to 1.
-#define ANY31(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
- void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
- const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 4]); \
- memset(temp, 0, 64 * 3); /* for YUY2 and msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, n); \
- } \
- memcpy(temp, y_buf + n, r); \
- memcpy(temp + 64, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- memcpy(temp + 128, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- ANY_SIMD(temp, temp + 64, temp + 128, temp + 192, MASK + 1); \
- memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 192, \
- SS(r, DUVSHIFT) * BPP); \
+#define ANY31(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
+ void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
+ const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \
+ SIMD_ALIGNED(uint8_t vin[64 * 3]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 3, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64 * 4]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ memset(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r); \
+ memcpy(vin + 64, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 128, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ ANY_SIMD(vin, vin + 64, vin + 128, vout, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
}
// Merge functions.
-#ifdef HAS_MERGERGBROW_SSSE3
-ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15)
+#ifdef HAS_MERGERGBROW_AVX2
+ANY31(MergeRGBRow_Any_AVX2, MergeRGBRow_AVX2, 0, 0, 3, 31)
#endif
#ifdef HAS_MERGERGBROW_NEON
ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15)
#endif
-#ifdef HAS_MERGERGBROW_MMI
-ANY31(MergeRGBRow_Any_MMI, MergeRGBRow_MMI, 0, 0, 3, 7)
+#ifdef HAS_MERGEXRGBROW_SSE2
+ANY31(MergeXRGBRow_Any_SSE2, MergeXRGBRow_SSE2, 0, 0, 4, 7)
+#endif
+#ifdef HAS_MERGEXRGBROW_AVX2
+ANY31(MergeXRGBRow_Any_AVX2, MergeXRGBRow_AVX2, 0, 0, 4, 15)
+#endif
+#ifdef HAS_MERGEXRGBROW_NEON
+ANY31(MergeXRGBRow_Any_NEON, MergeXRGBRow_NEON, 0, 0, 4, 15)
#endif
#ifdef HAS_I422TOYUY2ROW_SSE2
ANY31(I422ToYUY2Row_Any_SSE2, I422ToYUY2Row_SSE2, 1, 1, 4, 15)
@@ -106,20 +322,20 @@ ANY31(I422ToUYVYRow_Any_AVX2, I422ToUYVYRow_AVX2, 1, 1, 4, 31)
#ifdef HAS_I422TOYUY2ROW_NEON
ANY31(I422ToYUY2Row_Any_NEON, I422ToYUY2Row_NEON, 1, 1, 4, 15)
#endif
-#ifdef HAS_I422TOYUY2ROW_MSA
-ANY31(I422ToYUY2Row_Any_MSA, I422ToYUY2Row_MSA, 1, 1, 4, 31)
+#ifdef HAS_I422TOYUY2ROW_LSX
+ANY31(I422ToYUY2Row_Any_LSX, I422ToYUY2Row_LSX, 1, 1, 4, 15)
#endif
-#ifdef HAS_I422TOYUY2ROW_MMI
-ANY31(I422ToYUY2Row_Any_MMI, I422ToYUY2Row_MMI, 1, 1, 4, 7)
+#ifdef HAS_I422TOYUY2ROW_LASX
+ANY31(I422ToYUY2Row_Any_LASX, I422ToYUY2Row_LASX, 1, 1, 4, 31)
#endif
#ifdef HAS_I422TOUYVYROW_NEON
ANY31(I422ToUYVYRow_Any_NEON, I422ToUYVYRow_NEON, 1, 1, 4, 15)
#endif
-#ifdef HAS_I422TOUYVYROW_MSA
-ANY31(I422ToUYVYRow_Any_MSA, I422ToUYVYRow_MSA, 1, 1, 4, 31)
+#ifdef HAS_I422TOUYVYROW_LSX
+ANY31(I422ToUYVYRow_Any_LSX, I422ToUYVYRow_LSX, 1, 1, 4, 15)
#endif
-#ifdef HAS_I422TOUYVYROW_MMI
-ANY31(I422ToUYVYRow_Any_MMI, I422ToUYVYRow_MMI, 1, 1, 4, 7)
+#ifdef HAS_I422TOUYVYROW_LASX
+ANY31(I422ToUYVYRow_Any_LASX, I422ToUYVYRow_LASX, 1, 1, 4, 31)
#endif
#ifdef HAS_BLENDPLANEROW_AVX2
ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31)
@@ -127,41 +343,63 @@ ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31)
#ifdef HAS_BLENDPLANEROW_SSSE3
ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7)
#endif
-#ifdef HAS_BLENDPLANEROW_MMI
-ANY31(BlendPlaneRow_Any_MMI, BlendPlaneRow_MMI, 0, 0, 1, 7)
+#ifdef HAS_BLENDPLANEROW_NEON
+ANY31(BlendPlaneRow_Any_NEON, BlendPlaneRow_NEON, 0, 0, 1, 15)
+#endif
+#ifdef HAS_BLENDPLANEROW_WASMSIMD
+ANY31(BlendPlaneRow_Any_WASMSIMD, BlendPlaneRow_WASMSIMD, 0, 0, 1, 15)
#endif
#undef ANY31
// Note that odd width replication includes 444 due to implementation
// on arm that subsamples 444 to 422 internally.
// Any 3 planes to 1 with yuvconstants
-#define ANY31C(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
- void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
- const uint8_t* v_buf, uint8_t* dst_ptr, \
- const struct YuvConstants* yuvconstants, int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 4]); \
- memset(temp, 0, 128 * 3); /* for YUY2 and msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \
- } \
- memcpy(temp, y_buf + n, r); \
- memcpy(temp + 128, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- memcpy(temp + 256, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \
- if (width & 1) { \
- temp[128 + SS(r, UVSHIFT)] = temp[128 + SS(r, UVSHIFT) - 1]; \
- temp[256 + SS(r, UVSHIFT)] = temp[256 + SS(r, UVSHIFT) - 1]; \
- } \
- ANY_SIMD(temp, temp + 128, temp + 256, temp + 384, yuvconstants, \
- MASK + 1); \
- memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 384, \
- SS(r, DUVSHIFT) * BPP); \
+#define ANY31C(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \
+ void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \
+ const uint8_t* v_buf, uint8_t* dst_ptr, \
+ const struct YuvConstants* yuvconstants, int width) { \
+ SIMD_ALIGNED(uint8_t vin[128 * 3]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 3, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[128]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r); \
+ memcpy(vin + 128, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ memcpy(vin + 256, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT)); \
+ if (width & 1) { \
+ vin[128 + SS(r, UVSHIFT)] = vin[128 + SS(r, UVSHIFT) - 1]; \
+ vin[256 + SS(r, UVSHIFT)] = vin[256 + SS(r, UVSHIFT) - 1]; \
+ } \
+ ANY_SIMD(vin, vin + 128, vin + 256, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
}
#ifdef HAS_I422TOARGBROW_SSSE3
ANY31C(I422ToARGBRow_Any_SSSE3, I422ToARGBRow_SSSE3, 1, 0, 4, 7)
#endif
+#ifdef HAS_I422TORGBAROW_SSSE3
+ANY31C(I422ToRGBARow_Any_SSSE3, I422ToRGBARow_SSSE3, 1, 0, 4, 7)
+#endif
+#ifdef HAS_I422TOARGB4444ROW_SSSE3
+ANY31C(I422ToARGB4444Row_Any_SSSE3, I422ToARGB4444Row_SSSE3, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TOARGB1555ROW_SSSE3
+ANY31C(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TORGB565ROW_SSSE3
+ANY31C(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TORGB24ROW_SSSE3
+ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 7)
+#endif
#ifdef HAS_I422TOAR30ROW_SSSE3
ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7)
#endif
@@ -170,14 +408,12 @@ ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15)
#endif
#ifdef HAS_I444TOARGBROW_SSSE3
ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7)
-ANY31C(I422ToRGBARow_Any_SSSE3, I422ToRGBARow_SSSE3, 1, 0, 4, 7)
-ANY31C(I422ToARGB4444Row_Any_SSSE3, I422ToARGB4444Row_SSSE3, 1, 0, 2, 7)
-ANY31C(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7)
-ANY31C(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7)
-ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 15)
-#endif // HAS_I444TOARGBROW_SSSE3
+#endif
+#ifdef HAS_I444TORGB24ROW_SSSE3
+ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 7)
+#endif
#ifdef HAS_I422TORGB24ROW_AVX2
-ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 31)
+ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
#endif
#ifdef HAS_I422TOARGBROW_AVX2
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
@@ -188,6 +424,9 @@ ANY31C(I422ToRGBARow_Any_AVX2, I422ToRGBARow_AVX2, 1, 0, 4, 15)
#ifdef HAS_I444TOARGBROW_AVX2
ANY31C(I444ToARGBRow_Any_AVX2, I444ToARGBRow_AVX2, 0, 0, 4, 15)
#endif
+#ifdef HAS_I444TORGB24ROW_AVX2
+ANY31C(I444ToRGB24Row_Any_AVX2, I444ToRGB24Row_AVX2, 0, 0, 3, 31)
+#endif
#ifdef HAS_I422TOARGB4444ROW_AVX2
ANY31C(I422ToARGB4444Row_Any_AVX2, I422ToARGB4444Row_AVX2, 1, 0, 2, 15)
#endif
@@ -197,6 +436,12 @@ ANY31C(I422ToARGB1555Row_Any_AVX2, I422ToARGB1555Row_AVX2, 1, 0, 2, 15)
#ifdef HAS_I422TORGB565ROW_AVX2
ANY31C(I422ToRGB565Row_Any_AVX2, I422ToRGB565Row_AVX2, 1, 0, 2, 15)
#endif
+#ifdef HAS_I444TORGB24ROW_NEON
+ANY31C(I444ToRGB24Row_Any_NEON, I444ToRGB24Row_NEON, 0, 0, 3, 7)
+#endif
+#ifdef HAS_I422TOAR30ROW_NEON
+ANY31C(I422ToAR30Row_Any_NEON, I422ToAR30Row_NEON, 1, 0, 4, 7)
+#endif
#ifdef HAS_I422TOARGBROW_NEON
ANY31C(I444ToARGBRow_Any_NEON, I444ToARGBRow_NEON, 0, 0, 4, 7)
ANY31C(I422ToARGBRow_Any_NEON, I422ToARGBRow_NEON, 1, 0, 4, 7)
@@ -206,36 +451,73 @@ ANY31C(I422ToARGB4444Row_Any_NEON, I422ToARGB4444Row_NEON, 1, 0, 2, 7)
ANY31C(I422ToARGB1555Row_Any_NEON, I422ToARGB1555Row_NEON, 1, 0, 2, 7)
ANY31C(I422ToRGB565Row_Any_NEON, I422ToRGB565Row_NEON, 1, 0, 2, 7)
#endif
-#ifdef HAS_I422TOARGBROW_MSA
-ANY31C(I444ToARGBRow_Any_MSA, I444ToARGBRow_MSA, 0, 0, 4, 7)
-ANY31C(I422ToARGBRow_Any_MSA, I422ToARGBRow_MSA, 1, 0, 4, 7)
-ANY31C(I422ToRGBARow_Any_MSA, I422ToRGBARow_MSA, 1, 0, 4, 7)
-ANY31C(I422ToRGB24Row_Any_MSA, I422ToRGB24Row_MSA, 1, 0, 3, 15)
-ANY31C(I422ToARGB4444Row_Any_MSA, I422ToARGB4444Row_MSA, 1, 0, 2, 7)
-ANY31C(I422ToARGB1555Row_Any_MSA, I422ToARGB1555Row_MSA, 1, 0, 2, 7)
-ANY31C(I422ToRGB565Row_Any_MSA, I422ToRGB565Row_MSA, 1, 0, 2, 7)
+#ifdef HAS_I422TOARGBROW_LSX
+ANY31C(I422ToARGBRow_Any_LSX, I422ToARGBRow_LSX, 1, 0, 4, 7)
+ANY31C(I422ToRGBARow_Any_LSX, I422ToRGBARow_LSX, 1, 0, 4, 7)
+ANY31C(I422ToRGB24Row_Any_LSX, I422ToRGB24Row_LSX, 1, 0, 3, 7)
+ANY31C(I422ToRGB565Row_Any_LSX, I422ToRGB565Row_LSX, 1, 0, 2, 7)
+ANY31C(I422ToARGB4444Row_Any_LSX, I422ToARGB4444Row_LSX, 1, 0, 2, 7)
+ANY31C(I422ToARGB1555Row_Any_LSX, I422ToARGB1555Row_LSX, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TOARGBROW_LASX
+ANY31C(I422ToARGBRow_Any_LASX, I422ToARGBRow_LASX, 1, 0, 4, 15)
+ANY31C(I422ToRGBARow_Any_LASX, I422ToRGBARow_LASX, 1, 0, 4, 15)
+ANY31C(I422ToRGB24Row_Any_LASX, I422ToRGB24Row_LASX, 1, 0, 3, 15)
+ANY31C(I422ToRGB565Row_Any_LASX, I422ToRGB565Row_LASX, 1, 0, 2, 15)
+ANY31C(I422ToARGB4444Row_Any_LASX, I422ToARGB4444Row_LASX, 1, 0, 2, 15)
+ANY31C(I422ToARGB1555Row_Any_LASX, I422ToARGB1555Row_LASX, 1, 0, 2, 15)
+#endif
+#ifdef HAS_I444TOARGBROW_LSX
+ANY31C(I444ToARGBRow_Any_LSX, I444ToARGBRow_LSX, 0, 0, 4, 7)
+#endif
+#ifdef HAS_I444TOARGBROW_LASX
+ANY31C(I444ToARGBRow_Any_LASX, I444ToARGBRow_LASX, 0, 0, 4, 15)
+#endif
+#ifdef HAS_I422TOARGBROW_WASMSIMD
+ANY31C(I422ToARGBRow_Any_WASMSIMD, I422ToARGBRow_WASMSIMD, 1, 0, 4, 7)
+#endif
+#ifdef HAS_I422TOAR30ROW_WASMSIMD
+ANY31C(I422ToAR30Row_Any_WASMSIMD, I422ToAR30Row_WASMSIMD, 1, 0, 4, 7)
+#endif
+#ifdef HAS_I422TORGB24ROW_WASMSIMD
+ANY31C(I422ToRGB24Row_Any_WASMSIMD, I422ToRGB24Row_WASMSIMD, 1, 0, 3, 15)
+#endif
+#ifdef HAS_I422TORGB565ROW_WASMSIMD
+ANY31C(I422ToRGB565Row_Any_WASMSIMD, I422ToRGB565Row_WASMSIMD, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TOARGB1555ROW_WASMSIMD
+ANY31C(I422ToARGB1555Row_Any_WASMSIMD, I422ToARGB1555Row_WASMSIMD, 1, 0, 2, 7)
+#endif
+#ifdef HAS_I422TOARGB4444ROW_WASMSIMD
+ANY31C(I422ToARGB4444Row_Any_WASMSIMD, I422ToARGB4444Row_WASMSIMD, 1, 0, 2, 7)
#endif
#undef ANY31C
// Any 3 planes of 16 bit to 1 with yuvconstants
// TODO(fbarchard): consider sharing this code with ANY31C
-#define ANY31CT(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, T, SBPP, BPP, MASK) \
- void NAMEANY(const T* y_buf, const T* u_buf, const T* v_buf, \
- uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, \
- int width) { \
- SIMD_ALIGNED(T temp[16 * 3]); \
- SIMD_ALIGNED(uint8_t out[64]); \
- memset(temp, 0, 16 * 3 * SBPP); /* for YUY2 and msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \
- } \
- memcpy(temp, y_buf + n, r * SBPP); \
- memcpy(temp + 16, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
- memcpy(temp + 32, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
- ANY_SIMD(temp, temp + 16, temp + 32, out, yuvconstants, MASK + 1); \
- memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, out, SS(r, DUVSHIFT) * BPP); \
+#define ANY31CT(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, T, SBPP, BPP, MASK) \
+ void NAMEANY(const T* y_buf, const T* u_buf, const T* v_buf, \
+ uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, \
+ int width) { \
+ SIMD_ALIGNED(T vin[16 * 3]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 3, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r * SBPP); \
+ memcpy(vin + 16, u_buf + (np >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
+ memcpy(vin + 32, v_buf + (np >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \
+ ANY_SIMD(vin, vin + 16, vin + 32, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
}
#ifdef HAS_I210TOAR30ROW_SSSE3
@@ -250,24 +532,143 @@ ANY31CT(I210ToARGBRow_Any_AVX2, I210ToARGBRow_AVX2, 1, 0, uint16_t, 2, 4, 15)
#ifdef HAS_I210TOAR30ROW_AVX2
ANY31CT(I210ToAR30Row_Any_AVX2, I210ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
#endif
+#ifdef HAS_I410TOAR30ROW_SSSE3
+ANY31CT(I410ToAR30Row_Any_SSSE3, I410ToAR30Row_SSSE3, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOARGBROW_SSSE3
+ANY31CT(I410ToARGBRow_Any_SSSE3, I410ToARGBRow_SSSE3, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOARGBROW_AVX2
+ANY31CT(I410ToARGBRow_Any_AVX2, I410ToARGBRow_AVX2, 0, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_I410TOAR30ROW_AVX2
+ANY31CT(I410ToAR30Row_Any_AVX2, I410ToAR30Row_AVX2, 0, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_I212TOAR30ROW_SSSE3
+ANY31CT(I212ToAR30Row_Any_SSSE3, I212ToAR30Row_SSSE3, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOARGBROW_SSSE3
+ANY31CT(I212ToARGBRow_Any_SSSE3, I212ToARGBRow_SSSE3, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOARGBROW_AVX2
+ANY31CT(I212ToARGBRow_Any_AVX2, I212ToARGBRow_AVX2, 1, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_I212TOAR30ROW_AVX2
+ANY31CT(I212ToAR30Row_Any_AVX2, I212ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_I210TOARGBROW_NEON
+ANY31CT(I210ToARGBRow_Any_NEON, I210ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOARGBROW_NEON
+ANY31CT(I410ToARGBRow_Any_NEON, I410ToARGBRow_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I210TOAR30ROW_NEON
+ANY31CT(I210ToAR30Row_Any_NEON, I210ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOAR30ROW_NEON
+ANY31CT(I410ToAR30Row_Any_NEON, I410ToAR30Row_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOARGBROW_NEON
+ANY31CT(I212ToARGBRow_Any_NEON, I212ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOAR30ROW_NEON
+ANY31CT(I212ToAR30Row_Any_NEON, I212ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
#undef ANY31CT
+// Any 3 planes to 1 plane with parameter
+#define ANY31PT(NAMEANY, ANY_SIMD, STYPE, SBPP, DTYPE, BPP, MASK) \
+ void NAMEANY(const STYPE* r_buf, const STYPE* g_buf, const STYPE* b_buf, \
+ DTYPE* dst_ptr, int depth, int width) { \
+ SIMD_ALIGNED(STYPE vin[16 * 3]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 3, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(r_buf, g_buf, b_buf, dst_ptr, depth, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, r_buf + np, r * SBPP); \
+ memcpy(vin + 16, g_buf + np, r * SBPP); \
+ memcpy(vin + 32, b_buf + np, r * SBPP); \
+ ANY_SIMD(vin, vin + 16, vin + 32, vout, depth, MASK + 1); \
+ memcpy((uint8_t*)dst_ptr + np * BPP, vout, r * BPP); \
+ }
+
+#ifdef HAS_MERGEXR30ROW_AVX2
+ANY31PT(MergeXR30Row_Any_AVX2, MergeXR30Row_AVX2, uint16_t, 2, uint8_t, 4, 15)
+#endif
+
+#ifdef HAS_MERGEXR30ROW_NEON
+ANY31PT(MergeXR30Row_Any_NEON, MergeXR30Row_NEON, uint16_t, 2, uint8_t, 4, 3)
+ANY31PT(MergeXR30Row_10_Any_NEON,
+ MergeXR30Row_10_NEON,
+ uint16_t,
+ 2,
+ uint8_t,
+ 4,
+ 7)
+#endif
+
+#ifdef HAS_MERGEXR64ROW_AVX2
+ANY31PT(MergeXR64Row_Any_AVX2, MergeXR64Row_AVX2, uint16_t, 2, uint16_t, 8, 15)
+#endif
+
+#ifdef HAS_MERGEXR64ROW_NEON
+ANY31PT(MergeXR64Row_Any_NEON, MergeXR64Row_NEON, uint16_t, 2, uint16_t, 8, 7)
+#endif
+
+#ifdef HAS_MERGEXRGB16TO8ROW_AVX2
+ANY31PT(MergeXRGB16To8Row_Any_AVX2,
+ MergeXRGB16To8Row_AVX2,
+ uint16_t,
+ 2,
+ uint8_t,
+ 4,
+ 15)
+#endif
+
+#ifdef HAS_MERGEXRGB16TO8ROW_NEON
+ANY31PT(MergeXRGB16To8Row_Any_NEON,
+ MergeXRGB16To8Row_NEON,
+ uint16_t,
+ 2,
+ uint8_t,
+ 4,
+ 7)
+#endif
+
+#undef ANY31PT
+
// Any 2 planes to 1.
#define ANY21(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \
void NAMEANY(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, \
int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 3]); \
- memset(temp, 0, 64 * 2); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[128 * 2]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP2 <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[128]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(y_buf, uv_buf, dst_ptr, n); \
} \
- memcpy(temp, y_buf + n * SBPP, r * SBPP); \
- memcpy(temp + 64, uv_buf + (n >> UVSHIFT) * SBPP2, \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np * SBPP, r * SBPP); \
+ memcpy(vin + 128, uv_buf + (np >> UVSHIFT) * SBPP2, \
SS(r, UVSHIFT) * SBPP2); \
- ANY_SIMD(temp, temp + 64, temp + 128, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \
+ ANY_SIMD(vin, vin + 128, vout, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
// Merge functions.
@@ -275,20 +676,23 @@ ANY31CT(I210ToAR30Row_Any_AVX2, I210ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
ANY21(MergeUVRow_Any_SSE2, MergeUVRow_SSE2, 0, 1, 1, 2, 15)
#endif
#ifdef HAS_MERGEUVROW_AVX2
-ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 31)
+ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 15)
#endif
#ifdef HAS_MERGEUVROW_NEON
ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15)
#endif
-#ifdef HAS_MERGEUVROW_MSA
-ANY21(MergeUVRow_Any_MSA, MergeUVRow_MSA, 0, 1, 1, 2, 15)
+#ifdef HAS_MERGEUVROW_LSX
+ANY21(MergeUVRow_Any_LSX, MergeUVRow_LSX, 0, 1, 1, 2, 15)
#endif
-#ifdef HAS_MERGEUVROW_MMI
-ANY21(MergeUVRow_Any_MMI, MergeUVRow_MMI, 0, 1, 1, 2, 7)
+#ifdef HAS_MERGEUVROW_LASX
+ANY21(MergeUVRow_Any_LASX, MergeUVRow_LASX, 0, 1, 1, 2, 31)
#endif
#ifdef HAS_NV21TOYUV24ROW_NEON
ANY21(NV21ToYUV24Row_Any_NEON, NV21ToYUV24Row_NEON, 1, 1, 2, 3, 15)
#endif
+#ifdef HAS_NV21TOYUV24ROW_SSSE3
+ANY21(NV21ToYUV24Row_Any_SSSE3, NV21ToYUV24Row_SSSE3, 1, 1, 2, 3, 15)
+#endif
#ifdef HAS_NV21TOYUV24ROW_AVX2
ANY21(NV21ToYUV24Row_Any_AVX2, NV21ToYUV24Row_AVX2, 1, 1, 2, 3, 31)
#endif
@@ -320,78 +724,186 @@ ANY21(ARGBAddRow_Any_NEON, ARGBAddRow_NEON, 0, 4, 4, 4, 7)
#ifdef HAS_ARGBSUBTRACTROW_NEON
ANY21(ARGBSubtractRow_Any_NEON, ARGBSubtractRow_NEON, 0, 4, 4, 4, 7)
#endif
-#ifdef HAS_ARGBMULTIPLYROW_MSA
-ANY21(ARGBMultiplyRow_Any_MSA, ARGBMultiplyRow_MSA, 0, 4, 4, 4, 3)
+#ifdef HAS_ARGBBLENDROW_LSX
+ANY21(ARGBBlendRow_Any_LSX, ARGBBlendRow_LSX, 0, 4, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBBLENDROW_LASX
+ANY21(ARGBBlendRow_Any_LASX, ARGBBlendRow_LASX, 0, 4, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBMULTIPLYROW_LSX
+ANY21(ARGBMultiplyRow_Any_LSX, ARGBMultiplyRow_LSX, 0, 4, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBMULTIPLYROW_LASX
+ANY21(ARGBMultiplyRow_Any_LASX, ARGBMultiplyRow_LASX, 0, 4, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBADDROW_LSX
+ANY21(ARGBAddRow_Any_LSX, ARGBAddRow_LSX, 0, 4, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBADDROW_LASX
+ANY21(ARGBAddRow_Any_LASX, ARGBAddRow_LASX, 0, 4, 4, 4, 7)
#endif
-#ifdef HAS_ARGBMULTIPLYROW_MMI
-ANY21(ARGBMultiplyRow_Any_MMI, ARGBMultiplyRow_MMI, 0, 4, 4, 4, 1)
+#ifdef HAS_ARGBSUBTRACTROW_LSX
+ANY21(ARGBSubtractRow_Any_LSX, ARGBSubtractRow_LSX, 0, 4, 4, 4, 3)
#endif
-#ifdef HAS_ARGBADDROW_MSA
-ANY21(ARGBAddRow_Any_MSA, ARGBAddRow_MSA, 0, 4, 4, 4, 7)
+#ifdef HAS_ARGBSUBTRACTROW_LASX
+ANY21(ARGBSubtractRow_Any_LASX, ARGBSubtractRow_LASX, 0, 4, 4, 4, 7)
#endif
-#ifdef HAS_ARGBADDROW_MMI
-ANY21(ARGBAddRow_Any_MMI, ARGBAddRow_MMI, 0, 4, 4, 4, 1)
+#ifdef HAS_ARGBALPHABLENDROW_SSSE3
+ANY21(ARGBAlphaBlendRow_Any_SSSE3, ARGBAlphaBlendRow_SSSE3, 0, 4, 4, 4, 3)
#endif
-#ifdef HAS_ARGBSUBTRACTROW_MSA
-ANY21(ARGBSubtractRow_Any_MSA, ARGBSubtractRow_MSA, 0, 4, 4, 4, 7)
+#ifdef HAS_ARGBALPHABLENDROW_AVX2
+ANY21(ARGBAlphaBlendRow_Any_AVX2, ARGBAlphaBlendRow_AVX2, 0, 4, 4, 4, 7)
#endif
-#ifdef HAS_ARGBSUBTRACTROW_MMI
-ANY21(ARGBSubtractRow_Any_MMI, ARGBSubtractRow_MMI, 0, 4, 4, 4, 1)
+#ifdef HAS_ARGBALPHABLENDROW_NEON
+ANY21(ARGBAlphaBlendRow_Any_NEON, ARGBAlphaBlendRow_NEON, 0, 4, 4, 4, 7)
#endif
-#ifdef HAS_SOBELROW_SSE2
-ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15)
+#ifdef HAS_ARGBALPHABLENDROW_LSX
+ANY21(ARGBAlphaBlendRow_Any_LSX, ARGBAlphaBlendRow_LSX, 0, 4, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBALPHABLENDROW_LASX
+ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD
+ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3)
+#endif
+#ifdef HAS_SOBELROW_AVX2
+ANY21(SobelRow_Any_AVX2, SobelRow_AVX2, 0, 1, 1, 4, 15)
#endif
#ifdef HAS_SOBELROW_NEON
ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7)
#endif
-#ifdef HAS_SOBELROW_MSA
-ANY21(SobelRow_Any_MSA, SobelRow_MSA, 0, 1, 1, 4, 15)
+#ifdef HAS_SOBELROW_LSX
+ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 7)
#endif
-#ifdef HAS_SOBELROW_MMI
-ANY21(SobelRow_Any_MMI, SobelRow_MMI, 0, 1, 1, 4, 7)
+#ifdef HAS_SOBELROW_LASX
+ANY21(SobelRow_Any_LASX, SobelRow_LASX, 0, 1, 1, 4, 15)
#endif
-#ifdef HAS_SOBELTOPLANEROW_SSE2
-ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15)
+#ifdef HAS_SOBELTOPLANEROW_AVX2
+ANY21(SobelToPlaneRow_Any_AVX2, SobelToPlaneRow_AVX2, 0, 1, 1, 1, 31)
#endif
#ifdef HAS_SOBELTOPLANEROW_NEON
ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15)
#endif
-#ifdef HAS_SOBELTOPLANEROW_MSA
-ANY21(SobelToPlaneRow_Any_MSA, SobelToPlaneRow_MSA, 0, 1, 1, 1, 31)
+#ifdef HAS_SOBELTOPLANEROW_LSX
+ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 15)
#endif
-#ifdef HAS_SOBELTOPLANEROW_MMI
-ANY21(SobelToPlaneRow_Any_MMI, SobelToPlaneRow_MMI, 0, 1, 1, 1, 7)
+#ifdef HAS_SOBELTOPLANEROW_LASX
+ANY21(SobelToPlaneRow_Any_LASX, SobelToPlaneRow_LASX, 0, 1, 1, 1, 31)
#endif
-#ifdef HAS_SOBELXYROW_SSE2
-ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15)
+#ifdef HAS_SOBELXYROW_AVX2
+ANY21(SobelXYRow_Any_AVX2, SobelXYRow_AVX2, 0, 1, 1, 4, 15)
#endif
#ifdef HAS_SOBELXYROW_NEON
ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7)
#endif
-#ifdef HAS_SOBELXYROW_MSA
-ANY21(SobelXYRow_Any_MSA, SobelXYRow_MSA, 0, 1, 1, 4, 15)
+#ifdef HAS_SOBELXYROW_LSX
+ANY21(SobelXYRow_Any_LSX, SobelXYRow_LSX, 0, 1, 1, 4, 7)
+#endif
+#ifdef HAS_SOBELXYROW_LASX
+ANY21(SobelXYRow_Any_LASX, SobelXYRow_LASX, 0, 1, 1, 4, 15)
#endif
-#ifdef HAS_SOBELXYROW_MMI
-ANY21(SobelXYRow_Any_MMI, SobelXYRow_MMI, 0, 1, 1, 4, 7)
+#ifdef HAS_MERGEUVROW_WASMSIMD
+ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15)
#endif
#undef ANY21
+// Any 2 planes to 1 with stride
+// width is measured in source pixels. 4 bytes contains 2 pixels
+#define ANY21S(NAMEANY, ANY_SIMD, SBPP, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, \
+ int width) { \
+ SIMD_ALIGNED(uint8_t vin[(MASK + 1) * SBPP * 2]); \
+ SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int awidth = (width + 1) / 2; \
+ int r = awidth & MASK; \
+ int n = awidth & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_yuy2, stride_yuy2, dst_uv, n * 2); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_yuy2 + np * SBPP, r * SBPP); \
+ memcpy(vin + (MASK + 1) * SBPP, src_yuy2 + stride_yuy2 + np * SBPP, \
+ r * SBPP); \
+ ANY_SIMD(vin, (MASK + 1) * SBPP, vout, (MASK + 1) * 2); \
+ memcpy(dst_uv + np * BPP, vout, r * BPP); \
+ }
+
+#ifdef HAS_YUY2TONVUVROW_NEON
+ANY21S(YUY2ToNVUVRow_Any_NEON, YUY2ToNVUVRow_NEON, 4, 2, 7)
+#endif
+#ifdef HAS_YUY2TONVUVROW_SSE2
+ANY21S(YUY2ToNVUVRow_Any_SSE2, YUY2ToNVUVRow_SSE2, 4, 2, 7)
+#endif
+#ifdef HAS_YUY2TONVUVROW_AVX2
+ANY21S(YUY2ToNVUVRow_Any_AVX2, YUY2ToNVUVRow_AVX2, 4, 2, 15)
+#endif
+#undef ANY21S
+
+// Any 2 planes to 1 half-size merged UV row with stride (2 rows of U and V).
+#define ANY21HMS(NAMEANY, ANY_SIMD, MASK) \
+ void NAMEANY(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, \
+ int src_stride_v, uint8_t* dst_uv, int width) { \
+ SIMD_ALIGNED(uint8_t vin[64 * 4]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 4, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, 1) * 2 <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_u, src_stride_u, src_v, src_stride_v, dst_uv, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_u + np, r); \
+ memcpy(vin + 64, src_u + src_stride_u + np, r); \
+ memcpy(vin + 128, src_v + np, r); \
+ memcpy(vin + 192, src_v + src_stride_v + np, r); \
+ if (r & 1) { \
+ vin[r] = vin[r - 1]; \
+ vin[64 + r] = vin[64 + r - 1]; \
+ vin[128 + r] = vin[128 + r - 1]; \
+ vin[192 + r] = vin[192 + r - 1]; \
+ } \
+ ANY_SIMD(vin, 64, vin + 128, 64, vout, MASK + 1); \
+ memcpy(dst_uv + (np >> 1) * 2, vout, SS(r, 1) * 2); \
+ }
+
+#ifdef HAS_HALFMERGEUVROW_SSSE3
+ANY21HMS(HalfMergeUVRow_Any_SSSE3, HalfMergeUVRow_SSSE3, 15)
+#endif
+#ifdef HAS_HALFMERGEUVROW_AVX2
+ANY21HMS(HalfMergeUVRow_Any_AVX2, HalfMergeUVRow_AVX2, 31)
+#endif
+#undef ANY21HMS
+
// Any 2 planes to 1 with yuvconstants
#define ANY21C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \
void NAMEANY(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, \
const struct YuvConstants* yuvconstants, int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 3]); \
- memset(temp, 0, 128 * 2); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[128 * 2]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP2 <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[128]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(y_buf, uv_buf, dst_ptr, yuvconstants, n); \
} \
- memcpy(temp, y_buf + n * SBPP, r * SBPP); \
- memcpy(temp + 128, uv_buf + (n >> UVSHIFT) * SBPP2, \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np * SBPP, r * SBPP); \
+ memcpy(vin + 128, uv_buf + (np >> UVSHIFT) * SBPP2, \
SS(r, UVSHIFT) * SBPP2); \
- ANY_SIMD(temp, temp + 128, temp + 256, yuvconstants, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 256, r * BPP); \
+ ANY_SIMD(vin, vin + 128, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
// Biplanar to RGB.
@@ -404,8 +916,11 @@ ANY21C(NV12ToARGBRow_Any_AVX2, NV12ToARGBRow_AVX2, 1, 1, 2, 4, 15)
#ifdef HAS_NV12TOARGBROW_NEON
ANY21C(NV12ToARGBRow_Any_NEON, NV12ToARGBRow_NEON, 1, 1, 2, 4, 7)
#endif
-#ifdef HAS_NV12TOARGBROW_MSA
-ANY21C(NV12ToARGBRow_Any_MSA, NV12ToARGBRow_MSA, 1, 1, 2, 4, 7)
+#ifdef HAS_NV12TOARGBROW_LSX
+ANY21C(NV12ToARGBRow_Any_LSX, NV12ToARGBRow_LSX, 1, 1, 2, 4, 7)
+#endif
+#ifdef HAS_NV12TOARGBROW_LASX
+ANY21C(NV12ToARGBRow_Any_LASX, NV12ToARGBRow_LASX, 1, 1, 2, 4, 15)
#endif
#ifdef HAS_NV21TOARGBROW_SSSE3
ANY21C(NV21ToARGBRow_Any_SSSE3, NV21ToARGBRow_SSSE3, 1, 1, 2, 4, 7)
@@ -416,8 +931,11 @@ ANY21C(NV21ToARGBRow_Any_AVX2, NV21ToARGBRow_AVX2, 1, 1, 2, 4, 15)
#ifdef HAS_NV21TOARGBROW_NEON
ANY21C(NV21ToARGBRow_Any_NEON, NV21ToARGBRow_NEON, 1, 1, 2, 4, 7)
#endif
-#ifdef HAS_NV21TOARGBROW_MSA
-ANY21C(NV21ToARGBRow_Any_MSA, NV21ToARGBRow_MSA, 1, 1, 2, 4, 7)
+#ifdef HAS_NV21TOARGBROW_LSX
+ANY21C(NV21ToARGBRow_Any_LSX, NV21ToARGBRow_LSX, 1, 1, 2, 4, 7)
+#endif
+#ifdef HAS_NV21TOARGBROW_LASX
+ANY21C(NV21ToARGBRow_Any_LASX, NV21ToARGBRow_LASX, 1, 1, 2, 4, 15)
#endif
#ifdef HAS_NV12TORGB24ROW_NEON
ANY21C(NV12ToRGB24Row_Any_NEON, NV12ToRGB24Row_NEON, 1, 1, 2, 3, 7)
@@ -446,24 +964,138 @@ ANY21C(NV12ToRGB565Row_Any_AVX2, NV12ToRGB565Row_AVX2, 1, 1, 2, 2, 15)
#ifdef HAS_NV12TORGB565ROW_NEON
ANY21C(NV12ToRGB565Row_Any_NEON, NV12ToRGB565Row_NEON, 1, 1, 2, 2, 7)
#endif
-#ifdef HAS_NV12TORGB565ROW_MSA
-ANY21C(NV12ToRGB565Row_Any_MSA, NV12ToRGB565Row_MSA, 1, 1, 2, 2, 7)
+#ifdef HAS_NV12TORGB565ROW_LSX
+ANY21C(NV12ToRGB565Row_Any_LSX, NV12ToRGB565Row_LSX, 1, 1, 2, 2, 7)
+#endif
+#ifdef HAS_NV12TORGB565ROW_LASX
+ANY21C(NV12ToRGB565Row_Any_LASX, NV12ToRGB565Row_LASX, 1, 1, 2, 2, 15)
+#endif
+#ifdef HAS_NV12TOARGBROW_WASMSIMD
+ANY21C(NV12ToARGBRow_Any_WASMSIMD, NV12ToARGBRow_WASMSIMD, 1, 1, 2, 4, 7)
+#endif
+#ifdef HAS_NV21TOARGBROW_WASMSIMD
+ANY21C(NV21ToARGBRow_Any_WASMSIMD, NV21ToARGBRow_WASMSIMD, 1, 1, 2, 4, 7)
+#endif
+#ifdef HAS_NV12TORGB565ROW_WASMSIMD
+ANY21C(NV12ToRGB565Row_Any_WASMSIMD, NV12ToRGB565Row_WASMSIMD, 1, 1, 2, 2, 7)
#endif
#undef ANY21C
+// Any 2 planes of 16 bit to 1 with yuvconstants
+#define ANY21CT(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, T, SBPP, BPP, MASK) \
+ void NAMEANY(const T* y_buf, const T* uv_buf, uint8_t* dst_ptr, \
+ const struct YuvConstants* yuvconstants, int width) { \
+ SIMD_ALIGNED(T vin[32 * 2]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP * 2 <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(y_buf, uv_buf, dst_ptr, yuvconstants, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, y_buf + np, r * SBPP); \
+ memcpy(vin + 32, uv_buf + 2 * (np >> UVSHIFT), SS(r, UVSHIFT) * SBPP * 2); \
+ ANY_SIMD(vin, vin + 32, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + (np >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
+ }
+
+#ifdef HAS_P210TOAR30ROW_SSSE3
+ANY21CT(P210ToAR30Row_Any_SSSE3, P210ToAR30Row_SSSE3, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P210TOARGBROW_SSSE3
+ANY21CT(P210ToARGBRow_Any_SSSE3, P210ToARGBRow_SSSE3, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P210TOARGBROW_AVX2
+ANY21CT(P210ToARGBRow_Any_AVX2, P210ToARGBRow_AVX2, 1, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_P210TOAR30ROW_AVX2
+ANY21CT(P210ToAR30Row_Any_AVX2, P210ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_P210TOAR30ROW_NEON
+ANY21CT(P210ToAR30Row_Any_NEON, P210ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P210TOARGBROW_NEON
+ANY21CT(P210ToARGBRow_Any_NEON, P210ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P410TOAR30ROW_SSSE3
+ANY21CT(P410ToAR30Row_Any_SSSE3, P410ToAR30Row_SSSE3, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P410TOARGBROW_SSSE3
+ANY21CT(P410ToARGBRow_Any_SSSE3, P410ToARGBRow_SSSE3, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P410TOARGBROW_AVX2
+ANY21CT(P410ToARGBRow_Any_AVX2, P410ToARGBRow_AVX2, 0, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_P410TOAR30ROW_AVX2
+ANY21CT(P410ToAR30Row_Any_AVX2, P410ToAR30Row_AVX2, 0, 0, uint16_t, 2, 4, 15)
+#endif
+#ifdef HAS_P410TOAR30ROW_NEON
+ANY21CT(P410ToAR30Row_Any_NEON, P410ToAR30Row_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P410TOARGBROW_NEON
+ANY21CT(P410ToARGBRow_Any_NEON, P410ToARGBRow_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+
+#undef ANY21CT
+
+// Any 2 16 bit planes with parameter to 1
+#define ANY21PT(NAMEANY, ANY_SIMD, T, BPP, MASK) \
+ void NAMEANY(const T* src_u, const T* src_v, T* dst_uv, int depth, \
+ int width) { \
+ SIMD_ALIGNED(T vin[16 * 2]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[16]); \
+ static_assert((MASK + 1) * BPP * 2 <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_u, src_v, dst_uv, depth, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_u + np, r * BPP); \
+ memcpy(vin + 16, src_v + np, r * BPP); \
+ ANY_SIMD(vin, vin + 16, vout, depth, MASK + 1); \
+ memcpy(dst_uv + np * 2, vout, r * BPP * 2); \
+ }
+
+#ifdef HAS_MERGEUVROW_16_AVX2
+ANY21PT(MergeUVRow_16_Any_AVX2, MergeUVRow_16_AVX2, uint16_t, 2, 7)
+#endif
+#ifdef HAS_MERGEUVROW_16_NEON
+ANY21PT(MergeUVRow_16_Any_NEON, MergeUVRow_16_NEON, uint16_t, 2, 7)
+#endif
+
+#undef ANY21CT
+
// Any 1 to 1.
-#define ANY11(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 2]); \
- memset(temp, 0, 128); /* for YUY2 and msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(src_ptr, dst_ptr, n); \
- } \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
- ANY_SIMD(temp, temp + 128, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \
+#define ANY11(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
+ SIMD_ALIGNED(uint8_t vin[256]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
+ ANY_SIMD(vin, vout, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_COPYROW_AVX
@@ -475,32 +1107,59 @@ ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31)
#ifdef HAS_COPYROW_NEON
ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31)
#endif
+#ifdef HAS_COPYROW_LSX
+ANY11(CopyRow_Any_LSX, CopyRow_LSX, 0, 1, 1, 15)
+#endif
+#ifdef HAS_COPYROW_LASX
+ANY11(CopyRow_Any_LASX, CopyRow_LASX, 0, 1, 1, 31)
+#endif
#if defined(HAS_ARGBTORGB24ROW_SSSE3)
ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15)
ANY11(ARGBToRAWRow_Any_SSSE3, ARGBToRAWRow_SSSE3, 0, 4, 3, 15)
-ANY11(ARGBToRGB565Row_Any_SSE2, ARGBToRGB565Row_SSE2, 0, 4, 2, 3)
-ANY11(ARGBToARGB1555Row_Any_SSE2, ARGBToARGB1555Row_SSE2, 0, 4, 2, 3)
-ANY11(ARGBToARGB4444Row_Any_SSE2, ARGBToARGB4444Row_SSE2, 0, 4, 2, 3)
+
#endif
#if defined(HAS_ARGBTORGB24ROW_AVX2)
ANY11(ARGBToRGB24Row_Any_AVX2, ARGBToRGB24Row_AVX2, 0, 4, 3, 31)
#endif
-#if defined(HAS_ARGBTORGB24ROW_AVX512VBMI)
-ANY11(ARGBToRGB24Row_Any_AVX512VBMI, ARGBToRGB24Row_AVX512VBMI, 0, 4, 3, 31)
-#endif
#if defined(HAS_ARGBTORAWROW_AVX2)
ANY11(ARGBToRAWRow_Any_AVX2, ARGBToRAWRow_AVX2, 0, 4, 3, 31)
#endif
#if defined(HAS_ARGBTORGB565ROW_AVX2)
ANY11(ARGBToRGB565Row_Any_AVX2, ARGBToRGB565Row_AVX2, 0, 4, 2, 7)
#endif
-#if defined(HAS_ARGBTOARGB4444ROW_AVX2)
+#if defined(HAS_ARGBTOARGB1555ROW_AVX2)
ANY11(ARGBToARGB1555Row_Any_AVX2, ARGBToARGB1555Row_AVX2, 0, 4, 2, 7)
+#endif
+#if defined(HAS_ARGBTOARGB4444ROW_AVX2)
ANY11(ARGBToARGB4444Row_Any_AVX2, ARGBToARGB4444Row_AVX2, 0, 4, 2, 7)
#endif
#if defined(HAS_ABGRTOAR30ROW_SSSE3)
ANY11(ABGRToAR30Row_Any_SSSE3, ABGRToAR30Row_SSSE3, 0, 4, 4, 3)
#endif
+#if defined(HAS_ABGRTOAR30ROW_NEON)
+ANY11(ABGRToAR30Row_Any_NEON, ABGRToAR30Row_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_ARGBTOAR30ROW_NEON)
+ANY11(ARGBToAR30Row_Any_NEON, ARGBToAR30Row_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOARGBROW_NEON)
+ANY11(AR30ToARGBRow_Any_NEON, AR30ToARGBRow_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOABGRROW_NEON)
+ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOAB30ROW_NEON)
+ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOARGBROW_AVX2)
+ANY11(AR30ToARGBRow_Any_AVX2, AR30ToARGBRow_AVX2, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOABGRROW_AVX2)
+ANY11(AR30ToABGRRow_Any_AVX2, AR30ToABGRRow_AVX2, 0, 4, 4, 7)
+#endif
+#if defined(HAS_AR30TOAB30ROW_AVX2)
+ANY11(AR30ToAB30Row_Any_AVX2, AR30ToAB30Row_AVX2, 0, 4, 4, 7)
+#endif
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3)
#endif
@@ -510,28 +1169,30 @@ ANY11(ABGRToAR30Row_Any_AVX2, ABGRToAR30Row_AVX2, 0, 4, 4, 7)
#if defined(HAS_ARGBTOAR30ROW_AVX2)
ANY11(ARGBToAR30Row_Any_AVX2, ARGBToAR30Row_AVX2, 0, 4, 4, 7)
#endif
-#if defined(HAS_J400TOARGBROW_SSE2)
-ANY11(J400ToARGBRow_Any_SSE2, J400ToARGBRow_SSE2, 0, 1, 4, 7)
-#endif
+
#if defined(HAS_J400TOARGBROW_AVX2)
ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15)
#endif
-#if defined(HAS_I400TOARGBROW_SSE2)
-ANY11(I400ToARGBRow_Any_SSE2, I400ToARGBRow_SSE2, 0, 1, 4, 7)
-#endif
-#if defined(HAS_I400TOARGBROW_AVX2)
-ANY11(I400ToARGBRow_Any_AVX2, I400ToARGBRow_AVX2, 0, 1, 4, 15)
-#endif
#if defined(HAS_RGB24TOARGBROW_SSSE3)
ANY11(RGB24ToARGBRow_Any_SSSE3, RGB24ToARGBRow_SSSE3, 0, 3, 4, 15)
ANY11(RAWToARGBRow_Any_SSSE3, RAWToARGBRow_SSSE3, 0, 3, 4, 15)
-ANY11(RGB565ToARGBRow_Any_SSE2, RGB565ToARGBRow_SSE2, 0, 2, 4, 7)
-ANY11(ARGB1555ToARGBRow_Any_SSE2, ARGB1555ToARGBRow_SSE2, 0, 2, 4, 7)
-ANY11(ARGB4444ToARGBRow_Any_SSE2, ARGB4444ToARGBRow_SSE2, 0, 2, 4, 7)
+
+#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31)
+#endif
+#if defined(HAS_RGB24TOARGBROW_AVX2)
+ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31)
+#endif
+#if defined(HAS_RAWTORGBAROW_AVX2)
+ANY11(RAWToRGBARow_Any_AVX2, RAWToRGBARow_AVX2, 0, 3, 4, 15)
#endif
#if defined(HAS_RAWTORGB24ROW_SSSE3)
ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7)
#endif
+#if defined(HAS_RAWTORGB24ROW_AVX2)
+ANY11(RAWToRGB24Row_Any_AVX2, RAWToRGB24Row_AVX2, 0, 3, 3, 31)
+#endif
#if defined(HAS_RGB565TOARGBROW_AVX2)
ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15)
#endif
@@ -542,155 +1203,94 @@ ANY11(ARGB1555ToARGBRow_Any_AVX2, ARGB1555ToARGBRow_AVX2, 0, 2, 4, 15)
ANY11(ARGB4444ToARGBRow_Any_AVX2, ARGB4444ToARGBRow_AVX2, 0, 2, 4, 15)
#endif
#if defined(HAS_ARGBTORGB24ROW_NEON)
-ANY11(ARGBToRGB24Row_Any_NEON, ARGBToRGB24Row_NEON, 0, 4, 3, 7)
+ANY11(ARGBToRGB24Row_Any_NEON, ARGBToRGB24Row_NEON, 0, 4, 3, 15)
ANY11(ARGBToRAWRow_Any_NEON, ARGBToRAWRow_NEON, 0, 4, 3, 7)
ANY11(ARGBToRGB565Row_Any_NEON, ARGBToRGB565Row_NEON, 0, 4, 2, 7)
ANY11(ARGBToARGB1555Row_Any_NEON, ARGBToARGB1555Row_NEON, 0, 4, 2, 7)
ANY11(ARGBToARGB4444Row_Any_NEON, ARGBToARGB4444Row_NEON, 0, 4, 2, 7)
ANY11(J400ToARGBRow_Any_NEON, J400ToARGBRow_NEON, 0, 1, 4, 7)
-ANY11(I400ToARGBRow_Any_NEON, I400ToARGBRow_NEON, 0, 1, 4, 7)
-#endif
-#if defined(HAS_ARGBTORGB24ROW_MSA)
-ANY11(ARGBToRGB24Row_Any_MSA, ARGBToRGB24Row_MSA, 0, 4, 3, 15)
-ANY11(ARGBToRAWRow_Any_MSA, ARGBToRAWRow_MSA, 0, 4, 3, 15)
-ANY11(ARGBToRGB565Row_Any_MSA, ARGBToRGB565Row_MSA, 0, 4, 2, 7)
-ANY11(ARGBToARGB1555Row_Any_MSA, ARGBToARGB1555Row_MSA, 0, 4, 2, 7)
-ANY11(ARGBToARGB4444Row_Any_MSA, ARGBToARGB4444Row_MSA, 0, 4, 2, 7)
-ANY11(J400ToARGBRow_Any_MSA, J400ToARGBRow_MSA, 0, 1, 4, 15)
-ANY11(I400ToARGBRow_Any_MSA, I400ToARGBRow_MSA, 0, 1, 4, 15)
-#endif
-#if defined(HAS_ARGBTORGB24ROW_MMI)
-ANY11(ARGBToRGB24Row_Any_MMI, ARGBToRGB24Row_MMI, 0, 4, 3, 3)
-ANY11(ARGBToRAWRow_Any_MMI, ARGBToRAWRow_MMI, 0, 4, 3, 3)
-ANY11(ARGBToRGB565Row_Any_MMI, ARGBToRGB565Row_MMI, 0, 4, 2, 3)
-ANY11(ARGBToARGB1555Row_Any_MMI, ARGBToARGB1555Row_MMI, 0, 4, 2, 3)
-ANY11(ARGBToARGB4444Row_Any_MMI, ARGBToARGB4444Row_MMI, 0, 4, 2, 3)
-ANY11(J400ToARGBRow_Any_MMI, J400ToARGBRow_MMI, 0, 1, 4, 3)
-ANY11(I400ToARGBRow_Any_MMI, I400ToARGBRow_MMI, 0, 1, 4, 7)
#endif
-#if defined(HAS_RAWTORGB24ROW_NEON)
-ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 7)
+#if defined(HAS_ARGBTORGB24ROW_LSX)
+ANY11(ARGBToRGB24Row_Any_LSX, ARGBToRGB24Row_LSX, 0, 4, 3, 15)
+ANY11(ARGBToRAWRow_Any_LSX, ARGBToRAWRow_LSX, 0, 4, 3, 15)
+ANY11(ARGBToRGB565Row_Any_LSX, ARGBToRGB565Row_LSX, 0, 4, 2, 7)
+ANY11(ARGBToARGB1555Row_Any_LSX, ARGBToARGB1555Row_LSX, 0, 4, 2, 7)
+ANY11(ARGBToARGB4444Row_Any_LSX, ARGBToARGB4444Row_LSX, 0, 4, 2, 7)
+#endif
+#if defined(HAS_ARGBTORGB24ROW_LASX)
+ANY11(ARGBToRGB24Row_Any_LASX, ARGBToRGB24Row_LASX, 0, 4, 3, 31)
+ANY11(ARGBToRAWRow_Any_LASX, ARGBToRAWRow_LASX, 0, 4, 3, 31)
+ANY11(ARGBToRGB565Row_Any_LASX, ARGBToRGB565Row_LASX, 0, 4, 2, 15)
+ANY11(ARGBToARGB1555Row_Any_LASX, ARGBToARGB1555Row_LASX, 0, 4, 2, 15)
+ANY11(ARGBToARGB4444Row_Any_LASX, ARGBToARGB4444Row_LASX, 0, 4, 2, 15)
+#endif
+#if defined(HAS_J400TOARGBROW_LSX)
+ANY11(J400ToARGBRow_Any_LSX, J400ToARGBRow_LSX, 0, 1, 4, 7)
#endif
-#if defined(HAS_RAWTORGB24ROW_MSA)
-ANY11(RAWToRGB24Row_Any_MSA, RAWToRGB24Row_MSA, 0, 3, 3, 15)
+#if defined(HAS_J400TOARGBROW_LASX)
+ANY11(J400ToARGBRow_Any_LASX, J400ToARGBRow_LASX, 0, 1, 4, 15)
#endif
-#if defined(HAS_RAWTORGB24ROW_MMI)
-ANY11(RAWToRGB24Row_Any_MMI, RAWToRGB24Row_MMI, 0, 3, 3, 3)
+#if defined(HAS_RAWTORGB24ROW_NEON)
+ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 15)
#endif
-#ifdef HAS_ARGBTOYROW_AVX2
-ANY11(ARGBToYRow_Any_AVX2, ARGBToYRow_AVX2, 0, 4, 1, 31)
+#if defined(HAS_RAWTORGB24ROW_LSX)
+ANY11(RAWToRGB24Row_Any_LSX, RAWToRGB24Row_LSX, 0, 3, 3, 15)
#endif
-#ifdef HAS_ABGRTOYROW_AVX2
-ANY11(ABGRToYRow_Any_AVX2, ABGRToYRow_AVX2, 0, 4, 1, 31)
+#if defined(HAS_RAWTORGB24ROW_LASX)
+ANY11(RAWToRGB24Row_Any_LASX, RAWToRGB24Row_LASX, 0, 3, 3, 31)
#endif
-#ifdef HAS_ARGBTOYJROW_AVX2
-ANY11(ARGBToYJRow_Any_AVX2, ARGBToYJRow_AVX2, 0, 4, 1, 31)
+#if defined(HAS_RAWTORGB24ROW_WASMSIMD)
+ANY11(RAWToRGB24Row_Any_WASMSIMD, RAWToRGB24Row_WASMSIMD, 0, 3, 3, 15)
#endif
+
#ifdef HAS_UYVYTOYROW_AVX2
ANY11(UYVYToYRow_Any_AVX2, UYVYToYRow_AVX2, 0, 2, 1, 31)
#endif
#ifdef HAS_YUY2TOYROW_AVX2
ANY11(YUY2ToYRow_Any_AVX2, YUY2ToYRow_AVX2, 1, 4, 1, 31)
#endif
-#ifdef HAS_ARGBTOYROW_SSSE3
-ANY11(ARGBToYRow_Any_SSSE3, ARGBToYRow_SSSE3, 0, 4, 1, 15)
-#endif
-#ifdef HAS_BGRATOYROW_SSSE3
-ANY11(BGRAToYRow_Any_SSSE3, BGRAToYRow_SSSE3, 0, 4, 1, 15)
-ANY11(ABGRToYRow_Any_SSSE3, ABGRToYRow_SSSE3, 0, 4, 1, 15)
-ANY11(RGBAToYRow_Any_SSSE3, RGBAToYRow_SSSE3, 0, 4, 1, 15)
+#ifdef HAS_YUY2TOYROW_SSE2
ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15)
ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15)
#endif
-#ifdef HAS_ARGBTOYJROW_SSSE3
-ANY11(ARGBToYJRow_Any_SSSE3, ARGBToYJRow_SSSE3, 0, 4, 1, 15)
-#endif
-#ifdef HAS_ARGBTOYROW_NEON
-ANY11(ARGBToYRow_Any_NEON, ARGBToYRow_NEON, 0, 4, 1, 7)
+
+#ifdef HAS_RGB565TOYROW_AVX2
+ANY11(RGB565ToYRow_Any_AVX2, RGB565ToYRow_AVX2, 0, 2, 1, 31)
#endif
-#ifdef HAS_ARGBTOYROW_MSA
-ANY11(ARGBToYRow_Any_MSA, ARGBToYRow_MSA, 0, 4, 1, 15)
+#ifdef HAS_ARGB1555TOYROW_AVX2
+ANY11(ARGB1555ToYRow_Any_AVX2, ARGB1555ToYRow_AVX2, 0, 2, 1, 31)
#endif
-#ifdef HAS_ARGBTOYROW_MMI
-ANY11(ARGBToYRow_Any_MMI, ARGBToYRow_MMI, 0, 4, 1, 7)
+#ifdef HAS_ARGB4444TOYROW_AVX2
+ANY11(ARGB4444ToYRow_Any_AVX2, ARGB4444ToYRow_AVX2, 0, 2, 1, 31)
#endif
-#ifdef HAS_ARGBTOYJROW_NEON
-ANY11(ARGBToYJRow_Any_NEON, ARGBToYJRow_NEON, 0, 4, 1, 7)
+#ifdef HAS_RGB565TOYROW_NEON
+ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15)
#endif
-#ifdef HAS_ARGBTOYJROW_MSA
-ANY11(ARGBToYJRow_Any_MSA, ARGBToYJRow_MSA, 0, 4, 1, 15)
+#ifdef HAS_RGB565TOYROW_LSX
+ANY11(RGB565ToYRow_Any_LSX, RGB565ToYRow_LSX, 0, 2, 1, 7)
#endif
-#ifdef HAS_ARGBTOYJROW_MMI
-ANY11(ARGBToYJRow_Any_MMI, ARGBToYJRow_MMI, 0, 4, 1, 7)
+#ifdef HAS_RGB565TOYROW_LASX
+ANY11(RGB565ToYRow_Any_LASX, RGB565ToYRow_LASX, 0, 2, 1, 15)
#endif
-#ifdef HAS_BGRATOYROW_NEON
-ANY11(BGRAToYRow_Any_NEON, BGRAToYRow_NEON, 0, 4, 1, 7)
+#ifdef HAS_ARGB1555TOYROW_NEON
+#ifdef __aarch64__
+ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 15)
+#else
+ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7)
#endif
-#ifdef HAS_BGRATOYROW_MSA
-ANY11(BGRAToYRow_Any_MSA, BGRAToYRow_MSA, 0, 4, 1, 15)
#endif
-#ifdef HAS_BGRATOYROW_MMI
-ANY11(BGRAToYRow_Any_MMI, BGRAToYRow_MMI, 0, 4, 1, 7)
+#ifdef HAS_ARGB1555TOYROW_LSX
+ANY11(ARGB1555ToYRow_Any_LSX, ARGB1555ToYRow_LSX, 0, 2, 1, 7)
#endif
-#ifdef HAS_ABGRTOYROW_NEON
-ANY11(ABGRToYRow_Any_NEON, ABGRToYRow_NEON, 0, 4, 1, 7)
-#endif
-#ifdef HAS_ABGRTOYROW_MSA
-ANY11(ABGRToYRow_Any_MSA, ABGRToYRow_MSA, 0, 4, 1, 7)
-#endif
-#ifdef HAS_ABGRTOYROW_MMI
-ANY11(ABGRToYRow_Any_MMI, ABGRToYRow_MMI, 0, 4, 1, 7)
-#endif
-#ifdef HAS_RGBATOYROW_NEON
-ANY11(RGBAToYRow_Any_NEON, RGBAToYRow_NEON, 0, 4, 1, 7)
-#endif
-#ifdef HAS_RGBATOYROW_MSA
-ANY11(RGBAToYRow_Any_MSA, RGBAToYRow_MSA, 0, 4, 1, 15)
-#endif
-#ifdef HAS_RGBATOYROW_MMI
-ANY11(RGBAToYRow_Any_MMI, RGBAToYRow_MMI, 0, 4, 1, 7)
-#endif
-#ifdef HAS_RGB24TOYROW_NEON
-ANY11(RGB24ToYRow_Any_NEON, RGB24ToYRow_NEON, 0, 3, 1, 7)
-#endif
-#ifdef HAS_RGB24TOYROW_MSA
-ANY11(RGB24ToYRow_Any_MSA, RGB24ToYRow_MSA, 0, 3, 1, 15)
-#endif
-#ifdef HAS_RGB24TOYROW_MMI
-ANY11(RGB24ToYRow_Any_MMI, RGB24ToYRow_MMI, 0, 3, 1, 7)
-#endif
-#ifdef HAS_RAWTOYROW_NEON
-ANY11(RAWToYRow_Any_NEON, RAWToYRow_NEON, 0, 3, 1, 7)
-#endif
-#ifdef HAS_RAWTOYROW_MSA
-ANY11(RAWToYRow_Any_MSA, RAWToYRow_MSA, 0, 3, 1, 15)
-#endif
-#ifdef HAS_RAWTOYROW_MMI
-ANY11(RAWToYRow_Any_MMI, RAWToYRow_MMI, 0, 3, 1, 7)
-#endif
-#ifdef HAS_RGB565TOYROW_NEON
-ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 7)
-#endif
-#ifdef HAS_RGB565TOYROW_MSA
-ANY11(RGB565ToYRow_Any_MSA, RGB565ToYRow_MSA, 0, 2, 1, 15)
-#endif
-#ifdef HAS_RGB565TOYROW_MMI
-ANY11(RGB565ToYRow_Any_MMI, RGB565ToYRow_MMI, 0, 2, 1, 7)
-#endif
-#ifdef HAS_ARGB1555TOYROW_NEON
-ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7)
-#endif
-#ifdef HAS_ARGB1555TOYROW_MSA
-ANY11(ARGB1555ToYRow_Any_MSA, ARGB1555ToYRow_MSA, 0, 2, 1, 15)
-#endif
-#ifdef HAS_ARGB1555TOYROW_MMI
-ANY11(ARGB1555ToYRow_Any_MMI, ARGB1555ToYRow_MMI, 0, 2, 1, 7)
+#ifdef HAS_ARGB1555TOYROW_LASX
+ANY11(ARGB1555ToYRow_Any_LASX, ARGB1555ToYRow_LASX, 0, 2, 1, 15)
#endif
#ifdef HAS_ARGB4444TOYROW_NEON
+#ifdef __aarch64__
+ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 15)
+#else
ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 7)
#endif
-#ifdef HAS_ARGB4444TOYROW_MMI
-ANY11(ARGB4444ToYRow_Any_MMI, ARGB4444ToYRow_MMI, 0, 2, 1, 7)
#endif
#ifdef HAS_YUY2TOYROW_NEON
ANY11(YUY2ToYRow_Any_NEON, YUY2ToYRow_NEON, 1, 4, 1, 15)
@@ -698,17 +1298,20 @@ ANY11(YUY2ToYRow_Any_NEON, YUY2ToYRow_NEON, 1, 4, 1, 15)
#ifdef HAS_UYVYTOYROW_NEON
ANY11(UYVYToYRow_Any_NEON, UYVYToYRow_NEON, 1, 4, 1, 15)
#endif
-#ifdef HAS_YUY2TOYROW_MSA
-ANY11(YUY2ToYRow_Any_MSA, YUY2ToYRow_MSA, 1, 4, 1, 31)
+#ifdef HAS_YUY2TOYROW_LSX
+ANY11(YUY2ToYRow_Any_LSX, YUY2ToYRow_LSX, 1, 4, 1, 15)
+#endif
+#ifdef HAS_YUY2TOYROW_LASX
+ANY11(YUY2ToYRow_Any_LASX, YUY2ToYRow_LASX, 1, 4, 1, 31)
#endif
-#ifdef HAS_YUY2TOYROW_MMI
-ANY11(YUY2ToYRow_Any_MMI, YUY2ToYRow_MMI, 1, 4, 1, 7)
+#ifdef HAS_UYVYTOYROW_LSX
+ANY11(UYVYToYRow_Any_LSX, UYVYToYRow_LSX, 1, 4, 1, 15)
#endif
-#ifdef HAS_UYVYTOYROW_MSA
-ANY11(UYVYToYRow_Any_MSA, UYVYToYRow_MSA, 1, 4, 1, 31)
+#ifdef HAS_UYVYTOYROW_LASX
+ANY11(UYVYToYRow_Any_LASX, UYVYToYRow_LASX, 1, 4, 1, 31)
#endif
-#ifdef HAS_UYVYTOYROW_MMI
-ANY11(UYVYToYRow_Any_MMI, UYVYToYRow_MMI, 1, 4, 1, 15)
+#ifdef HAS_AYUVTOYROW_AVX2
+ANY11(AYUVToYRow_Any_AVX2, AYUVToYRow_AVX2, 0, 4, 1, 15)
#endif
#ifdef HAS_AYUVTOYROW_NEON
ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15)
@@ -725,47 +1328,50 @@ ANY11(SwapUVRow_Any_NEON, SwapUVRow_NEON, 0, 2, 2, 15)
#ifdef HAS_RGB24TOARGBROW_NEON
ANY11(RGB24ToARGBRow_Any_NEON, RGB24ToARGBRow_NEON, 0, 3, 4, 7)
#endif
-#ifdef HAS_RGB24TOARGBROW_MSA
-ANY11(RGB24ToARGBRow_Any_MSA, RGB24ToARGBRow_MSA, 0, 3, 4, 15)
+#ifdef HAS_RGB24TOARGBROW_LSX
+ANY11(RGB24ToARGBRow_Any_LSX, RGB24ToARGBRow_LSX, 0, 3, 4, 15)
#endif
-#ifdef HAS_RGB24TOARGBROW_MMI
-ANY11(RGB24ToARGBRow_Any_MMI, RGB24ToARGBRow_MMI, 0, 3, 4, 3)
+#ifdef HAS_RGB24TOARGBROW_LASX
+ANY11(RGB24ToARGBRow_Any_LASX, RGB24ToARGBRow_LASX, 0, 3, 4, 31)
#endif
#ifdef HAS_RAWTOARGBROW_NEON
ANY11(RAWToARGBRow_Any_NEON, RAWToARGBRow_NEON, 0, 3, 4, 7)
#endif
-#ifdef HAS_RAWTOARGBROW_MSA
-ANY11(RAWToARGBRow_Any_MSA, RAWToARGBRow_MSA, 0, 3, 4, 15)
+#ifdef HAS_RAWTORGBAROW_NEON
+ANY11(RAWToRGBARow_Any_NEON, RAWToRGBARow_NEON, 0, 3, 4, 7)
#endif
-#ifdef HAS_RAWTOARGBROW_MMI
-ANY11(RAWToARGBRow_Any_MMI, RAWToARGBRow_MMI, 0, 3, 4, 3)
+#ifdef HAS_RAWTOARGBROW_LSX
+ANY11(RAWToARGBRow_Any_LSX, RAWToARGBRow_LSX, 0, 3, 4, 15)
+#endif
+#ifdef HAS_RAWTOARGBROW_LASX
+ANY11(RAWToARGBRow_Any_LASX, RAWToARGBRow_LASX, 0, 3, 4, 31)
#endif
#ifdef HAS_RGB565TOARGBROW_NEON
-ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 7)
+ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 15)
#endif
-#ifdef HAS_RGB565TOARGBROW_MSA
-ANY11(RGB565ToARGBRow_Any_MSA, RGB565ToARGBRow_MSA, 0, 2, 4, 15)
+#ifdef HAS_RGB565TOARGBROW_LSX
+ANY11(RGB565ToARGBRow_Any_LSX, RGB565ToARGBRow_LSX, 0, 2, 4, 7)
#endif
-#ifdef HAS_RGB565TOARGBROW_MMI
-ANY11(RGB565ToARGBRow_Any_MMI, RGB565ToARGBRow_MMI, 0, 2, 4, 3)
+#ifdef HAS_RGB565TOARGBROW_LASX
+ANY11(RGB565ToARGBRow_Any_LASX, RGB565ToARGBRow_LASX, 0, 2, 4, 15)
#endif
#ifdef HAS_ARGB1555TOARGBROW_NEON
-ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 7)
+ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 15)
#endif
-#ifdef HAS_ARGB1555TOARGBROW_MSA
-ANY11(ARGB1555ToARGBRow_Any_MSA, ARGB1555ToARGBRow_MSA, 0, 2, 4, 15)
+#ifdef HAS_ARGB1555TOARGBROW_LSX
+ANY11(ARGB1555ToARGBRow_Any_LSX, ARGB1555ToARGBRow_LSX, 0, 2, 4, 7)
#endif
-#ifdef HAS_ARGB1555TOARGBROW_MMI
-ANY11(ARGB1555ToARGBRow_Any_MMI, ARGB1555ToARGBRow_MMI, 0, 2, 4, 3)
+#ifdef HAS_ARGB1555TOARGBROW_LASX
+ANY11(ARGB1555ToARGBRow_Any_LASX, ARGB1555ToARGBRow_LASX, 0, 2, 4, 15)
#endif
#ifdef HAS_ARGB4444TOARGBROW_NEON
ANY11(ARGB4444ToARGBRow_Any_NEON, ARGB4444ToARGBRow_NEON, 0, 2, 4, 7)
#endif
-#ifdef HAS_ARGB4444TOARGBROW_MSA
-ANY11(ARGB4444ToARGBRow_Any_MSA, ARGB4444ToARGBRow_MSA, 0, 2, 4, 15)
+#ifdef HAS_ARGB4444TOARGBROW_LSX
+ANY11(ARGB4444ToARGBRow_Any_LSX, ARGB4444ToARGBRow_LSX, 0, 2, 4, 7)
#endif
-#ifdef HAS_ARGB4444TOARGBROW_MMI
-ANY11(ARGB4444ToARGBRow_Any_MMI, ARGB4444ToARGBRow_MMI, 0, 2, 4, 3)
+#ifdef HAS_ARGB4444TOARGBROW_LASX
+ANY11(ARGB4444ToARGBRow_Any_LASX, ARGB4444ToARGBRow_LASX, 0, 2, 4, 15)
#endif
#ifdef HAS_ARGBATTENUATEROW_SSSE3
ANY11(ARGBAttenuateRow_Any_SSSE3, ARGBAttenuateRow_SSSE3, 0, 4, 4, 3)
@@ -782,11 +1388,14 @@ ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7)
#ifdef HAS_ARGBATTENUATEROW_NEON
ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7)
#endif
-#ifdef HAS_ARGBATTENUATEROW_MSA
-ANY11(ARGBAttenuateRow_Any_MSA, ARGBAttenuateRow_MSA, 0, 4, 4, 7)
+#ifdef HAS_ARGBUNATTENUATEROW_NEON
+ANY11(ARGBUnattenuateRow_Any_NEON, ARGBUnattenuateRow_NEON, 0, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBATTENUATEROW_LSX
+ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 3)
#endif
-#ifdef HAS_ARGBATTENUATEROW_MMI
-ANY11(ARGBAttenuateRow_Any_MMI, ARGBAttenuateRow_MMI, 0, 4, 4, 1)
+#ifdef HAS_ARGBATTENUATEROW_LASX
+ANY11(ARGBAttenuateRow_Any_LASX, ARGBAttenuateRow_LASX, 0, 4, 4, 7)
#endif
#ifdef HAS_ARGBEXTRACTALPHAROW_SSE2
ANY11(ARGBExtractAlphaRow_Any_SSE2, ARGBExtractAlphaRow_SSE2, 0, 4, 1, 7)
@@ -797,28 +1406,72 @@ ANY11(ARGBExtractAlphaRow_Any_AVX2, ARGBExtractAlphaRow_AVX2, 0, 4, 1, 31)
#ifdef HAS_ARGBEXTRACTALPHAROW_NEON
ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15)
#endif
-#ifdef HAS_ARGBEXTRACTALPHAROW_MSA
-ANY11(ARGBExtractAlphaRow_Any_MSA, ARGBExtractAlphaRow_MSA, 0, 4, 1, 15)
+#ifdef HAS_ARGBEXTRACTALPHAROW_LSX
+ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 7)
+#endif
+#ifdef HAS_ARGBEXTRACTALPHAROW_LASX
+ANY11(ARGBExtractAlphaRow_Any_LASX, ARGBExtractAlphaRow_LASX, 0, 4, 1, 15)
+#endif
+#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD
+ANY11(ARGBExtractAlphaRow_Any_WASMSIMD,
+ ARGBExtractAlphaRow_WASMSIMD,
+ 0,
+ 4,
+ 1,
+ 15)
+#endif
+#ifdef HAS_RAWTOARGBROW_WASMSIMD
+ANY11(RAWToARGBRow_Any_WASMSIMD, RAWToARGBRow_WASMSIMD, 0, 3, 4, 15)
+#endif
+#ifdef HAS_RGB24TOARGBROW_WASMSIMD
+ANY11(RGB24ToARGBRow_Any_WASMSIMD, RGB24ToARGBRow_WASMSIMD, 0, 3, 4, 15)
+#endif
+#ifdef HAS_SWAPUVROW_WASMSIMD
+ANY11(SwapUVRow_Any_WASMSIMD, SwapUVRow_WASMSIMD, 0, 2, 2, 15)
#endif
-#ifdef HAS_ARGBEXTRACTALPHAROW_MMI
-ANY11(ARGBExtractAlphaRow_Any_MMI, ARGBExtractAlphaRow_MMI, 0, 4, 1, 7)
+#ifdef HAS_J400TOARGBROW_WASMSIMD
+ANY11(J400ToARGBRow_Any_WASMSIMD, J400ToARGBRow_WASMSIMD, 0, 1, 4, 15)
+#endif
+#ifdef HAS_RGB565TOARGBROW_WASMSIMD
+ANY11(RGB565ToARGBRow_Any_WASMSIMD, RGB565ToARGBRow_WASMSIMD, 0, 2, 4, 7)
+#endif
+#ifdef HAS_ARGB1555TOARGBROW_WASMSIMD
+ANY11(ARGB1555ToARGBRow_Any_WASMSIMD, ARGB1555ToARGBRow_WASMSIMD, 0, 2, 4, 7)
+#endif
+#ifdef HAS_ARGB4444TOARGBROW_WASMSIMD
+ANY11(ARGB4444ToARGBRow_Any_WASMSIMD, ARGB4444ToARGBRow_WASMSIMD, 0, 2, 4, 7)
+#endif
+#ifdef HAS_ARGBTORGB565ROW_WASMSIMD
+ANY11(ARGBToRGB565Row_Any_WASMSIMD, ARGBToRGB565Row_WASMSIMD, 0, 4, 2, 7)
+#endif
+#ifdef HAS_ARGBTOARGB1555ROW_WASMSIMD
+ANY11(ARGBToARGB1555Row_Any_WASMSIMD, ARGBToARGB1555Row_WASMSIMD, 0, 4, 2, 7)
+#endif
+#ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD
+ANY11(ARGBToARGB4444Row_Any_WASMSIMD, ARGBToARGB4444Row_WASMSIMD, 0, 4, 2, 7)
#endif
#undef ANY11
// Any 1 to 1 blended. Destination is read, modify, write.
-#define ANY11B(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 2]); \
- memset(temp, 0, 64 * 2); /* for msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(src_ptr, dst_ptr, n); \
- } \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
- memcpy(temp + 64, dst_ptr + n * BPP, r * BPP); \
- ANY_SIMD(temp, temp + 64, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 64, r * BPP); \
+#define ANY11B(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
+ SIMD_ALIGNED(uint8_t vin[64]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
+ memcpy(vout, dst_ptr + np * BPP, r * BPP); \
+ ANY_SIMD(vin, vout, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_ARGBCOPYALPHAROW_AVX2
@@ -827,8 +1480,8 @@ ANY11B(ARGBCopyAlphaRow_Any_AVX2, ARGBCopyAlphaRow_AVX2, 0, 4, 4, 15)
#ifdef HAS_ARGBCOPYALPHAROW_SSE2
ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7)
#endif
-#ifdef HAS_ARGBCOPYALPHAROW_MMI
-ANY11B(ARGBCopyAlphaRow_Any_MMI, ARGBCopyAlphaRow_MMI, 0, 4, 4, 1)
+#ifdef HAS_ARGBCOPYALPHAROW_NEON
+ANY11B(ARGBCopyAlphaRow_Any_NEON, ARGBCopyAlphaRow_NEON, 0, 4, 4, 7)
#endif
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2
ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15)
@@ -836,34 +1489,78 @@ ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15)
#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2
ANY11B(ARGBCopyYToAlphaRow_Any_SSE2, ARGBCopyYToAlphaRow_SSE2, 0, 1, 4, 7)
#endif
-#ifdef HAS_ARGBCOPYYTOALPHAROW_MMI
-ANY11B(ARGBCopyYToAlphaRow_Any_MMI, ARGBCopyYToAlphaRow_MMI, 0, 1, 4, 7)
+#ifdef HAS_ARGBCOPYYTOALPHAROW_NEON
+ANY11B(ARGBCopyYToAlphaRow_Any_NEON, ARGBCopyYToAlphaRow_NEON, 0, 1, 4, 15)
+#endif
+#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
+ANY11B(ARGBCopyYToAlphaRow_Any_WASMSIMD,
+ ARGBCopyYToAlphaRow_WASMSIMD,
+ 0,
+ 1,
+ 4,
+ 15)
#endif
#undef ANY11B
// Any 1 to 1 with parameter.
#define ANY11P(NAMEANY, ANY_SIMD, T, SBPP, BPP, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, T param, int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 2]); \
- memset(temp, 0, 64); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[(MASK + 1) * SBPP]); \
+ SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, dst_ptr, param, n); \
} \
- memcpy(temp, src_ptr + n * SBPP, r * SBPP); \
- ANY_SIMD(temp, temp + 64, param, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 64, r * BPP); \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * SBPP, r * SBPP); \
+ ANY_SIMD(vin, vout, param, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
-#if defined(HAS_ARGBTORGB565DITHERROW_SSE2)
-ANY11P(ARGBToRGB565DitherRow_Any_SSE2,
- ARGBToRGB565DitherRow_SSE2,
- const uint32_t,
+#if defined(HAS_I400TOARGBROW_SSE2)
+ANY11P(I400ToARGBRow_Any_SSE2,
+ I400ToARGBRow_SSE2,
+ const struct YuvConstants*,
+ 1,
4,
- 2,
- 3)
+ 7)
+#endif
+#if defined(HAS_I400TOARGBROW_AVX2)
+ANY11P(I400ToARGBRow_Any_AVX2,
+ I400ToARGBRow_AVX2,
+ const struct YuvConstants*,
+ 1,
+ 4,
+ 15)
+#endif
+#if defined(HAS_I400TOARGBROW_NEON)
+ANY11P(I400ToARGBRow_Any_NEON,
+ I400ToARGBRow_NEON,
+ const struct YuvConstants*,
+ 1,
+ 4,
+ 7)
+#endif
+#if defined(HAS_I400TOARGBROW_LSX)
+ANY11P(I400ToARGBRow_Any_LSX,
+ I400ToARGBRow_LSX,
+ const struct YuvConstants*,
+ 1,
+ 4,
+ 7)
+#endif
+#if defined(HAS_I400TOARGBROW_LASX)
+ANY11P(I400ToARGBRow_Any_LASX,
+ I400ToARGBRow_LASX,
+ const struct YuvConstants*,
+ 1,
+ 4,
+ 15)
#endif
+
#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
ANY11P(ARGBToRGB565DitherRow_Any_AVX2,
ARGBToRGB565DitherRow_AVX2,
@@ -880,21 +1577,21 @@ ANY11P(ARGBToRGB565DitherRow_Any_NEON,
2,
7)
#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_MSA)
-ANY11P(ARGBToRGB565DitherRow_Any_MSA,
- ARGBToRGB565DitherRow_MSA,
+#if defined(HAS_ARGBTORGB565DITHERROW_LSX)
+ANY11P(ARGBToRGB565DitherRow_Any_LSX,
+ ARGBToRGB565DitherRow_LSX,
const uint32_t,
4,
2,
- 7)
+ 3)
#endif
-#if defined(HAS_ARGBTORGB565DITHERROW_MMI)
-ANY11P(ARGBToRGB565DitherRow_Any_MMI,
- ARGBToRGB565DitherRow_MMI,
+#if defined(HAS_ARGBTORGB565DITHERROW_LASX)
+ANY11P(ARGBToRGB565DitherRow_Any_LASX,
+ ARGBToRGB565DitherRow_LASX,
const uint32_t,
4,
2,
- 3)
+ 7)
#endif
#ifdef HAS_ARGBSHUFFLEROW_SSSE3
ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8_t*, 4, 4, 7)
@@ -905,29 +1602,165 @@ ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8_t*, 4, 4, 15)
#ifdef HAS_ARGBSHUFFLEROW_NEON
ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8_t*, 4, 4, 3)
#endif
-#ifdef HAS_ARGBSHUFFLEROW_MSA
-ANY11P(ARGBShuffleRow_Any_MSA, ARGBShuffleRow_MSA, const uint8_t*, 4, 4, 7)
+#ifdef HAS_ARGBSHUFFLEROW_LSX
+ANY11P(ARGBShuffleRow_Any_LSX, ARGBShuffleRow_LSX, const uint8_t*, 4, 4, 3)
#endif
-#ifdef HAS_ARGBSHUFFLEROW_MMI
-ANY11P(ARGBShuffleRow_Any_MMI, ARGBShuffleRow_MMI, const uint8_t*, 4, 4, 1)
+#ifdef HAS_ARGBSHUFFLEROW_LASX
+ANY11P(ARGBShuffleRow_Any_LASX, ARGBShuffleRow_LASX, const uint8_t*, 4, 4, 7)
+#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD)
+ANY11P(ARGBToRGB565DitherRow_Any_WASMSIMD,
+ ARGBToRGB565DitherRow_WASMSIMD,
+ const uint32_t,
+ 4,
+ 2,
+ 7)
+#endif
+#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD
+ANY11P(ARGBShuffleRow_Any_WASMSIMD,
+ ARGBShuffleRow_WASMSIMD,
+ const uint8_t*,
+ 4,
+ 4,
+ 3)
+#endif
+#ifdef HAS_ARGBORROW_SSE2
+ANY11P(ARGBOrRow_Any_SSE2, ARGBOrRow_SSE2, const uint32_t, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBORROW_AVX2
+ANY11P(ARGBOrRow_Any_AVX2, ARGBOrRow_AVX2, const uint32_t, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBORROW_NEON
+ANY11P(ARGBOrRow_Any_NEON, ARGBOrRow_NEON, const uint32_t, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBORROW_LSX
+ANY11P(ARGBOrRow_Any_LSX, ARGBOrRow_LSX, const uint32_t, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBORROW_LASX
+ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7)
+#endif
+#ifdef HAS_ARGBORROW_WASMSIMD
+ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3)
+#endif
+#ifdef HAS_ARGBPOLYNOMIALROW_SSE2
+ANY11P(ARGBPolynomialRow_Any_SSE2,
+ ARGBPolynomialRow_SSE2,
+ const float*,
+ 4,
+ 4,
+ 1)
+#endif
+#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
+ANY11P(ARGBPolynomialRow_Any_AVX2,
+ ARGBPolynomialRow_AVX2,
+ const float*,
+ 4,
+ 4,
+ 1)
+#endif
+#ifdef HAS_ARGBPOLYNOMIALROW_NEON
+ANY11P(ARGBPolynomialRow_Any_NEON,
+ ARGBPolynomialRow_NEON,
+ const float*,
+ 4,
+ 4,
+ 1)
#endif
#undef ANY11P
-#undef ANY11P
+
+// Any 1 to 1 with type
+#define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
+ void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int width) { \
+ SIMD_ALIGNED(uint8_t vin[(MASK + 1) * SBPP]); \
+ SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, (uint8_t*)(src_ptr) + np * SBPP, r * SBPP); \
+ ANY_SIMD((STYPE*)vin, (DTYPE*)vout, MASK + 1); \
+ memcpy((uint8_t*)(dst_ptr) + np * BPP, vout, r * BPP); \
+ }
+
+#ifdef HAS_ARGBTOAR64ROW_SSSE3
+ANY11T(ARGBToAR64Row_Any_SSSE3, ARGBToAR64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3)
+#endif
+
+#ifdef HAS_ARGBTOAB64ROW_SSSE3
+ANY11T(ARGBToAB64Row_Any_SSSE3, ARGBToAB64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3)
+#endif
+
+#ifdef HAS_AR64TOARGBROW_SSSE3
+ANY11T(AR64ToARGBRow_Any_SSSE3, AR64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
+#endif
+
+#ifdef HAS_AB64TOARGBROW_SSSE3
+ANY11T(AB64ToARGBRow_Any_SSSE3, AB64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
+#endif
+
+#ifdef HAS_ARGBTOAR64ROW_AVX2
+ANY11T(ARGBToAR64Row_Any_AVX2, ARGBToAR64Row_AVX2, 4, 8, uint8_t, uint16_t, 7)
+#endif
+
+#ifdef HAS_ARGBTOAB64ROW_AVX2
+ANY11T(ARGBToAB64Row_Any_AVX2, ARGBToAB64Row_AVX2, 4, 8, uint8_t, uint16_t, 7)
+#endif
+
+#ifdef HAS_AR64TOARGBROW_AVX2
+ANY11T(AR64ToARGBRow_Any_AVX2, AR64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
+#endif
+
+#ifdef HAS_AB64TOARGBROW_AVX2
+ANY11T(AB64ToARGBRow_Any_AVX2, AB64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
+#endif
+
+#ifdef HAS_ARGBTOAR64ROW_NEON
+ANY11T(ARGBToAR64Row_Any_NEON, ARGBToAR64Row_NEON, 4, 8, uint8_t, uint16_t, 7)
+#endif
+
+#ifdef HAS_ARGBTOAB64ROW_NEON
+ANY11T(ARGBToAB64Row_Any_NEON, ARGBToAB64Row_NEON, 4, 8, uint8_t, uint16_t, 7)
+#endif
+
+#ifdef HAS_AR64TOARGBROW_NEON
+ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
+#endif
+
+#ifdef HAS_AB64TOARGBROW_NEON
+ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
+#endif
+
+#ifdef HAS_COPYROW_16_AVX2
+ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31)
+#endif
+#ifdef HAS_COPYROW_16_NEON
+ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15)
+#endif
+
+#undef ANY11T
// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
-#define ANY11C(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
- void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int scale, int width) { \
- SIMD_ALIGNED(STYPE temp[32]); \
- SIMD_ALIGNED(DTYPE out[32]); \
- memset(temp, 0, 32 * SBPP); /* for msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(src_ptr, dst_ptr, scale, n); \
- } \
- memcpy(temp, src_ptr + n, r * SBPP); \
- ANY_SIMD(temp, out, scale, MASK + 1); \
- memcpy(dst_ptr + n, out, r * BPP); \
+#define ANY11C(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
+ void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int value, int width) { \
+ SIMD_ALIGNED(STYPE vin[64]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, value, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np, r * SBPP); \
+ ANY_SIMD(vin, vout, value, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * BPP); \
}
#ifdef HAS_CONVERT16TO8ROW_SSSE3
@@ -948,6 +1781,15 @@ ANY11C(Convert16To8Row_Any_AVX2,
uint8_t,
31)
#endif
+#ifdef HAS_CONVERT16TO8ROW_NEON
+ANY11C(Convert16To8Row_Any_NEON,
+ Convert16To8Row_NEON,
+ 2,
+ 1,
+ uint16_t,
+ uint8_t,
+ 15)
+#endif
#ifdef HAS_CONVERT8TO16ROW_SSE2
ANY11C(Convert8To16Row_Any_SSE2,
Convert8To16Row_SSE2,
@@ -966,27 +1808,120 @@ ANY11C(Convert8To16Row_Any_AVX2,
uint16_t,
31)
#endif
+#ifdef HAS_CONVERT8TO16ROW_NEON
+ANY11C(Convert8To16Row_Any_NEON,
+ Convert8To16Row_NEON,
+ 1,
+ 2,
+ uint8_t,
+ uint16_t,
+ 15)
+#endif
+#ifdef HAS_MULTIPLYROW_16_AVX2
+ANY11C(MultiplyRow_16_Any_AVX2,
+ MultiplyRow_16_AVX2,
+ 2,
+ 2,
+ uint16_t,
+ uint16_t,
+ 31)
+#endif
+#ifdef HAS_MULTIPLYROW_16_NEON
+ANY11C(MultiplyRow_16_Any_NEON,
+ MultiplyRow_16_NEON,
+ 2,
+ 2,
+ uint16_t,
+ uint16_t,
+ 15)
+#endif
+#ifdef HAS_DIVIDEROW_16_AVX2
+ANY11C(DivideRow_16_Any_AVX2, DivideRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31)
+#endif
+#ifdef HAS_DIVIDEROW_16_NEON
+ANY11C(DivideRow_16_Any_NEON, DivideRow_16_NEON, 2, 2, uint16_t, uint16_t, 15)
+#endif
+#ifdef HAS_CONVERT8TO16ROW_WASMSIMD
+ANY11C(Convert8To16Row_Any_WASMSIMD,
+ Convert8To16Row_WASMSIMD,
+ 1,
+ 2,
+ uint8_t,
+ uint16_t,
+ 15)
+#endif
+#ifdef HAS_MULTIPLYROW_16_WASMSIMD
+ANY11C(MultiplyRow_16_Any_WASMSIMD,
+ MultiplyRow_16_WASMSIMD,
+ 2,
+ 2,
+ uint16_t,
+ uint16_t,
+ 15)
+#endif
#undef ANY11C
-// Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts.
-#define ANY11P16(NAMEANY, ANY_SIMD, ST, T, SBPP, BPP, MASK) \
- void NAMEANY(const ST* src_ptr, T* dst_ptr, float param, int width) { \
- SIMD_ALIGNED(ST temp[32]); \
- SIMD_ALIGNED(T out[32]); \
- memset(temp, 0, SBPP * 32); /* for msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(src_ptr, dst_ptr, param, n); \
- } \
- memcpy(temp, src_ptr + n, r * SBPP); \
- ANY_SIMD(temp, out, param, MASK + 1); \
- memcpy(dst_ptr + n, out, r * BPP); \
+// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
+#define ANY11SB(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
+ void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int scale, int bias, \
+ int width) { \
+ SIMD_ALIGNED(STYPE vin[64]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, scale, bias, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np, r * SBPP); \
+ ANY_SIMD(vin, vout, scale, bias, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * BPP); \
}
-#ifdef HAS_HALFFLOATROW_SSE2
-ANY11P16(HalfFloatRow_Any_SSE2, HalfFloatRow_SSE2, uint16_t, uint16_t, 2, 2, 7)
+#ifdef HAS_CONVERT8TO8ROW_NEON
+ANY11SB(Convert8To8Row_Any_NEON,
+ Convert8To8Row_NEON,
+ 1,
+ 1,
+ uint8_t,
+ uint8_t,
+ 31)
+#endif
+#ifdef HAS_CONVERT8TO8ROW_AVX2
+ANY11SB(Convert8To8Row_Any_AVX2,
+ Convert8To8Row_AVX2,
+ 1,
+ 1,
+ uint8_t,
+ uint8_t,
+ 31)
#endif
+#undef ANY11B
+
+// Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts.
+#define ANY11P16(NAMEANY, ANY_SIMD, ST, T, SBPP, BPP, MASK) \
+ void NAMEANY(const ST* src_ptr, T* dst_ptr, float param, int width) { \
+ SIMD_ALIGNED(ST vin[32]); \
+ static_assert((MASK + 1) * SBPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[32]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, param, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np, r * SBPP); \
+ ANY_SIMD(vin, vout, param, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * BPP); \
+ }
+
#ifdef HAS_HALFFLOATROW_AVX2
ANY11P16(HalfFloatRow_Any_AVX2, HalfFloatRow_AVX2, uint16_t, uint16_t, 2, 2, 15)
#endif
@@ -1001,41 +1936,47 @@ ANY11P16(HalfFloat1Row_Any_F16C,
15)
#endif
#ifdef HAS_HALFFLOATROW_NEON
-ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 7)
-ANY11P16(HalfFloat1Row_Any_NEON,
- HalfFloat1Row_NEON,
- uint16_t,
- uint16_t,
- 2,
- 2,
- 7)
+ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15)
#endif
-#ifdef HAS_HALFFLOATROW_MSA
-ANY11P16(HalfFloatRow_Any_MSA, HalfFloatRow_MSA, uint16_t, uint16_t, 2, 2, 31)
+#ifdef HAS_BYTETOFLOATROW_AVX2
+ANY11P16(ByteToFloatRow_Any_AVX2, ByteToFloatRow_AVX2, uint8_t, float, 1, 4, 15)
#endif
#ifdef HAS_BYTETOFLOATROW_NEON
-ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 3, 7)
+ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7)
+#endif
+#ifdef HAS_HALFFLOATROW_LSX
+ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 7)
+#endif
+#ifdef HAS_HALFFLOATROW_LASX
+ANY11P16(HalfFloatRow_Any_LASX, HalfFloatRow_LASX, uint16_t, uint16_t, 2, 2, 15)
#endif
#undef ANY11P16
// Any 1 to 1 with yuvconstants
-#define ANY11C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, \
- const struct YuvConstants* yuvconstants, int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 2]); \
- memset(temp, 0, 128); /* for YUY2 and msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(src_ptr, dst_ptr, yuvconstants, n); \
- } \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
- ANY_SIMD(temp, temp + 128, yuvconstants, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \
+#define ANY11C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, \
+ const struct YuvConstants* yuvconstants, int width) { \
+ SIMD_ALIGNED(uint8_t vin[256]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * SBPP <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, yuvconstants, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \
+ ANY_SIMD(vin, vout, yuvconstants, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
+
#if defined(HAS_YUY2TOARGBROW_SSSE3)
-ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 15)
-ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 15)
+ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 7)
+ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 7)
#endif
#if defined(HAS_YUY2TOARGBROW_AVX2)
ANY11C(YUY2ToARGBRow_Any_AVX2, YUY2ToARGBRow_AVX2, 1, 4, 4, 31)
@@ -1045,59 +1986,229 @@ ANY11C(UYVYToARGBRow_Any_AVX2, UYVYToARGBRow_AVX2, 1, 4, 4, 31)
ANY11C(YUY2ToARGBRow_Any_NEON, YUY2ToARGBRow_NEON, 1, 4, 4, 7)
ANY11C(UYVYToARGBRow_Any_NEON, UYVYToARGBRow_NEON, 1, 4, 4, 7)
#endif
-#if defined(HAS_YUY2TOARGBROW_MSA)
-ANY11C(YUY2ToARGBRow_Any_MSA, YUY2ToARGBRow_MSA, 1, 4, 4, 7)
-ANY11C(UYVYToARGBRow_Any_MSA, UYVYToARGBRow_MSA, 1, 4, 4, 7)
+#if defined(HAS_YUY2TOARGBROW_LSX)
+ANY11C(YUY2ToARGBRow_Any_LSX, YUY2ToARGBRow_LSX, 1, 4, 4, 7)
+ANY11C(UYVYToARGBRow_Any_LSX, UYVYToARGBRow_LSX, 1, 4, 4, 7)
+#endif
+#if defined(HAS_YUY2TOARGBROW_LASX)
+ANY11C(YUY2ToARGBRow_Any_LASX, YUY2ToARGBRow_LASX, 1, 4, 4, 15)
+ANY11C(UYVYToARGBRow_Any_LASX, UYVYToARGBRow_LASX, 1, 4, 4, 15)
#endif
#undef ANY11C
// Any 1 to 1 interpolate. Takes 2 rows of source via stride.
-#define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, MASK) \
- void NAMEANY(uint8_t* dst_ptr, const uint8_t* src_ptr, \
- ptrdiff_t src_stride_ptr, int width, int source_y_fraction) { \
- SIMD_ALIGNED(uint8_t temp[64 * 3]); \
- memset(temp, 0, 64 * 2); /* for msan */ \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(dst_ptr, src_ptr, src_stride_ptr, n, source_y_fraction); \
- } \
- memcpy(temp, src_ptr + n * SBPP, r * SBPP); \
- memcpy(temp + 64, src_ptr + src_stride_ptr + n * SBPP, r * SBPP); \
- ANY_SIMD(temp + 128, temp, 64, MASK + 1, source_y_fraction); \
- memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \
+#define ANY11I(NAMEANY, ANY_SIMD, TD, TS, SBPP, BPP, MASK) \
+ void NAMEANY(TD* dst_ptr, const TS* src_ptr, ptrdiff_t src_stride, \
+ int width, int source_y_fraction) { \
+ SIMD_ALIGNED(TS vin[64 * 2]); \
+ static_assert((MASK + 1) * SBPP * sizeof(TS) <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(TD vout[64]); \
+ static_assert((MASK + 1) * BPP * sizeof(TD) <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(dst_ptr, src_ptr, src_stride, n, source_y_fraction); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * SBPP, r * SBPP * sizeof(TS)); \
+ if (source_y_fraction) { \
+ memcpy(vin + 64, src_ptr + src_stride + np * SBPP, \
+ r * SBPP * sizeof(TS)); \
+ } \
+ ANY_SIMD(vout, vin, 64, MASK + 1, source_y_fraction); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP * sizeof(TD)); \
}
-#ifdef HAS_INTERPOLATEROW_AVX2
-ANY11T(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, 1, 1, 31)
-#endif
-#ifdef HAS_INTERPOLATEROW_SSSE3
-ANY11T(InterpolateRow_Any_SSSE3, InterpolateRow_SSSE3, 1, 1, 15)
+#if defined(HAS_INTERPOLATEROW_AVX2)
+ANY11I(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, uint8_t, uint8_t, 1, 1, 31)
#endif
#ifdef HAS_INTERPOLATEROW_NEON
-ANY11T(InterpolateRow_Any_NEON, InterpolateRow_NEON, 1, 1, 15)
+ANY11I(InterpolateRow_Any_NEON, InterpolateRow_NEON, uint8_t, uint8_t, 1, 1, 15)
#endif
-#ifdef HAS_INTERPOLATEROW_MSA
-ANY11T(InterpolateRow_Any_MSA, InterpolateRow_MSA, 1, 1, 31)
+#ifdef HAS_INTERPOLATEROW_LSX
+ANY11I(InterpolateRow_Any_LSX, InterpolateRow_LSX, uint8_t, uint8_t, 1, 1, 15)
#endif
-#ifdef HAS_INTERPOLATEROW_MMI
-ANY11T(InterpolateRow_Any_MMI, InterpolateRow_MMI, 1, 1, 7)
+#ifdef HAS_INTERPOLATEROW_LASX
+ANY11I(InterpolateRow_Any_LASX, InterpolateRow_LASX, uint8_t, uint8_t, 1, 1, 31)
#endif
-#undef ANY11T
-// Any 1 to 1 mirror.
-#define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
- SIMD_ALIGNED(uint8_t temp[64 * 2]); \
- memset(temp, 0, 64); /* for msan */ \
+#ifdef HAS_INTERPOLATEROW_16_NEON
+ANY11I(InterpolateRow_16_Any_NEON,
+ InterpolateRow_16_NEON,
+ uint16_t,
+ uint16_t,
+ 1,
+ 1,
+ 7)
+#endif
+#ifdef HAS_INTERPOLATEROW_16_AVX2
+ANY11I(InterpolateRow_16_Any_AVX2,
+ InterpolateRow_16_AVX2,
+ uint16_t,
+ uint16_t,
+ 1,
+ 1,
+ 15)
+#endif
+#ifdef HAS_INTERPOLATEROW_WASMSIMD
+ANY11I(InterpolateRow_Any_WASMSIMD,
+ InterpolateRow_WASMSIMD,
+ uint8_t,
+ uint8_t,
+ 1,
+ 1,
+ 15)
+#endif
+#ifdef HAS_INTERPOLATEROW_16_WASMSIMD
+ANY11I(InterpolateRow_16_Any_WASMSIMD,
+ InterpolateRow_16_WASMSIMD,
+ uint16_t,
+ uint16_t,
+ 1,
+ 1,
+ 7)
+#endif
+#undef ANY11I
+
+// Any 1 to 1 half row
+#define ANY11H(NAMEANY, ANY_SIMD, T, MASK) \
+ void NAMEANY(const T* src_ptr, ptrdiff_t src_stride, T* dst_ptr, \
+ int width) { \
+ SIMD_ALIGNED(T vin[64 * 2]); \
+ static_assert((MASK + 1) * sizeof(T) <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[64]); \
+ static_assert((MASK + 1) * sizeof(T) <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, src_stride, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np, r * sizeof(T)); \
+ memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(T)); \
+ ANY_SIMD(vin, 64, vout, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * sizeof(T)); \
+ }
+
+#ifdef HAS_HALFROW_AVX2
+ANY11H(HalfRow_Any_AVX2, HalfRow_AVX2, uint8_t, 31)
+#endif
+#ifdef HAS_HALFROW_16_AVX2
+ANY11H(HalfRow_16_Any_AVX2, HalfRow_16_AVX2, uint16_t, 15)
+#endif
+
+#undef ANY11H
+
+// Any 1 to 1 half row with scale param
+#define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \
+ void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \
+ int scale, int width) { \
+ SIMD_ALIGNED(TS vin[64 * 2]); \
+ static_assert((MASK + 1) * sizeof(TS) <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(TD vout[64]); \
+ static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np, r * sizeof(TS)); \
+ memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(TS)); \
+ ANY_SIMD(vin, 64, vout, scale, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * sizeof(TD)); \
+ }
+
+#ifdef HAS_HALFROW_16TO8_SSSE3
+ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15)
+#endif
+#ifdef HAS_HALFROW_16TO8_AVX2
+ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31)
+#endif
+#ifdef HAS_HALFROW_16TO8_NEON
+ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15)
+#endif
+
+#undef ANY11HS
+
+// Any 2 to 1 half-width row with scale param
+#define ANY11HWS(NAMEANY, ANY_SIMD, TS, TD, MASK) \
+ void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \
+ int scale, int width) { \
+ SIMD_ALIGNED(TS vin[128 * 2]); \
+ static_assert((MASK + 1) * 2 * sizeof(TS) <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(TD vout[64]); \
+ static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
- ANY_SIMD(src_ptr + r * BPP, dst_ptr, n); \
+ ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \
} \
- memcpy(temp, src_ptr, r* BPP); \
- ANY_SIMD(temp, temp + 64, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp + 64 + (MASK + 1 - r) * BPP, r * BPP); \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * 2, r * 2 * sizeof(TS)); \
+ memcpy(vin + 128, src_ptr + src_stride + np * 2, r * 2 * sizeof(TS)); \
+ ANY_SIMD(vin, 128, vout, scale, MASK + 1); \
+ memcpy(dst_ptr + np, vout, r * sizeof(TD)); \
+ }
+
+#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3
+ANY11HWS(HalfWidthRow_16To8_Any_SSSE3,
+ HalfWidthRow_16To8_SSSE3,
+ uint16_t,
+ uint8_t,
+ 7)
+#endif
+#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
+ANY11HWS(HalfWidthRow_16To8_Any_AVX2,
+ HalfWidthRow_16To8_AVX2,
+ uint16_t,
+ uint8_t,
+ 15)
+#endif
+#ifdef HAS_HALFWIDTHROW_16TO8_NEON
+ANY11HWS(HalfWidthRow_16To8_Any_NEON,
+ HalfWidthRow_16To8_NEON,
+ uint16_t,
+ uint8_t,
+ 7)
+#endif
+
+#undef ANY11HWS
+
+// Any 1 to 1 mirror.
+#define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \
+ SIMD_ALIGNED(uint8_t vin[128]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[128]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr + r * BPP, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr, r * BPP); \
+ ANY_SIMD(vin, vout, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout + (MASK + 1 - r) * BPP, r * BPP); \
}
#ifdef HAS_MIRRORROW_AVX2
@@ -1107,13 +2218,28 @@ ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31)
ANY11M(MirrorRow_Any_SSSE3, MirrorRow_SSSE3, 1, 15)
#endif
#ifdef HAS_MIRRORROW_NEON
-ANY11M(MirrorRow_Any_NEON, MirrorRow_NEON, 1, 15)
+ANY11M(MirrorRow_Any_NEON, MirrorRow_NEON, 1, 31)
+#endif
+#ifdef HAS_MIRRORROW_LSX
+ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 15)
+#endif
+#ifdef HAS_MIRRORROW_LASX
+ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 31)
+#endif
+#ifdef HAS_MIRRORUVROW_AVX2
+ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15)
#endif
-#ifdef HAS_MIRRORROW_MSA
-ANY11M(MirrorRow_Any_MSA, MirrorRow_MSA, 1, 63)
+#ifdef HAS_MIRRORUVROW_SSSE3
+ANY11M(MirrorUVRow_Any_SSSE3, MirrorUVRow_SSSE3, 2, 7)
#endif
-#ifdef HAS_MIRRORROW_MMI
-ANY11M(MirrorRow_Any_MMI, MirrorRow_MMI, 1, 7)
+#ifdef HAS_MIRRORUVROW_NEON
+ANY11M(MirrorUVRow_Any_NEON, MirrorUVRow_NEON, 2, 31)
+#endif
+#ifdef HAS_MIRRORUVROW_LSX
+ANY11M(MirrorUVRow_Any_LSX, MirrorUVRow_LSX, 2, 7)
+#endif
+#ifdef HAS_MIRRORUVROW_LASX
+ANY11M(MirrorUVRow_Any_LASX, MirrorUVRow_LASX, 2, 15)
#endif
#ifdef HAS_ARGBMIRRORROW_AVX2
ANY11M(ARGBMirrorRow_Any_AVX2, ARGBMirrorRow_AVX2, 4, 7)
@@ -1122,40 +2248,131 @@ ANY11M(ARGBMirrorRow_Any_AVX2, ARGBMirrorRow_AVX2, 4, 7)
ANY11M(ARGBMirrorRow_Any_SSE2, ARGBMirrorRow_SSE2, 4, 3)
#endif
#ifdef HAS_ARGBMIRRORROW_NEON
-ANY11M(ARGBMirrorRow_Any_NEON, ARGBMirrorRow_NEON, 4, 3)
+ANY11M(ARGBMirrorRow_Any_NEON, ARGBMirrorRow_NEON, 4, 7)
+#endif
+#ifdef HAS_ARGBMIRRORROW_LSX
+ANY11M(ARGBMirrorRow_Any_LSX, ARGBMirrorRow_LSX, 4, 3)
+#endif
+#ifdef HAS_ARGBMIRRORROW_LASX
+ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 7)
+#endif
+#ifdef HAS_RGB24MIRRORROW_AVX2
+ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31)
+#endif
+#ifdef HAS_RGB24MIRRORROW_SSSE3
+ANY11M(RGB24MirrorRow_Any_SSSE3, RGB24MirrorRow_SSSE3, 3, 15)
+#endif
+#ifdef HAS_RGB24MIRRORROW_NEON
+ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15)
#endif
-#ifdef HAS_ARGBMIRRORROW_MSA
-ANY11M(ARGBMirrorRow_Any_MSA, ARGBMirrorRow_MSA, 4, 15)
+#ifdef HAS_MIRRORROW_WASMSIMD
+ANY11M(MirrorRow_Any_WASMSIMD, MirrorRow_WASMSIMD, 1, 15)
#endif
-#ifdef HAS_ARGBMIRRORROW_MMI
-ANY11M(ARGBMirrorRow_Any_MMI, ARGBMirrorRow_MMI, 4, 1)
+#ifdef HAS_MIRRORUVROW_WASMSIMD
+ANY11M(MirrorUVRow_Any_WASMSIMD, MirrorUVRow_WASMSIMD, 2, 7)
+#endif
+#ifdef HAS_ARGBMIRRORROW_WASMSIMD
+ANY11M(ARGBMirrorRow_Any_WASMSIMD, ARGBMirrorRow_WASMSIMD, 4, 3)
+#endif
+#ifdef HAS_RGB24MIRRORROW_WASMSIMD
+ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15)
#endif
#undef ANY11M
+// Any 1 to 1 mirror with type.
+#define ANY11MT(NAMEANY, ANY_SIMD, T, BPP, MASK) \
+ void NAMEANY(const T* src_ptr, T* dst_ptr, int width) { \
+ SIMD_ALIGNED(T vin[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr + r, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr, r * BPP); \
+ ANY_SIMD(vin, vout, MASK + 1); \
+ memcpy(dst_ptr + np, vout + (MASK + 1 - r), r * BPP); \
+ }
+
+#ifdef HAS_MIRRORROW_16_NEON
+ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15)
+#endif
+#undef ANY11MT
+
+// Any 1 to 2 mirror split UV.
+#define ANY12MR(NAMEANY, ANY_SIMD, MASK) \
+ void NAMEANY(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, \
+ int width) { \
+ SIMD_ALIGNED(uint8_t vin[128]); \
+ static_assert((MASK + 1) * 2 <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[64 * 2]); \
+ static_assert(MASK + 1 <= sizeof(vout) / 2, "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_uv + r * 2, dst_u, dst_v, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_uv, r * 2); \
+ ANY_SIMD(vin, vout, vout + 64, MASK + 1); \
+ memcpy(dst_u + np, vout + (MASK + 1 - r), r); \
+ memcpy(dst_v + np, vout + 64 + (MASK + 1 - r), r); \
+ }
+
+#ifdef HAS_MIRRORSPLITUVROW_SSSE3
+ANY12MR(MirrorSplitUVRow_Any_SSSE3, MirrorSplitUVRow_SSSE3, 7)
+#endif
+#ifdef HAS_MIRRORSPLITUVROW_AVX2
+ANY12MR(MirrorSplitUVRow_Any_AVX2, MirrorSplitUVRow_AVX2, 15)
+#endif
+#undef ANY12MR
+
// Any 1 plane. (memset)
-#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \
- void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \
- SIMD_ALIGNED(uint8_t temp[64]); \
- int r = width & MASK; \
- int n = width & ~MASK; \
- if (n > 0) { \
- ANY_SIMD(dst_ptr, v32, n); \
- } \
- ANY_SIMD(temp, v32, MASK + 1); \
- memcpy(dst_ptr + n * BPP, temp, r * BPP); \
+#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \
+ void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \
+ SIMD_ALIGNED(uint8_t vout[64]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(dst_ptr, v32, n); \
+ } \
+ ptrdiff_t np = n; \
+ ANY_SIMD(vout, v32, MASK + 1); \
+ memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
-#ifdef HAS_SETROW_X86
-ANY1(SetRow_Any_X86, SetRow_X86, uint8_t, 1, 3)
+#ifdef HAS_SETROW_AVX2
+ANY1(SetRow_Any_AVX2, SetRow_AVX2, uint8_t, 1, 31)
#endif
#ifdef HAS_SETROW_NEON
ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15)
#endif
+#ifdef HAS_SETROW_LSX
+ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15)
+#endif
+#ifdef HAS_SETROW_LASX
+ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31)
+#endif
+#ifdef HAS_ARGBSETROW_AVX2
+ANY1(ARGBSetRow_Any_AVX2, ARGBSetRow_AVX2, uint32_t, 4, 7)
+#endif
#ifdef HAS_ARGBSETROW_NEON
ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3)
#endif
-#ifdef HAS_ARGBSETROW_MSA
-ANY1(ARGBSetRow_Any_MSA, ARGBSetRow_MSA, uint32_t, 4, 3)
+#ifdef HAS_ARGBSETROW_LSX
+ANY1(ARGBSetRow_Any_LSX, ARGBSetRow_LSX, uint32_t, 4, 3)
+#endif
+#ifdef HAS_ARGBSETROW_LASX
+ANY1(ARGBSetRow_Any_LASX, ARGBSetRow_LASX, uint32_t, 4, 7)
#endif
#undef ANY1
@@ -1163,17 +2380,24 @@ ANY1(ARGBSetRow_Any_MSA, ARGBSetRow_MSA, uint32_t, 4, 3)
#define ANY12(NAMEANY, ANY_SIMD, UVSHIFT, BPP, DUVSHIFT, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, \
int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 3]); \
- memset(temp, 0, 128); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[256]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * BPP <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256 * 2]); \
+ static_assert(SS(MASK + 1, DUVSHIFT) <= sizeof(vout) / 2, \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, dst_u, dst_v, n); \
} \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
- ANY_SIMD(temp, temp + 128, temp + 256, MASK + 1); \
- memcpy(dst_u + (n >> DUVSHIFT), temp + 128, SS(r, DUVSHIFT)); \
- memcpy(dst_v + (n >> DUVSHIFT), temp + 256, SS(r, DUVSHIFT)); \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
+ ANY_SIMD(vin, vout, vout + 256, MASK + 1); \
+ memcpy(dst_u + (np >> DUVSHIFT), vout, SS(r, DUVSHIFT)); \
+ memcpy(dst_v + (np >> DUVSHIFT), vout + 256, SS(r, DUVSHIFT)); \
}
#ifdef HAS_SPLITUVROW_SSE2
@@ -1185,14 +2409,11 @@ ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31)
#ifdef HAS_SPLITUVROW_NEON
ANY12(SplitUVRow_Any_NEON, SplitUVRow_NEON, 0, 2, 0, 15)
#endif
-#ifdef HAS_SPLITUVROW_MSA
-ANY12(SplitUVRow_Any_MSA, SplitUVRow_MSA, 0, 2, 0, 31)
+#ifdef HAS_SPLITUVROW_LSX
+ANY12(SplitUVRow_Any_LSX, SplitUVRow_LSX, 0, 2, 0, 15)
#endif
-#ifdef HAS_SPLITUVROW_MMI
-ANY12(SplitUVRow_Any_MMI, SplitUVRow_MMI, 0, 2, 0, 7)
-#endif
-#ifdef HAS_ARGBTOUV444ROW_SSSE3
-ANY12(ARGBToUV444Row_Any_SSSE3, ARGBToUV444Row_SSSE3, 0, 4, 0, 15)
+#ifdef HAS_SPLITUVROW_LASX
+ANY12(SplitUVRow_Any_LASX, SplitUVRow_LASX, 0, 2, 0, 31)
#endif
#ifdef HAS_YUY2TOUV422ROW_AVX2
ANY12(YUY2ToUV422Row_Any_AVX2, YUY2ToUV422Row_AVX2, 1, 4, 1, 31)
@@ -1203,238 +2424,625 @@ ANY12(YUY2ToUV422Row_Any_SSE2, YUY2ToUV422Row_SSE2, 1, 4, 1, 15)
ANY12(UYVYToUV422Row_Any_SSE2, UYVYToUV422Row_SSE2, 1, 4, 1, 15)
#endif
#ifdef HAS_YUY2TOUV422ROW_NEON
-ANY12(ARGBToUV444Row_Any_NEON, ARGBToUV444Row_NEON, 0, 4, 0, 7)
ANY12(YUY2ToUV422Row_Any_NEON, YUY2ToUV422Row_NEON, 1, 4, 1, 15)
ANY12(UYVYToUV422Row_Any_NEON, UYVYToUV422Row_NEON, 1, 4, 1, 15)
#endif
-#ifdef HAS_YUY2TOUV422ROW_MSA
-ANY12(ARGBToUV444Row_Any_MSA, ARGBToUV444Row_MSA, 0, 4, 0, 15)
-ANY12(YUY2ToUV422Row_Any_MSA, YUY2ToUV422Row_MSA, 1, 4, 1, 31)
-ANY12(UYVYToUV422Row_Any_MSA, UYVYToUV422Row_MSA, 1, 4, 1, 31)
+#ifdef HAS_YUY2TOUV422ROW_LSX
+ANY12(YUY2ToUV422Row_Any_LSX, YUY2ToUV422Row_LSX, 1, 4, 1, 15)
+ANY12(UYVYToUV422Row_Any_LSX, UYVYToUV422Row_LSX, 1, 4, 1, 15)
#endif
-#ifdef HAS_YUY2TOUV422ROW_MMI
-ANY12(ARGBToUV444Row_Any_MMI, ARGBToUV444Row_MMI, 0, 4, 0, 7)
-ANY12(UYVYToUV422Row_Any_MMI, UYVYToUV422Row_MMI, 1, 4, 1, 15)
-ANY12(YUY2ToUV422Row_Any_MMI, YUY2ToUV422Row_MMI, 1, 4, 1, 15)
+#ifdef HAS_YUY2TOUV422ROW_LASX
+ANY12(YUY2ToUV422Row_Any_LASX, YUY2ToUV422Row_LASX, 1, 4, 1, 31)
+ANY12(UYVYToUV422Row_Any_LASX, UYVYToUV422Row_LASX, 1, 4, 1, 31)
#endif
#undef ANY12
+// Any 2 16 bit planes with parameter to 1
+#define ANY12PT(NAMEANY, ANY_SIMD, T, BPP, MASK) \
+ void NAMEANY(const T* src_uv, T* dst_u, T* dst_v, int depth, int width) { \
+ SIMD_ALIGNED(T vin[16 * 2]); \
+ static_assert((MASK + 1) * BPP * 2 <= sizeof(vin), \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[16 * 2]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout) / 2, \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_uv, dst_u, dst_v, depth, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_uv + np * 2, r * BPP * 2); \
+ ANY_SIMD(vin, vout, vout + 16, depth, MASK + 1); \
+ memcpy(dst_u + np, vout, r * BPP); \
+ memcpy(dst_v + np, vout + 16, r * BPP); \
+ }
+
+#ifdef HAS_SPLITUVROW_16_AVX2
+ANY12PT(SplitUVRow_16_Any_AVX2, SplitUVRow_16_AVX2, uint16_t, 2, 15)
+#endif
+
+#ifdef HAS_SPLITUVROW_16_NEON
+ANY12PT(SplitUVRow_16_Any_NEON, SplitUVRow_16_NEON, uint16_t, 2, 7)
+#endif
+
+#undef ANY21CT
+
// Any 1 to 3. Outputs RGB planes.
#define ANY13(NAMEANY, ANY_SIMD, BPP, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_r, uint8_t* dst_g, \
uint8_t* dst_b, int width) { \
- SIMD_ALIGNED(uint8_t temp[16 * 6]); \
- memset(temp, 0, 16 * 3); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[(MASK + 1) * BPP]); \
+ SIMD_ALIGNED(uint8_t vout[(MASK + 1) * 3]); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, dst_r, dst_g, dst_b, n); \
} \
- memcpy(temp, src_ptr + n * BPP, r * BPP); \
- ANY_SIMD(temp, temp + 16 * 3, temp + 16 * 4, temp + 16 * 5, MASK + 1); \
- memcpy(dst_r + n, temp + 16 * 3, r); \
- memcpy(dst_g + n, temp + 16 * 4, r); \
- memcpy(dst_b + n, temp + 16 * 5, r); \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * BPP, r * BPP); \
+ ANY_SIMD(vin, vout, vout + MASK + 1, vout + (MASK + 1) * 2, MASK + 1); \
+ memcpy(dst_r + np, vout, r); \
+ memcpy(dst_g + np, vout + MASK + 1, r); \
+ memcpy(dst_b + np, vout + (MASK + 1) * 2, r); \
}
#ifdef HAS_SPLITRGBROW_SSSE3
ANY13(SplitRGBRow_Any_SSSE3, SplitRGBRow_SSSE3, 3, 15)
#endif
+#ifdef HAS_SPLITRGBROW_SSE41
+ANY13(SplitRGBRow_Any_SSE41, SplitRGBRow_SSE41, 3, 15)
+#endif
+#ifdef HAS_SPLITRGBROW_AVX2
+ANY13(SplitRGBRow_Any_AVX2, SplitRGBRow_AVX2, 3, 31)
+#endif
#ifdef HAS_SPLITRGBROW_NEON
ANY13(SplitRGBRow_Any_NEON, SplitRGBRow_NEON, 3, 15)
#endif
-#ifdef HAS_SPLITRGBROW_MMI
-ANY13(SplitRGBRow_Any_MMI, SplitRGBRow_MMI, 3, 3)
+#ifdef HAS_SPLITXRGBROW_SSE2
+ANY13(SplitXRGBRow_Any_SSE2, SplitXRGBRow_SSE2, 4, 7)
+#endif
+#ifdef HAS_SPLITXRGBROW_SSSE3
+ANY13(SplitXRGBRow_Any_SSSE3, SplitXRGBRow_SSSE3, 4, 7)
+#endif
+#ifdef HAS_SPLITXRGBROW_AVX2
+ANY13(SplitXRGBRow_Any_AVX2, SplitXRGBRow_AVX2, 4, 15)
+#endif
+#ifdef HAS_SPLITXRGBROW_NEON
+ANY13(SplitXRGBRow_Any_NEON, SplitXRGBRow_NEON, 4, 15)
+#endif
+
+// Any 1 to 4. Outputs ARGB planes.
+#define ANY14(NAMEANY, ANY_SIMD, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_r, uint8_t* dst_g, \
+ uint8_t* dst_b, uint8_t* dst_a, int width) { \
+ SIMD_ALIGNED(uint8_t vin[16 * 4]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[16 * 4]); \
+ static_assert(MASK + 1 <= sizeof(vout) / 4, "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_r, dst_g, dst_b, dst_a, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * BPP, r * BPP); \
+ ANY_SIMD(vin, vout, vout + 16, vout + 32, vout + 48, MASK + 1); \
+ memcpy(dst_r + np, vout, r); \
+ memcpy(dst_g + np, vout + 16, r); \
+ memcpy(dst_b + np, vout + 32, r); \
+ memcpy(dst_a + np, vout + 48, r); \
+ }
+
+#ifdef HAS_SPLITARGBROW_SSE2
+ANY14(SplitARGBRow_Any_SSE2, SplitARGBRow_SSE2, 4, 7)
+#endif
+#ifdef HAS_SPLITARGBROW_SSSE3
+ANY14(SplitARGBRow_Any_SSSE3, SplitARGBRow_SSSE3, 4, 7)
+#endif
+#ifdef HAS_SPLITARGBROW_AVX2
+ANY14(SplitARGBRow_Any_AVX2, SplitARGBRow_AVX2, 4, 15)
+#endif
+#ifdef HAS_SPLITARGBROW_NEON
+ANY14(SplitARGBRow_Any_NEON, SplitARGBRow_NEON, 4, 15)
#endif
// Any 1 to 2 with source stride (2 rows of source). Outputs UV planes.
// 128 byte row allows for 32 avx ARGB pixels.
#define ANY12S(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_u, \
+ void NAMEANY(const uint8_t* src_ptr, int src_stride, uint8_t* dst_u, \
uint8_t* dst_v, int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 4]); \
- memset(temp, 0, 128 * 2); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[256 * 2]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * BPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256 * 2]); \
+ static_assert(SS(MASK + 1, 1) <= sizeof(vout) / 2, \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
- ANY_SIMD(src_ptr, src_stride_ptr, dst_u, dst_v, n); \
+ ANY_SIMD(src_ptr, src_stride, dst_u, dst_v, n); \
} \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
- memcpy(temp + 128, src_ptr + src_stride_ptr + (n >> UVSHIFT) * BPP, \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
+ memcpy(vin + 256, src_ptr + src_stride + (np >> UVSHIFT) * BPP, \
SS(r, UVSHIFT) * BPP); \
if ((width & 1) && UVSHIFT == 0) { /* repeat last pixel for subsample */ \
- memcpy(temp + SS(r, UVSHIFT) * BPP, temp + SS(r, UVSHIFT) * BPP - BPP, \
+ memcpy(vin + SS(r, UVSHIFT) * BPP, vin + SS(r, UVSHIFT) * BPP - BPP, \
BPP); \
- memcpy(temp + 128 + SS(r, UVSHIFT) * BPP, \
- temp + 128 + SS(r, UVSHIFT) * BPP - BPP, BPP); \
+ memcpy(vin + 256 + SS(r, UVSHIFT) * BPP, \
+ vin + 256 + SS(r, UVSHIFT) * BPP - BPP, BPP); \
} \
- ANY_SIMD(temp, 128, temp + 256, temp + 384, MASK + 1); \
- memcpy(dst_u + (n >> 1), temp + 256, SS(r, 1)); \
- memcpy(dst_v + (n >> 1), temp + 384, SS(r, 1)); \
+ ANY_SIMD(vin, 256, vout, vout + 256, MASK + 1); \
+ memcpy(dst_u + (np >> 1), vout, SS(r, 1)); \
+ memcpy(dst_v + (np >> 1), vout + 256, SS(r, 1)); \
}
-#ifdef HAS_ARGBTOUVROW_AVX2
-ANY12S(ARGBToUVRow_Any_AVX2, ARGBToUVRow_AVX2, 0, 4, 31)
+#define ANY12M(NAMEANY, ANY_SIMD, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, \
+ int width, const struct ArgbConstants* c) { \
+ SIMD_ALIGNED(uint8_t vin[256]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256 * 2]); \
+ static_assert(MASK + 1 <= sizeof(vout) / 2, "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_u, dst_v, n, c); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * BPP, r * BPP); \
+ ANY_SIMD(vin, vout, vout + 256, MASK + 1, c); \
+ memcpy(dst_u + np, vout, r); \
+ memcpy(dst_v + np, vout + 256, r); \
+ }
+
+#define ANY12MS(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, int src_stride, uint8_t* dst_u, \
+ uint8_t* dst_v, int width, const struct ArgbConstants* c) { \
+ SIMD_ALIGNED(uint8_t vin[256 * 2]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * BPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256 * 2]); \
+ static_assert(SS(MASK + 1, 1) <= sizeof(vout) / 2, \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, src_stride, dst_u, dst_v, n, c); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
+ memcpy(vin + 256, src_ptr + src_stride + (np >> UVSHIFT) * BPP, \
+ SS(r, UVSHIFT) * BPP); \
+ if ((width & 1) && UVSHIFT == 0) { /* repeat last pixel for subsample */ \
+ memcpy(vin + SS(r, UVSHIFT) * BPP, vin + SS(r, UVSHIFT) * BPP - BPP, \
+ BPP); \
+ memcpy(vin + 256 + SS(r, UVSHIFT) * BPP, \
+ vin + 256 + SS(r, UVSHIFT) * BPP - BPP, BPP); \
+ } \
+ ANY_SIMD(vin, 256, vout, vout + 256, MASK + 1, c); \
+ memcpy(dst_u + (np >> 1), vout, SS(r, 1)); \
+ memcpy(dst_v + (np >> 1), vout + 256, SS(r, 1)); \
+ }
+
+#ifdef HAS_ARGBTOUVMATRIXROW_NEON
+ANY12MS(ARGBToUVMatrixRow_Any_NEON, ARGBToUVMatrixRow_NEON, 0, 4, 15)
#endif
-#ifdef HAS_ABGRTOUVROW_AVX2
-ANY12S(ABGRToUVRow_Any_AVX2, ABGRToUVRow_AVX2, 0, 4, 31)
+#ifdef HAS_ARGBTOUVMATRIXROW_NEON_I8MM
+ANY12MS(ARGBToUVMatrixRow_Any_NEON_I8MM, ARGBToUVMatrixRow_NEON_I8MM, 0, 4, 15)
#endif
-#ifdef HAS_ARGBTOUVJROW_AVX2
-ANY12S(ARGBToUVJRow_Any_AVX2, ARGBToUVJRow_AVX2, 0, 4, 31)
+#ifdef HAS_RGBTOUVMATRIXROW_NEON
+ANY12MS(RGBToUVMatrixRow_Any_NEON, RGBToUVMatrixRow_NEON, 0, 3, 15)
#endif
-#ifdef HAS_ARGBTOUVROW_SSSE3
-ANY12S(ARGBToUVRow_Any_SSSE3, ARGBToUVRow_SSSE3, 0, 4, 15)
-ANY12S(ARGBToUVJRow_Any_SSSE3, ARGBToUVJRow_SSSE3, 0, 4, 15)
-ANY12S(BGRAToUVRow_Any_SSSE3, BGRAToUVRow_SSSE3, 0, 4, 15)
-ANY12S(ABGRToUVRow_Any_SSSE3, ABGRToUVRow_SSSE3, 0, 4, 15)
-ANY12S(RGBAToUVRow_Any_SSSE3, RGBAToUVRow_SSSE3, 0, 4, 15)
+#ifdef HAS_RGB565TOUVMATRIXROW_NEON
+ANY12MS(RGB565ToUVMatrixRow_Any_NEON, RGB565ToUVMatrixRow_NEON, 0, 2, 15)
#endif
-#ifdef HAS_YUY2TOUVROW_AVX2
-ANY12S(YUY2ToUVRow_Any_AVX2, YUY2ToUVRow_AVX2, 1, 4, 31)
-ANY12S(UYVYToUVRow_Any_AVX2, UYVYToUVRow_AVX2, 1, 4, 31)
+#ifdef HAS_ARGB1555TOUVMATRIXROW_NEON
+ANY12MS(ARGB1555ToUVMatrixRow_Any_NEON, ARGB1555ToUVMatrixRow_NEON, 0, 2, 15)
#endif
-#ifdef HAS_YUY2TOUVROW_SSE2
-ANY12S(YUY2ToUVRow_Any_SSE2, YUY2ToUVRow_SSE2, 1, 4, 15)
-ANY12S(UYVYToUVRow_Any_SSE2, UYVYToUVRow_SSE2, 1, 4, 15)
+#ifdef HAS_ARGB4444TOUVMATRIXROW_NEON
+ANY12MS(ARGB4444ToUVMatrixRow_Any_NEON, ARGB4444ToUVMatrixRow_NEON, 0, 2, 15)
+#endif
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
+ANY12MS(ARGBToUVMatrixRow_Any_AVX2, ARGBToUVMatrixRow_AVX2, 0, 4, 15)
+ANY12MS(RGBToUVMatrixRow_Any_AVX2, RGBToUVMatrixRow_AVX2, 0, 3, 31)
+ANY12MS(RGB565ToUVMatrixRow_Any_AVX2, RGB565ToUVMatrixRow_AVX2, 0, 2, 15)
+#ifdef HAS_ARGB1555TOARGBROW_AVX2
+ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 15)
+#endif
+#ifdef HAS_ARGB4444TOARGBROW_AVX2
+ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 15)
+#endif
+#endif
+#ifdef HAS_ARGBTOUVMATRIXROW_SSSE3
+ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7)
+#endif
+#ifdef HAS_RGBTOUVMATRIXROW_SSSE3
+ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 15)
+#endif
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX2
+ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31)
+#endif
+#ifdef HAS_RGBTOUV444MATRIXROW_AVX2
+ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31)
+#endif
+#ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3
+ANY12M(ARGBToUV444MatrixRow_Any_SSSE3, ARGBToUV444MatrixRow_SSSE3, 4, 15)
+#endif
+#ifdef HAS_RGBTOUV444MATRIXROW_SSSE3
+ANY12M(RGBToUV444MatrixRow_Any_SSSE3, RGBToUV444MatrixRow_SSSE3, 3, 15)
+#endif
+#ifdef HAS_ARGBTOUV444MATRIXROW_NEON
+ANY12M(ARGBToUV444MatrixRow_Any_NEON, ARGBToUV444MatrixRow_NEON, 4, 7)
+#endif
+#ifdef HAS_RGBTOUV444MATRIXROW_NEON
+ANY12M(RGBToUV444MatrixRow_Any_NEON, RGBToUV444MatrixRow_NEON, 3, 7)
+#endif
+#ifdef HAS_ARGBTOUV444MATRIXROW_NEON_I8MM
+ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, ARGBToUV444MatrixRow_NEON_I8MM, 4, 7)
+#endif
+#ifdef HAS_ARGBTOUVMATRIXROW_WASMSIMD
+ANY12MS(ARGBToUVMatrixRow_Any_WASMSIMD, ARGBToUVMatrixRow_WASMSIMD, 0, 4, 15)
+#endif
+#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD
+ANY12MS(RGBToUVMatrixRow_Any_WASMSIMD, RGBToUVMatrixRow_WASMSIMD, 0, 3, 15)
+#endif
+#ifdef HAS_RGB565TOUVMATRIXROW_WASMSIMD
+ANY12MS(RGB565ToUVMatrixRow_Any_WASMSIMD,
+ RGB565ToUVMatrixRow_WASMSIMD,
+ 0,
+ 2,
+ 15)
+#endif
+#ifdef HAS_ARGB1555TOUVMATRIXROW_WASMSIMD
+ANY12MS(ARGB1555ToUVMatrixRow_Any_WASMSIMD,
+ ARGB1555ToUVMatrixRow_WASMSIMD,
+ 0,
+ 2,
+ 15)
+#endif
+#ifdef HAS_ARGB4444TOUVMATRIXROW_WASMSIMD
+ANY12MS(ARGB4444ToUVMatrixRow_Any_WASMSIMD,
+ ARGB4444ToUVMatrixRow_WASMSIMD,
+ 0,
+ 2,
+ 15)
+#endif
+#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD
+ANY12M(ARGBToUV444MatrixRow_Any_WASMSIMD, ARGBToUV444MatrixRow_WASMSIMD, 4, 15)
+#endif
+#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD
+ANY12M(RGBToUV444MatrixRow_Any_WASMSIMD, RGBToUV444MatrixRow_WASMSIMD, 3, 15)
+#endif
+
+#define ANY11MC(NAMEANY, ANY_SIMD, BPP, MASK) \
+ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width, \
+ const struct ArgbConstants* c) { \
+ SIMD_ALIGNED(uint8_t vin[256]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[256]); \
+ static_assert(MASK + 1 <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, n, c); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + np * BPP, r * BPP); \
+ ANY_SIMD(vin, vout, MASK + 1, c); \
+ memcpy(dst_ptr + np, vout, r); \
+ }
+
+#ifdef HAS_ARGBTOYMATRIXROW_SSSE3
+ANY11MC(ARGBToYMatrixRow_Any_SSSE3, ARGBToYMatrixRow_SSSE3, 4, 15)
+ANY11MC(RGBToYMatrixRow_Any_SSSE3, RGBToYMatrixRow_SSSE3, 3, 15)
#endif
-#ifdef HAS_ARGBTOUVROW_NEON
-ANY12S(ARGBToUVRow_Any_NEON, ARGBToUVRow_NEON, 0, 4, 15)
+#ifdef HAS_ARGBTOYMATRIXROW_AVX2
+ANY11MC(ARGBToYMatrixRow_Any_AVX2, ARGBToYMatrixRow_AVX2, 4, 31)
+ANY11MC(RGBToYMatrixRow_Any_AVX2, RGBToYMatrixRow_AVX2, 3, 31)
+ANY11MC(RGB565ToYMatrixRow_Any_AVX2, RGB565ToYMatrixRow_AVX2, 2, 31)
+#ifdef HAS_ARGB1555TOYMATRIXROW_AVX2
+ANY11MC(ARGB1555ToYMatrixRow_Any_AVX2, ARGB1555ToYMatrixRow_AVX2, 2, 31)
#endif
-#ifdef HAS_ARGBTOUVROW_MSA
-ANY12S(ARGBToUVRow_Any_MSA, ARGBToUVRow_MSA, 0, 4, 31)
+#ifdef HAS_ARGB4444TOYMATRIXROW_AVX2
+ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31)
#endif
-#ifdef HAS_ARGBTOUVROW_MMI
-ANY12S(ARGBToUVRow_Any_MMI, ARGBToUVRow_MMI, 0, 4, 15)
#endif
-#ifdef HAS_ARGBTOUVJROW_NEON
-ANY12S(ARGBToUVJRow_Any_NEON, ARGBToUVJRow_NEON, 0, 4, 15)
+#ifdef HAS_ARGBTOYMATRIXROW_NEON
+ANY11MC(ARGBToYMatrixRow_Any_NEON, ARGBToYMatrixRow_NEON, 4, 15)
#endif
-#ifdef HAS_ARGBTOUVJROW_MSA
-ANY12S(ARGBToUVJRow_Any_MSA, ARGBToUVJRow_MSA, 0, 4, 31)
+#ifdef HAS_ARGBTOYMATRIXROW_NEON_DOTPROD
+ANY11MC(ARGBToYMatrixRow_Any_NEON_DotProd, ARGBToYMatrixRow_NEON_DotProd, 4, 15)
+ANY11MC(RGBToYMatrixRow_Any_NEON_DotProd, RGBToYMatrixRow_NEON_DotProd, 3, 15)
#endif
-#ifdef HAS_ARGBTOUVJROW_MMI
-ANY12S(ARGBToUVJRow_Any_MMI, ARGBToUVJRow_MMI, 0, 4, 15)
+#ifdef HAS_RGBTOYMATRIXROW_NEON
+ANY11MC(RGBToYMatrixRow_Any_NEON, RGBToYMatrixRow_NEON, 3, 15)
#endif
-#ifdef HAS_BGRATOUVROW_NEON
-ANY12S(BGRAToUVRow_Any_NEON, BGRAToUVRow_NEON, 0, 4, 15)
+#ifdef HAS_RGB565TOYMATRIXROW_NEON
+ANY11MC(RGB565ToYMatrixRow_Any_NEON, RGB565ToYMatrixRow_NEON, 2, 15)
#endif
-#ifdef HAS_BGRATOUVROW_MSA
-ANY12S(BGRAToUVRow_Any_MSA, BGRAToUVRow_MSA, 0, 4, 31)
+#ifdef HAS_ARGB1555TOYMATRIXROW_NEON
+ANY11MC(ARGB1555ToYMatrixRow_Any_NEON, ARGB1555ToYMatrixRow_NEON, 2, 15)
#endif
-#ifdef HAS_BGRATOUVROW_MMI
-ANY12S(BGRAToUVRow_Any_MMI, BGRAToUVRow_MMI, 0, 4, 15)
+#ifdef HAS_ARGB4444TOYMATRIXROW_NEON
+ANY11MC(ARGB4444ToYMatrixRow_Any_NEON, ARGB4444ToYMatrixRow_NEON, 2, 15)
#endif
-#ifdef HAS_ABGRTOUVROW_NEON
-ANY12S(ABGRToUVRow_Any_NEON, ABGRToUVRow_NEON, 0, 4, 15)
+#ifdef HAS_ARGBTOYMATRIXROW_LSX
+ANY11MC(ARGBToYMatrixRow_Any_LSX, ARGBToYMatrixRow_LSX, 4, 7)
#endif
-#ifdef HAS_ABGRTOUVROW_MSA
-ANY12S(ABGRToUVRow_Any_MSA, ABGRToUVRow_MSA, 0, 4, 31)
+#ifdef HAS_ARGBTOYMATRIXROW_LASX
+ANY11MC(ARGBToYMatrixRow_Any_LASX, ARGBToYMatrixRow_LASX, 4, 15)
#endif
-#ifdef HAS_ABGRTOUVROW_MMI
-ANY12S(ABGRToUVRow_Any_MMI, ABGRToUVRow_MMI, 0, 4, 15)
+#ifdef HAS_RGBTOYMATRIXROW_LSX
+ANY11MC(RGBToYMatrixRow_Any_LSX, RGBToYMatrixRow_LSX, 3, 15)
#endif
-#ifdef HAS_RGBATOUVROW_NEON
-ANY12S(RGBAToUVRow_Any_NEON, RGBAToUVRow_NEON, 0, 4, 15)
+#ifdef HAS_RGBTOYMATRIXROW_LASX
+ANY11MC(RGBToYMatrixRow_Any_LASX, RGBToYMatrixRow_LASX, 3, 31)
#endif
-#ifdef HAS_RGBATOUVROW_MSA
-ANY12S(RGBAToUVRow_Any_MSA, RGBAToUVRow_MSA, 0, 4, 31)
+#ifdef HAS_ARGBTOYMATRIXROW_RVV
+ANY11MC(ARGBToYMatrixRow_Any_RVV, ARGBToYMatrixRow_RVV, 4, 15)
#endif
-#ifdef HAS_RGBATOUVROW_MMI
-ANY12S(RGBAToUVRow_Any_MMI, RGBAToUVRow_MMI, 0, 4, 15)
+#ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD
+ANY11MC(ARGBToYMatrixRow_Any_WASMSIMD, ARGBToYMatrixRow_WASMSIMD, 4, 15)
#endif
+#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD
+ANY11MC(RGBToYMatrixRow_Any_WASMSIMD, RGBToYMatrixRow_WASMSIMD, 3, 15)
+#endif
+#ifdef HAS_RGB565TOYMATRIXROW_WASMSIMD
+ANY11MC(RGB565ToYMatrixRow_Any_WASMSIMD, RGB565ToYMatrixRow_WASMSIMD, 2, 15)
+#endif
+#ifdef HAS_ARGB1555TOYMATRIXROW_WASMSIMD
+ANY11MC(ARGB1555ToYMatrixRow_Any_WASMSIMD, ARGB1555ToYMatrixRow_WASMSIMD, 2, 15)
+#endif
+#ifdef HAS_ARGB4444TOYMATRIXROW_WASMSIMD
+ANY11MC(ARGB4444ToYMatrixRow_Any_WASMSIMD, ARGB4444ToYMatrixRow_WASMSIMD, 2, 15)
+#endif
+#undef ANY11MC
+
+#ifdef HAS_YUY2TOUVROW_AVX2
+ANY12S(YUY2ToUVRow_Any_AVX2, YUY2ToUVRow_AVX2, 1, 4, 31)
+ANY12S(UYVYToUVRow_Any_AVX2, UYVYToUVRow_AVX2, 1, 4, 31)
+#endif
+#ifdef HAS_YUY2TOUVROW_SSE2
+ANY12S(YUY2ToUVRow_Any_SSE2, YUY2ToUVRow_SSE2, 1, 4, 15)
+ANY12S(UYVYToUVRow_Any_SSE2, UYVYToUVRow_SSE2, 1, 4, 15)
+#endif
+
#ifdef HAS_RGB24TOUVROW_NEON
ANY12S(RGB24ToUVRow_Any_NEON, RGB24ToUVRow_NEON, 0, 3, 15)
#endif
-#ifdef HAS_RGB24TOUVROW_MSA
-ANY12S(RGB24ToUVRow_Any_MSA, RGB24ToUVRow_MSA, 0, 3, 15)
+#ifdef HAS_RGB24TOUVROW_LSX
+ANY12S(RGB24ToUVRow_Any_LSX, RGB24ToUVRow_LSX, 0, 3, 15)
#endif
-#ifdef HAS_RGB24TOUVROW_MMI
-ANY12S(RGB24ToUVRow_Any_MMI, RGB24ToUVRow_MMI, 0, 3, 15)
+#ifdef HAS_RGB24TOUVROW_LASX
+ANY12S(RGB24ToUVRow_Any_LASX, RGB24ToUVRow_LASX, 0, 3, 31)
#endif
#ifdef HAS_RAWTOUVROW_NEON
ANY12S(RAWToUVRow_Any_NEON, RAWToUVRow_NEON, 0, 3, 15)
#endif
-#ifdef HAS_RAWTOUVROW_MSA
-ANY12S(RAWToUVRow_Any_MSA, RAWToUVRow_MSA, 0, 3, 15)
+#ifdef HAS_RAWTOUVROW_LSX
+ANY12S(RAWToUVRow_Any_LSX, RAWToUVRow_LSX, 0, 3, 15)
#endif
-#ifdef HAS_RAWTOUVROW_MMI
-ANY12S(RAWToUVRow_Any_MMI, RAWToUVRow_MMI, 0, 3, 15)
+#ifdef HAS_RAWTOUVROW_LASX
+ANY12S(RAWToUVRow_Any_LASX, RAWToUVRow_LASX, 0, 3, 31)
+#endif
+#ifdef HAS_RGB565TOUVROW_AVX2
+ANY12S(RGB565ToUVRow_Any_AVX2, RGB565ToUVRow_AVX2, 0, 2, 31)
+#endif
+#ifdef HAS_ARGB1555TOUVROW_AVX2
+ANY12S(ARGB1555ToUVRow_Any_AVX2, ARGB1555ToUVRow_AVX2, 0, 2, 31)
+#endif
+#ifdef HAS_ARGB4444TOUVROW_AVX2
+ANY12S(ARGB4444ToUVRow_Any_AVX2, ARGB4444ToUVRow_AVX2, 0, 2, 31)
#endif
#ifdef HAS_RGB565TOUVROW_NEON
ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15)
#endif
-#ifdef HAS_RGB565TOUVROW_MSA
-ANY12S(RGB565ToUVRow_Any_MSA, RGB565ToUVRow_MSA, 0, 2, 15)
+#ifdef HAS_RGB565TOUVROW_LSX
+ANY12S(RGB565ToUVRow_Any_LSX, RGB565ToUVRow_LSX, 0, 2, 15)
#endif
-#ifdef HAS_RGB565TOUVROW_MMI
-ANY12S(RGB565ToUVRow_Any_MMI, RGB565ToUVRow_MMI, 0, 2, 15)
+#ifdef HAS_RGB565TOUVROW_LASX
+ANY12S(RGB565ToUVRow_Any_LASX, RGB565ToUVRow_LASX, 0, 2, 31)
#endif
#ifdef HAS_ARGB1555TOUVROW_NEON
ANY12S(ARGB1555ToUVRow_Any_NEON, ARGB1555ToUVRow_NEON, 0, 2, 15)
#endif
-#ifdef HAS_ARGB1555TOUVROW_MSA
-ANY12S(ARGB1555ToUVRow_Any_MSA, ARGB1555ToUVRow_MSA, 0, 2, 15)
+#ifdef HAS_ARGB1555TOUVROW_LSX
+ANY12S(ARGB1555ToUVRow_Any_LSX, ARGB1555ToUVRow_LSX, 0, 2, 15)
#endif
-#ifdef HAS_ARGB1555TOUVROW_MMI
-ANY12S(ARGB1555ToUVRow_Any_MMI, ARGB1555ToUVRow_MMI, 0, 2, 15)
+#ifdef HAS_ARGB1555TOUVROW_LASX
+ANY12S(ARGB1555ToUVRow_Any_LASX, ARGB1555ToUVRow_LASX, 0, 2, 31)
#endif
#ifdef HAS_ARGB4444TOUVROW_NEON
ANY12S(ARGB4444ToUVRow_Any_NEON, ARGB4444ToUVRow_NEON, 0, 2, 15)
#endif
-#ifdef HAS_ARGB4444TOUVROW_MMI
-ANY12S(ARGB4444ToUVRow_Any_MMI, ARGB4444ToUVRow_MMI, 0, 2, 15)
-#endif
#ifdef HAS_YUY2TOUVROW_NEON
ANY12S(YUY2ToUVRow_Any_NEON, YUY2ToUVRow_NEON, 1, 4, 15)
#endif
#ifdef HAS_UYVYTOUVROW_NEON
ANY12S(UYVYToUVRow_Any_NEON, UYVYToUVRow_NEON, 1, 4, 15)
#endif
-#ifdef HAS_YUY2TOUVROW_MSA
-ANY12S(YUY2ToUVRow_Any_MSA, YUY2ToUVRow_MSA, 1, 4, 31)
+#ifdef HAS_YUY2TOUVROW_LSX
+ANY12S(YUY2ToUVRow_Any_LSX, YUY2ToUVRow_LSX, 1, 4, 15)
#endif
-#ifdef HAS_YUY2TOUVROW_MMI
-ANY12S(YUY2ToUVRow_Any_MMI, YUY2ToUVRow_MMI, 1, 4, 15)
+#ifdef HAS_YUY2TOUVROW_LASX
+ANY12S(YUY2ToUVRow_Any_LASX, YUY2ToUVRow_LASX, 1, 4, 31)
#endif
-#ifdef HAS_UYVYTOUVROW_MSA
-ANY12S(UYVYToUVRow_Any_MSA, UYVYToUVRow_MSA, 1, 4, 31)
+#ifdef HAS_UYVYTOUVROW_LSX
+ANY12S(UYVYToUVRow_Any_LSX, UYVYToUVRow_LSX, 1, 4, 15)
#endif
-#ifdef HAS_UYVYTOUVROW_MMI
-ANY12S(UYVYToUVRow_Any_MMI, UYVYToUVRow_MMI, 1, 4, 15)
+#ifdef HAS_UYVYTOUVROW_LASX
+ANY12S(UYVYToUVRow_Any_LASX, UYVYToUVRow_LASX, 1, 4, 31)
#endif
#undef ANY12S
// Any 1 to 1 with source stride (2 rows of source). Outputs UV plane.
// 128 byte row allows for 32 avx ARGB pixels.
#define ANY11S(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \
- void NAMEANY(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_vu, \
+ void NAMEANY(const uint8_t* src_ptr, int src_stride, uint8_t* dst_vu, \
int width) { \
- SIMD_ALIGNED(uint8_t temp[128 * 3]); \
- memset(temp, 0, 128 * 2); /* for msan */ \
+ SIMD_ALIGNED(uint8_t vin[128 * 2]); \
+ static_assert(SS(MASK + 1, UVSHIFT) * BPP <= sizeof(vin) / 2, \
+ "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[128]); \
+ static_assert(SS(MASK + 1, 1) * 2 <= sizeof(vout), \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
- ANY_SIMD(src_ptr, src_stride_ptr, dst_vu, n); \
+ ANY_SIMD(src_ptr, src_stride, dst_vu, n); \
} \
- memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
- memcpy(temp + 128, src_ptr + src_stride_ptr + (n >> UVSHIFT) * BPP, \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_ptr + (np >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \
+ memcpy(vin + 128, src_ptr + src_stride + (np >> UVSHIFT) * BPP, \
SS(r, UVSHIFT) * BPP); \
if ((width & 1) && UVSHIFT == 0) { /* repeat last pixel for subsample */ \
- memcpy(temp + SS(r, UVSHIFT) * BPP, temp + SS(r, UVSHIFT) * BPP - BPP, \
+ memcpy(vin + SS(r, UVSHIFT) * BPP, vin + SS(r, UVSHIFT) * BPP - BPP, \
BPP); \
- memcpy(temp + 128 + SS(r, UVSHIFT) * BPP, \
- temp + 128 + SS(r, UVSHIFT) * BPP - BPP, BPP); \
+ memcpy(vin + 128 + SS(r, UVSHIFT) * BPP, \
+ vin + 128 + SS(r, UVSHIFT) * BPP - BPP, BPP); \
} \
- ANY_SIMD(temp, 128, temp + 256, MASK + 1); \
- memcpy(dst_vu + (n >> 1) * 2, temp + 256, SS(r, 1) * 2); \
+ ANY_SIMD(vin, 128, vout, MASK + 1); \
+ memcpy(dst_vu + (np >> 1) * 2, vout, SS(r, 1) * 2); \
}
+#ifdef HAS_AYUVTOUVROW_AVX2
+ANY11S(AYUVToUVRow_Any_AVX2, AYUVToUVRow_AVX2, 0, 4, 15)
+#endif
+#ifdef HAS_AYUVTOVUROW_AVX2
+ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15)
+#endif
#ifdef HAS_AYUVTOVUROW_NEON
ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15)
ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_AYUVTOUVROW_SVE2
+ANY11S(AYUVToUVRow_Any_SVE2, AYUVToUVRow_SVE2, 0, 4, 1)
+#endif
+#ifdef HAS_AYUVTOVUROW_SVE2
+ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1)
+#endif
#undef ANY11S
+#define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \
+ void NAMEANY(const T* src, ptrdiff_t src_tile_stride, T* dst, int width) { \
+ SIMD_ALIGNED(T vin[16]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(T vout[16]); \
+ static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src, src_tile_stride, dst, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src + (np / 16) * src_tile_stride, r * BPP); \
+ ANY_SIMD(vin, src_tile_stride, vout, MASK + 1); \
+ memcpy(dst + np, vout, r * BPP); \
+ }
+
+#ifdef HAS_DETILEROW_NEON
+ANYDETILE(DetileRow_Any_NEON, DetileRow_NEON, uint8_t, 1, 15)
+#endif
+#ifdef HAS_DETILEROW_AVX2
+ANYDETILE(DetileRow_Any_AVX2, DetileRow_AVX2, uint8_t, 1, 15)
+#endif
+#ifdef HAS_DETILEROW_16_NEON
+ANYDETILE(DetileRow_16_Any_NEON, DetileRow_16_NEON, uint16_t, 2, 15)
+#endif
+#ifdef HAS_DETILEROW_16_AVX2
+ANYDETILE(DetileRow_16_Any_AVX2, DetileRow_16_AVX2, uint16_t, 2, 15)
+#endif
+
+// DetileSplitUVRow width is in bytes
+#define ANYDETILESPLITUV(NAMEANY, ANY_SIMD, MASK) \
+ void NAMEANY(const uint8_t* src_uv, ptrdiff_t src_tile_stride, \
+ uint8_t* dst_u, uint8_t* dst_v, int width) { \
+ SIMD_ALIGNED(uint8_t vin[16]); \
+ static_assert(MASK + 1 <= sizeof(vin), "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[8 * 2]); \
+ static_assert((MASK + 1) / 2 <= sizeof(vout) / 2, \
+ "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_uv, src_tile_stride, dst_u, dst_v, n); \
+ } \
+ ptrdiff_t np = n; \
+ memcpy(vin, src_uv + (np / 16) * src_tile_stride, r); \
+ ANY_SIMD(vin, src_tile_stride, vout, vout + 8, r); \
+ memcpy(dst_u + np / 2, vout, (r + 1) / 2); \
+ memcpy(dst_v + np / 2, vout + 8, (r + 1) / 2); \
+ }
+
+#ifdef HAS_DETILESPLITUVROW_NEON
+ANYDETILESPLITUV(DetileSplitUVRow_Any_NEON, DetileSplitUVRow_NEON, 15)
+#endif
+#ifdef HAS_DETILESPLITUVROW_AVX2
+ANYDETILESPLITUV(DetileSplitUVRow_Any_AVX2, DetileSplitUVRow_AVX2, 15)
+#endif
+
+#define ANYDETILEMERGE(NAMEANY, ANY_SIMD, MASK) \
+ void NAMEANY(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, \
+ const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, \
+ uint8_t* dst_yuy2, int width) { \
+ SIMD_ALIGNED(uint8_t vin[32 * 2]); \
+ static_assert(MASK + 1 <= sizeof(vin) / 2, "vin buffer too small"); \
+ SIMD_ALIGNED(uint8_t vout[32 * 2]); \
+ static_assert(2 * (MASK + 1) <= sizeof(vout), "vout buffer too small"); \
+ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
+ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_y, src_y_tile_stride, src_uv, src_uv_tile_stride, dst_yuy2, \
+ n); \
+ } \
+ ptrdiff_t np = n; \
+ int r0 = (r < 16) ? r : 16; \
+ memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r0); \
+ memcpy(vin + 32, src_uv + (np / 16) * src_uv_tile_stride, r0); \
+ if (r > 16) { \
+ memcpy(vin + 16, src_y + (np / 16 + 1) * src_y_tile_stride, r - 16); \
+ memcpy(vin + 48, src_uv + (np / 16 + 1) * src_uv_tile_stride, r - 16); \
+ } \
+ ANY_SIMD(vin, 16, vin + 32, 16, vout, MASK + 1); \
+ memcpy(dst_yuy2 + 2 * np, vout, 2 * r); \
+ }
+
+#ifdef HAS_DETILETOYUY2_NEON
+ANYDETILEMERGE(DetileToYUY2_Any_NEON, DetileToYUY2_NEON, 15)
+#endif
+
+#ifdef HAS_DETILETOYUY2_AVX2
+ANYDETILEMERGE(DetileToYUY2_Any_AVX2, DetileToYUY2_AVX2, 31)
+#endif
+
+#undef ANYDETILEMERGE
+
+#undef MSAN_MEMSET
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/row_common.cc b/libs/libyuv/source/row_common.cc
index 9fa6e34ce5..48eda2282d 100644
--- a/libs/libyuv/source/row_common.cc
+++ b/libs/libyuv/source/row_common.cc
@@ -10,34 +10,55 @@
#include "libyuv/row.h"
-#include <stdio.h>
+#include <assert.h>
#include <string.h> // For memcpy and memset.
-#include "libyuv/basic_types.h"
+#include "libyuv/convert_argb.h" // For kYuvI601Constants
+#include "libyuv/convert_from_argb.h" // For ArgbConstants
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+#ifdef __cplusplus
+#define STATIC_CAST(type, expr) static_cast<type>(expr)
+#else
+#define STATIC_CAST(type, expr) (type)(expr)
+#endif
+
+// This macro controls YUV to RGB using unsigned math to extend range of
+// YUV to RGB coefficients to 0 to 4 instead of 0 to 2 for more accuracy on B:
+// LIBYUV_UNLIMITED_DATA
+
+// Macros to enable unlimited data for each colorspace
+// LIBYUV_UNLIMITED_BT601
+// LIBYUV_UNLIMITED_BT709
+// LIBYUV_UNLIMITED_BT2020
+
// llvm x86 is poor at ternary operator, so use branchless min/max.
#define USE_BRANCHLESS 1
-#if USE_BRANCHLESS
+#if defined(USE_BRANCHLESS)
static __inline int32_t clamp0(int32_t v) {
- return ((-(v) >> 31) & (v));
+ return -(v >= 0) & v;
}
-
+// TODO(fbarchard): make clamp255 preserve negative values.
static __inline int32_t clamp255(int32_t v) {
- return (((255 - (v)) >> 31) | (v)) & 255;
+ return (-(v >= 255) | v) & 255;
}
static __inline int32_t clamp1023(int32_t v) {
- return (((1023 - (v)) >> 31) | (v)) & 1023;
+ return (-(v >= 1023) | v) & 1023;
+}
+
+// clamp to max
+static __inline int32_t ClampMax(int32_t v, int32_t max) {
+ return (-(v >= max) | v) & max;
}
static __inline uint32_t Abs(int32_t v) {
- int m = v >> 31;
+ int m = -(v < 0);
return (v + m) ^ m;
}
#else // USE_BRANCHLESS
@@ -53,6 +74,10 @@ static __inline int32_t clamp1023(int32_t v) {
return (v > 1023) ? 1023 : v;
}
+static __inline int32_t ClampMax(int32_t v, int32_t max) {
+ return (v > max) ? max : v;
+}
+
static __inline uint32_t Abs(int32_t v) {
return (v < 0) ? -v : v;
}
@@ -71,7 +96,9 @@ static __inline uint32_t Clamp10(int32_t val) {
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
defined(_M_IX86) || defined(__arm__) || defined(_M_ARM) || \
(defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#define WRITEWORD(p, v) *(uint32_t*)(p) = v
+static inline void WRITEWORD(uint8_t* p, uint32_t v) {
+ memcpy(p, &v, 4);
+}
#else
static inline void WRITEWORD(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v & 255);
@@ -111,6 +138,21 @@ void RAWToARGBRow_C(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
}
}
+void RAWToRGBARow_C(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t r = src_raw[0];
+ uint8_t g = src_raw[1];
+ uint8_t b = src_raw[2];
+ dst_rgba[0] = 255u;
+ dst_rgba[1] = b;
+ dst_rgba[2] = g;
+ dst_rgba[3] = r;
+ dst_rgba += 4;
+ src_raw += 3;
+ }
+}
+
void RAWToRGB24Row_C(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
int x;
for (x = 0; x < width; ++x) {
@@ -130,12 +172,13 @@ void RGB565ToARGBRow_C(const uint8_t* src_rgb565,
int width) {
int x;
for (x = 0; x < width; ++x) {
- uint8_t b = src_rgb565[0] & 0x1f;
- uint8_t g = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3);
- uint8_t r = src_rgb565[1] >> 3;
- dst_argb[0] = (b << 3) | (b >> 2);
- dst_argb[1] = (g << 2) | (g >> 4);
- dst_argb[2] = (r << 3) | (r >> 2);
+ uint8_t b = STATIC_CAST(uint8_t, src_rgb565[0] & 0x1f);
+ uint8_t g = STATIC_CAST(
+ uint8_t, (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3));
+ uint8_t r = STATIC_CAST(uint8_t, src_rgb565[1] >> 3);
+ dst_argb[0] = STATIC_CAST(uint8_t, (b << 3) | (b >> 2));
+ dst_argb[1] = STATIC_CAST(uint8_t, (g << 2) | (g >> 4));
+ dst_argb[2] = STATIC_CAST(uint8_t, (r << 3) | (r >> 2));
dst_argb[3] = 255u;
dst_argb += 4;
src_rgb565 += 2;
@@ -147,13 +190,14 @@ void ARGB1555ToARGBRow_C(const uint8_t* src_argb1555,
int width) {
int x;
for (x = 0; x < width; ++x) {
- uint8_t b = src_argb1555[0] & 0x1f;
- uint8_t g = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3);
- uint8_t r = (src_argb1555[1] & 0x7c) >> 2;
- uint8_t a = src_argb1555[1] >> 7;
- dst_argb[0] = (b << 3) | (b >> 2);
- dst_argb[1] = (g << 3) | (g >> 2);
- dst_argb[2] = (r << 3) | (r >> 2);
+ uint8_t b = STATIC_CAST(uint8_t, src_argb1555[0] & 0x1f);
+ uint8_t g = STATIC_CAST(
+ uint8_t, (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3));
+ uint8_t r = STATIC_CAST(uint8_t, (src_argb1555[1] & 0x7c) >> 2);
+ uint8_t a = STATIC_CAST(uint8_t, src_argb1555[1] >> 7);
+ dst_argb[0] = STATIC_CAST(uint8_t, (b << 3) | (b >> 2));
+ dst_argb[1] = STATIC_CAST(uint8_t, (g << 3) | (g >> 2));
+ dst_argb[2] = STATIC_CAST(uint8_t, (r << 3) | (r >> 2));
dst_argb[3] = -a;
dst_argb += 4;
src_argb1555 += 2;
@@ -165,14 +209,14 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444,
int width) {
int x;
for (x = 0; x < width; ++x) {
- uint8_t b = src_argb4444[0] & 0x0f;
- uint8_t g = src_argb4444[0] >> 4;
- uint8_t r = src_argb4444[1] & 0x0f;
- uint8_t a = src_argb4444[1] >> 4;
- dst_argb[0] = (b << 4) | b;
- dst_argb[1] = (g << 4) | g;
- dst_argb[2] = (r << 4) | r;
- dst_argb[3] = (a << 4) | a;
+ uint8_t b = STATIC_CAST(uint8_t, src_argb4444[0] & 0x0f);
+ uint8_t g = STATIC_CAST(uint8_t, src_argb4444[0] >> 4);
+ uint8_t r = STATIC_CAST(uint8_t, src_argb4444[1] & 0x0f);
+ uint8_t a = STATIC_CAST(uint8_t, src_argb4444[1] >> 4);
+ dst_argb[0] = STATIC_CAST(uint8_t, (b << 4) | b);
+ dst_argb[1] = STATIC_CAST(uint8_t, (g << 4) | g);
+ dst_argb[2] = STATIC_CAST(uint8_t, (r << 4) | r);
+ dst_argb[3] = STATIC_CAST(uint8_t, (a << 4) | a);
dst_argb += 4;
src_argb4444 += 2;
}
@@ -181,7 +225,8 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444,
void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
int x;
for (x = 0; x < width; ++x) {
- uint32_t ar30 = *(const uint32_t*)src_ar30;
+ uint32_t ar30;
+ memcpy(&ar30, src_ar30, sizeof ar30);
uint32_t b = (ar30 >> 2) & 0xff;
uint32_t g = (ar30 >> 12) & 0xff;
uint32_t r = (ar30 >> 22) & 0xff;
@@ -195,7 +240,8 @@ void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
int x;
for (x = 0; x < width; ++x) {
- uint32_t ar30 = *(const uint32_t*)src_ar30;
+ uint32_t ar30;
+ memcpy(&ar30, src_ar30, sizeof ar30);
uint32_t b = (ar30 >> 2) & 0xff;
uint32_t g = (ar30 >> 12) & 0xff;
uint32_t r = (ar30 >> 22) & 0xff;
@@ -209,7 +255,8 @@ void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
int x;
for (x = 0; x < width; ++x) {
- uint32_t ar30 = *(const uint32_t*)src_ar30;
+ uint32_t ar30;
+ memcpy(&ar30, src_ar30, sizeof ar30);
uint32_t b = ar30 & 0x3ff;
uint32_t ga = ar30 & 0xc00ffc00;
uint32_t r = (ar30 >> 20) & 0x3ff;
@@ -219,6 +266,54 @@ void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
}
}
+void ARGBToABGRRow_C(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t b = src_argb[0];
+ uint8_t g = src_argb[1];
+ uint8_t r = src_argb[2];
+ uint8_t a = src_argb[3];
+ dst_abgr[0] = r;
+ dst_abgr[1] = g;
+ dst_abgr[2] = b;
+ dst_abgr[3] = a;
+ dst_abgr += 4;
+ src_argb += 4;
+ }
+}
+
+void ARGBToBGRARow_C(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t b = src_argb[0];
+ uint8_t g = src_argb[1];
+ uint8_t r = src_argb[2];
+ uint8_t a = src_argb[3];
+ dst_bgra[0] = a;
+ dst_bgra[1] = r;
+ dst_bgra[2] = g;
+ dst_bgra[3] = b;
+ dst_bgra += 4;
+ src_argb += 4;
+ }
+}
+
+void ARGBToRGBARow_C(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t b = src_argb[0];
+ uint8_t g = src_argb[1];
+ uint8_t r = src_argb[2];
+ uint8_t a = src_argb[3];
+ dst_rgba[0] = a;
+ dst_rgba[1] = b;
+ dst_rgba[2] = g;
+ dst_rgba[3] = r;
+ dst_rgba += 4;
+ src_argb += 4;
+ }
+}
+
void ARGBToRGB24Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
int x;
for (x = 0; x < width; ++x) {
@@ -247,6 +342,22 @@ void ARGBToRAWRow_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
}
}
+void RGBAToARGBRow_C(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t a = src_rgba[0];
+ uint8_t b = src_rgba[1];
+ uint8_t g = src_rgba[2];
+ uint8_t r = src_rgba[3];
+ dst_argb[0] = b;
+ dst_argb[1] = g;
+ dst_argb[2] = r;
+ dst_argb[3] = a;
+ dst_argb += 4;
+ src_rgba += 4;
+ }
+}
+
void ARGBToRGB565Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
int x;
for (x = 0; x < width - 1; x += 2) {
@@ -265,7 +376,7 @@ void ARGBToRGB565Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
uint8_t b0 = src_argb[0] >> 3;
uint8_t g0 = src_argb[1] >> 2;
uint8_t r0 = src_argb[2] >> 3;
- *(uint16_t*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 11);
+ *(uint16_t*)(dst_rgb) = STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 11));
}
}
@@ -279,29 +390,31 @@ void ARGBToRGB565Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
// or the upper byte for big endian.
void ARGBToRGB565DitherRow_C(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width) {
int x;
for (x = 0; x < width - 1; x += 2) {
int dither0 = ((const unsigned char*)(&dither4))[x & 3];
int dither1 = ((const unsigned char*)(&dither4))[(x + 1) & 3];
- uint8_t b0 = clamp255(src_argb[0] + dither0) >> 3;
- uint8_t g0 = clamp255(src_argb[1] + dither0) >> 2;
- uint8_t r0 = clamp255(src_argb[2] + dither0) >> 3;
- uint8_t b1 = clamp255(src_argb[4] + dither1) >> 3;
- uint8_t g1 = clamp255(src_argb[5] + dither1) >> 2;
- uint8_t r1 = clamp255(src_argb[6] + dither1) >> 3;
- WRITEWORD(dst_rgb, b0 | (g0 << 5) | (r0 << 11) | (b1 << 16) | (g1 << 21) |
- (r1 << 27));
+ uint8_t b0 = STATIC_CAST(uint8_t, clamp255(src_argb[0] + dither0) >> 3);
+ uint8_t g0 = STATIC_CAST(uint8_t, clamp255(src_argb[1] + dither0) >> 2);
+ uint8_t r0 = STATIC_CAST(uint8_t, clamp255(src_argb[2] + dither0) >> 3);
+ uint8_t b1 = STATIC_CAST(uint8_t, clamp255(src_argb[4] + dither1) >> 3);
+ uint8_t g1 = STATIC_CAST(uint8_t, clamp255(src_argb[5] + dither1) >> 2);
+ uint8_t r1 = STATIC_CAST(uint8_t, clamp255(src_argb[6] + dither1) >> 3);
+ *(uint16_t*)(dst_rgb + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 11));
+ *(uint16_t*)(dst_rgb + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 5) | (r1 << 11));
dst_rgb += 4;
src_argb += 8;
}
if (width & 1) {
int dither0 = ((const unsigned char*)(&dither4))[(width - 1) & 3];
- uint8_t b0 = clamp255(src_argb[0] + dither0) >> 3;
- uint8_t g0 = clamp255(src_argb[1] + dither0) >> 2;
- uint8_t r0 = clamp255(src_argb[2] + dither0) >> 3;
- *(uint16_t*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 11);
+ uint8_t b0 = STATIC_CAST(uint8_t, clamp255(src_argb[0] + dither0) >> 3);
+ uint8_t g0 = STATIC_CAST(uint8_t, clamp255(src_argb[1] + dither0) >> 2);
+ uint8_t r0 = STATIC_CAST(uint8_t, clamp255(src_argb[2] + dither0) >> 3);
+ *(uint16_t*)(dst_rgb) = STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 11));
}
}
@@ -316,8 +429,10 @@ void ARGBToARGB1555Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
uint8_t g1 = src_argb[5] >> 3;
uint8_t r1 = src_argb[6] >> 3;
uint8_t a1 = src_argb[7] >> 7;
- *(uint32_t*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 10) | (a0 << 15) |
- (b1 << 16) | (g1 << 21) | (r1 << 26) | (a1 << 31);
+ *(uint16_t*)(dst_rgb + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 10) | (a0 << 15));
+ *(uint16_t*)(dst_rgb + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 5) | (r1 << 10) | (a1 << 15));
dst_rgb += 4;
src_argb += 8;
}
@@ -326,7 +441,8 @@ void ARGBToARGB1555Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
uint8_t g0 = src_argb[1] >> 3;
uint8_t r0 = src_argb[2] >> 3;
uint8_t a0 = src_argb[3] >> 7;
- *(uint16_t*)(dst_rgb) = b0 | (g0 << 5) | (r0 << 10) | (a0 << 15);
+ *(uint16_t*)(dst_rgb) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 10) | (a0 << 15));
}
}
@@ -341,8 +457,10 @@ void ARGBToARGB4444Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
uint8_t g1 = src_argb[5] >> 4;
uint8_t r1 = src_argb[6] >> 4;
uint8_t a1 = src_argb[7] >> 4;
- *(uint32_t*)(dst_rgb) = b0 | (g0 << 4) | (r0 << 8) | (a0 << 12) |
- (b1 << 16) | (g1 << 20) | (r1 << 24) | (a1 << 28);
+ *(uint16_t*)(dst_rgb + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 4) | (r0 << 8) | (a0 << 12));
+ *(uint16_t*)(dst_rgb + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 4) | (r1 << 8) | (a1 << 12));
dst_rgb += 4;
src_argb += 8;
}
@@ -351,18 +469,20 @@ void ARGBToARGB4444Row_C(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
uint8_t g0 = src_argb[1] >> 4;
uint8_t r0 = src_argb[2] >> 4;
uint8_t a0 = src_argb[3] >> 4;
- *(uint16_t*)(dst_rgb) = b0 | (g0 << 4) | (r0 << 8) | (a0 << 12);
+ *(uint16_t*)(dst_rgb) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 4) | (r0 << 8) | (a0 << 12));
}
}
void ABGRToAR30Row_C(const uint8_t* src_abgr, uint8_t* dst_ar30, int width) {
int x;
for (x = 0; x < width; ++x) {
- uint32_t b0 = (src_abgr[0] >> 6) | ((uint32_t)(src_abgr[0]) << 2);
+ uint32_t r0 = (src_abgr[0] >> 6) | ((uint32_t)(src_abgr[0]) << 2);
uint32_t g0 = (src_abgr[1] >> 6) | ((uint32_t)(src_abgr[1]) << 2);
- uint32_t r0 = (src_abgr[2] >> 6) | ((uint32_t)(src_abgr[2]) << 2);
+ uint32_t b0 = (src_abgr[2] >> 6) | ((uint32_t)(src_abgr[2]) << 2);
uint32_t a0 = (src_abgr[3] >> 6);
- *(uint32_t*)(dst_ar30) = r0 | (g0 << 10) | (b0 << 20) | (a0 << 30);
+ *(uint32_t*)(dst_ar30) =
+ STATIC_CAST(uint32_t, b0 | (g0 << 10) | (r0 << 20) | (a0 << 30));
dst_ar30 += 4;
src_abgr += 4;
}
@@ -375,158 +495,269 @@ void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width) {
uint32_t g0 = (src_argb[1] >> 6) | ((uint32_t)(src_argb[1]) << 2);
uint32_t r0 = (src_argb[2] >> 6) | ((uint32_t)(src_argb[2]) << 2);
uint32_t a0 = (src_argb[3] >> 6);
- *(uint32_t*)(dst_ar30) = b0 | (g0 << 10) | (r0 << 20) | (a0 << 30);
+ *(uint32_t*)(dst_ar30) =
+ STATIC_CAST(uint32_t, b0 | (g0 << 10) | (r0 << 20) | (a0 << 30));
dst_ar30 += 4;
src_argb += 4;
}
}
-static __inline int RGBToY(uint8_t r, uint8_t g, uint8_t b) {
- return (66 * r + 129 * g + 25 * b + 0x1080) >> 8;
-}
-
-static __inline int RGBToU(uint8_t r, uint8_t g, uint8_t b) {
- return (112 * b - 74 * g - 38 * r + 0x8080) >> 8;
-}
-static __inline int RGBToV(uint8_t r, uint8_t g, uint8_t b) {
- return (112 * r - 94 * g - 18 * b + 0x8080) >> 8;
-}
-
-// ARGBToY_C and ARGBToUV_C
-#define MAKEROWY(NAME, R, G, B, BPP) \
- void NAME##ToYRow_C(const uint8_t* src_argb0, uint8_t* dst_y, int width) { \
- int x; \
- for (x = 0; x < width; ++x) { \
- dst_y[0] = RGBToY(src_argb0[R], src_argb0[G], src_argb0[B]); \
- src_argb0 += BPP; \
- dst_y += 1; \
- } \
- } \
- void NAME##ToUVRow_C(const uint8_t* src_rgb0, int src_stride_rgb, \
- uint8_t* dst_u, uint8_t* dst_v, int width) { \
- const uint8_t* src_rgb1 = src_rgb0 + src_stride_rgb; \
- int x; \
- for (x = 0; x < width - 1; x += 2) { \
- uint8_t ab = (src_rgb0[B] + src_rgb0[B + BPP] + src_rgb1[B] + \
- src_rgb1[B + BPP]) >> \
- 2; \
- uint8_t ag = (src_rgb0[G] + src_rgb0[G + BPP] + src_rgb1[G] + \
- src_rgb1[G + BPP]) >> \
- 2; \
- uint8_t ar = (src_rgb0[R] + src_rgb0[R + BPP] + src_rgb1[R] + \
- src_rgb1[R + BPP]) >> \
- 2; \
- dst_u[0] = RGBToU(ar, ag, ab); \
- dst_v[0] = RGBToV(ar, ag, ab); \
- src_rgb0 += BPP * 2; \
- src_rgb1 += BPP * 2; \
- dst_u += 1; \
- dst_v += 1; \
- } \
- if (width & 1) { \
- uint8_t ab = (src_rgb0[B] + src_rgb1[B]) >> 1; \
- uint8_t ag = (src_rgb0[G] + src_rgb1[G]) >> 1; \
- uint8_t ar = (src_rgb0[R] + src_rgb1[R]) >> 1; \
- dst_u[0] = RGBToU(ar, ag, ab); \
- dst_v[0] = RGBToV(ar, ag, ab); \
- } \
- }
-
-MAKEROWY(ARGB, 2, 1, 0, 4)
-MAKEROWY(BGRA, 1, 2, 3, 4)
-MAKEROWY(ABGR, 0, 1, 2, 4)
-MAKEROWY(RGBA, 3, 2, 1, 4)
-MAKEROWY(RGB24, 2, 1, 0, 3)
-MAKEROWY(RAW, 0, 1, 2, 3)
-#undef MAKEROWY
-
-// JPeg uses a variation on BT.601-1 full range
+void ARGBToAR64Row_C(const uint8_t* src_argb, uint16_t* dst_ar64, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint16_t b = src_argb[0] * 0x0101;
+ uint16_t g = src_argb[1] * 0x0101;
+ uint16_t r = src_argb[2] * 0x0101;
+ uint16_t a = src_argb[3] * 0x0101;
+ dst_ar64[0] = b;
+ dst_ar64[1] = g;
+ dst_ar64[2] = r;
+ dst_ar64[3] = a;
+ dst_ar64 += 4;
+ src_argb += 4;
+ }
+}
+
+void ARGBToAB64Row_C(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint16_t b = src_argb[0] * 0x0101;
+ uint16_t g = src_argb[1] * 0x0101;
+ uint16_t r = src_argb[2] * 0x0101;
+ uint16_t a = src_argb[3] * 0x0101;
+ dst_ab64[0] = r;
+ dst_ab64[1] = g;
+ dst_ab64[2] = b;
+ dst_ab64[3] = a;
+ dst_ab64 += 4;
+ src_argb += 4;
+ }
+}
+
+void AR64ToARGBRow_C(const uint16_t* src_ar64, uint8_t* dst_argb, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t b = src_ar64[0] >> 8;
+ uint8_t g = src_ar64[1] >> 8;
+ uint8_t r = src_ar64[2] >> 8;
+ uint8_t a = src_ar64[3] >> 8;
+ dst_argb[0] = b;
+ dst_argb[1] = g;
+ dst_argb[2] = r;
+ dst_argb[3] = a;
+ dst_argb += 4;
+ src_ar64 += 4;
+ }
+}
+
+void AB64ToARGBRow_C(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t r = src_ab64[0] >> 8;
+ uint8_t g = src_ab64[1] >> 8;
+ uint8_t b = src_ab64[2] >> 8;
+ uint8_t a = src_ab64[3] >> 8;
+ dst_argb[0] = b;
+ dst_argb[1] = g;
+ dst_argb[2] = r;
+ dst_argb[3] = a;
+ dst_argb += 4;
+ src_ab64 += 4;
+ }
+}
+
+void AR64ToAB64Row_C(const uint16_t* src_ar64, uint16_t* dst_ab64, int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint16_t b = src_ar64[0];
+ uint16_t g = src_ar64[1];
+ uint16_t r = src_ar64[2];
+ uint16_t a = src_ar64[3];
+ dst_ab64[0] = r;
+ dst_ab64[1] = g;
+ dst_ab64[2] = b;
+ dst_ab64[3] = a;
+ dst_ab64 += 4;
+ src_ar64 += 4;
+ }
+}
+
+// TODO(fbarchard): Make shuffle compatible with SIMD versions
+// width is in units of 32 bit ARGB pixels (2 per AR64 pixel) so that
+// AR64Shuffle can share the ARGBShuffleRow SIMD kernels.
+void AR64ShuffleRow_C(const uint8_t* src_ar64,
+ uint8_t* dst_ar64,
+ const uint8_t* shuffler,
+ int width) {
+ const uint16_t* src_ar64_16 = (const uint16_t*)src_ar64;
+ uint16_t* dst_ar64_16 = (uint16_t*)dst_ar64;
+ int index0 = shuffler[0] / 2;
+ int index1 = shuffler[2] / 2;
+ int index2 = shuffler[4] / 2;
+ int index3 = shuffler[6] / 2;
+ // Shuffle a row of AR64.
+ int x;
+ for (x = 0; x < width / 2; ++x) {
+ // To support in-place conversion.
+ uint16_t b = src_ar64_16[index0];
+ uint16_t g = src_ar64_16[index1];
+ uint16_t r = src_ar64_16[index2];
+ uint16_t a = src_ar64_16[index3];
+ dst_ar64_16[0] = b;
+ dst_ar64_16[1] = g;
+ dst_ar64_16[2] = r;
+ dst_ar64_16[3] = a;
+ src_ar64_16 += 4;
+ dst_ar64_16 += 4;
+ }
+}
+// BT601 8 bit Y:
+// b 0.114 * 219 = 24.966 = 25
+// g 0.587 * 219 = 128.553 = 129
+// r 0.299 * 219 = 65.481 = 66
+// BT601 8 bit U:
+// b 0.875 * 128 = 112.0 = 112
+// g -0.5781 * 128 = −73.9968 = -74
+// r -0.2969 * 128 = −38.0032 = -38
+// BT601 8 bit V:
+// b -0.1406 * 128 = −17.9968 = -18
+// g -0.7344 * 128 = −94.0032 = -94
+// r 0.875 * 128 = 112.0 = 112
+static __inline uint8_t RGBToY(uint8_t r, uint8_t g, uint8_t b) {
+ return STATIC_CAST(uint8_t, (66 * r + 129 * g + 25 * b + 0x1080) >> 8);
+}
+static __inline uint8_t RGBToU(uint8_t r, uint8_t g, uint8_t b) {
+ return STATIC_CAST(uint8_t, (112 * b - 74 * g - 38 * r + 0x8000) >> 8);
+}
+static __inline uint8_t RGBToV(uint8_t r, uint8_t g, uint8_t b) {
+ return STATIC_CAST(uint8_t, (112 * r - 94 * g - 18 * b + 0x8000) >> 8);
+}
+#define AVGB(a, b) (((a) + (b) + 1) >> 1)
+
+// JPeg uses BT.601-1 full range
// y = 0.29900 * r + 0.58700 * g + 0.11400 * b
// u = -0.16874 * r - 0.33126 * g + 0.50000 * b + center
// v = 0.50000 * r - 0.41869 * g - 0.08131 * b + center
-// BT.601 Mpeg range uses:
-// b 0.1016 * 255 = 25.908 = 25
-// g 0.5078 * 255 = 129.489 = 129
-// r 0.2578 * 255 = 65.739 = 66
-// JPeg 8 bit Y (not used):
+// JPeg 8 bit Y:
// b 0.11400 * 256 = 29.184 = 29
// g 0.58700 * 256 = 150.272 = 150
// r 0.29900 * 256 = 76.544 = 77
-// JPeg 7 bit Y:
-// b 0.11400 * 128 = 14.592 = 15
-// g 0.58700 * 128 = 75.136 = 75
-// r 0.29900 * 128 = 38.272 = 38
// JPeg 8 bit U:
-// b 0.50000 * 255 = 127.5 = 127
-// g -0.33126 * 255 = -84.4713 = -84
-// r -0.16874 * 255 = -43.0287 = -43
+// b 0.50000 * 256 = 128.0 = 128
+// g -0.33126 * 256 = −84.80256 = -85
+// r -0.16874 * 256 = −43.19744 = -43
// JPeg 8 bit V:
-// b -0.08131 * 255 = -20.73405 = -20
-// g -0.41869 * 255 = -106.76595 = -107
-// r 0.50000 * 255 = 127.5 = 127
-
-static __inline int RGBToYJ(uint8_t r, uint8_t g, uint8_t b) {
- return (38 * r + 75 * g + 15 * b + 64) >> 7;
+// b -0.08131 * 256 = −20.81536 = -21
+// g -0.41869 * 256 = −107.18464 = -107
+// r 0.50000 * 256 = 128.0 = 128
+
+// 8 bit
+static __inline uint8_t RGBToYJ(uint8_t r, uint8_t g, uint8_t b) {
+ return (77 * r + 150 * g + 29 * b + 128) >> 8;
+}
+
+static __inline uint8_t RGBToYMatrix(uint8_t b0,
+ uint8_t b1,
+ uint8_t b2,
+ uint8_t b3,
+ const struct ArgbConstants* c) {
+ return (c->kRGBToY[0] * b0 + c->kRGBToY[1] * b1 + c->kRGBToY[2] * b2 +
+ c->kRGBToY[3] * b3 + c->kAddY[0]) >>
+ 8;
+}
+static __inline uint8_t RGBToUMatrix(uint8_t b0,
+ uint8_t b1,
+ uint8_t b2,
+ uint8_t b3,
+ const struct ArgbConstants* c) {
+ return (c->kAddUV[0] - (c->kRGBToU[0] * b0 + c->kRGBToU[1] * b1 +
+ c->kRGBToU[2] * b2 + c->kRGBToU[3] * b3)) >>
+ 8;
+}
+static __inline uint8_t RGBToVMatrix(uint8_t b0,
+ uint8_t b1,
+ uint8_t b2,
+ uint8_t b3,
+ const struct ArgbConstants* c) {
+ return (c->kAddUV[0] - (c->kRGBToV[0] * b0 + c->kRGBToV[1] * b1 +
+ c->kRGBToV[2] * b2 + c->kRGBToV[3] * b3)) >>
+ 8;
+}
+
+void ARGBToYMatrixRow_C(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_y[0] =
+ RGBToYMatrix(src_argb[0], src_argb[1], src_argb[2], src_argb[3], c);
+ src_argb += 4;
+ dst_y += 1;
+ }
}
-static __inline int RGBToUJ(uint8_t r, uint8_t g, uint8_t b) {
- return (127 * b - 84 * g - 43 * r + 0x8080) >> 8;
-}
-static __inline int RGBToVJ(uint8_t r, uint8_t g, uint8_t b) {
- return (127 * r - 107 * g - 20 * b + 0x8080) >> 8;
+void ARGBToUVMatrixRow_C(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_argb1 = src_argb + src_stride_argb;
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ uint8_t b0 =
+ (src_argb[0] + src_argb[4] + src_argb1[0] + src_argb1[4] + 2) >> 2;
+ uint8_t b1 =
+ (src_argb[1] + src_argb[5] + src_argb1[1] + src_argb1[5] + 2) >> 2;
+ uint8_t b2 =
+ (src_argb[2] + src_argb[6] + src_argb1[2] + src_argb1[6] + 2) >> 2;
+ uint8_t b3 =
+ (src_argb[3] + src_argb[7] + src_argb1[3] + src_argb1[7] + 2) >> 2;
+ dst_u[0] = RGBToUMatrix(b0, b1, b2, b3, c);
+ dst_v[0] = RGBToVMatrix(b0, b1, b2, b3, c);
+ src_argb += 8;
+ src_argb1 += 8;
+ dst_u += 1;
+ dst_v += 1;
+ }
+ if (width & 1) {
+ uint8_t b0 = (src_argb[0] + src_argb1[0] + 1) >> 1;
+ uint8_t b1 = (src_argb[1] + src_argb1[1] + 1) >> 1;
+ uint8_t b2 = (src_argb[2] + src_argb1[2] + 1) >> 1;
+ uint8_t b3 = (src_argb[3] + src_argb1[3] + 1) >> 1;
+ dst_u[0] = RGBToUMatrix(b0, b1, b2, b3, c);
+ dst_v[0] = RGBToVMatrix(b0, b1, b2, b3, c);
+ }
}
-#define AVGB(a, b) (((a) + (b) + 1) >> 1)
-
-// ARGBToYJ_C and ARGBToUVJ_C
-#define MAKEROWYJ(NAME, R, G, B, BPP) \
- void NAME##ToYJRow_C(const uint8_t* src_argb0, uint8_t* dst_y, int width) { \
- int x; \
- for (x = 0; x < width; ++x) { \
- dst_y[0] = RGBToYJ(src_argb0[R], src_argb0[G], src_argb0[B]); \
- src_argb0 += BPP; \
- dst_y += 1; \
- } \
- } \
- void NAME##ToUVJRow_C(const uint8_t* src_rgb0, int src_stride_rgb, \
- uint8_t* dst_u, uint8_t* dst_v, int width) { \
- const uint8_t* src_rgb1 = src_rgb0 + src_stride_rgb; \
- int x; \
- for (x = 0; x < width - 1; x += 2) { \
- uint8_t ab = AVGB(AVGB(src_rgb0[B], src_rgb1[B]), \
- AVGB(src_rgb0[B + BPP], src_rgb1[B + BPP])); \
- uint8_t ag = AVGB(AVGB(src_rgb0[G], src_rgb1[G]), \
- AVGB(src_rgb0[G + BPP], src_rgb1[G + BPP])); \
- uint8_t ar = AVGB(AVGB(src_rgb0[R], src_rgb1[R]), \
- AVGB(src_rgb0[R + BPP], src_rgb1[R + BPP])); \
- dst_u[0] = RGBToUJ(ar, ag, ab); \
- dst_v[0] = RGBToVJ(ar, ag, ab); \
- src_rgb0 += BPP * 2; \
- src_rgb1 += BPP * 2; \
- dst_u += 1; \
- dst_v += 1; \
- } \
- if (width & 1) { \
- uint8_t ab = AVGB(src_rgb0[B], src_rgb1[B]); \
- uint8_t ag = AVGB(src_rgb0[G], src_rgb1[G]); \
- uint8_t ar = AVGB(src_rgb0[R], src_rgb1[R]); \
- dst_u[0] = RGBToUJ(ar, ag, ab); \
- dst_v[0] = RGBToVJ(ar, ag, ab); \
- } \
- }
-
-MAKEROWYJ(ARGB, 2, 1, 0, 4)
-#undef MAKEROWYJ
+void ARGBToUV444MatrixRow_C(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_u[0] =
+ RGBToUMatrix(src_argb[0], src_argb[1], src_argb[2], src_argb[3], c);
+ dst_v[0] =
+ RGBToVMatrix(src_argb[0], src_argb[1], src_argb[2], src_argb[3], c);
+ src_argb += 4;
+ dst_u += 1;
+ dst_v += 1;
+ }
+}
void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
int x;
for (x = 0; x < width; ++x) {
uint8_t b = src_rgb565[0] & 0x1f;
- uint8_t g = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3);
+ uint8_t g = STATIC_CAST(
+ uint8_t, (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3));
uint8_t r = src_rgb565[1] >> 3;
- b = (b << 3) | (b >> 2);
- g = (g << 2) | (g >> 4);
- r = (r << 3) | (r >> 2);
+ b = STATIC_CAST(uint8_t, (b << 3) | (b >> 2));
+ g = STATIC_CAST(uint8_t, (g << 2) | (g >> 4));
+ r = STATIC_CAST(uint8_t, (r << 3) | (r >> 2));
dst_y[0] = RGBToY(r, g, b);
src_rgb565 += 2;
dst_y += 1;
@@ -537,11 +768,12 @@ void ARGB1555ToYRow_C(const uint8_t* src_argb1555, uint8_t* dst_y, int width) {
int x;
for (x = 0; x < width; ++x) {
uint8_t b = src_argb1555[0] & 0x1f;
- uint8_t g = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3);
+ uint8_t g = STATIC_CAST(
+ uint8_t, (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3));
uint8_t r = (src_argb1555[1] & 0x7c) >> 2;
- b = (b << 3) | (b >> 2);
- g = (g << 3) | (g >> 2);
- r = (r << 3) | (r >> 2);
+ b = STATIC_CAST(uint8_t, (b << 3) | (b >> 2));
+ g = STATIC_CAST(uint8_t, (g << 3) | (g >> 2));
+ r = STATIC_CAST(uint8_t, (r << 3) | (r >> 2));
dst_y[0] = RGBToY(r, g, b);
src_argb1555 += 2;
dst_y += 1;
@@ -554,9 +786,9 @@ void ARGB4444ToYRow_C(const uint8_t* src_argb4444, uint8_t* dst_y, int width) {
uint8_t b = src_argb4444[0] & 0x0f;
uint8_t g = src_argb4444[0] >> 4;
uint8_t r = src_argb4444[1] & 0x0f;
- b = (b << 4) | b;
- g = (g << 4) | g;
- r = (r << 4) | r;
+ b = STATIC_CAST(uint8_t, (b << 4) | b);
+ g = STATIC_CAST(uint8_t, (g << 4) | g);
+ r = STATIC_CAST(uint8_t, (r << 4) | r);
dst_y[0] = RGBToY(r, g, b);
src_argb4444 += 2;
dst_y += 1;
@@ -571,45 +803,68 @@ void RGB565ToUVRow_C(const uint8_t* src_rgb565,
const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565;
int x;
for (x = 0; x < width - 1; x += 2) {
- uint8_t b0 = src_rgb565[0] & 0x1f;
- uint8_t g0 = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3);
- uint8_t r0 = src_rgb565[1] >> 3;
- uint8_t b1 = src_rgb565[2] & 0x1f;
- uint8_t g1 = (src_rgb565[2] >> 5) | ((src_rgb565[3] & 0x07) << 3);
- uint8_t r1 = src_rgb565[3] >> 3;
- uint8_t b2 = next_rgb565[0] & 0x1f;
- uint8_t g2 = (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3);
- uint8_t r2 = next_rgb565[1] >> 3;
- uint8_t b3 = next_rgb565[2] & 0x1f;
- uint8_t g3 = (next_rgb565[2] >> 5) | ((next_rgb565[3] & 0x07) << 3);
- uint8_t r3 = next_rgb565[3] >> 3;
- uint8_t b = (b0 + b1 + b2 + b3); // 565 * 4 = 787.
- uint8_t g = (g0 + g1 + g2 + g3);
- uint8_t r = (r0 + r1 + r2 + r3);
- b = (b << 1) | (b >> 6); // 787 -> 888.
- r = (r << 1) | (r >> 6);
+ uint8_t b0 = STATIC_CAST(uint8_t, src_rgb565[0] & 0x1f);
+ uint8_t g0 = STATIC_CAST(
+ uint8_t, (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3));
+ uint8_t r0 = STATIC_CAST(uint8_t, src_rgb565[1] >> 3);
+ uint8_t b1 = STATIC_CAST(uint8_t, src_rgb565[2] & 0x1f);
+ uint8_t g1 = STATIC_CAST(
+ uint8_t, (src_rgb565[2] >> 5) | ((src_rgb565[3] & 0x07) << 3));
+ uint8_t r1 = STATIC_CAST(uint8_t, src_rgb565[3] >> 3);
+ uint8_t b2 = STATIC_CAST(uint8_t, next_rgb565[0] & 0x1f);
+ uint8_t g2 = STATIC_CAST(
+ uint8_t, (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3));
+ uint8_t r2 = STATIC_CAST(uint8_t, next_rgb565[1] >> 3);
+ uint8_t b3 = STATIC_CAST(uint8_t, next_rgb565[2] & 0x1f);
+ uint8_t g3 = STATIC_CAST(
+ uint8_t, (next_rgb565[2] >> 5) | ((next_rgb565[3] & 0x07) << 3));
+ uint8_t r3 = STATIC_CAST(uint8_t, next_rgb565[3] >> 3);
+
+ b0 = STATIC_CAST(uint8_t, (b0 << 3) | (b0 >> 2));
+ g0 = STATIC_CAST(uint8_t, (g0 << 2) | (g0 >> 4));
+ r0 = STATIC_CAST(uint8_t, (r0 << 3) | (r0 >> 2));
+ b1 = STATIC_CAST(uint8_t, (b1 << 3) | (b1 >> 2));
+ g1 = STATIC_CAST(uint8_t, (g1 << 2) | (g1 >> 4));
+ r1 = STATIC_CAST(uint8_t, (r1 << 3) | (r1 >> 2));
+ b2 = STATIC_CAST(uint8_t, (b2 << 3) | (b2 >> 2));
+ g2 = STATIC_CAST(uint8_t, (g2 << 2) | (g2 >> 4));
+ r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
+ b3 = STATIC_CAST(uint8_t, (b3 << 3) | (b3 >> 2));
+ g3 = STATIC_CAST(uint8_t, (g3 << 2) | (g3 >> 4));
+ r3 = STATIC_CAST(uint8_t, (r3 << 3) | (r3 >> 2));
+
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
dst_u[0] = RGBToU(r, g, b);
dst_v[0] = RGBToV(r, g, b);
+
src_rgb565 += 4;
next_rgb565 += 4;
dst_u += 1;
dst_v += 1;
}
if (width & 1) {
- uint8_t b0 = src_rgb565[0] & 0x1f;
- uint8_t g0 = (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3);
- uint8_t r0 = src_rgb565[1] >> 3;
- uint8_t b2 = next_rgb565[0] & 0x1f;
- uint8_t g2 = (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3);
- uint8_t r2 = next_rgb565[1] >> 3;
- uint8_t b = (b0 + b2); // 565 * 2 = 676.
- uint8_t g = (g0 + g2);
- uint8_t r = (r0 + r2);
- b = (b << 2) | (b >> 4); // 676 -> 888
- g = (g << 1) | (g >> 6);
- r = (r << 2) | (r >> 4);
- dst_u[0] = RGBToU(r, g, b);
- dst_v[0] = RGBToV(r, g, b);
+ uint8_t b0 = STATIC_CAST(uint8_t, src_rgb565[0] & 0x1f);
+ uint8_t g0 = STATIC_CAST(
+ uint8_t, (src_rgb565[0] >> 5) | ((src_rgb565[1] & 0x07) << 3));
+ uint8_t r0 = STATIC_CAST(uint8_t, src_rgb565[1] >> 3);
+ uint8_t b2 = STATIC_CAST(uint8_t, next_rgb565[0] & 0x1f);
+ uint8_t g2 = STATIC_CAST(
+ uint8_t, (next_rgb565[0] >> 5) | ((next_rgb565[1] & 0x07) << 3));
+ uint8_t r2 = STATIC_CAST(uint8_t, next_rgb565[1] >> 3);
+ b0 = STATIC_CAST(uint8_t, (b0 << 3) | (b0 >> 2));
+ g0 = STATIC_CAST(uint8_t, (g0 << 2) | (g0 >> 4));
+ r0 = STATIC_CAST(uint8_t, (r0 << 3) | (r0 >> 2));
+ b2 = STATIC_CAST(uint8_t, (b2 << 3) | (b2 >> 2));
+ g2 = STATIC_CAST(uint8_t, (g2 << 2) | (g2 >> 4));
+ r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
+
+ uint8_t ab = AVGB(b0, b2);
+ uint8_t ag = AVGB(g0, g2);
+ uint8_t ar = AVGB(r0, r2);
+ dst_u[0] = RGBToU(ar, ag, ab);
+ dst_v[0] = RGBToV(ar, ag, ab);
}
}
@@ -621,46 +876,69 @@ void ARGB1555ToUVRow_C(const uint8_t* src_argb1555,
const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555;
int x;
for (x = 0; x < width - 1; x += 2) {
- uint8_t b0 = src_argb1555[0] & 0x1f;
- uint8_t g0 = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3);
- uint8_t r0 = (src_argb1555[1] & 0x7c) >> 2;
- uint8_t b1 = src_argb1555[2] & 0x1f;
- uint8_t g1 = (src_argb1555[2] >> 5) | ((src_argb1555[3] & 0x03) << 3);
- uint8_t r1 = (src_argb1555[3] & 0x7c) >> 2;
- uint8_t b2 = next_argb1555[0] & 0x1f;
- uint8_t g2 = (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3);
- uint8_t r2 = (next_argb1555[1] & 0x7c) >> 2;
- uint8_t b3 = next_argb1555[2] & 0x1f;
- uint8_t g3 = (next_argb1555[2] >> 5) | ((next_argb1555[3] & 0x03) << 3);
- uint8_t r3 = (next_argb1555[3] & 0x7c) >> 2;
- uint8_t b = (b0 + b1 + b2 + b3); // 555 * 4 = 777.
- uint8_t g = (g0 + g1 + g2 + g3);
- uint8_t r = (r0 + r1 + r2 + r3);
- b = (b << 1) | (b >> 6); // 777 -> 888.
- g = (g << 1) | (g >> 6);
- r = (r << 1) | (r >> 6);
+ uint8_t b0 = STATIC_CAST(uint8_t, src_argb1555[0] & 0x1f);
+ uint8_t g0 = STATIC_CAST(
+ uint8_t, (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3));
+ uint8_t r0 = STATIC_CAST(uint8_t, (src_argb1555[1] & 0x7c) >> 2);
+ uint8_t b1 = STATIC_CAST(uint8_t, src_argb1555[2] & 0x1f);
+ uint8_t g1 = STATIC_CAST(
+ uint8_t, (src_argb1555[2] >> 5) | ((src_argb1555[3] & 0x03) << 3));
+ uint8_t r1 = STATIC_CAST(uint8_t, (src_argb1555[3] & 0x7c) >> 2);
+ uint8_t b2 = STATIC_CAST(uint8_t, next_argb1555[0] & 0x1f);
+ uint8_t g2 = STATIC_CAST(
+ uint8_t, (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3));
+ uint8_t r2 = STATIC_CAST(uint8_t, (next_argb1555[1] & 0x7c) >> 2);
+ uint8_t b3 = STATIC_CAST(uint8_t, next_argb1555[2] & 0x1f);
+ uint8_t g3 = STATIC_CAST(
+ uint8_t, (next_argb1555[2] >> 5) | ((next_argb1555[3] & 0x03) << 3));
+ uint8_t r3 = STATIC_CAST(uint8_t, (next_argb1555[3] & 0x7c) >> 2);
+
+ b0 = STATIC_CAST(uint8_t, (b0 << 3) | (b0 >> 2));
+ g0 = STATIC_CAST(uint8_t, (g0 << 3) | (g0 >> 2));
+ r0 = STATIC_CAST(uint8_t, (r0 << 3) | (r0 >> 2));
+ b1 = STATIC_CAST(uint8_t, (b1 << 3) | (b1 >> 2));
+ g1 = STATIC_CAST(uint8_t, (g1 << 3) | (g1 >> 2));
+ r1 = STATIC_CAST(uint8_t, (r1 << 3) | (r1 >> 2));
+ b2 = STATIC_CAST(uint8_t, (b2 << 3) | (b2 >> 2));
+ g2 = STATIC_CAST(uint8_t, (g2 << 3) | (g2 >> 2));
+ r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
+ b3 = STATIC_CAST(uint8_t, (b3 << 3) | (b3 >> 2));
+ g3 = STATIC_CAST(uint8_t, (g3 << 3) | (g3 >> 2));
+ r3 = STATIC_CAST(uint8_t, (r3 << 3) | (r3 >> 2));
+
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
dst_u[0] = RGBToU(r, g, b);
dst_v[0] = RGBToV(r, g, b);
+
src_argb1555 += 4;
next_argb1555 += 4;
dst_u += 1;
dst_v += 1;
}
if (width & 1) {
- uint8_t b0 = src_argb1555[0] & 0x1f;
- uint8_t g0 = (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3);
- uint8_t r0 = (src_argb1555[1] & 0x7c) >> 2;
- uint8_t b2 = next_argb1555[0] & 0x1f;
- uint8_t g2 = (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3);
- uint8_t r2 = next_argb1555[1] >> 3;
- uint8_t b = (b0 + b2); // 555 * 2 = 666.
- uint8_t g = (g0 + g2);
- uint8_t r = (r0 + r2);
- b = (b << 2) | (b >> 4); // 666 -> 888.
- g = (g << 2) | (g >> 4);
- r = (r << 2) | (r >> 4);
- dst_u[0] = RGBToU(r, g, b);
- dst_v[0] = RGBToV(r, g, b);
+ uint8_t b0 = STATIC_CAST(uint8_t, src_argb1555[0] & 0x1f);
+ uint8_t g0 = STATIC_CAST(
+ uint8_t, (src_argb1555[0] >> 5) | ((src_argb1555[1] & 0x03) << 3));
+ uint8_t r0 = STATIC_CAST(uint8_t, (src_argb1555[1] & 0x7c) >> 2);
+ uint8_t b2 = STATIC_CAST(uint8_t, next_argb1555[0] & 0x1f);
+ uint8_t g2 = STATIC_CAST(
+ uint8_t, (next_argb1555[0] >> 5) | ((next_argb1555[1] & 0x03) << 3));
+ uint8_t r2 = STATIC_CAST(uint8_t, (next_argb1555[1] & 0x7c) >> 2);
+
+ b0 = STATIC_CAST(uint8_t, (b0 << 3) | (b0 >> 2));
+ g0 = STATIC_CAST(uint8_t, (g0 << 3) | (g0 >> 2));
+ r0 = STATIC_CAST(uint8_t, (r0 << 3) | (r0 >> 2));
+ b2 = STATIC_CAST(uint8_t, (b2 << 3) | (b2 >> 2));
+ g2 = STATIC_CAST(uint8_t, (g2 << 3) | (g2 >> 2));
+ r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
+
+ uint8_t ab = AVGB(b0, b2);
+ uint8_t ag = AVGB(g0, g2);
+ uint8_t ar = AVGB(r0, r2);
+ dst_u[0] = RGBToU(ar, ag, ab);
+ dst_v[0] = RGBToV(ar, ag, ab);
}
}
@@ -684,14 +962,26 @@ void ARGB4444ToUVRow_C(const uint8_t* src_argb4444,
uint8_t b3 = next_argb4444[2] & 0x0f;
uint8_t g3 = next_argb4444[2] >> 4;
uint8_t r3 = next_argb4444[3] & 0x0f;
- uint8_t b = (b0 + b1 + b2 + b3); // 444 * 4 = 666.
- uint8_t g = (g0 + g1 + g2 + g3);
- uint8_t r = (r0 + r1 + r2 + r3);
- b = (b << 2) | (b >> 4); // 666 -> 888.
- g = (g << 2) | (g >> 4);
- r = (r << 2) | (r >> 4);
+
+ b0 = STATIC_CAST(uint8_t, (b0 << 4) | b0);
+ g0 = STATIC_CAST(uint8_t, (g0 << 4) | g0);
+ r0 = STATIC_CAST(uint8_t, (r0 << 4) | r0);
+ b1 = STATIC_CAST(uint8_t, (b1 << 4) | b1);
+ g1 = STATIC_CAST(uint8_t, (g1 << 4) | g1);
+ r1 = STATIC_CAST(uint8_t, (r1 << 4) | r1);
+ b2 = STATIC_CAST(uint8_t, (b2 << 4) | b2);
+ g2 = STATIC_CAST(uint8_t, (g2 << 4) | g2);
+ r2 = STATIC_CAST(uint8_t, (r2 << 4) | r2);
+ b3 = STATIC_CAST(uint8_t, (b3 << 4) | b3);
+ g3 = STATIC_CAST(uint8_t, (g3 << 4) | g3);
+ r3 = STATIC_CAST(uint8_t, (r3 << 4) | r3);
+
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
dst_u[0] = RGBToU(r, g, b);
dst_v[0] = RGBToV(r, g, b);
+
src_argb4444 += 4;
next_argb4444 += 4;
dst_u += 1;
@@ -704,31 +994,19 @@ void ARGB4444ToUVRow_C(const uint8_t* src_argb4444,
uint8_t b2 = next_argb4444[0] & 0x0f;
uint8_t g2 = next_argb4444[0] >> 4;
uint8_t r2 = next_argb4444[1] & 0x0f;
- uint8_t b = (b0 + b2); // 444 * 2 = 555.
- uint8_t g = (g0 + g2);
- uint8_t r = (r0 + r2);
- b = (b << 3) | (b >> 2); // 555 -> 888.
- g = (g << 3) | (g >> 2);
- r = (r << 3) | (r >> 2);
- dst_u[0] = RGBToU(r, g, b);
- dst_v[0] = RGBToV(r, g, b);
- }
-}
-void ARGBToUV444Row_C(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- for (x = 0; x < width; ++x) {
- uint8_t ab = src_argb[0];
- uint8_t ag = src_argb[1];
- uint8_t ar = src_argb[2];
+ b0 = STATIC_CAST(uint8_t, (b0 << 4) | b0);
+ g0 = STATIC_CAST(uint8_t, (g0 << 4) | g0);
+ r0 = STATIC_CAST(uint8_t, (r0 << 4) | r0);
+ b2 = STATIC_CAST(uint8_t, (b2 << 4) | b2);
+ g2 = STATIC_CAST(uint8_t, (g2 << 4) | g2);
+ r2 = STATIC_CAST(uint8_t, (r2 << 4) | r2);
+
+ uint8_t ab = AVGB(b0, b2);
+ uint8_t ag = AVGB(g0, g2);
+ uint8_t ar = AVGB(r0, r2);
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
- src_argb += 4;
- dst_u += 1;
- dst_v += 1;
}
}
@@ -754,9 +1032,9 @@ void ARGBSepiaRow_C(uint8_t* dst_argb, int width) {
int sg = (b * 22 + g * 88 + r * 45) >> 7;
int sr = (b * 24 + g * 98 + r * 50) >> 7;
// b does not over flow. a is preserved from original.
- dst_argb[0] = sb;
- dst_argb[1] = clamp255(sg);
- dst_argb[2] = clamp255(sr);
+ dst_argb[0] = STATIC_CAST(uint8_t, sb);
+ dst_argb[1] = STATIC_CAST(uint8_t, clamp255(sg));
+ dst_argb[2] = STATIC_CAST(uint8_t, clamp255(sr));
dst_argb += 4;
}
}
@@ -785,10 +1063,10 @@ void ARGBColorMatrixRow_C(const uint8_t* src_argb,
int sa = (b * matrix_argb[12] + g * matrix_argb[13] + r * matrix_argb[14] +
a * matrix_argb[15]) >>
6;
- dst_argb[0] = Clamp(sb);
- dst_argb[1] = Clamp(sg);
- dst_argb[2] = Clamp(sr);
- dst_argb[3] = Clamp(sa);
+ dst_argb[0] = STATIC_CAST(uint8_t, Clamp(sb));
+ dst_argb[1] = STATIC_CAST(uint8_t, Clamp(sg));
+ dst_argb[2] = STATIC_CAST(uint8_t, Clamp(sr));
+ dst_argb[3] = STATIC_CAST(uint8_t, Clamp(sa));
src_argb += 4;
dst_argb += 4;
}
@@ -838,9 +1116,12 @@ void ARGBQuantizeRow_C(uint8_t* dst_argb,
int b = dst_argb[0];
int g = dst_argb[1];
int r = dst_argb[2];
- dst_argb[0] = (b * scale >> 16) * interval_size + interval_offset;
- dst_argb[1] = (g * scale >> 16) * interval_size + interval_offset;
- dst_argb[2] = (r * scale >> 16) * interval_size + interval_offset;
+ dst_argb[0] = STATIC_CAST(
+ uint8_t, (b * scale >> 16) * interval_size + interval_offset);
+ dst_argb[1] = STATIC_CAST(
+ uint8_t, (g * scale >> 16) * interval_size + interval_offset);
+ dst_argb[2] = STATIC_CAST(
+ uint8_t, (r * scale >> 16) * interval_size + interval_offset);
dst_argb += 4;
}
}
@@ -874,56 +1155,51 @@ void ARGBShadeRow_C(const uint8_t* src_argb,
#undef REPEAT8
#undef SHADE
-#define REPEAT8(v) (v) | ((v) << 8)
-#define SHADE(f, v) v* f >> 16
-
-void ARGBMultiplyRow_C(const uint8_t* src_argb0,
+void ARGBMultiplyRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
int i;
for (i = 0; i < width; ++i) {
- const uint32_t b = REPEAT8(src_argb0[0]);
- const uint32_t g = REPEAT8(src_argb0[1]);
- const uint32_t r = REPEAT8(src_argb0[2]);
- const uint32_t a = REPEAT8(src_argb0[3]);
+ const uint32_t b = src_argb[0];
+ const uint32_t g = src_argb[1];
+ const uint32_t r = src_argb[2];
+ const uint32_t a = src_argb[3];
const uint32_t b_scale = src_argb1[0];
const uint32_t g_scale = src_argb1[1];
const uint32_t r_scale = src_argb1[2];
const uint32_t a_scale = src_argb1[3];
- dst_argb[0] = SHADE(b, b_scale);
- dst_argb[1] = SHADE(g, g_scale);
- dst_argb[2] = SHADE(r, r_scale);
- dst_argb[3] = SHADE(a, a_scale);
- src_argb0 += 4;
+ dst_argb[0] = STATIC_CAST(uint8_t, (b * b_scale + 128) >> 8);
+ dst_argb[1] = STATIC_CAST(uint8_t, (g * g_scale + 128) >> 8);
+ dst_argb[2] = STATIC_CAST(uint8_t, (r * r_scale + 128) >> 8);
+ dst_argb[3] = STATIC_CAST(uint8_t, (a * a_scale + 128) >> 8);
+ src_argb += 4;
src_argb1 += 4;
dst_argb += 4;
}
}
-#undef REPEAT8
-#undef SHADE
#define SHADE(f, v) clamp255(v + f)
-void ARGBAddRow_C(const uint8_t* src_argb0,
+void ARGBAddRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
int i;
for (i = 0; i < width; ++i) {
- const int b = src_argb0[0];
- const int g = src_argb0[1];
- const int r = src_argb0[2];
- const int a = src_argb0[3];
+ const int b = src_argb[0];
+ const int g = src_argb[1];
+ const int r = src_argb[2];
+ const int a = src_argb[3];
const int b_add = src_argb1[0];
const int g_add = src_argb1[1];
const int r_add = src_argb1[2];
const int a_add = src_argb1[3];
- dst_argb[0] = SHADE(b, b_add);
- dst_argb[1] = SHADE(g, g_add);
- dst_argb[2] = SHADE(r, r_add);
- dst_argb[3] = SHADE(a, a_add);
- src_argb0 += 4;
+ dst_argb[0] = STATIC_CAST(uint8_t, SHADE(b, b_add));
+ dst_argb[1] = STATIC_CAST(uint8_t, SHADE(g, g_add));
+ dst_argb[2] = STATIC_CAST(uint8_t, SHADE(r, r_add));
+ dst_argb[3] = STATIC_CAST(uint8_t, SHADE(a, a_add));
+ src_argb += 4;
src_argb1 += 4;
dst_argb += 4;
}
@@ -932,25 +1208,25 @@ void ARGBAddRow_C(const uint8_t* src_argb0,
#define SHADE(f, v) clamp0(f - v)
-void ARGBSubtractRow_C(const uint8_t* src_argb0,
+void ARGBSubtractRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
int i;
for (i = 0; i < width; ++i) {
- const int b = src_argb0[0];
- const int g = src_argb0[1];
- const int r = src_argb0[2];
- const int a = src_argb0[3];
+ const int b = src_argb[0];
+ const int g = src_argb[1];
+ const int r = src_argb[2];
+ const int a = src_argb[3];
const int b_sub = src_argb1[0];
const int g_sub = src_argb1[1];
const int r_sub = src_argb1[2];
const int a_sub = src_argb1[3];
- dst_argb[0] = SHADE(b, b_sub);
- dst_argb[1] = SHADE(g, g_sub);
- dst_argb[2] = SHADE(r, r_sub);
- dst_argb[3] = SHADE(a, a_sub);
- src_argb0 += 4;
+ dst_argb[0] = STATIC_CAST(uint8_t, SHADE(b, b_sub));
+ dst_argb[1] = STATIC_CAST(uint8_t, SHADE(g, g_sub));
+ dst_argb[2] = STATIC_CAST(uint8_t, SHADE(r, r_sub));
+ dst_argb[3] = STATIC_CAST(uint8_t, SHADE(a, a_sub));
+ src_argb += 4;
src_argb1 += 4;
dst_argb += 4;
}
@@ -1058,257 +1334,384 @@ void J400ToARGBRow_C(const uint8_t* src_y, uint8_t* dst_argb, int width) {
}
}
-// TODO(fbarchard): Unify these structures to be platform independent.
-// TODO(fbarchard): Generate SIMD structures from float matrix.
+// Macros to create SIMD specific yuv to rgb conversion constants.
-// BT.601 YUV to RGB reference
-// R = (Y - 16) * 1.164 - V * -1.596
-// G = (Y - 16) * 1.164 - U * 0.391 - V * 0.813
-// B = (Y - 16) * 1.164 - U * -2.018
+// clang-format off
-// Y contribution to R,G,B. Scale and bias.
-#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */
-#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+// Bias values include subtract 128 from U and V, bias from Y and rounding.
+// For B and R bias is negative. For G bias is positive.
+#define YUVCONSTANTSBODY(YG, YB, UB, UG, VG, VR) \
+ {{UB, VR, UG, VG, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \
+ {YG, (UB * 128 - YB), (UG * 128 + VG * 128 + YB), (VR * 128 - YB), YB, 0, \
+ 0, 0}}
+#else
+#define YUVCONSTANTSBODY(YG, YB, UB, UG, VG, VR) \
+ {{UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, \
+ UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0}, \
+ {UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, \
+ UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG}, \
+ {0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, \
+ 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR}, \
+ {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}, \
+ {YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB, YB}}
+#endif
+
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+#define ARGBCONSTANTSBODY(Y0, Y1, Y2, Y3, U0, U1, U2, U3, V0, V1, V2, V3, AY, \
+ AUV) \
+ {{Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3}, \
+ {U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3}, \
+ {V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3}, \
+ {AY, AY, AY, AY, AY, AY, AY, AY}, \
+ {AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV}}
+#else
+#define ARGBCONSTANTSBODY(Y0, Y1, Y2, Y3, U0, U1, U2, U3, V0, V1, V2, V3, AY, \
+ AUV) \
+ {{Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, \
+ Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3, Y0, Y1, Y2, Y3}, \
+ {U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3, \
+ U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3, U0, U1, U2, U3}, \
+ {V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3, \
+ V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3, V0, V1, V2, V3}, \
+ {AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY, AY}, \
+ {AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, AUV, \
+ AUV, AUV}}
+#endif
+
+// clang-format on
+
+#define MAKEYUVCONSTANTS(name, YG, YB, UB, UG, VG, VR) \
+ const struct YuvConstants SIMD_ALIGNED(kYuv##name##Constants) = \
+ YUVCONSTANTSBODY(YG, YB, UB, UG, VG, VR); \
+ const struct YuvConstants SIMD_ALIGNED(kYvu##name##Constants) = \
+ YUVCONSTANTSBODY(YG, YB, VR, VG, UG, UB);
+
+#define MAKEARGBCONSTANTS(name, RY, GY, BY, RU, GU, BU, RV, GV, BV, AY, AUV) \
+ extern const struct ArgbConstants SIMD_ALIGNED(kArgb##name##Constants) = \
+ ARGBCONSTANTSBODY(BY, GY, RY, 0, -(BU), -(GU), -(RU), 0, -(BV), -(GV), \
+ -(RV), 0, AY, AUV); \
+ extern const struct ArgbConstants SIMD_ALIGNED(kAbgr##name##Constants) = \
+ ARGBCONSTANTSBODY(RY, GY, BY, 0, -(RU), -(GU), -(BU), 0, -(RV), -(GV), \
+ -(BV), 0, AY, AUV); \
+ extern const struct ArgbConstants SIMD_ALIGNED(kRgba##name##Constants) = \
+ ARGBCONSTANTSBODY(0, BY, GY, RY, 0, -(BU), -(GU), -(RU), 0, -(BV), \
+ -(GV), -(RV), AY, AUV); \
+ extern const struct ArgbConstants SIMD_ALIGNED(kBgra##name##Constants) = \
+ ARGBCONSTANTSBODY(0, RY, GY, BY, 0, -(RU), -(GU), -(BU), 0, -(RV), \
+ -(GV), -(BV), AY, AUV);
+
+// BT.601 limited range RGB to YUV coefficients
+// RY = round(0.299 * 219 / 255 * 256) = 66
+// GY = round(0.587 * 219 / 255 * 256) = 129
+// BY = round(0.114 * 219 / 255 * 256) = 25
+// BU = round(0.500 * 224 / 255 * 256) = 112
+// RU = round(-0.299 / (1 - 0.114) * 112.4) = -38
+// GU = round(-0.587 / (1 - 0.114) * 112.4) = -74
+// RV = 112
+// GV = round(-0.587 / (1 - 0.299) * 112.4) = -94
+// BV = round(-0.114 / (1 - 0.299) * 112.4) = -18
+// AY = 16 * 256 + 128 = 4224
+// AUV = 128 * 256 = 32768
+MAKEARGBCONSTANTS(I601, 66, 129, 25, -38, -74, 112, 112, -94, -18, 4224, 32768)
+
+// BT.601 full range RGB to YUV coefficients (aka JPEG)
+// RY = round(0.299 * 256) = 77
+// GY = round(0.587 * 256) = 150
+// BY = round(0.114 * 256) = 29
+// BU = 128
+// RU = round(-0.299 / (1 - 0.114) * 128) = -43
+// GU = round(-0.587 / (1 - 0.114) * 128) = -85
+// RV = 128
+// GV = round(-0.587 / (1 - 0.299) * 128) = -107
+// BV = round(-0.114 / (1 - 0.299) * 128) = -21
+// AY = 128
+// AUV = 32768
+MAKEARGBCONSTANTS(JPEG, 77, 150, 29, -43, -85, 128, 128, -107, -21, 128, 32768)
+
+// BT.709 limited range RGB to YUV coefficients
+// RY = round(0.2126 * 219 / 255 * 256) = 47
+// GY = round(0.7152 * 219 / 255 * 256) = 157
+// BY = round(0.0722 * 219 / 255 * 256) = 16
+// BU = round(0.500 * 224 / 255 * 256) = 112
+// RU = round(-0.2126 / (1 - 0.0722) * 112.4) = -26
+// GU = round(-0.7152 / (1 - 0.0722) * 112.4) = -86
+// RV = 112
+// GV = round(-0.7152 / (1 - 0.2126) * 112.4) = -102
+// BV = round(-0.0722 / (1 - 0.2126) * 112.4) = -10
+// AY = 16 * 256 + 128 = 4224
+// AUV = 128 * 256 = 32768
+MAKEARGBCONSTANTS(H709, 47, 157, 16, -26, -86, 112, 112, -102, -10, 4224, 32768)
+
+// BT.709 full range RGB to YUV coefficients
+// RY = round(0.2126 * 256) = 54
+// GY = round(0.7152 * 256) = 183
+// BY = round(0.0722 * 256) = 19
+// BU = 128
+// RU = round(-0.2126 / (1 - 0.0722) * 128) = -29
+// GU = round(-0.7152 / (1 - 0.0722) * 128) = -99
+// RV = 128
+// GV = round(-0.7152 / (1 - 0.2126) * 128) = -116
+// BV = round(-0.0722 / (1 - 0.2126) * 128) = -12
+// AY = 128
+// AUV = 32768
+MAKEARGBCONSTANTS(F709, 54, 183, 19, -29, -99, 128, 128, -116, -12, 128, 32768)
+
+// BT.2020 limited range RGB to YUV coefficients
+// RY = round(0.2627 * 219 / 255 * 256) = 58
+// GY = round(0.6780 * 219 / 255 * 256) = 149
+// BY = round(0.0593 * 219 / 255 * 256) = 13
+// BU = 112
+// RU = round(-0.2627 / (1 - 0.0593) * 112.4) = -31
+// GU = round(-0.6780 / (1 - 0.0593) * 112.4) = -81
+// RV = 112
+// GV = round(-0.6780 / (1 - 0.2627) * 112.4) = -103
+// BV = round(-0.0593 / (1 - 0.2627) * 112.4) = -9
+// AY = 16 * 256 + 128 = 4224
+// AUV = 128 * 256 = 32768
+MAKEARGBCONSTANTS(U2020, 59, 148, 13, -31, -81, 112, 112, -103, -9, 4224, 32768)
+
+// BT.2020 full range RGB to YUV coefficients
+// RY = round(0.2627 * 256) = 67
+// GY = round(0.6780 * 256) = 174
+// BY = round(0.0593 * 256) = 15
+// BU = 128
+// RU = round(-0.2627 / (1 - 0.0593) * 128) = -36
+// GU = round(-0.6780 / (1 - 0.0593) * 128) = -92
+// RV = 128
+// GV = round(-0.6780 / (1 - 0.2627) * 128) = -118
+// BV = round(-0.0593 / (1 - 0.2627) * 128) = -10
+// AY = 128
+// AUV = 32768
+MAKEARGBCONSTANTS(V2020, 67, 174, 15, -36, -92, 128, 128, -118, -10, 128, 32768)
+
+// TODO(fbarchard): Generate SIMD structures from float matrix.
+
+// BT.601 limited range YUV to RGB reference
+// R = (Y - 16) * 1.164 + V * 1.596
+// G = (Y - 16) * 1.164 - U * 0.391 - V * 0.813
+// B = (Y - 16) * 1.164 + U * 2.018
+// KR = 0.299; KB = 0.114
// U and V contributions to R,G,B.
-#define UB -128 /* max(-128, round(-2.018 * 64)) */
-#define UG 25 /* round(0.391 * 64) */
-#define VG 52 /* round(0.813 * 64) */
-#define VR -102 /* round(-1.596 * 64) */
-
-// Bias values to subtract 16 from Y and 128 from U and V.
-#define BB (UB * 128 + YGB)
-#define BG (UG * 128 + VG * 128 + YGB)
-#define BR (VR * 128 + YGB)
-
-#if defined(__aarch64__) // 64 bit arm
-const struct YuvConstants SIMD_ALIGNED(kYuvI601Constants) = {
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuI601Constants) = {
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-#elif defined(__arm__) // 32 bit arm
-const struct YuvConstants SIMD_ALIGNED(kYuvI601Constants) = {
- {-UB, -UB, -UB, -UB, -VR, -VR, -VR, -VR, 0, 0, 0, 0, 0, 0, 0, 0},
- {UG, UG, UG, UG, VG, VG, VG, VG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuI601Constants) = {
- {-VR, -VR, -VR, -VR, -UB, -UB, -UB, -UB, 0, 0, 0, 0, 0, 0, 0, 0},
- {VG, VG, VG, VG, UG, UG, UG, UG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
+#if defined(LIBYUV_UNLIMITED_DATA) || defined(LIBYUV_UNLIMITED_BT601)
+#define UB 129 /* round(2.018 * 64) */
#else
-const struct YuvConstants SIMD_ALIGNED(kYuvI601Constants) = {
- {UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0,
- UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0},
- {UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG,
- UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG},
- {0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR,
- 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
-const struct YuvConstants SIMD_ALIGNED(kYvuI601Constants) = {
- {VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0,
- VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0},
- {VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG,
- VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG},
- {0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB,
- 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
+#define UB 128 /* max(128, round(2.018 * 64)) */
#endif
+#define UG 25 /* round(0.391 * 64) */
+#define VG 52 /* round(0.813 * 64) */
+#define VR 102 /* round(1.596 * 64) */
-#undef BB
-#undef BG
-#undef BR
-#undef YGB
+// Y contribution to R,G,B. Scale and bias.
+#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */
+#define YB -1160 /* 1.164 * 64 * -16 + 64 / 2 */
+
+MAKEYUVCONSTANTS(I601, YG, YB, UB, UG, VG, VR)
+
+#undef YG
+#undef YB
#undef UB
#undef UG
#undef VG
#undef VR
-#undef YG
-// JPEG YUV to RGB reference
-// * R = Y - V * -1.40200
-// * G = Y - U * 0.34414 - V * 0.71414
-// * B = Y - U * -1.77200
+// BT.601 full range YUV to RGB reference (aka JPEG)
+// * R = Y + V * 1.40200
+// * G = Y - U * 0.34414 - V * 0.71414
+// * B = Y + U * 1.77200
+// KR = 0.299; KB = 0.114
+
+// U and V contributions to R,G,B.
+#define UB 113 /* round(1.77200 * 64) */
+#define UG 22 /* round(0.34414 * 64) */
+#define VG 46 /* round(0.71414 * 64) */
+#define VR 90 /* round(1.40200 * 64) */
// Y contribution to R,G,B. Scale and bias.
#define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
-#define YGB 32 /* 64 / 2 */
+#define YB 32 /* 64 / 2 */
+
+MAKEYUVCONSTANTS(JPEG, YG, YB, UB, UG, VG, VR)
+
+#undef YG
+#undef YB
+#undef UB
+#undef UG
+#undef VG
+#undef VR
+
+// BT.709 limited range YUV to RGB reference
+// R = (Y - 16) * 1.164 + V * 1.793
+// G = (Y - 16) * 1.164 - U * 0.213 - V * 0.533
+// B = (Y - 16) * 1.164 + U * 2.112
+// KR = 0.2126, KB = 0.0722
// U and V contributions to R,G,B.
-#define UB -113 /* round(-1.77200 * 64) */
-#define UG 22 /* round(0.34414 * 64) */
-#define VG 46 /* round(0.71414 * 64) */
-#define VR -90 /* round(-1.40200 * 64) */
-
-// Bias values to round, and subtract 128 from U and V.
-#define BB (UB * 128 + YGB)
-#define BG (UG * 128 + VG * 128 + YGB)
-#define BR (VR * 128 + YGB)
-
-#if defined(__aarch64__)
-const struct YuvConstants SIMD_ALIGNED(kYuvJPEGConstants) = {
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuJPEGConstants) = {
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-#elif defined(__arm__)
-const struct YuvConstants SIMD_ALIGNED(kYuvJPEGConstants) = {
- {-UB, -UB, -UB, -UB, -VR, -VR, -VR, -VR, 0, 0, 0, 0, 0, 0, 0, 0},
- {UG, UG, UG, UG, VG, VG, VG, VG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuJPEGConstants) = {
- {-VR, -VR, -VR, -VR, -UB, -UB, -UB, -UB, 0, 0, 0, 0, 0, 0, 0, 0},
- {VG, VG, VG, VG, UG, UG, UG, UG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
+#if defined(LIBYUV_UNLIMITED_DATA) || defined(LIBYUV_UNLIMITED_BT709)
+#define UB 135 /* round(2.112 * 64) */
#else
-const struct YuvConstants SIMD_ALIGNED(kYuvJPEGConstants) = {
- {UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0,
- UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0},
- {UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG,
- UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG},
- {0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR,
- 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
-const struct YuvConstants SIMD_ALIGNED(kYvuJPEGConstants) = {
- {VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0,
- VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0},
- {VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG,
- VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG},
- {0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB,
- 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
+#define UB 128 /* max(128, round(2.112 * 64)) */
#endif
+#define UG 14 /* round(0.213 * 64) */
+#define VG 34 /* round(0.533 * 64) */
+#define VR 115 /* round(1.793 * 64) */
-#undef BB
-#undef BG
-#undef BR
-#undef YGB
+// Y contribution to R,G,B. Scale and bias.
+#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */
+#define YB -1160 /* 1.164 * 64 * -16 + 64 / 2 */
+
+MAKEYUVCONSTANTS(H709, YG, YB, UB, UG, VG, VR)
+
+#undef YG
+#undef YB
#undef UB
#undef UG
#undef VG
#undef VR
-#undef YG
-// BT.709 YUV to RGB reference
-// R = (Y - 16) * 1.164 - V * -1.793
-// G = (Y - 16) * 1.164 - U * 0.213 - V * 0.533
-// B = (Y - 16) * 1.164 - U * -2.112
-// See also http://www.equasys.de/colorconversion.html
+// BT.709 full range YUV to RGB reference
+// R = Y + V * 1.5748
+// G = Y - U * 0.18732 - V * 0.46812
+// B = Y + U * 1.8556
+// KR = 0.2126, KB = 0.0722
-// Y contribution to R,G,B. Scale and bias.
-#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */
-#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */
+// U and V contributions to R,G,B.
+#define UB 119 /* round(1.8556 * 64) */
+#define UG 12 /* round(0.18732 * 64) */
+#define VG 30 /* round(0.46812 * 64) */
+#define VR 101 /* round(1.5748 * 64) */
+
+// Y contribution to R,G,B. Scale and bias. (same as jpeg)
+#define YG 16320 /* round(1 * 64 * 256 * 256 / 257) */
+#define YB 32 /* 64 / 2 */
+
+MAKEYUVCONSTANTS(F709, YG, YB, UB, UG, VG, VR)
+
+#undef YG
+#undef YB
+#undef UB
+#undef UG
+#undef VG
+#undef VR
+
+// BT.2020 limited range YUV to RGB reference
+// R = (Y - 16) * 1.164384 + V * 1.67867
+// G = (Y - 16) * 1.164384 - U * 0.187326 - V * 0.65042
+// B = (Y - 16) * 1.164384 + U * 2.14177
+// KR = 0.2627; KB = 0.0593
-// TODO(fbarchard): Find way to express 2.112 instead of 2.0.
// U and V contributions to R,G,B.
-#define UB -128 /* max(-128, round(-2.112 * 64)) */
-#define UG 14 /* round(0.213 * 64) */
-#define VG 34 /* round(0.533 * 64) */
-#define VR -115 /* round(-1.793 * 64) */
-
-// Bias values to round, and subtract 128 from U and V.
-#define BB (UB * 128 + YGB)
-#define BG (UG * 128 + VG * 128 + YGB)
-#define BR (VR * 128 + YGB)
-
-#if defined(__aarch64__)
-const struct YuvConstants SIMD_ALIGNED(kYuvH709Constants) = {
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {-UB, -VR, -UB, -VR, -UB, -VR, -UB, -VR},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {UG, VG, UG, VG, UG, VG, UG, VG},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuH709Constants) = {
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {-VR, -UB, -VR, -UB, -VR, -UB, -VR, -UB},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {VG, UG, VG, UG, VG, UG, VG, UG},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-#elif defined(__arm__)
-const struct YuvConstants SIMD_ALIGNED(kYuvH709Constants) = {
- {-UB, -UB, -UB, -UB, -VR, -VR, -VR, -VR, 0, 0, 0, 0, 0, 0, 0, 0},
- {UG, UG, UG, UG, VG, VG, VG, VG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BB, BG, BR, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
-const struct YuvConstants SIMD_ALIGNED(kYvuH709Constants) = {
- {-VR, -VR, -VR, -VR, -UB, -UB, -UB, -UB, 0, 0, 0, 0, 0, 0, 0, 0},
- {VG, VG, VG, VG, UG, UG, UG, UG, 0, 0, 0, 0, 0, 0, 0, 0},
- {BR, BG, BB, 0, 0, 0, 0, 0},
- {0x0101 * YG, 0, 0, 0}};
+#if defined(LIBYUV_UNLIMITED_DATA) || defined(LIBYUV_UNLIMITED_BT2020)
+#define UB 137 /* round(2.142 * 64) */
#else
-const struct YuvConstants SIMD_ALIGNED(kYuvH709Constants) = {
- {UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0,
- UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0},
- {UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG,
- UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG},
- {0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR,
- 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
-const struct YuvConstants SIMD_ALIGNED(kYvuH709Constants) = {
- {VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0,
- VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0, VR, 0},
- {VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG,
- VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG},
- {0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB,
- 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB, 0, UB},
- {BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR, BR},
- {BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG, BG},
- {BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB, BB},
- {YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG, YG}};
+#define UB 128 /* max(128, round(2.142 * 64)) */
#endif
+#define UG 12 /* round(0.187326 * 64) */
+#define VG 42 /* round(0.65042 * 64) */
+#define VR 107 /* round(1.67867 * 64) */
-#undef BB
-#undef BG
-#undef BR
-#undef YGB
+// Y contribution to R,G,B. Scale and bias.
+#define YG 19003 /* round(1.164384 * 64 * 256 * 256 / 257) */
+#define YB -1160 /* 1.164384 * 64 * -16 + 64 / 2 */
+
+MAKEYUVCONSTANTS(2020, YG, YB, UB, UG, VG, VR)
+
+#undef YG
+#undef YB
#undef UB
#undef UG
#undef VG
#undef VR
+
+// BT.2020 full range YUV to RGB reference
+// R = Y + V * 1.474600
+// G = Y - U * 0.164553 - V * 0.571353
+// B = Y + U * 1.881400
+// KR = 0.2627; KB = 0.0593
+
+#define UB 120 /* round(1.881400 * 64) */
+#define UG 11 /* round(0.164553 * 64) */
+#define VG 37 /* round(0.571353 * 64) */
+#define VR 94 /* round(1.474600 * 64) */
+
+// Y contribution to R,G,B. Scale and bias. (same as jpeg)
+#define YG 16320 /* round(1 * 64 * 256 * 256 / 257) */
+#define YB 32 /* 64 / 2 */
+
+MAKEYUVCONSTANTS(V2020, YG, YB, UB, UG, VG, VR)
+
#undef YG
+#undef YB
+#undef UB
+#undef UG
+#undef VG
+#undef VR
+
+#undef BB
+#undef BG
+#undef BR
+
+#undef MAKEYUVCONSTANTS
+
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+#define LOAD_YUV_CONSTANTS \
+ int ub = yuvconstants->kUVCoeff[0]; \
+ int vr = yuvconstants->kUVCoeff[1]; \
+ int ug = yuvconstants->kUVCoeff[2]; \
+ int vg = yuvconstants->kUVCoeff[3]; \
+ int yg = yuvconstants->kRGBCoeffBias[0]; \
+ int bb = yuvconstants->kRGBCoeffBias[1]; \
+ int bg = yuvconstants->kRGBCoeffBias[2]; \
+ int br = yuvconstants->kRGBCoeffBias[3]
+
+#define CALC_RGB16 \
+ int32_t y1 = (uint32_t)(y32 * yg) >> 16; \
+ int b16 = y1 + (u * ub) - bb; \
+ int g16 = y1 + bg - (u * ug + v * vg); \
+ int r16 = y1 + (v * vr) - br
+#else
+#define LOAD_YUV_CONSTANTS \
+ int ub = yuvconstants->kUVToB[0]; \
+ int ug = yuvconstants->kUVToG[0]; \
+ int vg = yuvconstants->kUVToG[1]; \
+ int vr = yuvconstants->kUVToR[1]; \
+ int yg = yuvconstants->kYToRgb[0]; \
+ int yb = yuvconstants->kYBiasToRgb[0]
+
+#define CALC_RGB16 \
+ int32_t y1 = ((uint32_t)(y32 * yg) >> 16) + yb; \
+ int8_t ui = (int8_t)u; \
+ int8_t vi = (int8_t)v; \
+ ui -= 0x80; \
+ vi -= 0x80; \
+ int b16 = y1 + (ui * ub); \
+ int g16 = y1 - (ui * ug + vi * vg); \
+ int r16 = y1 + (vi * vr)
+#endif
+
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+#define LOAD_YUV_CONSTANTS_AR30 \
+ int ub = yuvconstants->kUVCoeff[0]; \
+ int vr = yuvconstants->kUVCoeff[1]; \
+ int ug = yuvconstants->kUVCoeff[2]; \
+ int vg = yuvconstants->kUVCoeff[3]; \
+ int yg = yuvconstants->kRGBCoeffBias[0]; \
+ int bb = yuvconstants->kRGBCoeffBias[1] + 24; \
+ int bg = yuvconstants->kRGBCoeffBias[2] - 24; \
+ int br = yuvconstants->kRGBCoeffBias[3] + 24
+#else
+#define LOAD_YUV_CONSTANTS_AR30 \
+ int ub = yuvconstants->kUVToB[0]; \
+ int ug = yuvconstants->kUVToG[0]; \
+ int vg = yuvconstants->kUVToG[1]; \
+ int vr = yuvconstants->kUVToR[1]; \
+ int yg = yuvconstants->kYToRgb[0]; \
+ int yb = yuvconstants->kYBiasToRgb[0] - 24
+#endif
// C reference code that mimics the YUV assembly.
// Reads 8 bit YUV and leaves result as 16 bit.
-
static __inline void YuvPixel(uint8_t y,
uint8_t u,
uint8_t v,
@@ -1316,39 +1719,12 @@ static __inline void YuvPixel(uint8_t y,
uint8_t* g,
uint8_t* r,
const struct YuvConstants* yuvconstants) {
-#if defined(__aarch64__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = -yuvconstants->kUVToRB[1];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#elif defined(__arm__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[4];
- int vr = -yuvconstants->kUVToRB[4];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#else
- int ub = yuvconstants->kUVToB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = yuvconstants->kUVToR[1];
- int bb = yuvconstants->kUVBiasB[0];
- int bg = yuvconstants->kUVBiasG[0];
- int br = yuvconstants->kUVBiasR[0];
- int yg = yuvconstants->kYToRgb[0];
-#endif
-
- uint32_t y1 = (uint32_t)(y * 0x0101 * yg) >> 16;
- *b = Clamp((int32_t)(-(u * ub) + y1 + bb) >> 6);
- *g = Clamp((int32_t)(-(u * ug + v * vg) + y1 + bg) >> 6);
- *r = Clamp((int32_t)(-(v * vr) + y1 + br) >> 6);
+ LOAD_YUV_CONSTANTS;
+ uint32_t y32 = y * 0x0101;
+ CALC_RGB16;
+ *b = STATIC_CAST(uint8_t, Clamp((int32_t)(b16) >> 6));
+ *g = STATIC_CAST(uint8_t, Clamp((int32_t)(g16) >> 6));
+ *r = STATIC_CAST(uint8_t, Clamp((int32_t)(r16) >> 6));
}
// Reads 8 bit YUV and leaves result as 16 bit.
@@ -1359,85 +1735,50 @@ static __inline void YuvPixel8_16(uint8_t y,
int* g,
int* r,
const struct YuvConstants* yuvconstants) {
-#if defined(__aarch64__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = -yuvconstants->kUVToRB[1];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#elif defined(__arm__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[4];
- int vr = -yuvconstants->kUVToRB[4];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#else
- int ub = yuvconstants->kUVToB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = yuvconstants->kUVToR[1];
- int bb = yuvconstants->kUVBiasB[0];
- int bg = yuvconstants->kUVBiasG[0];
- int br = yuvconstants->kUVBiasR[0];
- int yg = yuvconstants->kYToRgb[0];
-#endif
-
- uint32_t y1 = (uint32_t)(y * 0x0101 * yg) >> 16;
- *b = (int)(-(u * ub) + y1 + bb);
- *g = (int)(-(u * ug + v * vg) + y1 + bg);
- *r = (int)(-(v * vr) + y1 + br);
+ LOAD_YUV_CONSTANTS_AR30;
+ uint32_t y32 = y * 0x0101;
+ CALC_RGB16;
+ *b = b16;
+ *g = g16;
+ *r = r16;
}
// C reference code that mimics the YUV 16 bit assembly.
// Reads 10 bit YUV and leaves result as 16 bit.
-static __inline void YuvPixel16(int16_t y,
- int16_t u,
- int16_t v,
- int* b,
- int* g,
- int* r,
- const struct YuvConstants* yuvconstants) {
-#if defined(__aarch64__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = -yuvconstants->kUVToRB[1];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#elif defined(__arm__)
- int ub = -yuvconstants->kUVToRB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[4];
- int vr = -yuvconstants->kUVToRB[4];
- int bb = yuvconstants->kUVBiasBGR[0];
- int bg = yuvconstants->kUVBiasBGR[1];
- int br = yuvconstants->kUVBiasBGR[2];
- int yg = yuvconstants->kYToRgb[0] / 0x0101;
-#else
- int ub = yuvconstants->kUVToB[0];
- int ug = yuvconstants->kUVToG[0];
- int vg = yuvconstants->kUVToG[1];
- int vr = yuvconstants->kUVToR[1];
- int bb = yuvconstants->kUVBiasB[0];
- int bg = yuvconstants->kUVBiasG[0];
- int br = yuvconstants->kUVBiasR[0];
- int yg = yuvconstants->kYToRgb[0];
-#endif
+static __inline void YuvPixel10_16(uint16_t y,
+ uint16_t u,
+ uint16_t v,
+ int* b,
+ int* g,
+ int* r,
+ const struct YuvConstants* yuvconstants) {
+ LOAD_YUV_CONSTANTS;
+ uint32_t y32 = (y << 6) | (y >> 4);
+ u = STATIC_CAST(uint8_t, clamp255(u >> 2));
+ v = STATIC_CAST(uint8_t, clamp255(v >> 2));
+ CALC_RGB16;
+ *b = b16;
+ *g = g16;
+ *r = r16;
+}
- uint32_t y1 = (uint32_t)((y << 6) * yg) >> 16;
- u = clamp255(u >> 2);
- v = clamp255(v >> 2);
- *b = (int)(-(u * ub) + y1 + bb);
- *g = (int)(-(u * ug + v * vg) + y1 + bg);
- *r = (int)(-(v * vr) + y1 + br);
+// C reference code that mimics the YUV 16 bit assembly.
+// Reads 12 bit YUV and leaves result as 16 bit.
+static __inline void YuvPixel12_16(int16_t y,
+ int16_t u,
+ int16_t v,
+ int* b,
+ int* g,
+ int* r,
+ const struct YuvConstants* yuvconstants) {
+ LOAD_YUV_CONSTANTS;
+ uint32_t y32 = (y << 4) | (y >> 8);
+ u = STATIC_CAST(uint8_t, clamp255(u >> 4));
+ v = STATIC_CAST(uint8_t, clamp255(v >> 4));
+ CALC_RGB16;
+ *b = b16;
+ *g = g16;
+ *r = r16;
}
// C reference code that mimics the YUV 10 bit assembly.
@@ -1452,31 +1793,89 @@ static __inline void YuvPixel10(uint16_t y,
int b16;
int g16;
int r16;
- YuvPixel16(y, u, v, &b16, &g16, &r16, yuvconstants);
- *b = Clamp(b16 >> 6);
- *g = Clamp(g16 >> 6);
- *r = Clamp(r16 >> 6);
+ YuvPixel10_16(y, u, v, &b16, &g16, &r16, yuvconstants);
+ *b = STATIC_CAST(uint8_t, Clamp(b16 >> 6));
+ *g = STATIC_CAST(uint8_t, Clamp(g16 >> 6));
+ *r = STATIC_CAST(uint8_t, Clamp(r16 >> 6));
}
-// Y contribution to R,G,B. Scale and bias.
-#define YG 18997 /* round(1.164 * 64 * 256 * 256 / 257) */
-#define YGB -1160 /* 1.164 * 64 * -16 + 64 / 2 */
+// C reference code that mimics the YUV 12 bit assembly.
+// Reads 12 bit YUV and clamps down to 8 bit RGB.
+static __inline void YuvPixel12(uint16_t y,
+ uint16_t u,
+ uint16_t v,
+ uint8_t* b,
+ uint8_t* g,
+ uint8_t* r,
+ const struct YuvConstants* yuvconstants) {
+ int b16;
+ int g16;
+ int r16;
+ YuvPixel12_16(y, u, v, &b16, &g16, &r16, yuvconstants);
+ *b = STATIC_CAST(uint8_t, Clamp(b16 >> 6));
+ *g = STATIC_CAST(uint8_t, Clamp(g16 >> 6));
+ *r = STATIC_CAST(uint8_t, Clamp(r16 >> 6));
+}
-// C reference code that mimics the YUV assembly.
-static __inline void YPixel(uint8_t y, uint8_t* b, uint8_t* g, uint8_t* r) {
- uint32_t y1 = (uint32_t)(y * 0x0101 * YG) >> 16;
- *b = Clamp((int32_t)(y1 + YGB) >> 6);
- *g = Clamp((int32_t)(y1 + YGB) >> 6);
- *r = Clamp((int32_t)(y1 + YGB) >> 6);
+// C reference code that mimics the YUV 16 bit assembly.
+// Reads 16 bit YUV and leaves result as 8 bit.
+static __inline void YuvPixel16_8(uint16_t y,
+ uint16_t u,
+ uint16_t v,
+ uint8_t* b,
+ uint8_t* g,
+ uint8_t* r,
+ const struct YuvConstants* yuvconstants) {
+ LOAD_YUV_CONSTANTS;
+ uint32_t y32 = y;
+ u = STATIC_CAST(uint16_t, clamp255(u >> 8));
+ v = STATIC_CAST(uint16_t, clamp255(v >> 8));
+ CALC_RGB16;
+ *b = STATIC_CAST(uint8_t, Clamp((int32_t)(b16) >> 6));
+ *g = STATIC_CAST(uint8_t, Clamp((int32_t)(g16) >> 6));
+ *r = STATIC_CAST(uint8_t, Clamp((int32_t)(r16) >> 6));
}
-#undef YG
-#undef YGB
+// C reference code that mimics the YUV 16 bit assembly.
+// Reads 16 bit YUV and leaves result as 16 bit.
+static __inline void YuvPixel16_16(uint16_t y,
+ uint16_t u,
+ uint16_t v,
+ int* b,
+ int* g,
+ int* r,
+ const struct YuvConstants* yuvconstants) {
+ LOAD_YUV_CONSTANTS;
+ uint32_t y32 = y;
+ u = STATIC_CAST(uint16_t, clamp255(u >> 8));
+ v = STATIC_CAST(uint16_t, clamp255(v >> 8));
+ CALC_RGB16;
+ *b = b16;
+ *g = g16;
+ *r = r16;
+}
+
+// C reference code that mimics the YUV assembly.
+// Reads 8 bit YUV and leaves result as 8 bit.
+static __inline void YPixel(uint8_t y,
+ uint8_t* b,
+ uint8_t* g,
+ uint8_t* r,
+ const struct YuvConstants* yuvconstants) {
+#if defined(__aarch64__) || defined(__arm__) || defined(__riscv)
+ int yg = yuvconstants->kRGBCoeffBias[0];
+ int ygb = yuvconstants->kRGBCoeffBias[4];
+#else
+ int ygb = yuvconstants->kYBiasToRgb[0];
+ int yg = yuvconstants->kYToRgb[0];
+#endif
+ uint32_t y1 = (uint32_t)(y * 0x0101 * yg) >> 16;
+ uint8_t b8 = STATIC_CAST(uint8_t, Clamp(((int32_t)(y1) + ygb) >> 6));
+ *b = b8;
+ *g = b8;
+ *r = b8;
+}
-#if !defined(LIBYUV_DISABLE_NEON) && \
- (defined(__ARM_NEON__) || defined(__aarch64__) || defined(LIBYUV_NEON))
-// C mimic assembly.
-// TODO(fbarchard): Remove subsampling from Neon.
void I444ToARGBRow_C(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -1484,45 +1883,33 @@ void I444ToARGBRow_C(const uint8_t* src_y,
const struct YuvConstants* yuvconstants,
int width) {
int x;
- for (x = 0; x < width - 1; x += 2) {
- uint8_t u = (src_u[0] + src_u[1] + 1) >> 1;
- uint8_t v = (src_v[0] + src_v[1] + 1) >> 1;
- YuvPixel(src_y[0], u, v, rgb_buf + 0, rgb_buf + 1, rgb_buf + 2,
- yuvconstants);
- rgb_buf[3] = 255;
- YuvPixel(src_y[1], u, v, rgb_buf + 4, rgb_buf + 5, rgb_buf + 6,
- yuvconstants);
- rgb_buf[7] = 255;
- src_y += 2;
- src_u += 2;
- src_v += 2;
- rgb_buf += 8; // Advance 2 pixels.
- }
- if (width & 1) {
+ for (x = 0; x < width; ++x) {
YuvPixel(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
rgb_buf + 2, yuvconstants);
rgb_buf[3] = 255;
+ src_y += 1;
+ src_u += 1;
+ src_v += 1;
+ rgb_buf += 4; // Advance 1 pixel.
}
}
-#else
-void I444ToARGBRow_C(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* rgb_buf,
- const struct YuvConstants* yuvconstants,
- int width) {
+
+void I444ToRGB24Row_C(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
int x;
for (x = 0; x < width; ++x) {
YuvPixel(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
rgb_buf + 2, yuvconstants);
- rgb_buf[3] = 255;
src_y += 1;
src_u += 1;
src_v += 1;
- rgb_buf += 4; // Advance 1 pixel.
+ rgb_buf += 3; // Advance 1 pixel.
}
}
-#endif
// Also used for 420
void I422ToARGBRow_C(const uint8_t* src_y,
@@ -1578,9 +1965,102 @@ void I210ToARGBRow_C(const uint16_t* src_y,
}
}
+void I410ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ YuvPixel10(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = 255;
+ src_y += 1;
+ src_u += 1;
+ src_v += 1;
+ rgb_buf += 4; // Advance 1 pixels.
+ }
+}
+
+void I210AlphaToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ YuvPixel10(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = STATIC_CAST(uint8_t, clamp255(src_a[0] >> 2));
+ YuvPixel10(src_y[1], src_u[0], src_v[0], rgb_buf + 4, rgb_buf + 5,
+ rgb_buf + 6, yuvconstants);
+ rgb_buf[7] = STATIC_CAST(uint8_t, clamp255(src_a[1] >> 2));
+ src_y += 2;
+ src_u += 1;
+ src_v += 1;
+ src_a += 2;
+ rgb_buf += 8; // Advance 2 pixels.
+ }
+ if (width & 1) {
+ YuvPixel10(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = STATIC_CAST(uint8_t, clamp255(src_a[0] >> 2));
+ }
+}
+
+void I410AlphaToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ YuvPixel10(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = STATIC_CAST(uint8_t, clamp255(src_a[0] >> 2));
+ src_y += 1;
+ src_u += 1;
+ src_v += 1;
+ src_a += 1;
+ rgb_buf += 4; // Advance 1 pixels.
+ }
+}
+
+// 12 bit YUV to ARGB
+void I212ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ YuvPixel12(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = 255;
+ YuvPixel12(src_y[1], src_u[0], src_v[0], rgb_buf + 4, rgb_buf + 5,
+ rgb_buf + 6, yuvconstants);
+ rgb_buf[7] = 255;
+ src_y += 2;
+ src_u += 1;
+ src_v += 1;
+ rgb_buf += 8; // Advance 2 pixels.
+ }
+ if (width & 1) {
+ YuvPixel12(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = 255;
+ }
+}
+
static void StoreAR30(uint8_t* rgb_buf, int b, int g, int r) {
uint32_t ar30;
- b = b >> 4; // convert 10.6 to 10 bit.
+ b = b >> 4; // convert 8 bit 10.6 to 10 bit.
g = g >> 4;
r = r >> 4;
b = Clamp10(b);
@@ -1602,9 +2082,36 @@ void I210ToAR30Row_C(const uint16_t* src_y,
int g;
int r;
for (x = 0; x < width - 1; x += 2) {
- YuvPixel16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ YuvPixel10_16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ StoreAR30(rgb_buf, b, g, r);
+ YuvPixel10_16(src_y[1], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ StoreAR30(rgb_buf + 4, b, g, r);
+ src_y += 2;
+ src_u += 1;
+ src_v += 1;
+ rgb_buf += 8; // Advance 2 pixels.
+ }
+ if (width & 1) {
+ YuvPixel10_16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ StoreAR30(rgb_buf, b, g, r);
+ }
+}
+
+// 12 bit YUV to 10 bit AR30
+void I212ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ int b;
+ int g;
+ int r;
+ for (x = 0; x < width - 1; x += 2) {
+ YuvPixel12_16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
StoreAR30(rgb_buf, b, g, r);
- YuvPixel16(src_y[1], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ YuvPixel12_16(src_y[1], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
StoreAR30(rgb_buf + 4, b, g, r);
src_y += 2;
src_u += 1;
@@ -1612,8 +2119,111 @@ void I210ToAR30Row_C(const uint16_t* src_y,
rgb_buf += 8; // Advance 2 pixels.
}
if (width & 1) {
- YuvPixel16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ YuvPixel12_16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
+ StoreAR30(rgb_buf, b, g, r);
+ }
+}
+
+void I410ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ int b;
+ int g;
+ int r;
+ for (x = 0; x < width; ++x) {
+ YuvPixel10_16(src_y[0], src_u[0], src_v[0], &b, &g, &r, yuvconstants);
StoreAR30(rgb_buf, b, g, r);
+ src_y += 1;
+ src_u += 1;
+ src_v += 1;
+ rgb_buf += 4; // Advance 1 pixel.
+ }
+}
+
+// P210 has 10 bits in msb of 16 bit NV12 style layout.
+void P210ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ YuvPixel16_8(src_y[0], src_uv[0], src_uv[1], dst_argb + 0, dst_argb + 1,
+ dst_argb + 2, yuvconstants);
+ dst_argb[3] = 255;
+ YuvPixel16_8(src_y[1], src_uv[0], src_uv[1], dst_argb + 4, dst_argb + 5,
+ dst_argb + 6, yuvconstants);
+ dst_argb[7] = 255;
+ src_y += 2;
+ src_uv += 2;
+ dst_argb += 8; // Advance 2 pixels.
+ }
+ if (width & 1) {
+ YuvPixel16_8(src_y[0], src_uv[0], src_uv[1], dst_argb + 0, dst_argb + 1,
+ dst_argb + 2, yuvconstants);
+ dst_argb[3] = 255;
+ }
+}
+
+void P410ToARGBRow_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ YuvPixel16_8(src_y[0], src_uv[0], src_uv[1], dst_argb + 0, dst_argb + 1,
+ dst_argb + 2, yuvconstants);
+ dst_argb[3] = 255;
+ src_y += 1;
+ src_uv += 2;
+ dst_argb += 4; // Advance 1 pixels.
+ }
+}
+
+void P210ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ int b;
+ int g;
+ int r;
+ for (x = 0; x < width - 1; x += 2) {
+ YuvPixel16_16(src_y[0], src_uv[0], src_uv[1], &b, &g, &r, yuvconstants);
+ StoreAR30(dst_ar30, b, g, r);
+ YuvPixel16_16(src_y[1], src_uv[0], src_uv[1], &b, &g, &r, yuvconstants);
+ StoreAR30(dst_ar30 + 4, b, g, r);
+ src_y += 2;
+ src_uv += 2;
+ dst_ar30 += 8; // Advance 2 pixels.
+ }
+ if (width & 1) {
+ YuvPixel16_16(src_y[0], src_uv[0], src_uv[1], &b, &g, &r, yuvconstants);
+ StoreAR30(dst_ar30, b, g, r);
+ }
+}
+
+void P410ToAR30Row_C(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ int b;
+ int g;
+ int r;
+ for (x = 0; x < width; ++x) {
+ YuvPixel16_16(src_y[0], src_uv[0], src_uv[1], &b, &g, &r, yuvconstants);
+ StoreAR30(dst_ar30, b, g, r);
+ src_y += 1;
+ src_uv += 2;
+ dst_ar30 += 4; // Advance 1 pixel.
}
}
@@ -1645,6 +2255,26 @@ void I422ToAR30Row_C(const uint8_t* src_y,
}
}
+void I444AlphaToARGBRow_C(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ YuvPixel(src_y[0], src_u[0], src_v[0], rgb_buf + 0, rgb_buf + 1,
+ rgb_buf + 2, yuvconstants);
+ rgb_buf[3] = src_a[0];
+ src_y += 1;
+ src_u += 1;
+ src_v += 1;
+ src_a += 1;
+ rgb_buf += 4; // Advance 1 pixel.
+ }
+}
+
void I422AlphaToARGBRow_C(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -1718,8 +2348,10 @@ void I422ToARGB4444Row_C(const uint8_t* src_y,
b1 = b1 >> 4;
g1 = g1 >> 4;
r1 = r1 >> 4;
- *(uint32_t*)(dst_argb4444) = b0 | (g0 << 4) | (r0 << 8) | (b1 << 16) |
- (g1 << 20) | (r1 << 24) | 0xf000f000;
+ *(uint16_t*)(dst_argb4444 + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 4) | (r0 << 8) | 0xf000);
+ *(uint16_t*)(dst_argb4444 + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 4) | (r1 << 8) | 0xf000);
src_y += 2;
src_u += 1;
src_v += 1;
@@ -1730,7 +2362,8 @@ void I422ToARGB4444Row_C(const uint8_t* src_y,
b0 = b0 >> 4;
g0 = g0 >> 4;
r0 = r0 >> 4;
- *(uint16_t*)(dst_argb4444) = b0 | (g0 << 4) | (r0 << 8) | 0xf000;
+ *(uint16_t*)(dst_argb4444) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 4) | (r0 << 8) | 0xf000);
}
}
@@ -1756,8 +2389,10 @@ void I422ToARGB1555Row_C(const uint8_t* src_y,
b1 = b1 >> 3;
g1 = g1 >> 3;
r1 = r1 >> 3;
- *(uint32_t*)(dst_argb1555) = b0 | (g0 << 5) | (r0 << 10) | (b1 << 16) |
- (g1 << 21) | (r1 << 26) | 0x80008000;
+ *(uint16_t*)(dst_argb1555 + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 10) | 0x8000);
+ *(uint16_t*)(dst_argb1555 + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 5) | (r1 << 10) | 0x8000);
src_y += 2;
src_u += 1;
src_v += 1;
@@ -1768,7 +2403,8 @@ void I422ToARGB1555Row_C(const uint8_t* src_y,
b0 = b0 >> 3;
g0 = g0 >> 3;
r0 = r0 >> 3;
- *(uint16_t*)(dst_argb1555) = b0 | (g0 << 5) | (r0 << 10) | 0x8000;
+ *(uint16_t*)(dst_argb1555) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 10) | 0x8000);
}
}
@@ -1794,8 +2430,10 @@ void I422ToRGB565Row_C(const uint8_t* src_y,
b1 = b1 >> 3;
g1 = g1 >> 2;
r1 = r1 >> 3;
- *(uint32_t*)(dst_rgb565) =
- b0 | (g0 << 5) | (r0 << 11) | (b1 << 16) | (g1 << 21) | (r1 << 27);
+ *(uint16_t*)(dst_rgb565 + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 11));
+ *(uint16_t*)(dst_rgb565 + 2) =
+ STATIC_CAST(uint16_t, b1 | (g1 << 5) | (r1 << 11));
src_y += 2;
src_u += 1;
src_v += 1;
@@ -1806,7 +2444,8 @@ void I422ToRGB565Row_C(const uint8_t* src_y,
b0 = b0 >> 3;
g0 = g0 >> 2;
r0 = r0 >> 3;
- *(uint16_t*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11);
+ *(uint16_t*)(dst_rgb565 + 0) =
+ STATIC_CAST(uint16_t, b0 | (g0 << 5) | (r0 << 11));
}
}
@@ -1921,8 +2560,12 @@ void NV12ToRGB565Row_C(const uint8_t* src_y,
b1 = b1 >> 3;
g1 = g1 >> 2;
r1 = r1 >> 3;
- *(uint32_t*)(dst_rgb565) =
- b0 | (g0 << 5) | (r0 << 11) | (b1 << 16) | (g1 << 21) | (r1 << 27);
+ *(uint16_t*)(dst_rgb565 + 0) = STATIC_CAST(uint16_t, b0) |
+ STATIC_CAST(uint16_t, g0 << 5) |
+ STATIC_CAST(uint16_t, r0 << 11);
+ *(uint16_t*)(dst_rgb565 + 2) = STATIC_CAST(uint16_t, b1) |
+ STATIC_CAST(uint16_t, g1 << 5) |
+ STATIC_CAST(uint16_t, r1 << 11);
src_y += 2;
src_uv += 2;
dst_rgb565 += 4; // Advance 2 pixels.
@@ -1932,7 +2575,9 @@ void NV12ToRGB565Row_C(const uint8_t* src_y,
b0 = b0 >> 3;
g0 = g0 >> 2;
r0 = r0 >> 3;
- *(uint16_t*)(dst_rgb565) = b0 | (g0 << 5) | (r0 << 11);
+ *(uint16_t*)(dst_rgb565) = STATIC_CAST(uint16_t, b0) |
+ STATIC_CAST(uint16_t, g0 << 5) |
+ STATIC_CAST(uint16_t, r0 << 11);
}
}
@@ -2006,18 +2651,21 @@ void I422ToRGBARow_C(const uint8_t* src_y,
}
}
-void I400ToARGBRow_C(const uint8_t* src_y, uint8_t* rgb_buf, int width) {
+void I400ToARGBRow_C(const uint8_t* src_y,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width) {
int x;
for (x = 0; x < width - 1; x += 2) {
- YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2);
+ YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2, yuvconstants);
rgb_buf[3] = 255;
- YPixel(src_y[1], rgb_buf + 4, rgb_buf + 5, rgb_buf + 6);
+ YPixel(src_y[1], rgb_buf + 4, rgb_buf + 5, rgb_buf + 6, yuvconstants);
rgb_buf[7] = 255;
src_y += 2;
rgb_buf += 8; // Advance 2 pixels.
}
if (width & 1) {
- YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2);
+ YPixel(src_y[0], rgb_buf + 0, rgb_buf + 1, rgb_buf + 2, yuvconstants);
rgb_buf[3] = 255;
}
}
@@ -2035,15 +2683,39 @@ void MirrorRow_C(const uint8_t* src, uint8_t* dst, int width) {
}
}
-void MirrorUVRow_C(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width) {
int x;
- src_uv += (width - 1) << 1;
+ src += width - 1;
for (x = 0; x < width - 1; x += 2) {
- dst_u[x] = src_uv[0];
- dst_u[x + 1] = src_uv[-2];
+ dst[x] = src[0];
+ dst[x + 1] = src[-1];
+ src -= 2;
+ }
+ if (width & 1) {
+ dst[width - 1] = src[0];
+ }
+}
+
+void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ int x;
+ src_uv += (width - 1) << 1;
+ for (x = 0; x < width; ++x) {
+ dst_uv[0] = src_uv[0];
+ dst_uv[1] = src_uv[1];
+ src_uv -= 2;
+ dst_uv += 2;
+ }
+}
+
+void MirrorSplitUVRow_C(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int x;
+ src_uv += (width - 1) << 1;
+ for (x = 0; x < width - 1; x += 2) {
+ dst_u[x] = src_uv[0];
+ dst_u[x + 1] = src_uv[-2];
dst_v[x] = src_uv[1];
dst_v[x + 1] = src_uv[-2 + 1];
src_uv -= 4;
@@ -2056,16 +2728,30 @@ void MirrorUVRow_C(const uint8_t* src_uv,
void ARGBMirrorRow_C(const uint8_t* src, uint8_t* dst, int width) {
int x;
- const uint32_t* src32 = (const uint32_t*)(src);
- uint32_t* dst32 = (uint32_t*)(dst);
- src32 += width - 1;
- for (x = 0; x < width - 1; x += 2) {
- dst32[x] = src32[0];
- dst32[x + 1] = src32[-1];
- src32 -= 2;
+ const uint8_t* s = src + (ptrdiff_t)(width - 1) * 4;
+ uint8_t* d = dst;
+ for (x = 0; x < width; ++x) {
+ d[0] = s[0];
+ d[1] = s[1];
+ d[2] = s[2];
+ d[3] = s[3];
+ s -= 4;
+ d += 4;
}
- if (width & 1) {
- dst32[width - 1] = src32[0];
+}
+
+void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width) {
+ int x;
+ src_rgb24 += (ptrdiff_t)width * 3 - 3;
+ for (x = 0; x < width; ++x) {
+ uint8_t b = src_rgb24[0];
+ uint8_t g = src_rgb24[1];
+ uint8_t r = src_rgb24[2];
+ dst_rgb24[0] = b;
+ dst_rgb24[1] = g;
+ dst_rgb24[2] = r;
+ src_rgb24 -= 3;
+ dst_rgb24 += 3;
}
}
@@ -2105,6 +2791,111 @@ void MergeUVRow_C(const uint8_t* src_u,
}
}
+void DetileRow_C(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width) {
+ int x;
+ for (x = 0; x < width - 15; x += 16) {
+ memcpy(dst, src, 16);
+ dst += 16;
+ src += src_tile_stride;
+ }
+ if (width & 15) {
+ memcpy(dst, src, width & 15);
+ }
+}
+
+void DetileRow_16_C(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width) {
+ int x;
+ for (x = 0; x < width - 15; x += 16) {
+ memcpy(dst, src, 16 * sizeof(uint16_t));
+ dst += 16;
+ src += src_tile_stride;
+ }
+ if (width & 15) {
+ memcpy(dst, src, (width & 15) * sizeof(uint16_t));
+ }
+}
+
+void DetileSplitUVRow_C(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int x;
+ for (x = 0; x < width - 15; x += 16) {
+ SplitUVRow_C(src_uv, dst_u, dst_v, 8);
+ dst_u += 8;
+ dst_v += 8;
+ src_uv += src_tile_stride;
+ }
+ if (width & 15) {
+ SplitUVRow_C(src_uv, dst_u, dst_v, ((width & 15) + 1) / 2);
+ }
+}
+
+void DetileToYUY2_C(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ int x;
+ for (x = 0; x < width - 15; x += 16) {
+ int i;
+ for (i = 0; i < 8; i++) {
+ dst_yuy2[0] = src_y[0];
+ dst_yuy2[1] = src_uv[0];
+ dst_yuy2[2] = src_y[1];
+ dst_yuy2[3] = src_uv[1];
+ dst_yuy2 += 4;
+ src_y += 2;
+ src_uv += 2;
+ }
+ src_y += src_y_tile_stride - 16;
+ src_uv += src_uv_tile_stride - 16;
+ }
+ // Remaining pixels of a partial 16 pixel tile.
+ for (x = 0; x < (width & 15); x += 2) {
+ dst_yuy2[0] = src_y[0];
+ dst_yuy2[1] = src_uv[0];
+ dst_yuy2[2] = src_y[1];
+ dst_yuy2[3] = src_uv[1];
+ dst_yuy2 += 4;
+ src_y += 2;
+ src_uv += 2;
+ }
+}
+
+// Unpack MT2T into tiled P010 64 pixels at a time. MT2T's bitstream is encoded
+// in 80 byte blocks representing 64 pixels each. The first 16 bytes of the
+// block contain all of the lower 2 bits of each pixel packed together, and the
+// next 64 bytes represent all the upper 8 bits of the pixel. The lower bits are
+// packed into 1x4 blocks, whereas the upper bits are packed in normal raster
+// order.
+void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size) {
+ while (size >= 80) {
+ const uint8_t* src_lower_bits = src;
+ const uint8_t* src_upper_bits = src + 16;
+
+ for (int j = 0; j < 4; j++) {
+ for (int k = 0; k < 16; k++) {
+ *dst++ = ((src_lower_bits[k] >> (j * 2)) & 0x3) << 6 |
+ (uint16_t)*src_upper_bits << 8 |
+ (uint16_t)*src_upper_bits >> 2;
+ src_upper_bits++;
+ }
+ }
+
+ src += 80;
+ size -= 80;
+ }
+}
+
void SplitRGBRow_C(const uint8_t* src_rgb,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -2133,27 +2924,197 @@ void MergeRGBRow_C(const uint8_t* src_r,
}
}
-// Use scale to convert lsb formats to msb, depending how many bits there are:
-// 128 = 9 bits
-// 64 = 10 bits
-// 16 = 12 bits
-// 1 = 16 bits
+void SplitARGBRow_C(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_b[x] = src_argb[0];
+ dst_g[x] = src_argb[1];
+ dst_r[x] = src_argb[2];
+ dst_a[x] = src_argb[3];
+ src_argb += 4;
+ }
+}
+
+void MergeARGBRow_C(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_argb[0] = src_b[x];
+ dst_argb[1] = src_g[x];
+ dst_argb[2] = src_r[x];
+ dst_argb[3] = src_a[x];
+ dst_argb += 4;
+ }
+}
+
+void MergeXR30Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ assert(depth >= 10);
+ assert(depth <= 16);
+ int x;
+ int shift = depth - 10;
+ uint32_t* dst_ar30_32 = (uint32_t*)dst_ar30;
+ for (x = 0; x < width; ++x) {
+ uint32_t r = clamp1023(src_r[x] >> shift);
+ uint32_t g = clamp1023(src_g[x] >> shift);
+ uint32_t b = clamp1023(src_b[x] >> shift);
+ dst_ar30_32[x] = b | (g << 10) | (r << 20) | 0xc0000000;
+ }
+}
+
+void MergeAR64Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ assert(depth >= 1);
+ assert(depth <= 16);
+ int x;
+ int shift = 16 - depth;
+ int max = (1 << depth) - 1;
+ for (x = 0; x < width; ++x) {
+ dst_ar64[0] = STATIC_CAST(uint16_t, ClampMax(src_b[x], max) << shift);
+ dst_ar64[1] = STATIC_CAST(uint16_t, ClampMax(src_g[x], max) << shift);
+ dst_ar64[2] = STATIC_CAST(uint16_t, ClampMax(src_r[x], max) << shift);
+ dst_ar64[3] = STATIC_CAST(uint16_t, ClampMax(src_a[x], max) << shift);
+ dst_ar64 += 4;
+ }
+}
+
+void MergeARGB16To8Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ assert(depth >= 8);
+ assert(depth <= 16);
+ int x;
+ int shift = depth - 8;
+ for (x = 0; x < width; ++x) {
+ dst_argb[0] = STATIC_CAST(uint8_t, clamp255(src_b[x] >> shift));
+ dst_argb[1] = STATIC_CAST(uint8_t, clamp255(src_g[x] >> shift));
+ dst_argb[2] = STATIC_CAST(uint8_t, clamp255(src_r[x] >> shift));
+ dst_argb[3] = STATIC_CAST(uint8_t, clamp255(src_a[x] >> shift));
+ dst_argb += 4;
+ }
+}
+
+void MergeXR64Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ assert(depth >= 1);
+ assert(depth <= 16);
+ int x;
+ int shift = 16 - depth;
+ int max = (1 << depth) - 1;
+ for (x = 0; x < width; ++x) {
+ dst_ar64[0] = STATIC_CAST(uint16_t, ClampMax(src_b[x], max) << shift);
+ dst_ar64[1] = STATIC_CAST(uint16_t, ClampMax(src_g[x], max) << shift);
+ dst_ar64[2] = STATIC_CAST(uint16_t, ClampMax(src_r[x], max) << shift);
+ dst_ar64[3] = 0xffff;
+ dst_ar64 += 4;
+ }
+}
+
+void MergeXRGB16To8Row_C(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ assert(depth >= 8);
+ assert(depth <= 16);
+ int x;
+ int shift = depth - 8;
+ for (x = 0; x < width; ++x) {
+ dst_argb[0] = STATIC_CAST(uint8_t, clamp255(src_b[x] >> shift));
+ dst_argb[1] = STATIC_CAST(uint8_t, clamp255(src_g[x] >> shift));
+ dst_argb[2] = STATIC_CAST(uint8_t, clamp255(src_r[x] >> shift));
+ dst_argb[3] = 0xff;
+ dst_argb += 4;
+ }
+}
+
+void SplitXRGBRow_C(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_b[x] = src_argb[0];
+ dst_g[x] = src_argb[1];
+ dst_r[x] = src_argb[2];
+ src_argb += 4;
+ }
+}
+
+void MergeXRGBRow_C(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_argb[0] = src_b[x];
+ dst_argb[1] = src_g[x];
+ dst_argb[2] = src_r[x];
+ dst_argb[3] = 255;
+ dst_argb += 4;
+ }
+}
+
+// Convert lsb formats to msb, depending on sample depth.
void MergeUVRow_16_C(const uint16_t* src_u,
const uint16_t* src_v,
uint16_t* dst_uv,
- int scale,
+ int depth,
int width) {
+ int shift = 16 - depth;
+ assert(depth >= 8);
+ assert(depth <= 16);
int x;
- for (x = 0; x < width - 1; x += 2) {
- dst_uv[0] = src_u[x] * scale;
- dst_uv[1] = src_v[x] * scale;
- dst_uv[2] = src_u[x + 1] * scale;
- dst_uv[3] = src_v[x + 1] * scale;
- dst_uv += 4;
+ for (x = 0; x < width; ++x) {
+ dst_uv[0] = STATIC_CAST(uint16_t, src_u[x] << shift);
+ dst_uv[1] = STATIC_CAST(uint16_t, src_v[x] << shift);
+ dst_uv += 2;
}
- if (width & 1) {
- dst_uv[0] = src_u[width - 1] * scale;
- dst_uv[1] = src_v[width - 1] * scale;
+}
+
+// Convert msb formats to lsb, depending on sample depth.
+void SplitUVRow_16_C(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ int x;
+ assert(depth >= 8);
+ assert(depth <= 16);
+ for (x = 0; x < width; ++x) {
+ dst_u[x] = src_uv[0] >> shift;
+ dst_v[x] = src_uv[1] >> shift;
+ src_uv += 2;
}
}
@@ -2163,7 +3124,19 @@ void MultiplyRow_16_C(const uint16_t* src_y,
int width) {
int x;
for (x = 0; x < width; ++x) {
- dst_y[x] = src_y[x] * scale;
+ dst_y[x] = STATIC_CAST(uint16_t, src_y[x] * scale);
+ }
+}
+
+void DivideRow_16_C(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ int x;
+ // scale can be 65536 so multiply unsigned to avoid signed overflow.
+ for (x = 0; x < width; ++x) {
+ dst_y[x] =
+ STATIC_CAST(uint16_t, ((uint32_t)src_y[x] * (uint32_t)scale) >> 16);
}
}
@@ -2172,26 +3145,50 @@ void MultiplyRow_16_C(const uint16_t* src_y,
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
+// TODO(fbarchard): change scale to bits
+#define C16TO8(v, scale) \
+ clamp255((int32_t)(((uint32_t)(v) * (uint32_t)(scale)) >> 16))
+
void Convert16To8Row_C(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width) {
int x;
+ assert(scale >= 256);
+ assert(scale <= 32768);
+
for (x = 0; x < width; ++x) {
- dst_y[x] = clamp255((src_y[x] * scale) >> 16);
+ dst_y[x] = STATIC_CAST(uint8_t, C16TO8(src_y[x], scale));
}
}
-// Use scale to convert lsb formats to msb, depending how many bits there are:
-// 1024 = 10 bits
+// Convert 8 bit values to 10, 12 or 16 bits.
void Convert8To16Row_C(const uint8_t* src_y,
uint16_t* dst_y,
- int scale,
+ int bits,
int width) {
int x;
- scale *= 0x0101; // replicates the byte.
+ int shift = 16 - bits;
+ for (x = 0; x < width; ++x) {
+ dst_y[x] = (src_y[x] * 0x0101) >> shift;
+ }
+}
+
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_C(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ int x;
+ assert(scale >= 0);
+ assert(scale <= 255);
+
for (x = 0; x < width; ++x) {
- dst_y[x] = (src_y[x] * scale) >> 16;
+ dst_y[x] = ((src_y[x] * scale) >> 8) + bias;
}
}
@@ -2200,7 +3197,7 @@ void CopyRow_C(const uint8_t* src, uint8_t* dst, int count) {
}
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count) {
- memcpy(dst, src, count * 2);
+ memcpy(dst, src, (size_t)count * 2);
}
void SetRow_C(uint8_t* dst, uint8_t v8, int width) {
@@ -2208,10 +3205,9 @@ void SetRow_C(uint8_t* dst, uint8_t v8, int width) {
}
void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width) {
- uint32_t* d = (uint32_t*)(dst_argb);
int x;
for (x = 0; x < width; ++x) {
- d[x] = v32;
+ memcpy(dst_argb + x * sizeof v32, &v32, sizeof v32);
}
}
@@ -2224,14 +3220,31 @@ void YUY2ToUVRow_C(const uint8_t* src_yuy2,
// Output a row of UV values, filtering 2 rows of YUY2.
int x;
for (x = 0; x < width; x += 2) {
- dst_u[0] = (src_yuy2[1] + src_yuy2[src_stride_yuy2 + 1] + 1) >> 1;
- dst_v[0] = (src_yuy2[3] + src_yuy2[src_stride_yuy2 + 3] + 1) >> 1;
+ const uint8_t* src_yuy2_next = src_yuy2 + src_stride_yuy2;
+ dst_u[0] = (src_yuy2[1] + src_yuy2_next[1] + 1) >> 1;
+ dst_v[0] = (src_yuy2[3] + src_yuy2_next[3] + 1) >> 1;
src_yuy2 += 4;
dst_u += 1;
dst_v += 1;
}
}
+// Filter 2 rows of YUY2 UV's (422) into UV (NV12).
+void YUY2ToNVUVRow_C(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ // Output a row of UV values, filtering 2 rows of YUY2.
+ int x;
+ for (x = 0; x < width; x += 2) {
+ const uint8_t* src_yuy2_next = src_yuy2 + src_stride_yuy2;
+ dst_uv[0] = (src_yuy2[1] + src_yuy2_next[1] + 1) >> 1;
+ dst_uv[1] = (src_yuy2[3] + src_yuy2_next[3] + 1) >> 1;
+ src_yuy2 += 4;
+ dst_uv += 2;
+ }
+}
+
// Copy row of YUY2 UV's (422) into U and V (422).
void YUY2ToUV422Row_C(const uint8_t* src_yuy2,
uint8_t* dst_u,
@@ -2271,8 +3284,9 @@ void UYVYToUVRow_C(const uint8_t* src_uyvy,
// Output a row of UV values.
int x;
for (x = 0; x < width; x += 2) {
- dst_u[0] = (src_uyvy[0] + src_uyvy[src_stride_uyvy + 0] + 1) >> 1;
- dst_v[0] = (src_uyvy[2] + src_uyvy[src_stride_uyvy + 2] + 1) >> 1;
+ const uint8_t* src_uyvy_next = src_uyvy + src_stride_uyvy;
+ dst_u[0] = (src_uyvy[0] + src_uyvy_next[0] + 1) >> 1;
+ dst_v[0] = (src_uyvy[2] + src_uyvy_next[2] + 1) >> 1;
src_uyvy += 4;
dst_u += 1;
dst_v += 1;
@@ -2309,62 +3323,123 @@ void UYVYToYRow_C(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
}
}
-#define BLEND(f, b, a) (((256 - a) * b) >> 8) + f
+#define BLEND(f, b, a) clamp255((((256 - a) * b) >> 8) + f)
-// Blend src_argb0 over src_argb1 and store to dst_argb.
-// dst_argb may be src_argb0 or src_argb1.
+// Blend src_argb over src_argb1 and store to dst_argb.
+// dst_argb may be src_argb or src_argb1.
// This code mimics the SSSE3 version for better testability.
-void ARGBBlendRow_C(const uint8_t* src_argb0,
+void ARGBBlendRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
int x;
for (x = 0; x < width - 1; x += 2) {
- uint32_t fb = src_argb0[0];
- uint32_t fg = src_argb0[1];
- uint32_t fr = src_argb0[2];
- uint32_t a = src_argb0[3];
+ uint32_t fb = src_argb[0];
+ uint32_t fg = src_argb[1];
+ uint32_t fr = src_argb[2];
+ uint32_t a = src_argb[3];
uint32_t bb = src_argb1[0];
uint32_t bg = src_argb1[1];
uint32_t br = src_argb1[2];
- dst_argb[0] = BLEND(fb, bb, a);
- dst_argb[1] = BLEND(fg, bg, a);
- dst_argb[2] = BLEND(fr, br, a);
+ dst_argb[0] = STATIC_CAST(uint8_t, BLEND(fb, bb, a));
+ dst_argb[1] = STATIC_CAST(uint8_t, BLEND(fg, bg, a));
+ dst_argb[2] = STATIC_CAST(uint8_t, BLEND(fr, br, a));
dst_argb[3] = 255u;
- fb = src_argb0[4 + 0];
- fg = src_argb0[4 + 1];
- fr = src_argb0[4 + 2];
- a = src_argb0[4 + 3];
+ fb = src_argb[4 + 0];
+ fg = src_argb[4 + 1];
+ fr = src_argb[4 + 2];
+ a = src_argb[4 + 3];
bb = src_argb1[4 + 0];
bg = src_argb1[4 + 1];
br = src_argb1[4 + 2];
- dst_argb[4 + 0] = BLEND(fb, bb, a);
- dst_argb[4 + 1] = BLEND(fg, bg, a);
- dst_argb[4 + 2] = BLEND(fr, br, a);
+ dst_argb[4 + 0] = STATIC_CAST(uint8_t, BLEND(fb, bb, a));
+ dst_argb[4 + 1] = STATIC_CAST(uint8_t, BLEND(fg, bg, a));
+ dst_argb[4 + 2] = STATIC_CAST(uint8_t, BLEND(fr, br, a));
dst_argb[4 + 3] = 255u;
- src_argb0 += 8;
+ src_argb += 8;
src_argb1 += 8;
dst_argb += 8;
}
if (width & 1) {
- uint32_t fb = src_argb0[0];
- uint32_t fg = src_argb0[1];
- uint32_t fr = src_argb0[2];
- uint32_t a = src_argb0[3];
+ uint32_t fb = src_argb[0];
+ uint32_t fg = src_argb[1];
+ uint32_t fr = src_argb[2];
+ uint32_t a = src_argb[3];
uint32_t bb = src_argb1[0];
uint32_t bg = src_argb1[1];
uint32_t br = src_argb1[2];
- dst_argb[0] = BLEND(fb, bb, a);
- dst_argb[1] = BLEND(fg, bg, a);
- dst_argb[2] = BLEND(fr, br, a);
+ dst_argb[0] = STATIC_CAST(uint8_t, BLEND(fb, bb, a));
+ dst_argb[1] = STATIC_CAST(uint8_t, BLEND(fg, bg, a));
+ dst_argb[2] = STATIC_CAST(uint8_t, BLEND(fr, br, a));
dst_argb[3] = 255u;
}
}
#undef BLEND
-#define UBLEND(f, b, a) (((a)*f) + ((255 - a) * b) + 255) >> 8
+// Blend unattenuated src_argb0 over src_argb1 and store to dst_argb.
+void ARGBAlphaBlendRow_C(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ int sa = src_argb0[3];
+ int da = src_argb1[3];
+ if (da == 0) {
+ dst_argb[0] = src_argb0[0];
+ dst_argb[1] = src_argb0[1];
+ dst_argb[2] = src_argb0[2];
+ dst_argb[3] = STATIC_CAST(uint8_t, sa);
+ } else if (sa == 0) {
+ dst_argb[0] = src_argb1[0];
+ dst_argb[1] = src_argb1[1];
+ dst_argb[2] = src_argb1[2];
+ dst_argb[3] = STATIC_CAST(uint8_t, da);
+ } else {
+ int dst_alpha = da + sa - da * sa / 255;
+ int alpha_ratio = sa * 255 / dst_alpha;
+ dst_argb[0] = STATIC_CAST(
+ uint8_t,
+ (src_argb1[0] * (255 - alpha_ratio) + src_argb0[0] * alpha_ratio) /
+ 255);
+ dst_argb[1] = STATIC_CAST(
+ uint8_t,
+ (src_argb1[1] * (255 - alpha_ratio) + src_argb0[1] * alpha_ratio) /
+ 255);
+ dst_argb[2] = STATIC_CAST(
+ uint8_t,
+ (src_argb1[2] * (255 - alpha_ratio) + src_argb0[2] * alpha_ratio) /
+ 255);
+ dst_argb[3] = STATIC_CAST(uint8_t, dst_alpha);
+ }
+ src_argb0 += 4;
+ src_argb1 += 4;
+ dst_argb += 4;
+ }
+}
+
+void ARGBOrRow_C(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ const uint8_t v0 = value & 0xff;
+ const uint8_t v1 = (value >> 8) & 0xff;
+ const uint8_t v2 = (value >> 16) & 0xff;
+ const uint8_t v3 = (value >> 24) & 0xff;
+ int i;
+ for (i = 0; i < width; ++i) {
+ dst_argb[0] = src_argb[0] | v0;
+ dst_argb[1] = src_argb[1] | v1;
+ dst_argb[2] = src_argb[2] | v2;
+ dst_argb[3] = src_argb[3] | v3;
+ src_argb += 4;
+ dst_argb += 4;
+ }
+}
+
+#define UBLEND(f, b, a) (((a) * f) + ((255 - a) * b) + 255) >> 8
void BlendPlaneRow_C(const uint8_t* src0,
const uint8_t* src1,
const uint8_t* alpha,
@@ -2385,10 +3460,9 @@ void BlendPlaneRow_C(const uint8_t* src0,
}
#undef UBLEND
-#define ATTENUATE(f, a) (a | (a << 8)) * (f | (f << 8)) >> 24
+#define ATTENUATE(f, a) (f * a + 255) >> 8
// Multiply source RGB by alpha and store to destination.
-// This code mimics the SSSE3 version for better testability.
void ARGBAttenuateRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
int i;
for (i = 0; i < width - 1; i += 2) {
@@ -2399,7 +3473,7 @@ void ARGBAttenuateRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
dst_argb[0] = ATTENUATE(b, a);
dst_argb[1] = ATTENUATE(g, a);
dst_argb[2] = ATTENUATE(r, a);
- dst_argb[3] = a;
+ dst_argb[3] = STATIC_CAST(uint8_t, a);
b = src_argb[4];
g = src_argb[5];
r = src_argb[6];
@@ -2407,7 +3481,7 @@ void ARGBAttenuateRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
dst_argb[4] = ATTENUATE(b, a);
dst_argb[5] = ATTENUATE(g, a);
dst_argb[6] = ATTENUATE(r, a);
- dst_argb[7] = a;
+ dst_argb[7] = STATIC_CAST(uint8_t, a);
src_argb += 8;
dst_argb += 8;
}
@@ -2420,7 +3494,7 @@ void ARGBAttenuateRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
dst_argb[0] = ATTENUATE(b, a);
dst_argb[1] = ATTENUATE(g, a);
dst_argb[2] = ATTENUATE(r, a);
- dst_argb[3] = a;
+ dst_argb[3] = STATIC_CAST(uint8_t, a);
}
}
#undef ATTENUATE
@@ -2472,6 +3546,10 @@ const uint32_t fixed_invtbl8[256] = {
T(0xfc), T(0xfd), T(0xfe), 0x01000100};
#undef T
+// This code mimics the Intel SIMD version for better testability.
+#define UNATTENUATE(f, ia) clamp255(((f | (f << 8)) * ia) >> 16)
+
+// mimics the Intel SIMD code for exactness.
void ARGBUnattenuateRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
@@ -2482,14 +3560,12 @@ void ARGBUnattenuateRow_C(const uint8_t* src_argb,
uint32_t r = src_argb[2];
const uint32_t a = src_argb[3];
const uint32_t ia = fixed_invtbl8[a] & 0xffff; // 8.8 fixed point
- b = (b * ia) >> 8;
- g = (g * ia) >> 8;
- r = (r * ia) >> 8;
+
// Clamping should not be necessary but is free in assembly.
- dst_argb[0] = clamp255(b);
- dst_argb[1] = clamp255(g);
- dst_argb[2] = clamp255(r);
- dst_argb[3] = a;
+ dst_argb[0] = STATIC_CAST(uint8_t, UNATTENUATE(b, ia));
+ dst_argb[1] = STATIC_CAST(uint8_t, UNATTENUATE(g, ia));
+ dst_argb[2] = STATIC_CAST(uint8_t, UNATTENUATE(r, ia));
+ dst_argb[3] = STATIC_CAST(uint8_t, a);
src_argb += 4;
dst_argb += 4;
}
@@ -2519,13 +3595,24 @@ void CumulativeSumToAverageRow_C(const int32_t* tl,
int area,
uint8_t* dst,
int count) {
- float ooa = 1.0f / area;
+ float ooa;
int i;
+ assert(area != 0);
+
+ ooa = 1.0f / STATIC_CAST(float, area);
for (i = 0; i < count; ++i) {
- dst[0] = (uint8_t)((bl[w + 0] + tl[0] - bl[0] - tl[w + 0]) * ooa);
- dst[1] = (uint8_t)((bl[w + 1] + tl[1] - bl[1] - tl[w + 1]) * ooa);
- dst[2] = (uint8_t)((bl[w + 2] + tl[2] - bl[2] - tl[w + 2]) * ooa);
- dst[3] = (uint8_t)((bl[w + 3] + tl[3] - bl[3] - tl[w + 3]) * ooa);
+ dst[0] =
+ (uint8_t)(STATIC_CAST(float, bl[w + 0] + tl[0] - bl[0] - tl[w + 0]) *
+ ooa);
+ dst[1] =
+ (uint8_t)(STATIC_CAST(float, bl[w + 1] + tl[1] - bl[1] - tl[w + 1]) *
+ ooa);
+ dst[2] =
+ (uint8_t)(STATIC_CAST(float, bl[w + 2] + tl[2] - bl[2] - tl[w + 2]) *
+ ooa);
+ dst[3] =
+ (uint8_t)(STATIC_CAST(float, bl[w + 3] + tl[3] - bl[3] - tl[w + 3]) *
+ ooa);
dst += 4;
tl += 4;
bl += 4;
@@ -2548,7 +3635,8 @@ void ARGBAffineRow_C(const uint8_t* src_argb,
int x = (int)(uv[0]);
int y = (int)(uv[1]);
*(uint32_t*)(dst_argb) =
- *(const uint32_t*)(src_argb + y * src_argb_stride + x * 4);
+ *(const uint32_t*)(src_argb + (ptrdiff_t)y * src_argb_stride +
+ (ptrdiff_t)x * 4);
dst_argb += 4;
uv[0] += uv_dudv[2];
uv[1] += uv_dudv[3];
@@ -2556,26 +3644,90 @@ void ARGBAffineRow_C(const uint8_t* src_argb,
}
// Blend 2 rows into 1.
-static void HalfRow_C(const uint8_t* src_uv,
- ptrdiff_t src_uv_stride,
- uint8_t* dst_uv,
- int width) {
+void HalfRow_C(const uint8_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int width) {
int x;
for (x = 0; x < width; ++x) {
dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1;
}
}
-static void HalfRow_16_C(const uint16_t* src_uv,
- ptrdiff_t src_uv_stride,
- uint16_t* dst_uv,
- int width) {
+void HalfRow_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint16_t* dst_uv,
+ int width) {
int x;
for (x = 0; x < width; ++x) {
dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1;
}
}
+void HalfRow_16To8_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ dst_uv[x] = STATIC_CAST(
+ uint8_t,
+ C16TO8((src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1, scale));
+ }
+}
+
+void HalfWidthRow_16To8_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* s = src_uv;
+ const uint16_t* t = src_uv + src_uv_stride;
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ dst_uv[0] = STATIC_CAST(
+ uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
+ dst_uv[1] = STATIC_CAST(
+ uint8_t, C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
+ dst_uv += 2;
+ s += 4;
+ t += 4;
+ }
+ if (width & 1) {
+ dst_uv[0] = STATIC_CAST(
+ uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
+ }
+}
+
+void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* s = src_uv;
+ const uint16_t* t = src_uv + src_uv_stride;
+ int x;
+ width -= 1;
+ for (x = 0; x < width - 1; x += 2) {
+ dst_uv[0] = STATIC_CAST(
+ uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
+ dst_uv[1] = STATIC_CAST(
+ uint8_t, C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
+ dst_uv += 2;
+ s += 4;
+ t += 4;
+ }
+ if (width & 1) {
+ dst_uv[0] = STATIC_CAST(
+ uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
+ dst_uv += 1;
+ s += 2;
+ t += 2;
+ }
+ dst_uv[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + t[0] + 1) >> 1, scale));
+}
+
// C version 2x2 -> 2x1.
void InterpolateRow_C(uint8_t* dst_ptr,
const uint8_t* src_ptr,
@@ -2586,6 +3738,9 @@ void InterpolateRow_C(uint8_t* dst_ptr,
int y0_fraction = 256 - y1_fraction;
const uint8_t* src_ptr1 = src_ptr + src_stride;
int x;
+ assert(source_y_fraction >= 0);
+ assert(source_y_fraction < 256);
+
if (y1_fraction == 0) {
memcpy(dst_ptr, src_ptr, width);
return;
@@ -2594,21 +3749,17 @@ void InterpolateRow_C(uint8_t* dst_ptr,
HalfRow_C(src_ptr, src_stride, dst_ptr, width);
return;
}
- for (x = 0; x < width - 1; x += 2) {
- dst_ptr[0] =
- (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8;
- dst_ptr[1] =
- (src_ptr[1] * y0_fraction + src_ptr1[1] * y1_fraction + 128) >> 8;
- src_ptr += 2;
- src_ptr1 += 2;
- dst_ptr += 2;
- }
- if (width & 1) {
- dst_ptr[0] =
- (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8;
+ for (x = 0; x < width; ++x) {
+ dst_ptr[0] = STATIC_CAST(
+ uint8_t,
+ (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8);
+ ++src_ptr;
+ ++src_ptr1;
+ ++dst_ptr;
}
}
+// C version 2x2 -> 2x1.
void InterpolateRow_16_C(uint16_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
@@ -2618,23 +3769,24 @@ void InterpolateRow_16_C(uint16_t* dst_ptr,
int y0_fraction = 256 - y1_fraction;
const uint16_t* src_ptr1 = src_ptr + src_stride;
int x;
- if (source_y_fraction == 0) {
- memcpy(dst_ptr, src_ptr, width * 2);
+ assert(source_y_fraction >= 0);
+ assert(source_y_fraction < 256);
+
+ if (y1_fraction == 0) {
+ memcpy(dst_ptr, src_ptr, (size_t)width * 2);
return;
}
- if (source_y_fraction == 128) {
+ if (y1_fraction == 128) {
HalfRow_16_C(src_ptr, src_stride, dst_ptr, width);
return;
}
- for (x = 0; x < width - 1; x += 2) {
- dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8;
- dst_ptr[1] = (src_ptr[1] * y0_fraction + src_ptr1[1] * y1_fraction) >> 8;
- src_ptr += 2;
- src_ptr1 += 2;
- dst_ptr += 2;
- }
- if (width & 1) {
- dst_ptr[0] = (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction) >> 8;
+ for (x = 0; x < width; ++x) {
+ dst_ptr[0] = STATIC_CAST(
+ uint16_t,
+ (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8);
+ ++src_ptr;
+ ++src_ptr1;
+ ++dst_ptr;
}
}
@@ -2743,10 +3895,10 @@ void ARGBPolynomialRow_C(const uint8_t* src_argb,
dr += poly[14] * r3;
da += poly[15] * a3;
- dst_argb[0] = Clamp((int32_t)(db));
- dst_argb[1] = Clamp((int32_t)(dg));
- dst_argb[2] = Clamp((int32_t)(dr));
- dst_argb[3] = Clamp((int32_t)(da));
+ dst_argb[0] = STATIC_CAST(uint8_t, Clamp((int32_t)(db)));
+ dst_argb[1] = STATIC_CAST(uint8_t, Clamp((int32_t)(dg)));
+ dst_argb[2] = STATIC_CAST(uint8_t, Clamp((int32_t)(dr)));
+ dst_argb[3] = STATIC_CAST(uint8_t, Clamp((int32_t)(da)));
src_argb += 4;
dst_argb += 4;
}
@@ -2774,9 +3926,15 @@ void HalfFloatRow_C(const uint16_t* src,
float scale,
int width) {
int i;
- float mult = 1.9259299444e-34f * scale;
+ // Multiply by scale first, then by 2^-112 to move the exponent into half
+ // float position. Doing the scale and bias as two multiplies keeps every
+ // intermediate normal; a combined (scale * 2^-112) constant is a denormal
+ // for scale < 2^-14 (e.g. 1/65535), which costs a microcode assist per
+ // multiply on Intel CPUs (50x slower). 2^-112 is a power of 2 so the
+ // result is identical to the single multiply when that was representable.
+ const float kScaleBias = 1.9259299444e-34f; // 2^-112
for (i = 0; i < width; ++i) {
- float value = src[i] * mult;
+ float value = (src[i] * scale) * kScaleBias;
dst[i] = (uint16_t)((*(const uint32_alias_t*)&value) >> 13);
}
}
@@ -2785,7 +3943,7 @@ void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width) {
int i;
for (i = 0; i < width; ++i) {
float value = src[i] * scale;
- dst[i] = value;
+ memcpy(&dst[i], &value, sizeof(value));
}
}
@@ -2873,8 +4031,8 @@ void ARGBCopyYToAlphaRow_C(const uint8_t* src, uint8_t* dst, int width) {
// Maximum temporary width for wrappers to process at a time, in pixels.
#define MAXTWIDTH 2048
-#if !(defined(_MSC_VER) && defined(_M_IX86)) && \
- defined(HAS_I422TORGB565ROW_SSSE3)
+#if !(defined(_MSC_VER) && !defined(__clang__) && defined(_M_IX86)) && \
+ defined(HAS_I422TORGB565ROW_SSSE3) && !defined(LIBYUV_ENABLE_ROWWIN)
// row_win.cc has asm version, but GCC uses 2 step wrapper.
void I422ToRGB565Row_SSSE3(const uint8_t* src_y,
const uint8_t* src_u,
@@ -2886,7 +4044,7 @@ void I422ToRGB565Row_SSSE3(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, yuvconstants, twidth);
- ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth);
+ ARGBToRGB565Row_C(row, dst_rgb565, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -2908,7 +4066,7 @@ void I422ToARGB1555Row_SSSE3(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, yuvconstants, twidth);
- ARGBToARGB1555Row_SSE2(row, dst_argb1555, twidth);
+ ARGBToARGB1555Row_C(row, dst_argb1555, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -2930,7 +4088,7 @@ void I422ToARGB4444Row_SSSE3(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_SSSE3(src_y, src_u, src_v, row, yuvconstants, twidth);
- ARGBToARGB4444Row_SSE2(row, dst_argb4444, twidth);
+ ARGBToARGB4444Row_C(row, dst_argb4444, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -2951,7 +4109,7 @@ void NV12ToRGB565Row_SSSE3(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
NV12ToARGBRow_SSSE3(src_y, src_uv, row, yuvconstants, twidth);
- ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth);
+ ARGBToRGB565Row_C(row, dst_rgb565, twidth);
src_y += twidth;
src_uv += twidth;
dst_rgb565 += twidth * 2;
@@ -3000,7 +4158,7 @@ void NV21ToRGB24Row_SSSE3(const uint8_t* src_y,
}
#endif
-#if defined(HAS_NV12TORGB24ROW_AVX2)
+#if defined(HAS_NV12TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
void NV12ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb24,
@@ -3011,11 +4169,7 @@ void NV12ToRGB24Row_AVX2(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
NV12ToARGBRow_AVX2(src_y, src_uv, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTORGB24ROW_AVX2)
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
-#else
- ARGBToRGB24Row_SSSE3(row, dst_rgb24, twidth);
-#endif
src_y += twidth;
src_uv += twidth;
dst_rgb24 += twidth * 3;
@@ -3024,7 +4178,7 @@ void NV12ToRGB24Row_AVX2(const uint8_t* src_y,
}
#endif
-#if defined(HAS_NV21TORGB24ROW_AVX2)
+#if defined(HAS_NV21TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
void NV21ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_rgb24,
@@ -3035,11 +4189,7 @@ void NV21ToRGB24Row_AVX2(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
NV21ToARGBRow_AVX2(src_y, src_vu, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTORGB24ROW_AVX2)
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
-#else
- ARGBToRGB24Row_SSSE3(row, dst_rgb24, twidth);
-#endif
src_y += twidth;
src_vu += twidth;
dst_rgb24 += twidth * 3;
@@ -3048,22 +4198,18 @@ void NV21ToRGB24Row_AVX2(const uint8_t* src_y,
}
#endif
-#if defined(HAS_I422TORGB565ROW_AVX2)
-void I422ToRGB565Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width) {
+#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB565ROW_AVX512BW)
+void I422ToRGB565Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
- I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTORGB565ROW_AVX2)
- ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth);
-#else
- ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth);
-#endif
+ I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
+ ARGBToRGB565Row_AVX512BW(row, dst_rgb565, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -3073,7 +4219,78 @@ void I422ToRGB565Row_AVX2(const uint8_t* src_y,
}
#endif
-#if defined(HAS_I422TOARGB1555ROW_AVX2)
+#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB1555ROW_AVX2)
+void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
+ if (IS_ALIGNED(twidth, 8)) {
+ ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth);
+ } else {
+ ARGBToARGB1555Row_Any_AVX2(row, dst_argb1555, twidth);
+ }
+ src_y += twidth;
+ src_u += twidth / 2;
+ src_v += twidth / 2;
+ dst_argb1555 += twidth * 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB4444ROW_AVX2)
+void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
+ if (IS_ALIGNED(twidth, 8)) {
+ ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth);
+ } else {
+ ARGBToARGB4444Row_Any_AVX2(row, dst_argb4444, twidth);
+ }
+ src_y += twidth;
+ src_u += twidth / 2;
+ src_v += twidth / 2;
+ dst_argb4444 += twidth * 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB565ROW_AVX2)
+void I422ToRGB565Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
+ ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth);
+ src_y += twidth;
+ src_u += twidth / 2;
+ src_v += twidth / 2;
+ dst_rgb565 += twidth * 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTOARGB1555ROW_AVX2)
void I422ToARGB1555Row_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -3085,11 +4302,7 @@ void I422ToARGB1555Row_AVX2(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTOARGB1555ROW_AVX2)
ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth);
-#else
- ARGBToARGB1555Row_SSE2(row, dst_argb1555, twidth);
-#endif
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -3099,7 +4312,7 @@ void I422ToARGB1555Row_AVX2(const uint8_t* src_y,
}
#endif
-#if defined(HAS_I422TOARGB4444ROW_AVX2)
+#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTOARGB4444ROW_AVX2)
void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -3111,11 +4324,7 @@ void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTOARGB4444ROW_AVX2)
ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth);
-#else
- ARGBToARGB4444Row_SSE2(row, dst_argb4444, twidth);
-#endif
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
@@ -3125,8 +4334,8 @@ void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
}
#endif
-#if defined(HAS_I422TORGB24ROW_AVX2)
-void I422ToRGB24Row_AVX2(const uint8_t* src_y,
+#if defined(HAS_I444TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
+void I444ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
@@ -3136,22 +4345,18 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
- I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTORGB24ROW_AVX2)
+ I444ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
-#else
- ARGBToRGB24Row_SSSE3(row, dst_rgb24, twidth);
-#endif
src_y += twidth;
- src_u += twidth / 2;
- src_v += twidth / 2;
+ src_u += twidth;
+ src_v += twidth;
dst_rgb24 += twidth * 3;
width -= twidth;
}
}
#endif
-#if defined(HAS_NV12TORGB565ROW_AVX2)
+#if defined(HAS_NV12TOARGBROW_AVX2) && defined(HAS_ARGBTORGB565ROW_AVX2)
void NV12ToRGB565Row_AVX2(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb565,
@@ -3162,11 +4367,7 @@ void NV12ToRGB565Row_AVX2(const uint8_t* src_y,
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
NV12ToARGBRow_AVX2(src_y, src_uv, row, yuvconstants, twidth);
-#if defined(HAS_ARGBTORGB565ROW_AVX2)
ARGBToRGB565Row_AVX2(row, dst_rgb565, twidth);
-#else
- ARGBToRGB565Row_SSE2(row, dst_rgb565, twidth);
-#endif
src_y += twidth;
src_uv += twidth;
dst_rgb565 += twidth * 2;
@@ -3178,9 +4379,6 @@ void NV12ToRGB565Row_AVX2(const uint8_t* src_y,
float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) {
float fsum = 0.f;
int i;
-#if defined(__clang__)
-#pragma clang loop vectorize_width(4)
-#endif
for (i = 0; i < width; ++i) {
float v = *src++;
fsum += v * v;
@@ -3211,8 +4409,9 @@ void ScaleSamples_C(const float* src, float* dst, float scale, int width) {
void GaussRow_C(const uint32_t* src, uint16_t* dst, int width) {
int i;
for (i = 0; i < width; ++i) {
- *dst++ =
- (src[0] + src[1] * 4 + src[2] * 6 + src[3] * 4 + src[4] + 128) >> 8;
+ *dst++ = STATIC_CAST(
+ uint16_t,
+ (src[0] + src[1] * 4 + src[2] * 6 + src[3] * 4 + src[4] + 128) >> 8);
++src;
}
}
@@ -3231,6 +4430,29 @@ void GaussCol_C(const uint16_t* src0,
}
}
+void GaussRow_F32_C(const float* src, float* dst, int width) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ *dst++ = (src[0] + src[1] * 4 + src[2] * 6 + src[3] * 4 + src[4]) *
+ (1.0f / 256.0f);
+ ++src;
+ }
+}
+
+// filter 5 rows with 1, 4, 6, 4, 1 coefficients to produce 1 row.
+void GaussCol_F32_C(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width) {
+ int i;
+ for (i = 0; i < width; ++i) {
+ *dst++ = *src0++ + *src1++ * 4 + *src2++ * 6 + *src3++ * 4 + *src4++;
+ }
+}
+
// Convert biplanar NV21 to packed YUV24
void NV21ToYUV24Row_C(const uint8_t* src_y,
const uint8_t* src_vu,
@@ -3256,29 +4478,28 @@ void NV21ToYUV24Row_C(const uint8_t* src_y,
}
// Filter 2 rows of AYUV UV's (444) into UV (420).
+// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
void AYUVToUVRow_C(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_uv,
int width) {
// Output a row of UV values, filtering 2x2 rows of AYUV.
int x;
- for (x = 0; x < width; x += 2) {
- dst_uv[0] = (src_ayuv[1] + src_ayuv[5] + src_ayuv[src_stride_ayuv + 1] +
- src_ayuv[src_stride_ayuv + 5] + 2) >>
- 2;
- dst_uv[1] = (src_ayuv[0] + src_ayuv[4] + src_ayuv[src_stride_ayuv + 0] +
- src_ayuv[src_stride_ayuv + 4] + 2) >>
- 2;
+ for (x = 0; x < width - 1; x += 2) {
+ const uint8_t* src_ayuv_next = src_ayuv + src_stride_ayuv;
+ dst_uv[0] =
+ (src_ayuv[1] + src_ayuv[5] + src_ayuv_next[1] + src_ayuv_next[5] + 2) >>
+ 2;
+ dst_uv[1] =
+ (src_ayuv[0] + src_ayuv[4] + src_ayuv_next[0] + src_ayuv_next[4] + 2) >>
+ 2;
src_ayuv += 8;
dst_uv += 2;
}
if (width & 1) {
- dst_uv[0] = (src_ayuv[0] + src_ayuv[0] + src_ayuv[src_stride_ayuv + 0] +
- src_ayuv[src_stride_ayuv + 0] + 2) >>
- 2;
- dst_uv[1] = (src_ayuv[1] + src_ayuv[1] + src_ayuv[src_stride_ayuv + 1] +
- src_ayuv[src_stride_ayuv + 1] + 2) >>
- 2;
+ const uint8_t* src_ayuv_next = src_ayuv + src_stride_ayuv;
+ dst_uv[0] = (src_ayuv[1] + src_ayuv_next[1] + 1) >> 1;
+ dst_uv[1] = (src_ayuv[0] + src_ayuv_next[0] + 1) >> 1;
}
}
@@ -3289,23 +4510,21 @@ void AYUVToVURow_C(const uint8_t* src_ayuv,
int width) {
// Output a row of VU values, filtering 2x2 rows of AYUV.
int x;
- for (x = 0; x < width; x += 2) {
- dst_vu[0] = (src_ayuv[0] + src_ayuv[4] + src_ayuv[src_stride_ayuv + 0] +
- src_ayuv[src_stride_ayuv + 4] + 2) >>
- 2;
- dst_vu[1] = (src_ayuv[1] + src_ayuv[5] + src_ayuv[src_stride_ayuv + 1] +
- src_ayuv[src_stride_ayuv + 5] + 2) >>
- 2;
+ for (x = 0; x < width - 1; x += 2) {
+ const uint8_t* src_ayuv_next = src_ayuv + src_stride_ayuv;
+ dst_vu[0] =
+ (src_ayuv[0] + src_ayuv[4] + src_ayuv_next[0] + src_ayuv_next[4] + 2) >>
+ 2;
+ dst_vu[1] =
+ (src_ayuv[1] + src_ayuv[5] + src_ayuv_next[1] + src_ayuv_next[5] + 2) >>
+ 2;
src_ayuv += 8;
dst_vu += 2;
}
if (width & 1) {
- dst_vu[0] = (src_ayuv[0] + src_ayuv[0] + src_ayuv[src_stride_ayuv + 0] +
- src_ayuv[src_stride_ayuv + 0] + 2) >>
- 2;
- dst_vu[1] = (src_ayuv[1] + src_ayuv[1] + src_ayuv[src_stride_ayuv + 1] +
- src_ayuv[src_stride_ayuv + 1] + 2) >>
- 2;
+ const uint8_t* src_ayuv_next = src_ayuv + src_stride_ayuv;
+ dst_vu[0] = (src_ayuv[0] + src_ayuv_next[0] + 1) >> 1;
+ dst_vu[1] = (src_ayuv[1] + src_ayuv_next[1] + 1) >> 1;
}
}
@@ -3332,6 +4551,638 @@ void SwapUVRow_C(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
}
}
+void HalfMergeUVRow_C(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ int x;
+ for (x = 0; x < width - 1; x += 2) {
+ const uint8_t* src_u_next = src_u + src_stride_u;
+ dst_uv[0] = (src_u[0] + src_u[1] + src_u_next[0] + src_u_next[1] + 2) >> 2;
+ const uint8_t* src_v_next = src_v + src_stride_v;
+ dst_uv[1] = (src_v[0] + src_v[1] + src_v_next[0] + src_v_next[1] + 2) >> 2;
+ src_u += 2;
+ src_v += 2;
+ dst_uv += 2;
+ }
+ if (width & 1) {
+ const uint8_t* src_u_next = src_u + src_stride_u;
+ dst_uv[0] = (src_u[0] + src_u_next[0] + 1) >> 1;
+ const uint8_t* src_v_next = src_v + src_stride_v;
+ dst_uv[1] = (src_v[0] + src_v_next[0] + 1) >> 1;
+ }
+}
+
+#undef STATIC_CAST
+
+void RGBToYMatrixRow_C(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_C(src_rgb, row, twidth);
+ ARGBToYMatrixRow_C(row, dst_y, twidth, c);
+ src_rgb += twidth * 3;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+
+void RGBToUVMatrixRow_C(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_C(src_rgb, row, twidth);
+ RGB24ToARGBRow_C(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_C(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+
+void RGBToUV444MatrixRow_C(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_C(src_rgb, row, twidth);
+ ARGBToUV444MatrixRow_C(row, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth;
+ dst_v += twidth;
+ width -= twidth;
+ }
+}
+
+#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3)
+void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
+ ARGBToYMatrixRow_SSSE3(row, dst_y, twidth, c);
+ src_rgb += twidth * 3;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2)
+void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_AVX2(src_rgb, row, twidth);
+ ARGBToYMatrixRow_AVX2(row, dst_y, twidth, c);
+ src_rgb += twidth * 3;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3)
+void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
+ RGB24ToARGBRow_SSSE3(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_SSSE3(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2)
+void RGBToUVMatrixRow_AVX2(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_AVX2(src_rgb, row, twidth);
+ RGB24ToARGBRow_AVX2(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_AVX2(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) && \
+ defined(HAS_RGB24TOARGBROW_AVX512BW)
+void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_AVX512BW(src_rgb, row, twidth);
+ RGB24ToARGBRow_AVX512BW(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_AVX512BW(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3)
+void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
+ ARGBToUV444MatrixRow_SSSE3(row, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth;
+ dst_v += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2)
+void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_AVX2(src_rgb, row, twidth);
+ ARGBToUV444MatrixRow_AVX2(row, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth;
+ dst_v += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) && \
+ defined(HAS_RGB24TOARGBROW_AVX512BW)
+void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB24ToARGBRow_AVX512BW(src_rgb, row, twidth);
+ ARGBToUV444MatrixRow_AVX512BW(row, dst_u, dst_v, twidth, c);
+ src_rgb += twidth * 3;
+ dst_u += twidth;
+ dst_v += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+void RGB565ToYMatrixRow_C(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_C(src_rgb565, row, twidth);
+ ARGBToYMatrixRow_C(row, dst_y, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+
+void RGB565ToUVMatrixRow_C(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_C(src_rgb565, row, twidth);
+ RGB565ToARGBRow_C(src_rgb565 + src_stride_rgb565, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_C(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2) && defined(HAS_RGB565TOARGBROW_AVX2)
+void RGB565ToYMatrixRow_AVX2(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_AVX2(src_rgb565, row, twidth);
+ ARGBToYMatrixRow_AVX2(row, dst_y, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2) && defined(HAS_RGB565TOARGBROW_AVX2)
+void RGB565ToUVMatrixRow_AVX2(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_AVX2(src_rgb565, row, twidth);
+ RGB565ToARGBRow_AVX2(src_rgb565 + src_stride_rgb565, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_AVX2(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_RGB565TOARGBROW_NEON) && defined(HAS_ARGBTOYMATRIXROW_NEON)
+void RGB565ToYMatrixRow_NEON(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_NEON(src_rgb565, row, twidth);
+ ARGBToYMatrixRow_NEON(row, dst_y, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_RGB565TOARGBROW_NEON) && defined(HAS_ARGBTOUVMATRIXROW_NEON)
+void RGB565ToUVMatrixRow_NEON(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ RGB565ToARGBRow_NEON(src_rgb565, row, twidth);
+ RGB565ToARGBRow_NEON(src_rgb565 + src_stride_rgb565, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_NEON(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_rgb565 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+void ARGB1555ToYMatrixRow_C(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_C(src_argb1555, row, twidth);
+ ARGBToYMatrixRow_C(row, dst_y, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+
+void ARGB1555ToUVMatrixRow_C(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_C(src_argb1555, row, twidth);
+ ARGB1555ToARGBRow_C(src_argb1555 + src_stride_argb1555, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_C(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+
+void ARGB4444ToYMatrixRow_C(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_C(src_argb4444, row, twidth);
+ ARGBToYMatrixRow_C(row, dst_y, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+
+void ARGB4444ToUVMatrixRow_C(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_C(src_argb4444, row, twidth);
+ ARGB4444ToARGBRow_C(src_argb4444 + src_stride_argb4444, row + MAXTWIDTH * 4,
+ twidth);
+ ARGBToUVMatrixRow_C(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+#if defined(HAS_ARGB1555TOARGBROW_AVX2)
+void ARGB1555ToYMatrixRow_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_AVX2(src_argb1555, row, twidth);
+ ARGBToYMatrixRow_AVX2(row, dst_y, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGB4444TOARGBROW_AVX2)
+void ARGB4444ToYMatrixRow_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_AVX2(src_argb4444, row, twidth);
+ ARGBToYMatrixRow_AVX2(row, dst_y, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
+#if defined(HAS_ARGB1555TOARGBROW_AVX2)
+void ARGB1555ToUVMatrixRow_AVX2(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_AVX2(src_argb1555, row, twidth);
+ ARGB1555ToARGBRow_AVX2(src_argb1555 + src_stride_argb1555,
+ row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_AVX2(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGB4444TOARGBROW_AVX2)
+void ARGB4444ToUVMatrixRow_AVX2(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_AVX2(src_argb4444, row, twidth);
+ ARGB4444ToARGBRow_AVX2(src_argb4444 + src_stride_argb4444,
+ row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_AVX2(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+#endif
+
+#if defined(HAS_ARGBTOYMATRIXROW_NEON) && defined(HAS_ARGB1555TOARGBROW_NEON)
+void ARGB1555ToYMatrixRow_NEON(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_NEON(src_argb1555, row, twidth);
+ ARGBToYMatrixRow_NEON(row, dst_y, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOYMATRIXROW_NEON) && defined(HAS_ARGB4444TOARGBROW_NEON)
+void ARGB4444ToYMatrixRow_NEON(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_NEON(src_argb4444, row, twidth);
+ ARGBToYMatrixRow_NEON(row, dst_y, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_y += twidth;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON) && defined(HAS_ARGB1555TOARGBROW_NEON)
+void ARGB1555ToUVMatrixRow_NEON(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB1555ToARGBRow_NEON(src_argb1555, row, twidth);
+ ARGB1555ToARGBRow_NEON(src_argb1555 + src_stride_argb1555,
+ row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_NEON(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb1555 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON) && defined(HAS_ARGB4444TOARGBROW_NEON)
+void ARGB4444ToUVMatrixRow_NEON(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
+ while (width > 0) {
+ int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+ ARGB4444ToARGBRow_NEON(src_argb4444, row, twidth);
+ ARGB4444ToARGBRow_NEON(src_argb4444 + src_stride_argb4444,
+ row + MAXTWIDTH * 4, twidth);
+ ARGBToUVMatrixRow_NEON(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
+ src_argb4444 += twidth * 2;
+ dst_u += twidth / 2;
+ dst_v += twidth / 2;
+ width -= twidth;
+ }
+}
+#endif
+
+#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_RGB565TOYMATRIXROW_AVX2)
+void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
+ RGB565ToYMatrixRow_AVX2(src_rgb565, dst_y, width, &kArgbI601Constants);
+}
+#endif
+
+#if defined(HAS_ARGB1555TOYROW_AVX2) && defined(HAS_ARGB1555TOYMATRIXROW_AVX2)
+void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width) {
+ ARGB1555ToYMatrixRow_AVX2(src_argb1555, dst_y, width, &kArgbI601Constants);
+}
+#endif
+
+#if defined(HAS_ARGB4444TOYROW_AVX2) && defined(HAS_ARGB4444TOYMATRIXROW_AVX2)
+void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width) {
+ ARGB4444ToYMatrixRow_AVX2(src_argb4444, dst_y, width, &kArgbI601Constants);
+}
+#endif
+
+#if defined(HAS_RGB565TOUVROW_AVX2) && defined(HAS_RGB565TOUVMATRIXROW_AVX2)
+void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ RGB565ToUVMatrixRow_AVX2(src_rgb565, src_stride_rgb565, dst_u, dst_v, width,
+ &kArgbI601Constants);
+}
+#endif
+
+#if defined(HAS_ARGB1555TOUVROW_AVX2) && defined(HAS_ARGB1555TOUVMATRIXROW_AVX2)
+void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGB1555ToUVMatrixRow_AVX2(src_argb1555, src_stride_argb1555, dst_u, dst_v,
+ width, &kArgbI601Constants);
+}
+#endif
+
+#if defined(HAS_ARGB4444TOUVROW_AVX2) && defined(HAS_ARGB4444TOUVMATRIXROW_AVX2)
+void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGB4444ToUVMatrixRow_AVX2(src_argb4444, src_stride_argb4444, dst_u, dst_v,
+ width, &kArgbI601Constants);
+}
+#endif
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/row_gcc.cc b/libs/libyuv/source/row_gcc.cc
index ae672d625c..3abe073b02 100644
--- a/libs/libyuv/source/row_gcc.cc
+++ b/libs/libyuv/source/row_gcc.cc
@@ -8,6 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <string.h> // For memcpy
+
+#include "libyuv/convert_from_argb.h" // For ArgbConstants
#include "libyuv/row.h"
#ifdef __cplusplus
@@ -15,88 +18,114 @@ namespace libyuv {
extern "C" {
#endif
-// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
-
-#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3)
-
-// Constants for ARGB
-static const vec8 kARGBToY = {13, 65, 33, 0, 13, 65, 33, 0,
- 13, 65, 33, 0, 13, 65, 33, 0};
-
-// JPeg full range.
-static const vec8 kARGBToYJ = {15, 75, 38, 0, 15, 75, 38, 0,
- 15, 75, 38, 0, 15, 75, 38, 0};
-#endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3)
-
-#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3)
-
-static const vec8 kARGBToU = {112, -74, -38, 0, 112, -74, -38, 0,
- 112, -74, -38, 0, 112, -74, -38, 0};
-
-static const vec8 kARGBToUJ = {127, -84, -43, 0, 127, -84, -43, 0,
- 127, -84, -43, 0, 127, -84, -43, 0};
-
-static const vec8 kARGBToV = {-18, -94, 112, 0, -18, -94, 112, 0,
- -18, -94, 112, 0, -18, -94, 112, 0};
-
-static const vec8 kARGBToVJ = {-20, -107, 127, 0, -20, -107, 127, 0,
- -20, -107, 127, 0, -20, -107, 127, 0};
-
-// Constants for BGRA
-static const vec8 kBGRAToY = {0, 33, 65, 13, 0, 33, 65, 13,
- 0, 33, 65, 13, 0, 33, 65, 13};
-
-static const vec8 kBGRAToU = {0, -38, -74, 112, 0, -38, -74, 112,
- 0, -38, -74, 112, 0, -38, -74, 112};
-
-static const vec8 kBGRAToV = {0, 112, -94, -18, 0, 112, -94, -18,
- 0, 112, -94, -18, 0, 112, -94, -18};
-
-// Constants for ABGR
-static const vec8 kABGRToY = {33, 65, 13, 0, 33, 65, 13, 0,
- 33, 65, 13, 0, 33, 65, 13, 0};
-
-static const vec8 kABGRToU = {-38, -74, 112, 0, -38, -74, 112, 0,
- -38, -74, 112, 0, -38, -74, 112, 0};
-
-static const vec8 kABGRToV = {112, -94, -18, 0, 112, -94, -18, 0,
- 112, -94, -18, 0, 112, -94, -18, 0};
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
-// Constants for RGBA.
-static const vec8 kRGBAToY = {0, 13, 65, 33, 0, 13, 65, 33,
- 0, 13, 65, 33, 0, 13, 65, 33};
-
-static const vec8 kRGBAToU = {0, 112, -74, -38, 0, 112, -74, -38,
- 0, 112, -74, -38, 0, 112, -74, -38};
-
-static const vec8 kRGBAToV = {0, -18, -94, 112, 0, -18, -94, 112,
- 0, -18, -94, 112, 0, -18, -94, 112};
-
-static const uvec8 kAddY16 = {16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u,
- 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u};
-
-// 7 bit fixed point 0.5.
-static const vec16 kAddYJ64 = {64, 64, 64, 64, 64, 64, 64, 64};
-
-static const uvec8 kAddUV128 = {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
-
-static const uvec16 kAddUVJ128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u,
- 0x8080u, 0x8080u, 0x8080u, 0x8080u};
-#endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3)
+// This module is for GCC x86 and x64.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
-#ifdef HAS_RGB24TOARGBROW_SSSE3
+// Note: for avx and avx512 declare clobber as xmm registers due to
+// clang for windows needing to preserve xmm registers but not saving
+// them if declared as ymm or zmm.
+#if defined(HAS_RGB24TOARGBROW_SSSE3) || defined(HAS_RGB24TOARGBROW_AVX2) || \
+ defined(HAS_RGB24TOARGBROW_AVX512BW) || \
+ defined(HAS_RGBTOYMATRIXROW_AVX512BW)
// Shuffle table for converting RGB24 to ARGB.
-static const uvec8 kShuffleMaskRGB24ToARGB = {
- 0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u};
-
+static const uvec8 kShuffleMaskRGB24ToARGB[2] = {
+ {0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
+ {4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
+ 128u}};
+#endif // defined(HAS_RGB24TOARGBROW_SSSE3) ||
+ // defined(HAS_RGB24TOARGBROW_AVX2) ||
+ // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
+ // defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+
+#if defined(HAS_RAWTOARGBROW_SSSE3) || defined(HAS_RAWTOARGBROW_AVX2)
// Shuffle table for converting RAW to ARGB.
-static const uvec8 kShuffleMaskRAWToARGB = {2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u,
- 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u};
-
+static const uvec8 kShuffleMaskRAWToARGB = {
+ 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u};
+// Shuffle table for converting RAW to ARGB. Last 12
+static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u,
+ 7u, 128u, 12u, 11u, 10u, 128u,
+ 15u, 14u, 13u, 128u};
+#endif // defined(HAS_RAWTOARGBROW_SSSE3) || defined(HAS_RAWTOARGBROW_AVX2)
+#if defined(HAS_RAWTOARGBROW_AVX512BW)
+static const uvec8 kShuffleMaskRAWToARGB_AVX512BW[2] = {
+ {2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u},
+ {6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u,
+ 128u}};
+#endif // defined(HAS_RAWTOARGBROW_AVX512BW)
+#if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
+static const uvec8 kShuffleMaskRAWToRGB24_AVX2[2] = {
+ {2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, 9u, 128u, 128u, 128u, 128u},
+ {6u, 5u, 4u, 9u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u,
+ 128u}};
+#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
+ // defined(HAS_RAWTORGB24ROW_AVX512BW)
+#if defined(HAS_RAWTORGB24ROW_AVX2)
+static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 3, 7};
+static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1};
+static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4};
+#endif // defined(HAS_RAWTORGB24ROW_AVX2)
+#if defined(HAS_RAWTORGB24ROW_AVX512BW) || \
+ defined(HAS_ARGBTORGB24ROW_AVX512BW) || \
+ defined(HAS_ARGBTORAWROW_AVX512BW)
+static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
+ 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
+static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
+ 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
+static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
+ 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
+#endif // defined(HAS_RAWTORGB24ROW_AVX512BW) ||
+ // defined(HAS_ARGBTORGB24ROW_AVX512BW) ||
+ // defined(HAS_ARGBTORAWROW_AVX512BW)
+#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
+static const uint8_t kPermRAWToRGB24_0[64] = {
+ 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a,
+ 0x09, 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17,
+ 0x16, 0x15, 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e,
+ 0x23, 0x22, 0x21, 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b,
+ 0x2a, 0x2f, 0x2e, 0x2d, 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38,
+ 0x37, 0x36, 0x3b, 0x3a, 0x39, 0x3e, 0x3d, 0x3c, 0x41};
+static const uint8_t kPermRAWToRGB24_1[64] = {
+ 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08,
+ 0x0d, 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15,
+ 0x14, 0x19, 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22,
+ 0x21, 0x20, 0x25, 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29,
+ 0x2e, 0x2d, 0x2c, 0x31, 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36,
+ 0x35, 0x3a, 0x39, 0x38, 0x3d, 0x3c, 0x3b, 0x00, 0x3f};
+static const uint8_t kPermRAWToRGB24_2[64] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
+ 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20,
+ 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
+ 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36,
+ 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x40, 0x3f};
+static const uint8_t kPermRAWToRGB24_3[64] = {
+ 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c,
+ 0x4b, 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53,
+ 0x58, 0x57, 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60,
+ 0x5f, 0x64, 0x63, 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d,
+ 0x6c, 0x6b, 0x70, 0x6f, 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74,
+ 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d};
+#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
+
+#if defined(HAS_RAWTORGBAROW_AVX2)
+// Shuffle table for converting RAW to RGBA.
+static const uvec8 kShuffleMaskRAWToRGBA_AVX2[2] = {
+ {128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u},
+ {128u, 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u,
+ 13u}};
+#endif // defined(HAS_RAWTORGBAROW_AVX2)
+
+#if defined(HAS_RAWTORGB24ROW_SSSE3)
// Shuffle table for converting RAW to RGB24. First 8.
static const uvec8 kShuffleMaskRAWToRGB24_0 = {
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u,
@@ -111,177 +140,503 @@ static const uvec8 kShuffleMaskRAWToRGB24_1 = {
static const uvec8 kShuffleMaskRAWToRGB24_2 = {
8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u,
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+#endif // defined(HAS_RAWTORGB24ROW_SSSE3)
// Shuffle table for converting ARGB to RGB24.
-static const uvec8 kShuffleMaskARGBToRGB24 = {
- 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u};
+// and for I422ToRGB24. First 8 + next 4
+static const uvec8 kShuffleMaskARGBToRGB24[2] = {
+ {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
+ 128u},
+ {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
+ 14u}};
// Shuffle table for converting ARGB to RAW.
static const uvec8 kShuffleMaskARGBToRAW = {
2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u};
-// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4
-static const uvec8 kShuffleMaskARGBToRGB24_0 = {
- 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
-
// YUY2 shuf 16 Y to 32 Y.
-static const lvec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10,
- 10, 12, 12, 14, 14, 0, 0, 2, 2, 4, 4,
- 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
+static const vec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6,
+ 8, 8, 10, 10, 12, 12, 14, 14};
// YUY2 shuf 8 UV to 16 UV.
-static const lvec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7, 9, 11, 9,
- 11, 13, 15, 13, 15, 1, 3, 1, 3, 5, 7,
- 5, 7, 9, 11, 9, 11, 13, 15, 13, 15};
+static const vec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7,
+ 9, 11, 9, 11, 13, 15, 13, 15};
// UYVY shuf 16 Y to 32 Y.
-static const lvec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11,
- 11, 13, 13, 15, 15, 1, 1, 3, 3, 5, 5,
- 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
+static const vec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7,
+ 9, 9, 11, 11, 13, 13, 15, 15};
// UYVY shuf 8 UV to 16 UV.
-static const lvec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6, 8, 10, 8,
- 10, 12, 14, 12, 14, 0, 2, 0, 2, 4, 6,
- 4, 6, 8, 10, 8, 10, 12, 14, 12, 14};
+static const vec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6,
+ 8, 10, 8, 10, 12, 14, 12, 14};
// NV21 shuf 8 VU to 16 UV.
static const lvec8 kShuffleNV21 = {
1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
};
-#endif // HAS_RGB24TOARGBROW_SSSE3
-#ifdef HAS_J400TOARGBROW_SSE2
-void J400ToARGBRow_SSE2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+#if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW)
+static const uint8_t kShuffleMaskJ400ToARGB[64] = {
+ 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u,
+ 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u,
+ 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u,
+ 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u, 13u, 128u,
+ 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
+#endif // defined(HAS_J400TOARGBROW_AVX2) ||
+ // defined(HAS_J400TOARGBROW_AVX512BW)
+
+#ifdef HAS_J400TOARGBROW_AVX2
+void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "pslld $0x18,%%xmm5 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpslld $0x18,%%ymm7,%%ymm7 \n"
+ "vmovdqu (%3),%%ymm5 \n"
+ "vmovdqu 0x20(%3),%%ymm6 \n"
LABELALIGN
- "1: \n"
- "movq (%0),%%xmm0 \n"
- "lea 0x8(%0),%0 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklwd %%xmm0,%%xmm0 \n"
- "punpckhwd %%xmm1,%%xmm1 \n"
- "por %%xmm5,%%xmm0 \n"
- "por %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm5");
+ "1: \n"
+ "vbroadcasti128 (%0),%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm0,%%ymm2 \n"
+ "vpor %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm7,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm1,(%1) \n"
+ "vmovdqu %%ymm2,0x20(%1) \n"
+ "lea 0x10(%0),%0 \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(kShuffleMaskJ400ToARGB) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5", "xmm6", "xmm7");
+}
+#endif // HAS_J400TOARGBROW_AVX2
+
+#ifdef HAS_J400TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff
+ "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000
+ "vmovdqu64 %[shuf],%%zmm5 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vbroadcasti32x4 (%0),%%zmm0 \n"
+ "vbroadcasti32x4 0x10(%0),%%zmm1 \n"
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm16,%%zmm1,%%zmm1 \n"
+ "vmovdqu64 %%zmm0,(%1) \n"
+ "vmovdqu64 %%zmm1,0x40(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "vmovdqu8 (%0),%%ymm0%{%%k1%}%{z%} \n"
+ "vshufi32x4 $0x55,%%zmm0,%%zmm0,%%zmm1 \n"
+ "vshufi32x4 $0x00,%%zmm0,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm16,%%zmm1,%%zmm1 \n"
+ "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
+ "vmovdqu32 %%zmm1,0x40(%1)%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [shuf] "m"(kShuffleMaskJ400ToARGB) // %[shuf]
+ : "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm16", "k1", "k2");
}
-#endif // HAS_J400TOARGBROW_SSE2
+#endif // HAS_J400TOARGBROW_AVX512BW
#ifdef HAS_RGB24TOARGBROW_SSSE3
void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n" // 0xff000000
- "pslld $0x18,%%xmm5 \n"
- "movdqa %3,%%xmm4 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // 0xff000000
+ "pslld $0x18,%%xmm5 \n"
+ "movdqa %3,%%xmm4 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm3 \n"
- "lea 0x30(%0),%0 \n"
- "movdqa %%xmm3,%%xmm2 \n"
- "palignr $0x8,%%xmm1,%%xmm2 \n"
- "pshufb %%xmm4,%%xmm2 \n"
- "por %%xmm5,%%xmm2 \n"
- "palignr $0xc,%%xmm0,%%xmm1 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "por %%xmm5,%%xmm0 \n"
- "pshufb %%xmm4,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "palignr $0x4,%%xmm3,%%xmm3 \n"
- "pshufb %%xmm4,%%xmm3 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "por %%xmm5,%%xmm3 \n"
- "movdqu %%xmm3,0x30(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_rgb24), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskRGB24ToARGB) // %3
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm3 \n"
+ "lea 0x30(%0),%0 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "palignr $0x8,%%xmm1,%%xmm2 \n"
+ "pshufb %%xmm4,%%xmm2 \n"
+ "por %%xmm5,%%xmm2 \n"
+ "palignr $0xc,%%xmm0,%%xmm1 \n"
+ "pshufb %%xmm4,%%xmm0 \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "por %%xmm5,%%xmm0 \n"
+ "pshufb %%xmm4,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "por %%xmm5,%%xmm1 \n"
+ "palignr $0x4,%%xmm3,%%xmm3 \n"
+ "pshufb %%xmm4,%%xmm3 \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "por %%xmm5,%%xmm3 \n"
+ "movdqu %%xmm3,0x30(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRGB24ToARGB[0]) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
+#endif // HAS_RGB24TOARGBROW_SSSE3
+
+#ifdef HAS_RGB24TOARGBROW_AVX2
+void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0xff000000
+ "vpslld $0x18,%%ymm6,%%ymm6 \n"
+ "vbroadcasti128 %3,%%ymm4 \n"
+ "vbroadcasti128 %4,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // first 12
+ "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12
+ "vmovdqu 24(%0),%%xmm1 \n" // third 12
+ "vinserti128 $1,36(%0),%%ymm1,%%ymm1 \n" // forth 12
+ "vmovdqu 48(%0),%%xmm2 \n" // fifth 12
+ "vinserti128 $1,60(%0),%%ymm2,%%ymm2 \n" // sixth 12
+ "vmovdqu 68(%0),%%xmm3 \n" // seventh 12
+ "vinserti128 $1,80(%0),%%ymm3,%%ymm3 \n" // eighth 12
+ "lea 96(%0),%0 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm6,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "vmovdqu %%ymm3,0x60(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRGB24ToARGB[0]), // %3
+ "m"(kShuffleMaskRGB24ToARGB[1]) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_RGB24TOARGBROW_AVX2
+#ifdef HAS_RAWTOARGBROW_SSSE3
void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n" // 0xff000000
- "pslld $0x18,%%xmm5 \n"
- "movdqa %3,%%xmm4 \n"
+ "pcmpeqb %%xmm6,%%xmm6 \n" // 0xff000000
+ "pslld $0x18,%%xmm6 \n"
+ "movdqa %3,%%xmm4 \n"
+ "movdqa %4,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm3 \n"
- "lea 0x30(%0),%0 \n"
- "movdqa %%xmm3,%%xmm2 \n"
- "palignr $0x8,%%xmm1,%%xmm2 \n"
- "pshufb %%xmm4,%%xmm2 \n"
- "por %%xmm5,%%xmm2 \n"
- "palignr $0xc,%%xmm0,%%xmm1 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "por %%xmm5,%%xmm0 \n"
- "pshufb %%xmm4,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "palignr $0x4,%%xmm3,%%xmm3 \n"
- "pshufb %%xmm4,%%xmm3 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "por %%xmm5,%%xmm3 \n"
- "movdqu %%xmm3,0x30(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_raw), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskRAWToARGB) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 12(%0),%%xmm1 \n"
+ "movdqu 24(%0),%%xmm2 \n"
+ "movdqu 32(%0),%%xmm3 \n"
+ "lea 0x30(%0),%0 \n"
+ "pshufb %%xmm4,%%xmm0 \n"
+ "pshufb %%xmm4,%%xmm1 \n"
+ "pshufb %%xmm4,%%xmm2 \n"
+ "pshufb %%xmm5,%%xmm3 \n"
+ "por %%xmm6,%%xmm0 \n"
+ "por %%xmm6,%%xmm1 \n"
+ "por %%xmm6,%%xmm2 \n"
+ "por %%xmm6,%%xmm3 \n"
+ "movdqu %%xmm0,0x00(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "movdqu %%xmm3,0x30(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToARGB), // %3
+ "m"(kShuffleMaskRAWToARGB_0) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_RAWTOARGBROW_SSSE3
+
+#ifdef HAS_RAWTOARGBROW_AVX2
+void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0xff000000
+ "vpslld $0x18,%%ymm6,%%ymm6 \n"
+ "vbroadcasti128 %3,%%ymm4 \n" //
+ "vbroadcasti128 %4,%%ymm5 \n" //
+
+ LABELALIGN //
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // first 12
+ "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12
+ "vmovdqu 24(%0),%%xmm1 \n" // third 12
+ "vinserti128 $1,36(%0),%%ymm1,%%ymm1 \n" // forth 12
+ "vmovdqu 48(%0),%%xmm2 \n" // fifth 12
+ "vinserti128 $1,60(%0),%%ymm2,%%ymm2 \n" // sixth 12
+ "vmovdqu 68(%0),%%xmm3 \n" // seventh 12
+ "vinserti128 $1,80(%0),%%ymm3,%%ymm3 \n" // eighth 12
+ "lea 96(%0),%0 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm6,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "vmovdqu %%ymm3,0x60(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToARGB), // %3
+ "m"(kShuffleMaskRAWToARGB_0) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_RAWTOARGBROW_AVX2
+
+#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
+ defined(HAS_RGB24TOARGBROW_AVX512BW) || \
+ defined(HAS_RAWTORGB24ROW_AVX512BW) || \
+ defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, 3, 4, 4, 5};
+static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
+ 9, 10, 10, 11};
+static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, 7, 8, 8, 9};
+static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7};
+#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
+ // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
+ // defined(HAS_RAWTORGB24ROW_AVX512BW) ||
+ // defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+
+#ifdef HAS_RAWTOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+static inline void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ const uint32_t* shuffler,
+ int width) {
+ uintptr_t temp;
+ uintptr_t mask;
+ asm volatile(
+ "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
+ "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
+ "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
+ "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
+ "vbroadcasti64x4 (%[shuffler]),%%zmm4 \n" // shuffler (32 bytes)
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff
+ "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0 \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1 \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2 \n"
+ "lea 192(%[src_raw]),%[src_raw] \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vmovdqa64 %%zmm0,%%zmm19 \n"
+ "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n"
+ "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm19,%%zmm19 \n"
+ "vpshufb %%zmm4,%%zmm1,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm16,%%zmm19,%%zmm19 \n"
+ "vpord %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm16,%%zmm3,%%zmm3 \n"
+ "vmovdqu32 %%zmm0,(%[dst_argb]) \n"
+ "vmovdqu32 %%zmm19,0x40(%[dst_argb]) \n"
+ "vmovdqu32 %%zmm2,0x80(%[dst_argb]) \n"
+ "vmovdqu32 %%zmm3,0xc0(%[dst_argb]) \n"
+ "lea 0x100(%[dst_argb]),%[dst_argb]\n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k5 \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0%{%%k5%}%{z%} \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k5 \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k5%}%{z%} \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k5 \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k5%}%{z%} \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vpermi2q %%zmm2,%%zmm1,%%zmm22 \n"
+ "vpermi2q %%zmm1,%%zmm0,%%zmm21 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm21,%%zmm1 \n"
+ "vpshufb %%zmm4,%%zmm22,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm16,%%zmm3,%%zmm3 \n"
+ "vmovdqu32 %%zmm0,(%[dst_argb])%{%%k1%} \n"
+ "vmovdqu32 %%zmm1,0x40(%[dst_argb])%{%%k2%} \n"
+ "vmovdqu32 %%zmm2,0x80(%[dst_argb])%{%%k3%} \n"
+ "vmovdqu32 %%zmm3,0xc0(%[dst_argb])%{%%k4%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_raw] "+r"(src_raw), // %[src_raw]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [temp] "=&r"(temp), // %[temp]
+ [mask] "=&r"(mask) // %[mask]
+ : [shuffler] "r"(shuffler), // %[shuffler]
+ [q0] "m"(kPermqRGB24ToARGB0_AVX512BW), // %[q0]
+ [q1] "m"(kPermqRGB24ToARGB1_AVX512BW), // %[q1]
+ [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2]
+ [q3] "m"(kPermqRGB24ToARGB3_AVX512BW) // %[q3]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
+ "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", "k4",
+ "k5");
+}
+
+LIBYUV_TARGET_AVX512BW
+void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ int width) {
+ RGBToARGBRow_AVX512BW(src_raw, dst_argb,
+ (const uint32_t*)&kShuffleMaskRAWToARGB_AVX512BW[0],
+ width);
+}
+
+LIBYUV_TARGET_AVX512BW
+void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ RGBToARGBRow_AVX512BW(src_rgb24, dst_argb,
+ (const uint32_t*)&kShuffleMaskRGB24ToARGB[0], width);
+}
+#endif // HAS_RAWTOARGBROW_AVX512BW
+
+#ifdef HAS_RAWTORGBAROW_AVX2
+void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrld $24,%%ymm5,%%ymm5 \n" // 0x000000ff
+ "vbroadcasti128 %3,%%ymm3 \n"
+ "vmovdqu %3,%%ymm4 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // first 12 bytes
+ "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12 bytes
+ "vmovdqu 24(%0),%%xmm1 \n" // third 12 bytes
+ "vinserti128 $1,32(%0),%%ymm1,%%ymm1 \n" // fourth 12 bytes
+ "lea 0x30(%0),%0 \n"
+ "vpshufb %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_rgba), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToRGBA_AVX2) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm3", "xmm4", "xmm5");
}
+#endif // HAS_RAWTORGBAROW_AVX2
+#ifdef HAS_RAWTORGB24ROW_SSSE3
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
asm volatile(
- "movdqa %3,%%xmm3 \n"
- "movdqa %4,%%xmm4 \n"
- "movdqa %5,%%xmm5 \n"
+ "movdqa %3,%%xmm3 \n"
+ "movdqa %4,%%xmm4 \n"
+ "movdqa %5,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x4(%0),%%xmm1 \n"
- "movdqu 0x8(%0),%%xmm2 \n"
- "lea 0x18(%0),%0 \n"
- "pshufb %%xmm3,%%xmm0 \n"
- "pshufb %%xmm4,%%xmm1 \n"
- "pshufb %%xmm5,%%xmm2 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x8(%1) \n"
- "movq %%xmm2,0x10(%1) \n"
- "lea 0x18(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x4(%0),%%xmm1 \n"
+ "movdqu 0x8(%0),%%xmm2 \n"
+ "lea 0x18(%0),%0 \n"
+ "pshufb %%xmm3,%%xmm0 \n"
+ "pshufb %%xmm4,%%xmm1 \n"
+ "pshufb %%xmm5,%%xmm2 \n"
+ "movq %%xmm0,(%1) \n"
+ "movq %%xmm1,0x8(%1) \n"
+ "movq %%xmm2,0x10(%1) \n"
+ "lea 0x18(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
@@ -290,47 +645,286 @@ void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
"m"(kShuffleMaskRAWToRGB24_2) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
+#endif // HAS_RAWTORGB24ROW_SSSE3
-void RGB565ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_RAWTORGB24ROW_AVX2
+void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
asm volatile(
- "mov $0x1080108,%%eax \n"
- "movd %%eax,%%xmm5 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "mov $0x20802080,%%eax \n"
- "movd %%eax,%%xmm6 \n"
- "pshufd $0x0,%%xmm6,%%xmm6 \n"
- "pcmpeqb %%xmm3,%%xmm3 \n"
- "psllw $0xb,%%xmm3 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psllw $0xa,%%xmm4 \n"
- "psrlw $0x5,%%xmm4 \n"
- "pcmpeqb %%xmm7,%%xmm7 \n"
- "psllw $0x8,%%xmm7 \n"
- "sub %0,%1 \n"
- "sub %0,%1 \n"
+ "vmovdqu %3,%%ymm4 \n" // shuffler (32 bytes)
+ "vmovdqu %4,%%ymm5 \n" // kPermd0_AVX2
+ "vmovdqu %5,%%ymm6 \n" // kPermd1_AVX2
+ "vmovdqu %6,%%ymm7 \n" // kPermd2_AVX2
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "pand %%xmm3,%%xmm1 \n"
- "psllw $0xb,%%xmm2 \n"
- "pmulhuw %%xmm5,%%xmm1 \n"
- "pmulhuw %%xmm5,%%xmm2 \n"
- "psllw $0x8,%%xmm1 \n"
- "por %%xmm2,%%xmm1 \n"
- "pand %%xmm4,%%xmm0 \n"
- "pmulhuw %%xmm6,%%xmm0 \n"
- "por %%xmm7,%%xmm0 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "punpcklbw %%xmm0,%%xmm1 \n"
- "punpckhbw %%xmm0,%%xmm2 \n"
- "movdqu %%xmm1,0x00(%1,%0,2) \n"
- "movdqu %%xmm2,0x10(%1,%0,2) \n"
- "lea 0x10(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // raw0
+ "vmovdqu 32(%0),%%ymm1 \n" // raw1
+ "vmovdqu 64(%0),%%ymm2 \n" // raw2
+ "lea 96(%0),%0 \n"
+
+ "vperm2i128 $0x21,%%ymm1,%%ymm0,%%ymm3 \n" // m01
+ "vpermq $0x94,%%ymm0,%%ymm0 \n" // b0
+ "vperm2i128 $0x21,%%ymm2,%%ymm1,%%ymm1 \n" // m12
+ "vpermq $0xe9,%%ymm2,%%ymm2 \n" // b3
+ "vpermq $0xe9,%%ymm3,%%ymm3 \n" // b1
+ "vpermq $0x94,%%ymm1,%%ymm1 \n" // b2
+
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm2,%%ymm2 \n"
+
+ "vpermd %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpermd %%ymm3,%%ymm6,%%ymm3 \n"
+ "vpermd %%ymm1,%%ymm7,%%ymm1 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+
+ // Dst0
+ "vpblendd $0xc0,%%ymm3,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ // Dst1
+ "vpblendd $0xf0,%%ymm1,%%ymm3,%%ymm3 \n"
+ "vpermq $0x90,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm3,32(%1) \n"
+
+ // Dst2
+ "vpblendd $0xfc,%%ymm2,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm1,64(%1) \n"
+
+ "lea 96(%1),%1 \n"
+ "sub $32,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToRGB24_AVX2), // %3
+ "m"(kPermd0_AVX2), // %4
+ "m"(kPermd1_AVX2), // %5
+ "m"(kPermd2_AVX2) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_RAWTORGB24ROW_AVX2
+
+#ifdef HAS_RAWTORGB24ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ uintptr_t temp;
+ uintptr_t mask;
+ asm volatile(
+ "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
+ "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
+ "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
+ "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
+ "vbroadcasti64x4 %[shuf],%%zmm4 \n" // shuffler (32 bytes)
+ "vmovdqu32 %[d0],%%zmm24 \n" // kPermdRGB24_0_AVX512BW
+ "vmovdqu32 %[d1],%%zmm25 \n" // kPermdRGB24_1_AVX512BW
+ "vmovdqu32 %[d2],%%zmm26 \n" // kPermdRGB24_2_AVX512BW
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0 \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1 \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2 \n"
+ "lea 192(%[src_raw]),%[src_raw] \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vmovdqa64 %%zmm0,%%zmm19 \n"
+ "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n"
+ "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm19,%%zmm19 \n"
+ "vpshufb %%zmm4,%%zmm1,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpermt2d %%zmm19,%%zmm24,%%zmm0 \n"
+ "vpermt2d %%zmm2,%%zmm25,%%zmm19 \n"
+ "vpermt2d %%zmm3,%%zmm26,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_rgb24]) \n"
+ "vmovdqu8 %%zmm19,0x40(%[dst_rgb24]) \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst_rgb24]) \n"
+ "lea 0xc0(%[dst_rgb24]),%[dst_rgb24]\n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k2 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k3 \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k3%}%{z%} \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vmovdqa64 %%zmm0,%%zmm19 \n"
+ "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n"
+ "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm19,%%zmm19 \n"
+ "vpshufb %%zmm4,%%zmm1,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpermt2d %%zmm19,%%zmm24,%%zmm0 \n"
+ "vpermt2d %%zmm2,%%zmm25,%%zmm19 \n"
+ "vpermt2d %%zmm3,%%zmm26,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_rgb24])%{%%k1%} \n"
+ "vmovdqu8 %%zmm19,0x40(%[dst_rgb24])%{%%k2%} \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst_rgb24])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_raw] "+r"(src_raw), // %[src_raw]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width), // %[width]
+ [temp] "=&r"(temp), // %[temp]
+ [mask] "=&r"(mask) // %[mask]
+ : [q0] "m"(kPermqRGB24ToARGB0_AVX512BW), // %[q0]
+ [q1] "m"(kPermqRGB24ToARGB1_AVX512BW), // %[q1]
+ [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2]
+ [q3] "m"(kPermqRGB24ToARGB3_AVX512BW), // %[q3]
+ [shuf] "m"(kShuffleMaskRAWToRGB24_AVX2), // %[shuf]
+ [d0] "m"(kPermdRGB24_0_AVX512BW), // %[d0]
+ [d1] "m"(kPermdRGB24_1_AVX512BW), // %[d1]
+ [d2] "m"(kPermdRGB24_2_AVX512BW) // %[d2]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm19",
+ "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", "k1",
+ "k2", "k3");
+}
+#endif // HAS_RAWTORGB24ROW_AVX512BW
+
+#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ uintptr_t temp;
+ uintptr_t mask;
+ asm volatile(
+ "vmovdqu32 %[kPermRAWToRGB24_0],%%zmm20 \n"
+ "vmovdqu32 %[kPermRAWToRGB24_1],%%zmm21 \n"
+ "vmovdqu32 %[kPermRAWToRGB24_2],%%zmm22 \n"
+ "vmovdqu32 %[kPermRAWToRGB24_3],%%zmm23 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0 \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1 \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2 \n"
+ "lea 192(%[src_raw]),%[src_raw] \n"
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+ "vpermt2b %%zmm0,%%zmm21,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm20,%%zmm0 \n"
+ "vpermt2b %%zmm2,%%zmm23,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm22,%%zmm1 \n"
+ "vmovdqu8 %%zmm0,(%[dst_rgb24]) \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst_rgb24]) \n"
+ "vmovdqu8 %%zmm3,0x80(%[dst_rgb24]) \n"
+ "lea 0xc0(%[dst_rgb24]),%[dst_rgb24]\n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k2 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k3 \n"
+ "vmovdqu8 (%[src_raw]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+ "vpermt2b %%zmm0,%%zmm21,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm20,%%zmm0 \n"
+ "vpermt2b %%zmm2,%%zmm23,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm22,%%zmm1 \n"
+ "vmovdqu8 %%zmm0,(%[dst_rgb24])%{%%k1%} \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst_rgb24])%{%%k2%} \n"
+ "vmovdqu8 %%zmm3,0x80(%[dst_rgb24])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_raw] "+r"(src_raw), [dst_rgb24] "+r"(dst_rgb24),
+ [width] "+r"(width), [temp] "=&r"(temp), [mask] "=&r"(mask)
+ : [kPermRAWToRGB24_0] "m"(kPermRAWToRGB24_0),
+ [kPermRAWToRGB24_1] "m"(kPermRAWToRGB24_1),
+ [kPermRAWToRGB24_2] "m"(kPermRAWToRGB24_2),
+ [kPermRAWToRGB24_3] "m"(kPermRAWToRGB24_3)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm20", "xmm21",
+ "xmm22", "xmm23", "k1", "k2", "k3");
+}
+#endif // HAS_RAWTORGB24ROW_AVX512VBMI
+
+#ifdef HAS_RGB565TOARGBROW_AVX2
+void RGB565ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "mov $0x1080108,%%eax \n"
+ "vmovd %%eax,%%xmm5 \n"
+ "vpbroadcastd %%xmm5,%%ymm5 \n"
+ "mov $0x20802080,%%eax \n"
+ "vmovd %%eax,%%xmm6 \n"
+ "vpbroadcastd %%xmm6,%%ymm6 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsllw $0xb,%%ymm3,%%ymm3 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsllw $10,%%ymm4,%%ymm4 \n"
+ "vpsrlw $5,%%ymm4,%%ymm4 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsllw $0x8,%%ymm7,%%ymm7 \n"
+ "sub %0,%1 \n"
+ "sub %0,%1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpand %%ymm3,%%ymm0,%%ymm1 \n"
+ "vpsllw $0xb,%%ymm0,%%ymm2 \n"
+ "vpmulhuw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpmulhuw %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpunpcklbw %%ymm0,%%ymm1,%%ymm2 \n"
+ "vpunpckhbw %%ymm0,%%ymm1,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm2,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm2,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1,%0,2) \n"
+ "vmovdqu %%ymm1,0x20(%1,%0,2) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -338,50 +932,50 @@ void RGB565ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
+#endif // HAS_RGB565TOARGBROW_AVX2
-void ARGB1555ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_ARGB1555TOARGBROW_AVX2
+void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "mov $0x1080108,%%eax \n"
- "movd %%eax,%%xmm5 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "mov $0x42004200,%%eax \n"
- "movd %%eax,%%xmm6 \n"
- "pshufd $0x0,%%xmm6,%%xmm6 \n"
- "pcmpeqb %%xmm3,%%xmm3 \n"
- "psllw $0xb,%%xmm3 \n"
- "movdqa %%xmm3,%%xmm4 \n"
- "psrlw $0x6,%%xmm4 \n"
- "pcmpeqb %%xmm7,%%xmm7 \n"
- "psllw $0x8,%%xmm7 \n"
- "sub %0,%1 \n"
- "sub %0,%1 \n"
+ "mov $0x1080108,%%eax \n"
+ "vmovd %%eax,%%xmm5 \n"
+ "vpbroadcastd %%xmm5,%%ymm5 \n"
+ "mov $0x42004200,%%eax \n"
+ "vmovd %%eax,%%xmm6 \n"
+ "vpbroadcastd %%xmm6,%%ymm6 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsllw $0xb,%%ymm3,%%ymm3 \n"
+ "vpsrlw $0x6,%%ymm3,%%ymm4 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsllw $0x8,%%ymm7,%%ymm7 \n"
+ "sub %0,%1 \n"
+ "sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "psllw $0x1,%%xmm1 \n"
- "psllw $0xb,%%xmm2 \n"
- "pand %%xmm3,%%xmm1 \n"
- "pmulhuw %%xmm5,%%xmm2 \n"
- "pmulhuw %%xmm5,%%xmm1 \n"
- "psllw $0x8,%%xmm1 \n"
- "por %%xmm2,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "pand %%xmm4,%%xmm0 \n"
- "psraw $0x8,%%xmm2 \n"
- "pmulhuw %%xmm6,%%xmm0 \n"
- "pand %%xmm7,%%xmm2 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "punpcklbw %%xmm0,%%xmm1 \n"
- "punpckhbw %%xmm0,%%xmm2 \n"
- "movdqu %%xmm1,0x00(%1,%0,2) \n"
- "movdqu %%xmm2,0x10(%1,%0,2) \n"
- "lea 0x10(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsllw $0x1,%%ymm0,%%ymm1 \n"
+ "vpsllw $0xb,%%ymm0,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpmulhuw %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpmulhuw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpsraw $0x8,%%ymm0,%%ymm2 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpunpcklbw %%ymm0,%%ymm1,%%ymm2 \n"
+ "vpunpckhbw %%ymm0,%%ymm1,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm2,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm2,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1,%0,2) \n"
+ "vmovdqu %%ymm1,0x20(%1,%0,2) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -389,121 +983,216 @@ void ARGB1555ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
+#endif // HAS_ARGB1555TOARGBROW_AVX2
-void ARGB4444ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_ARGB1555TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ uintptr_t temp;
asm volatile(
- "mov $0xf0f0f0f,%%eax \n"
- "movd %%eax,%%xmm4 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "pslld $0x4,%%xmm5 \n"
- "sub %0,%1 \n"
- "sub %0,%1 \n"
+ "mov $0x01080108,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm5 \n"
+ "mov $0x42004200,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm6 \n"
+ "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n"
+ "vpsllw $0xb,%%zmm3,%%zmm3 \n"
+ "vpsrlw $0x6,%%zmm3,%%zmm4 \n"
+ "vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n"
+ "vpsllw $0x8,%%zmm7,%%zmm7 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "pand %%xmm4,%%xmm0 \n"
- "pand %%xmm5,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "psllw $0x4,%%xmm1 \n"
- "psrlw $0x4,%%xmm3 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm3,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm2,%%xmm0 \n"
- "punpckhbw %%xmm2,%%xmm1 \n"
- "movdqu %%xmm0,0x00(%1,%0,2) \n"
- "movdqu %%xmm1,0x10(%1,%0,2) \n"
- "lea 0x10(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsllw $0x1,%%zmm0,%%zmm1 \n"
+ "vpsllw $0xb,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpsraw $0x8,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n"
+ "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n"
+ "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n"
+ "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n"
+ "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n"
+ "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n"
+ "vmovdqu32 %%zmm0,(%1) \n"
+ "vmovdqu32 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsllw $0x1,%%zmm0,%%zmm1 \n"
+ "vpsllw $0xb,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpsraw $0x8,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n"
+ "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n"
+ "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n"
+ "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n"
+ "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n"
+ "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n"
+ "cmp $0x10,%2 \n"
+ "jle 3f \n"
+ "vmovdqu32 %%zmm0,(%1) \n"
+ "kshiftrd $0x10,%%k1,%%k1 \n"
+ "vmovdqu32 %%zmm1,0x40(%1)%{%%k1%} \n"
+ "jmp 99f \n"
+
+ "3: \n"
+ "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "k1");
+}
+#endif // HAS_ARGB1555TOARGBROW_AVX512BW
+
+#ifdef HAS_ARGB4444TOARGBROW_AVX2
+void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "mov $0x0f0f0f0f,%%eax \n"
+ "vmovd %%eax,%%xmm4 \n"
+ "vpbroadcastd %%xmm4,%%ymm4 \n"
+ "vpslld $0x4,%%ymm4,%%ymm5 \n"
+ "sub %0,%1 \n"
+ "sub %0,%1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm2 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpsllw $0x4,%%ymm0,%%ymm1 \n"
+ "vpsrlw $0x4,%%ymm2,%%ymm3 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpunpckhbw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm1 \n"
+ "vmovdqu %%ymm2,(%1,%0,2) \n"
+ "vmovdqu %%ymm1,0x20(%1,%0,2) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
:
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
+#endif // HAS_ARGB4444TOARGBROW_AVX2
+#ifdef HAS_ARGBTORGB24ROW_SSSE3
void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
-
- "movdqa %3,%%xmm6 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "pshufb %%xmm6,%%xmm0 \n"
- "pshufb %%xmm6,%%xmm1 \n"
- "pshufb %%xmm6,%%xmm2 \n"
- "pshufb %%xmm6,%%xmm3 \n"
- "movdqa %%xmm1,%%xmm4 \n"
- "psrldq $0x4,%%xmm1 \n"
- "pslldq $0xc,%%xmm4 \n"
- "movdqa %%xmm2,%%xmm5 \n"
- "por %%xmm4,%%xmm0 \n"
- "pslldq $0x8,%%xmm5 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "psrldq $0x8,%%xmm2 \n"
- "pslldq $0x4,%%xmm3 \n"
- "por %%xmm3,%%xmm2 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "lea 0x30(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskARGBToRGB24) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ asm volatile("movdqa %3,%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "pshufb %%xmm6,%%xmm0 \n"
+ "pshufb %%xmm6,%%xmm1 \n"
+ "pshufb %%xmm6,%%xmm2 \n"
+ "pshufb %%xmm6,%%xmm3 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "psrldq $0x4,%%xmm1 \n"
+ "pslldq $0xc,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "por %%xmm4,%%xmm0 \n"
+ "pslldq $0x8,%%xmm5 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "por %%xmm5,%%xmm1 \n"
+ "psrldq $0x8,%%xmm2 \n"
+ "pslldq $0x4,%%xmm3 \n"
+ "por %%xmm3,%%xmm2 \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "lea 0x30(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskARGBToRGB24[0]) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
+#endif // HAS_ARGBTORGB24ROW_SSSE3
+#ifdef HAS_ARGBTORAWROW_SSSE3
void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
-
- "movdqa %3,%%xmm6 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "pshufb %%xmm6,%%xmm0 \n"
- "pshufb %%xmm6,%%xmm1 \n"
- "pshufb %%xmm6,%%xmm2 \n"
- "pshufb %%xmm6,%%xmm3 \n"
- "movdqa %%xmm1,%%xmm4 \n"
- "psrldq $0x4,%%xmm1 \n"
- "pslldq $0xc,%%xmm4 \n"
- "movdqa %%xmm2,%%xmm5 \n"
- "por %%xmm4,%%xmm0 \n"
- "pslldq $0x8,%%xmm5 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "psrldq $0x8,%%xmm2 \n"
- "pslldq $0x4,%%xmm3 \n"
- "por %%xmm3,%%xmm2 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "lea 0x30(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskARGBToRAW) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ asm volatile("movdqa %3,%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "pshufb %%xmm6,%%xmm0 \n"
+ "pshufb %%xmm6,%%xmm1 \n"
+ "pshufb %%xmm6,%%xmm2 \n"
+ "pshufb %%xmm6,%%xmm3 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "psrldq $0x4,%%xmm1 \n"
+ "pslldq $0xc,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "por %%xmm4,%%xmm0 \n"
+ "pslldq $0x8,%%xmm5 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "por %%xmm5,%%xmm1 \n"
+ "psrldq $0x8,%%xmm2 \n"
+ "pslldq $0x4,%%xmm3 \n"
+ "por %%xmm3,%%xmm2 \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "lea 0x30(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskARGBToRAW) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
+#endif // HAS_ARGBTORAWROW_SSSE3
#ifdef HAS_ARGBTORGB24ROW_AVX2
// vpermd for 12+12 to 24
@@ -511,133 +1200,187 @@ static const lvec32 kPermdRGB24_AVX = {0, 1, 2, 4, 5, 6, 3, 7};
void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vbroadcastf128 %3,%%ymm6 \n"
- "vmovdqa %4,%%ymm7 \n"
+ "vbroadcasti128 %3,%%ymm6 \n"
+ "vmovdqa %4,%%ymm7 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" // xxx0yyy0
- "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
- "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
- "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
- "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // pack to 24 bytes
- "vpermd %%ymm1,%%ymm7,%%ymm1 \n"
- "vpermd %%ymm2,%%ymm7,%%ymm2 \n"
- "vpermd %%ymm3,%%ymm7,%%ymm3 \n"
- "vpermq $0x3f,%%ymm1,%%ymm4 \n" // combine 24 + 8
- "vpor %%ymm4,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vpermq $0xf9,%%ymm1,%%ymm1 \n" // combine 16 + 16
- "vpermq $0x4f,%%ymm2,%%ymm4 \n"
- "vpor %%ymm4,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "vpermq $0xfe,%%ymm2,%%ymm2 \n" // combine 8 + 24
- "vpermq $0x93,%%ymm3,%%ymm3 \n"
- "vpor %%ymm3,%%ymm2,%%ymm2 \n"
- "vmovdqu %%ymm2,0x40(%1) \n"
- "lea 0x60(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskARGBToRGB24), // %3
- "m"(kPermdRGB24_AVX) // %4
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm3 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" // xxx0yyy0
+ "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // pack to 24 bytes
+ "vpermd %%ymm1,%%ymm7,%%ymm1 \n"
+ "vpermd %%ymm2,%%ymm7,%%ymm2 \n"
+ "vpermd %%ymm3,%%ymm7,%%ymm3 \n"
+ "vpermq $0x3f,%%ymm1,%%ymm4 \n" // combine 24 + 8
+ "vpor %%ymm4,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vpermq $0xf9,%%ymm1,%%ymm1 \n" // combine 16 + 16
+ "vpermq $0x4f,%%ymm2,%%ymm4 \n"
+ "vpor %%ymm4,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vpermq $0xfe,%%ymm2,%%ymm2 \n" // combine 8 + 24
+ "vpermq $0x93,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "lea 0x60(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskARGBToRGB24[0]), // %3
+ "m"(kPermdRGB24_AVX) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-#endif
-
-#ifdef HAS_ARGBTORGB24ROW_AVX512VBMI
-// Shuffle table for converting ARGBToRGB24
-static const ulvec8 kPermARGBToRGB24_0 = {
- 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u,
- 14u, 16u, 17u, 18u, 20u, 21u, 22u, 24u, 25u, 26u, 28u,
- 29u, 30u, 32u, 33u, 34u, 36u, 37u, 38u, 40u, 41u};
-static const ulvec8 kPermARGBToRGB24_1 = {
- 10u, 12u, 13u, 14u, 16u, 17u, 18u, 20u, 21u, 22u, 24u,
- 25u, 26u, 28u, 29u, 30u, 32u, 33u, 34u, 36u, 37u, 38u,
- 40u, 41u, 42u, 44u, 45u, 46u, 48u, 49u, 50u, 52u};
-static const ulvec8 kPermARGBToRGB24_2 = {
- 21u, 22u, 24u, 25u, 26u, 28u, 29u, 30u, 32u, 33u, 34u,
- 36u, 37u, 38u, 40u, 41u, 42u, 44u, 45u, 46u, 48u, 49u,
- 50u, 52u, 53u, 54u, 56u, 57u, 58u, 60u, 61u, 62u};
-
-void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) {
+#endif // HAS_ARGBTORGB24ROW_AVX2
+
+#if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ int width,
+ const uvec8& shuffler) {
+ uintptr_t temp;
asm volatile(
- "vmovdqa %3,%%ymm5 \n"
- "vmovdqa %4,%%ymm6 \n"
- "vmovdqa %5,%%ymm7 \n"
+ "vbroadcasti32x4 %[shuffler],%%zmm4 \n"
+ "vmovdqu32 %[kPermdRGB24_0],%%zmm5 \n"
+ "vmovdqu32 %[kPermdRGB24_1],%%zmm16 \n"
+ "vmovdqu32 %[kPermdRGB24_2],%%zmm17 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vpermt2b %%ymm1,%%ymm5,%%ymm0 \n"
- "vpermt2b %%ymm2,%%ymm6,%%ymm1 \n"
- "vpermt2b %%ymm3,%%ymm7,%%ymm2 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "vmovdqu %%ymm2,0x40(%1) \n"
- "lea 0x60(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kPermARGBToRGB24_0), // %3
- "m"(kPermARGBToRGB24_1), // %4
- "m"(kPermARGBToRGB24_2) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7");
+ "1: \n"
+ "vmovdqu8 (%[src]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src]),%%zmm1 \n"
+ "vmovdqu8 0x80(%[src]),%%zmm2 \n"
+ "vmovdqu8 0xc0(%[src]),%%zmm3 \n"
+ "lea 0x100(%[src]),%[src] \n"
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpermt2d %%zmm1,%%zmm5,%%zmm0 \n"
+ "vpermt2d %%zmm2,%%zmm16,%%zmm1 \n"
+ "vpermt2d %%zmm3,%%zmm17,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst]) \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst]) \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst]) \n"
+ "lea 0xc0(%[dst]),%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "shr $0x10,%[temp] \n"
+ "kmovw %k[temp],%%k2 \n"
+ "shr $0x10,%[temp] \n"
+ "kmovw %k[temp],%%k3 \n"
+ "shr $0x10,%[temp] \n"
+ "kmovw %k[temp],%%k4 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src]),%%zmm3%{%%k4%}%{z%} \n"
+
+ "mov $-1,%[temp] \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[src] \n"
+ "kmovq %[src],%%k1 \n"
+ "xor %k[src],%k[src] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[src] \n"
+ "bzhi %q[width],%[src],%[src] \n"
+ "kmovq %[src],%%k2 \n"
+ "xor %k[src],%k[src] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[src] \n"
+ "bzhi %q[width],%[src],%[src] \n"
+ "kmovq %[src],%%k3 \n"
+
+ "vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpermt2d %%zmm1,%%zmm5,%%zmm0 \n"
+ "vpermt2d %%zmm2,%%zmm16,%%zmm1 \n"
+ "vpermt2d %%zmm3,%%zmm17,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst])%{%%k2%} \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [temp] "=&r"(temp) // %[temp]
+ : [shuffler] "m"(shuffler), [kPermdRGB24_0] "m"(kPermdRGB24_0_AVX512BW),
+ [kPermdRGB24_1] "m"(kPermdRGB24_1_AVX512BW),
+ [kPermdRGB24_2] "m"(kPermdRGB24_2_AVX512BW)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "k1", "k2", "k3", "k4");
}
-#endif
+#endif // defined(HAS_ARGBTORGB24ROW_AVX512BW) ||
+ // defined(HAS_ARGBTORAWROW_AVX512BW)
+
+#ifdef HAS_ARGBTORGB24ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]);
+}
+#endif // HAS_ARGBTORGB24ROW_AVX512BW
#ifdef HAS_ARGBTORAWROW_AVX2
void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vbroadcastf128 %3,%%ymm6 \n"
- "vmovdqa %4,%%ymm7 \n"
+ "vbroadcasti128 %3,%%ymm6 \n"
+ "vmovdqa %4,%%ymm7 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" // xxx0yyy0
- "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
- "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
- "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
- "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // pack to 24 bytes
- "vpermd %%ymm1,%%ymm7,%%ymm1 \n"
- "vpermd %%ymm2,%%ymm7,%%ymm2 \n"
- "vpermd %%ymm3,%%ymm7,%%ymm3 \n"
- "vpermq $0x3f,%%ymm1,%%ymm4 \n" // combine 24 + 8
- "vpor %%ymm4,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vpermq $0xf9,%%ymm1,%%ymm1 \n" // combine 16 + 16
- "vpermq $0x4f,%%ymm2,%%ymm4 \n"
- "vpor %%ymm4,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "vpermq $0xfe,%%ymm2,%%ymm2 \n" // combine 8 + 24
- "vpermq $0x93,%%ymm3,%%ymm3 \n"
- "vpor %%ymm3,%%ymm2,%%ymm2 \n"
- "vmovdqu %%ymm2,0x40(%1) \n"
- "lea 0x60(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm3 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" // xxx0yyy0
+ "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // pack to 24 bytes
+ "vpermd %%ymm1,%%ymm7,%%ymm1 \n"
+ "vpermd %%ymm2,%%ymm7,%%ymm2 \n"
+ "vpermd %%ymm3,%%ymm7,%%ymm3 \n"
+ "vpermq $0x3f,%%ymm1,%%ymm4 \n" // combine 24 + 8
+ "vpor %%ymm4,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vpermq $0xf9,%%ymm1,%%ymm1 \n" // combine 16 + 16
+ "vpermq $0x4f,%%ymm2,%%ymm4 \n"
+ "vpor %%ymm4,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vpermq $0xfe,%%ymm2,%%ymm2 \n" // combine 8 + 24
+ "vpermq $0x93,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "lea 0x60(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -646,84 +1389,52 @@ void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-#endif
-
-void ARGBToRGB565Row_SSE2(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
- "pcmpeqb %%xmm3,%%xmm3 \n"
- "psrld $0x1b,%%xmm3 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrld $0x1a,%%xmm4 \n"
- "pslld $0x5,%%xmm4 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "pslld $0xb,%%xmm5 \n"
+#endif // HAS_ARGBTORAWROW_AVX2
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "pslld $0x8,%%xmm0 \n"
- "psrld $0x3,%%xmm1 \n"
- "psrld $0x5,%%xmm2 \n"
- "psrad $0x10,%%xmm0 \n"
- "pand %%xmm3,%%xmm1 \n"
- "pand %%xmm4,%%xmm2 \n"
- "pand %%xmm5,%%xmm0 \n"
- "por %%xmm2,%%xmm1 \n"
- "por %%xmm1,%%xmm0 \n"
- "packssdw %%xmm0,%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+#ifdef HAS_ARGBTORAWROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW);
}
+#endif // HAS_ARGBTORAWROW_AVX512BW
-void ARGBToRGB565DitherRow_SSE2(const uint8_t* src,
+#ifdef HAS_ARGBTORGB565DITHERROW_AVX2
+void ARGBToRGB565DitherRow_AVX2(const uint8_t* src,
uint8_t* dst,
- const uint32_t dither4,
+ uint32_t dither4,
int width) {
asm volatile(
- "movd %3,%%xmm6 \n"
- "punpcklbw %%xmm6,%%xmm6 \n"
- "movdqa %%xmm6,%%xmm7 \n"
- "punpcklwd %%xmm6,%%xmm6 \n"
- "punpckhwd %%xmm7,%%xmm7 \n"
- "pcmpeqb %%xmm3,%%xmm3 \n"
- "psrld $0x1b,%%xmm3 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrld $0x1a,%%xmm4 \n"
- "pslld $0x5,%%xmm4 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "pslld $0xb,%%xmm5 \n"
+ "vbroadcastss %3,%%xmm6 \n"
+ "vpunpcklbw %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpermq $0xd8,%%ymm6,%%ymm6 \n"
+ "vpunpcklwd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsrld $0x1b,%%ymm3,%%ymm3 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $0x1a,%%ymm4,%%ymm4 \n"
+ "vpslld $0x5,%%ymm4,%%ymm4 \n"
+ "vpslld $0xb,%%ymm3,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "paddusb %%xmm6,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "pslld $0x8,%%xmm0 \n"
- "psrld $0x3,%%xmm1 \n"
- "psrld $0x5,%%xmm2 \n"
- "psrad $0x10,%%xmm0 \n"
- "pand %%xmm3,%%xmm1 \n"
- "pand %%xmm4,%%xmm2 \n"
- "pand %%xmm5,%%xmm0 \n"
- "por %%xmm2,%%xmm1 \n"
- "por %%xmm1,%%xmm0 \n"
- "packssdw %%xmm0,%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpaddusb %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x5,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x3,%%ymm0,%%ymm1 \n"
+ "vpsrld $0x8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -731,123 +1442,249 @@ void ARGBToRGB565DitherRow_SSE2(const uint8_t* src,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
+#endif // HAS_ARGBTORGB565DITHERROW_AVX2
-#ifdef HAS_ARGBTORGB565DITHERROW_AVX2
-void ARGBToRGB565DitherRow_AVX2(const uint8_t* src,
- uint8_t* dst,
- const uint32_t dither4,
- int width) {
+#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ uint32_t dither4,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastss %3,%%xmm6 \n"
- "vpunpcklbw %%xmm6,%%xmm6,%%xmm6 \n"
- "vpermq $0xd8,%%ymm6,%%ymm6 \n"
- "vpunpcklwd %%ymm6,%%ymm6,%%ymm6 \n"
- "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
- "vpsrld $0x1b,%%ymm3,%%ymm3 \n"
- "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrld $0x1a,%%ymm4,%%ymm4 \n"
- "vpslld $0x5,%%ymm4,%%ymm4 \n"
- "vpslld $0xb,%%ymm3,%%ymm5 \n"
+ "vmovd %[dither4],%%xmm6 \n"
+ "vpunpcklbw %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpunpcklwd %%xmm6,%%xmm6,%%xmm6 \n"
+ "vshufi32x4 $0x00,%%zmm6,%%zmm6,%%zmm6 \n"
+ "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n"
+ "vpsrld $0x1b,%%zmm3,%%zmm3 \n"
+ "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
+ "vpsrld $0x1a,%%zmm4,%%zmm4 \n"
+ "vpslld $0x5,%%zmm4,%%zmm4 \n"
+ "vpslld $0xb,%%zmm3,%%zmm5 \n"
+ "sub $0x10,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vpaddusb %%ymm6,%%ymm0,%%ymm0 \n"
- "vpsrld $0x5,%%ymm0,%%ymm2 \n"
- "vpsrld $0x3,%%ymm0,%%ymm1 \n"
- "vpsrld $0x8,%%ymm0,%%ymm0 \n"
- "vpand %%ymm4,%%ymm2,%%ymm2 \n"
- "vpand %%ymm3,%%ymm1,%%ymm1 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpor %%ymm2,%%ymm1,%%ymm1 \n"
- "vpor %%ymm1,%%ymm0,%%ymm0 \n"
- "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "lea 0x20(%0),%0 \n"
- "vmovdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(dither4) // %3
+ "1: \n"
+ "vmovdqu32 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpsrld $0x3,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x5,%%zmm0,%%zmm2 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n"
+ "vpmovdw %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpsrld $0x3,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x5,%%zmm0,%%zmm2 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n"
+ "vpmovdw %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [dither4] "r"(dither4) // %[dither4]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ "k1");
}
-#endif // HAS_ARGBTORGB565DITHERROW_AVX2
+#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW
-void ARGBToARGB1555Row_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_ARGBTORGB565ROW_AVX2
+void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrld $0x1b,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "pslld $0x5,%%xmm5 \n"
- "movdqa %%xmm4,%%xmm6 \n"
- "pslld $0xa,%%xmm6 \n"
- "pcmpeqb %%xmm7,%%xmm7 \n"
- "pslld $0xf,%%xmm7 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsrld $0x1b,%%ymm3,%%ymm3 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $0x1a,%%ymm4,%%ymm4 \n"
+ "vpslld $0x5,%%ymm4,%%ymm4 \n"
+ "vpslld $0xb,%%ymm3,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm3 \n"
- "psrad $0x10,%%xmm0 \n"
- "psrld $0x3,%%xmm1 \n"
- "psrld $0x6,%%xmm2 \n"
- "psrld $0x9,%%xmm3 \n"
- "pand %%xmm7,%%xmm0 \n"
- "pand %%xmm4,%%xmm1 \n"
- "pand %%xmm5,%%xmm2 \n"
- "pand %%xmm6,%%xmm3 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm3,%%xmm2 \n"
- "por %%xmm2,%%xmm0 \n"
- "packssdw %%xmm0,%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsrld $0x5,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x3,%%ymm0,%%ymm1 \n"
+ "vpsrld $0x8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
+#endif // HAS_ARGBTORGB565ROW_AVX2
-void ARGBToARGB4444Row_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_ARGBTORGB565ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB565Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ uintptr_t temp;
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psllw $0xc,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm3 \n"
- "psrlw $0x8,%%xmm3 \n"
+ "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n"
+ "vpsrld $0x1b,%%zmm3,%%zmm3 \n"
+ "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
+ "vpsrld $0x1a,%%zmm4,%%zmm4 \n"
+ "vpslld $0x5,%%zmm4,%%zmm4 \n"
+ "vpslld $0xb,%%zmm3,%%zmm5 \n"
+ "sub $0x10,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pand %%xmm3,%%xmm0 \n"
- "pand %%xmm4,%%xmm1 \n"
- "psrlq $0x4,%%xmm0 \n"
- "psrlq $0x8,%%xmm1 \n"
- "por %%xmm1,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu32 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x3,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x5,%%zmm0,%%zmm2 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n"
+ "vpmovdw %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrld $0x3,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x5,%%zmm0,%%zmm2 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n"
+ "vpmovdw %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1");
+}
+#endif // HAS_ARGBTORGB565ROW_AVX512BW
+
+#ifdef HAS_ARGBTOARGB1555ROW_AVX2
+void ARGBToARGB1555Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpslld $0x1f,%%ymm3,%%ymm6 \n"
+ "vpsrld $0x10,%%ymm6,%%ymm6 \n"
+ "vpsrld $0x1b,%%ymm3,%%ymm3 \n"
+ "vpslld $0x5,%%ymm3,%%ymm4 \n"
+ "vpslld $0xa,%%ymm3,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsrld $0x6,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x3,%%ymm0,%%ymm1 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm7 \n"
+ "vpsrld $0x9,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm7,%%ymm7 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-#endif // HAS_RGB24TOARGBROW_SSSE3
+#endif // HAS_ARGBTOARGB1555ROW_AVX2
+
+#ifdef HAS_ARGBTOARGB4444ROW_AVX2
+void ARGBToARGB4444Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsrld $0x1c,%%ymm3,%%ymm3 \n"
+ "vpslld $0x4,%%ymm3,%%ymm4 \n"
+ "vpslld $0x8,%%ymm3,%%ymm5 \n"
+ "vpslld $0xc,%%ymm3,%%ymm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsrld $0x8,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x4,%%ymm0,%%ymm1 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm7 \n"
+ "vpsrld $0xc,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm7,%%ymm7 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_ARGBTOARGB4444ROW_AVX2
/*
@@ -882,33 +1719,34 @@ static const uint32_t kMaskRB10 = 0x3ff003ff;
static const uint32_t kMaskAG10 = 0xc000ff00;
static const uint32_t kMulAG10 = 64 * 65536 + 1028;
+#ifdef HAS_ARGBTOAR30ROW_SSSE3
void ARGBToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "movdqa %3,%%xmm2 \n" // shuffler for RB
- "movd %4,%%xmm3 \n" // multipler for RB
- "movd %5,%%xmm4 \n" // mask for R10 B10
- "movd %6,%%xmm5 \n" // mask for AG
- "movd %7,%%xmm6 \n" // multipler for AG
- "pshufd $0x0,%%xmm3,%%xmm3 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "pshufd $0x0,%%xmm6,%%xmm6 \n"
- "sub %0,%1 \n"
-
- "1: \n"
- "movdqu (%0),%%xmm0 \n" // fetch 4 ARGB pixels
- "movdqa %%xmm0,%%xmm1 \n"
- "pshufb %%xmm2,%%xmm1 \n" // R0B0
- "pand %%xmm5,%%xmm0 \n" // A0G0
- "pmulhuw %%xmm3,%%xmm1 \n" // X2 R16 X4 B10
- "pmulhuw %%xmm6,%%xmm0 \n" // X10 A2 X10 G10
- "pand %%xmm4,%%xmm1 \n" // X2 R10 X10 B10
- "pslld $10,%%xmm0 \n" // A2 x10 G10 x10
- "por %%xmm1,%%xmm0 \n" // A2 R10 G10 B10
- "movdqu %%xmm0,(%1,%0) \n" // store 4 AR30 pixels
- "add $0x10,%0 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "movdqa %3,%%xmm2 \n" // shuffler for RB
+ "movd %4,%%xmm3 \n" // multipler for RB
+ "movd %5,%%xmm4 \n" // mask for R10 B10
+ "movd %6,%%xmm5 \n" // mask for AG
+ "movd %7,%%xmm6 \n" // multipler for AG
+ "pshufd $0x0,%%xmm3,%%xmm3 \n"
+ "pshufd $0x0,%%xmm4,%%xmm4 \n"
+ "pshufd $0x0,%%xmm5,%%xmm5 \n"
+ "pshufd $0x0,%%xmm6,%%xmm6 \n"
+ "sub %0,%1 \n"
+
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // fetch 4 ARGB pixels
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pshufb %%xmm2,%%xmm1 \n" // R0B0
+ "pand %%xmm5,%%xmm0 \n" // A0G0
+ "pmulhuw %%xmm3,%%xmm1 \n" // X2 R16 X4 B10
+ "pmulhuw %%xmm6,%%xmm0 \n" // X10 A2 X10 G10
+ "pand %%xmm4,%%xmm1 \n" // X2 R10 X10 B10
+ "pslld $10,%%xmm0 \n" // A2 x10 G10 x10
+ "por %%xmm1,%%xmm0 \n" // A2 R10 G10 B10
+ "movdqu %%xmm0,(%1,%0) \n" // store 4 AR30 pixels
+ "add $0x10,%0 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -920,34 +1758,36 @@ void ARGBToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
+#endif // HAS_ARGBTOAR30ROW_SSSE3
+#ifdef HAS_ABGRTOAR30ROW_SSSE3
void ABGRToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "movdqa %3,%%xmm2 \n" // shuffler for RB
- "movd %4,%%xmm3 \n" // multipler for RB
- "movd %5,%%xmm4 \n" // mask for R10 B10
- "movd %6,%%xmm5 \n" // mask for AG
- "movd %7,%%xmm6 \n" // multipler for AG
- "pshufd $0x0,%%xmm3,%%xmm3 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "pshufd $0x0,%%xmm6,%%xmm6 \n"
- "sub %0,%1 \n"
-
- "1: \n"
- "movdqu (%0),%%xmm0 \n" // fetch 4 ABGR pixels
- "movdqa %%xmm0,%%xmm1 \n"
- "pshufb %%xmm2,%%xmm1 \n" // R0B0
- "pand %%xmm5,%%xmm0 \n" // A0G0
- "pmulhuw %%xmm3,%%xmm1 \n" // X2 R16 X4 B10
- "pmulhuw %%xmm6,%%xmm0 \n" // X10 A2 X10 G10
- "pand %%xmm4,%%xmm1 \n" // X2 R10 X10 B10
- "pslld $10,%%xmm0 \n" // A2 x10 G10 x10
- "por %%xmm1,%%xmm0 \n" // A2 R10 G10 B10
- "movdqu %%xmm0,(%1,%0) \n" // store 4 AR30 pixels
- "add $0x10,%0 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "movdqa %3,%%xmm2 \n" // shuffler for RB
+ "movd %4,%%xmm3 \n" // multipler for RB
+ "movd %5,%%xmm4 \n" // mask for R10 B10
+ "movd %6,%%xmm5 \n" // mask for AG
+ "movd %7,%%xmm6 \n" // multipler for AG
+ "pshufd $0x0,%%xmm3,%%xmm3 \n"
+ "pshufd $0x0,%%xmm4,%%xmm4 \n"
+ "pshufd $0x0,%%xmm5,%%xmm5 \n"
+ "pshufd $0x0,%%xmm6,%%xmm6 \n"
+ "sub %0,%1 \n"
+
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // fetch 4 ABGR pixels
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pshufb %%xmm2,%%xmm1 \n" // R0B0
+ "pand %%xmm5,%%xmm0 \n" // A0G0
+ "pmulhuw %%xmm3,%%xmm1 \n" // X2 R16 X4 B10
+ "pmulhuw %%xmm6,%%xmm0 \n" // X10 A2 X10 G10
+ "pand %%xmm4,%%xmm1 \n" // X2 R10 X10 B10
+ "pslld $10,%%xmm0 \n" // A2 x10 G10 x10
+ "por %%xmm1,%%xmm0 \n" // A2 R10 G10 B10
+ "movdqu %%xmm0,(%1,%0) \n" // store 4 AR30 pixels
+ "add $0x10,%0 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -959,31 +1799,32 @@ void ABGRToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
+#endif // HAS_ABGRTOAR30ROW_SSSE3
#ifdef HAS_ARGBTOAR30ROW_AVX2
void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vbroadcastf128 %3,%%ymm2 \n" // shuffler for RB
- "vbroadcastss %4,%%ymm3 \n" // multipler for RB
- "vbroadcastss %5,%%ymm4 \n" // mask for R10 B10
- "vbroadcastss %6,%%ymm5 \n" // mask for AG
- "vbroadcastss %7,%%ymm6 \n" // multipler for AG
- "sub %0,%1 \n"
+ "vbroadcasti128 %3,%%ymm2 \n" // shuffler for RB
+ "vbroadcastss %4,%%ymm3 \n" // multipler for RB
+ "vbroadcastss %5,%%ymm4 \n" // mask for R10 B10
+ "vbroadcastss %6,%%ymm5 \n" // mask for AG
+ "vbroadcastss %7,%%ymm6 \n" // multipler for AG
+ "sub %0,%1 \n"
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n" // fetch 8 ARGB pixels
- "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
- "vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" // X2 R16 X4 B10
- "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" // X10 A2 X10 G10
- "vpand %%ymm4,%%ymm1,%%ymm1 \n" // X2 R10 X10 B10
- "vpslld $10,%%ymm0,%%ymm0 \n" // A2 x10 G10 x10
- "vpor %%ymm1,%%ymm0,%%ymm0 \n" // A2 R10 G10 B10
- "vmovdqu %%ymm0,(%1,%0) \n" // store 8 AR30 pixels
- "add $0x20,%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // fetch 8 ARGB pixels
+ "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
+ "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" // X2 R16 X4 B10
+ "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" // X10 A2 X10 G10
+ "vpand %%ymm4,%%ymm1,%%ymm1 \n" // X2 R10 X10 B10
+ "vpslld $10,%%ymm0,%%ymm0 \n" // A2 x10 G10 x10
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n" // A2 R10 G10 B10
+ "vmovdqu %%ymm0,(%1,%0) \n" // store 8 AR30 pixels
+ "add $0x20,%0 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -995,32 +1836,32 @@ void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
-#endif
+#endif // HAS_ARGBTOAR30ROW_AVX2
#ifdef HAS_ABGRTOAR30ROW_AVX2
void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vbroadcastf128 %3,%%ymm2 \n" // shuffler for RB
- "vbroadcastss %4,%%ymm3 \n" // multipler for RB
- "vbroadcastss %5,%%ymm4 \n" // mask for R10 B10
- "vbroadcastss %6,%%ymm5 \n" // mask for AG
- "vbroadcastss %7,%%ymm6 \n" // multipler for AG
- "sub %0,%1 \n"
+ "vbroadcasti128 %3,%%ymm2 \n" // shuffler for RB
+ "vbroadcastss %4,%%ymm3 \n" // multipler for RB
+ "vbroadcastss %5,%%ymm4 \n" // mask for R10 B10
+ "vbroadcastss %6,%%ymm5 \n" // mask for AG
+ "vbroadcastss %7,%%ymm6 \n" // multipler for AG
+ "sub %0,%1 \n"
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n" // fetch 8 ABGR pixels
- "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
- "vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" // X2 R16 X4 B10
- "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" // X10 A2 X10 G10
- "vpand %%ymm4,%%ymm1,%%ymm1 \n" // X2 R10 X10 B10
- "vpslld $10,%%ymm0,%%ymm0 \n" // A2 x10 G10 x10
- "vpor %%ymm1,%%ymm0,%%ymm0 \n" // A2 R10 G10 B10
- "vmovdqu %%ymm0,(%1,%0) \n" // store 8 AR30 pixels
- "add $0x20,%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // fetch 8 ABGR pixels
+ "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
+ "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" // X2 R16 X4 B10
+ "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" // X10 A2 X10 G10
+ "vpand %%ymm4,%%ymm1,%%ymm1 \n" // X2 R10 X10 B10
+ "vpslld $10,%%ymm0,%%ymm0 \n" // A2 x10 G10 x10
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n" // A2 R10 G10 B10
+ "vmovdqu %%ymm0,(%1,%0) \n" // store 8 AR30 pixels
+ "add $0x20,%0 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -1032,932 +1873,2120 @@ void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
+#endif // HAS_ABGRTOAR30ROW_AVX2
+
+#if defined(HAS_AB64TOARGBROW_SSSE3) || defined(HAS_AB64TOARGBROW_AVX2)
+static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
#endif
-#ifdef HAS_ARGBTOYROW_SSSE3
-// Convert 16 ARGB pixels (64 bytes) to 16 Y values.
-void ARGBToYRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_y, int width) {
- asm volatile(
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
+#if defined(HAS_ARGBTOAB64ROW_SSSE3) || defined(HAS_ARGBTOAB64ROW_AVX2)
+static const uvec8 kShuffleARGBToAB64Lo = {2, 2, 1, 1, 0, 0, 3, 3,
+ 6, 6, 5, 5, 4, 4, 7, 7};
+static const uvec8 kShuffleARGBToAB64Hi = {10, 10, 9, 9, 8, 8, 11, 11,
+ 14, 14, 13, 13, 12, 12, 15, 15};
+#endif
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm3,%%xmm2 \n"
- "psrlw $0x7,%%xmm0 \n"
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+#ifdef HAS_ARGBTOAR64ROW_SSSE3
+void ARGBToAR64Row_SSSE3(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm0,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
: "+r"(src_argb), // %0
- "+r"(dst_y), // %1
+ "+r"(dst_ar64), // %1
"+r"(width) // %2
- : "m"(kARGBToY), // %3
- "m"(kAddY16) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
-#endif // HAS_ARGBTOYROW_SSSE3
+#endif // HAS_ARGBTOAR64ROW_SSSE3
-#ifdef HAS_ARGBTOYJROW_SSSE3
-// Convert 16 ARGB pixels (64 bytes) to 16 YJ values.
-// Same as ARGBToYRow but different coefficients, no add 16, but do rounding.
-void ARGBToYJRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+#ifdef HAS_ARGBTOAB64ROW_SSSE3
+void ARGBToAB64Row_SSSE3(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
asm volatile(
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
+ "movdqa %3,%%xmm2 \n"
+ "movdqa %4,%%xmm3 \n" LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pshufb %%xmm2,%%xmm0 \n"
+ "pshufb %%xmm3,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ab64), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToAB64Lo), // %3
+ "m"(kShuffleARGBToAB64Hi) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_ARGBTOAB64ROW_SSSE3
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm3,%%xmm2 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "paddw %%xmm5,%%xmm2 \n"
- "psrlw $0x7,%%xmm0 \n"
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+#ifdef HAS_AR64TOARGBROW_SSSE3
+void AR64ToARGBRow_SSSE3(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "psrlw $8,%%xmm0 \n"
+ "psrlw $8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ar64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ ::"memory",
+ "cc", "xmm0", "xmm1");
+}
+#endif // HAS_AR64TOARGBROW_SSSE3
+
+#ifdef HAS_AB64TOARGBROW_SSSE3
+void AB64ToARGBRow_SSSE3(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile("movdqa %3,%%xmm2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "psrlw $8,%%xmm0 \n"
+ "psrlw $8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "pshufb %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToABGR) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_AB64TOARGBROW_SSSE3
+
+#ifdef HAS_ARGBTOAR64ROW_AVX2
+void ARGBToAR64Row_AVX2(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
- "+r"(dst_y), // %1
+ "+r"(dst_ar64), // %1
"+r"(width) // %2
- : "m"(kARGBToYJ), // %3
- "m"(kAddYJ64) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
+}
+#endif // HAS_ARGBTOAR64ROW_AVX2
+
+#ifdef HAS_ARGBTOAB64ROW_AVX2
+void ARGBToAB64Row_AVX2(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 %3,%%ymm2 \n"
+ "vbroadcasti128 %4,%%ymm3 \n" LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm3,%%ymm0,%%ymm1 \n"
+ "vpshufb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ab64), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToAB64Lo), // %3
+ "m"(kShuffleARGBToAB64Hi) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_ARGBTOAB64ROW_AVX2
+
+#ifdef HAS_AR64TOARGBROW_AVX2
+void AR64ToARGBRow_AVX2(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpsrlw $8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x40(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ar64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
-#endif // HAS_ARGBTOYJROW_SSSE3
+#endif // HAS_AR64TOARGBROW_AVX2
-#ifdef HAS_ARGBTOYROW_AVX2
+#ifdef HAS_AB64TOARGBROW_AVX2
+void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile("vbroadcasti128 %3,%%ymm2 \n" LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpsrlw $8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x40(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToABGR) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_AB64TOARGBROW_AVX2
+
+// clang-format off
+
+// TODO(mraptis): Consider passing R, G, B multipliers as parameter.
+// round parameter is register containing value to add before shift.
+#define RGBTOY(round) \
+ "1: \n" \
+ "movdqu (%0),%%xmm0 \n" \
+ "movdqu 0x10(%0),%%xmm1 \n" \
+ "movdqu 0x20(%0),%%xmm2 \n" \
+ "movdqu 0x30(%0),%%xmm3 \n" \
+ "psubb %%xmm5,%%xmm0 \n" \
+ "psubb %%xmm5,%%xmm1 \n" \
+ "psubb %%xmm5,%%xmm2 \n" \
+ "psubb %%xmm5,%%xmm3 \n" \
+ "movdqu %%xmm4,%%xmm6 \n" \
+ "pmaddubsw %%xmm0,%%xmm6 \n" \
+ "movdqu %%xmm4,%%xmm0 \n" \
+ "pmaddubsw %%xmm1,%%xmm0 \n" \
+ "movdqu %%xmm4,%%xmm1 \n" \
+ "pmaddubsw %%xmm2,%%xmm1 \n" \
+ "movdqu %%xmm4,%%xmm2 \n" \
+ "pmaddubsw %%xmm3,%%xmm2 \n" \
+ "lea 0x40(%0),%0 \n" \
+ "phaddw %%xmm0,%%xmm6 \n" \
+ "phaddw %%xmm2,%%xmm1 \n" \
+ "prefetcht0 1280(%0) \n" \
+ "paddw %%" #round ",%%xmm6 \n" \
+ "paddw %%" #round ",%%xmm1 \n" \
+ "psrlw $0x8,%%xmm6 \n" \
+ "psrlw $0x8,%%xmm1 \n" \
+ "packuswb %%xmm1,%%xmm6 \n" \
+ "movdqu %%xmm6,(%1) \n" \
+ "lea 0x10(%1),%1 \n" \
+ "sub $0x10,%2 \n" \
+ "jg 1b \n"
+
+#define RGBTOY_AVX2(round) \
+ "1: \n" \
+ "vmovdqu (%0),%%ymm0 \n" \
+ "vmovdqu 0x20(%0),%%ymm1 \n" \
+ "vmovdqu 0x40(%0),%%ymm2 \n" \
+ "vmovdqu 0x60(%0),%%ymm3 \n" \
+ "vpsubb %%ymm5, %%ymm0, %%ymm0 \n" \
+ "vpsubb %%ymm5, %%ymm1, %%ymm1 \n" \
+ "vpsubb %%ymm5, %%ymm2, %%ymm2 \n" \
+ "vpsubb %%ymm5, %%ymm3, %%ymm3 \n" \
+ "vpmaddubsw %%ymm0,%%ymm4,%%ymm0 \n" \
+ "vpmaddubsw %%ymm1,%%ymm4,%%ymm1 \n" \
+ "vpmaddubsw %%ymm2,%%ymm4,%%ymm2 \n" \
+ "vpmaddubsw %%ymm3,%%ymm4,%%ymm3 \n" \
+ "lea 0x80(%0),%0 \n" \
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" /* mutates. */ \
+ "vphaddw %%ymm3,%%ymm2,%%ymm2 \n" \
+ "prefetcht0 1280(%0) \n" \
+ "vpaddw %%" #round ",%%ymm0,%%ymm0 \n" /* Add 16 */ \
+ "vpaddw %%" #round ",%%ymm2,%%ymm2 \n" \
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n" \
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" /* mutates. */ \
+ "vpermd %%ymm0,%%ymm6,%%ymm0 \n" /* unmutate. */ \
+ "vmovdqu %%ymm0,(%1) \n" \
+ "lea 0x20(%1),%1 \n" \
+ "sub $0x20,%2 \n" \
+ "jg 1b \n"
+
+// clang-format on
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX2) || \
+ defined(HAS_ARGBEXTRACTALPHAROW_AVX2)
// vpermd for vphaddw + vpackuswb vpermd.
static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7};
-
-// Convert 32 ARGB pixels (128 bytes) to 32 Y values.
-void ARGBToYRow_AVX2(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
+ // defined(HAS_ARGBTOUV444MATRIXROW_AVX2) ||
+ // defined(HAS_ARGBEXTRACTALPHAROW_AVX2)
+
+#ifdef HAS_ARGBTOYMATRIXROW_SSSE3
+void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "vbroadcastf128 %3,%%ymm4 \n"
- "vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psllw $15,%%xmm5 \n"
+ "packsswb %%xmm5,%%xmm5 \n"
+ "movdqu 0(%3),%%xmm4 \n"
+ "movdqu 0x60(%3),%%xmm7 \n"
+ "movdqa %%xmm4,%%xmm6 \n"
+ "pmaddubsw %%xmm5,%%xmm6 \n"
+ "phaddw %%xmm6,%%xmm6 \n"
+ "psubw %%xmm6,%%xmm7 \n" LABELALIGN "" RGBTOY(xmm7)
+ : "+r"(src_argb), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(c) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_ARGBTOYMATRIXROW_SSSE3
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates.
- "vphaddw %%ymm3,%%ymm2,%%ymm2 \n"
- "vpsrlw $0x7,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x7,%%ymm2,%%ymm2 \n"
- "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates.
- "vpermd %%ymm0,%%ymm6,%%ymm0 \n" // unmutate.
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n" // add 16 for Y
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+#ifdef HAS_ARGBTOYMATRIXROW_AVX2
+void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsllw $15,%%ymm5,%%ymm5 \n"
+ "vpacksswb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vbroadcasti128 0(%3),%%ymm4 \n"
+ "vbroadcasti128 0x60(%3),%%ymm7 \n"
+ "vpmaddubsw %%ymm5,%%ymm4,%%ymm6 \n"
+ "vphaddw %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsubw %%ymm6,%%ymm7,%%ymm7 \n"
+ "vmovdqa %4,%%ymm6 \n" LABELALIGN
+ "" RGBTOY_AVX2(ymm7) "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
- : "m"(kARGBToY), // %3
- "m"(kAddY16), // %4
- "m"(kPermdARGBToY_AVX) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ : "r"(c), // %3
+ "m"(kPermdARGBToY_AVX) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-#endif // HAS_ARGBTOYROW_AVX2
-
-#ifdef HAS_ABGRTOYROW_AVX2
-// Convert 32 ABGR pixels (128 bytes) to 32 Y values.
-void ABGRToYRow_AVX2(const uint8_t* src_abgr, uint8_t* dst_y, int width) {
+#endif // HAS_ARGBTOYMATRIXROW_AVX2
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || \
+ defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+#endif // defined(HAS_ARGBTOYMATRIXROW_AVX512BW) ||
+ // defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) ||
+ // defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) ||
+ // defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) ||
+ // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) ||
+ // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) ||
+ // defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+
+#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
+ 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
+#endif // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) ||
+ // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) ||
+ // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastf128 %3,%%ymm4 \n"
- "vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n"
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpsllw $15,%%zmm16,%%zmm5 \n"
+ "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
+ "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1
+ "vbroadcasti32x4 0(%[c]),%%zmm4 \n"
+ "vbroadcasti32x4 0x60(%[c]),%%zmm17 \n"
+ "vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n"
+ "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n"
+ "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n"
+ "vpsubw %%zmm18,%%zmm17,%%zmm17 \n"
+ "vmovdqu32 %[perm],%%zmm18 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates.
- "vphaddw %%ymm3,%%ymm2,%%ymm2 \n"
- "vpsrlw $0x7,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x7,%%ymm2,%%ymm2 \n"
- "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates.
- "vpermd %%ymm0,%%ymm6,%%ymm0 \n" // unmutate.
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n" // add 16 for Y
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kABGRToY), // %3
- "m"(kAddY16), // %4
- "m"(kPermdARGBToY_AVX) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "1: \n"
+ "vmovdqu8 (%[src_argb]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n"
+ "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n"
+ "vmovdqu8 0xc0(%[src_argb]),%%zmm3 \n"
+ "vpsubb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsubb %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n"
+ "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y]) \n"
+ "lea 0x40(%[dst_y]),%[dst_y] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n"
+ "vpsubb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsubb %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n"
+ "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "k1", "k2", "k3", "k4");
}
-#endif // HAS_ABGRTOYROW_AVX2
-
+#endif // HAS_ARGBTOYMATRIXROW_AVX512BW
-#ifdef HAS_ARGBTOYJROW_AVX2
-// Convert 32 ARGB pixels (128 bytes) to 32 Y values.
-void ARGBToYJRow_AVX2(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastf128 %3,%%ymm4 \n"
- "vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n"
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpsllw $15,%%zmm16,%%zmm5 \n"
+ "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
+ "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // kRGBToY (unsigned)
+ "vpmovzxwd 0x60(%[c]),%%zmm17 \n" // kAddY (32-bit)
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c)
+ "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c)
+ "vmovdqu32 %[perm],%%zmm16 \n" // kPermdARGBToY_AVX512BW
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates.
- "vphaddw %%ymm3,%%ymm2,%%ymm2 \n"
- "vpaddw %%ymm5,%%ymm0,%%ymm0 \n" // Add .5 for rounding.
- "vpaddw %%ymm5,%%ymm2,%%ymm2 \n"
- "vpsrlw $0x7,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x7,%%ymm2,%%ymm2 \n"
- "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates.
- "vpermd %%ymm0,%%ymm6,%%ymm0 \n" // unmutate.
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kARGBToYJ), // %3
- "m"(kAddYJ64), // %4
- "m"(kPermdARGBToY_AVX) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "1: \n"
+ "vmovdqu8 (%[src_argb]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n"
+ "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n"
+ "vmovdqu8 0xc0(%[src_argb]),%%zmm3 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vpxorq %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm5,%%zmm3,%%zmm3 \n"
+ "vmovdqa64 %%zmm17,%%zmm18 \n"
+ "vmovdqa64 %%zmm17,%%zmm19 \n"
+ "vmovdqa64 %%zmm17,%%zmm20 \n"
+ "vmovdqa64 %%zmm17,%%zmm21 \n"
+ "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n"
+ "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n"
+ "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n"
+ "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n"
+ "vpackusdw %%zmm19,%%zmm18,%%zmm0 \n"
+ "vpackusdw %%zmm21,%%zmm20,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm16,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y]) \n"
+ "lea 0x40(%[dst_y]),%[dst_y] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n"
+ "vpxorq %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm5,%%zmm3,%%zmm3 \n"
+ "vmovdqa64 %%zmm17,%%zmm18 \n"
+ "vmovdqa64 %%zmm17,%%zmm19 \n"
+ "vmovdqa64 %%zmm17,%%zmm20 \n"
+ "vmovdqa64 %%zmm17,%%zmm21 \n"
+ "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n"
+ "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n"
+ "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n"
+ "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n"
+ "vpackusdw %%zmm19,%%zmm18,%%zmm0 \n"
+ "vpackusdw %%zmm21,%%zmm20,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm16,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
}
-#endif // HAS_ARGBTOYJROW_AVX2
+#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+// AVX512VNNIVBMI kernels use both AVX512VNNI (vpdpbusd) and AVX512VBMI
+// (vpermt2b) and require both CPU features. There are no AVX512VBMI
+// versions without AVX512VNNI.
+
+// vpermt2b indices selecting byte 1 of each int32 from 2 vectors. The 32
+// byte pattern is broadcast so each 256 bit half holds the same 32 results.
+static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = {
+ 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61,
+ 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125};
+#endif
-#ifdef HAS_ARGBTOUVROW_SSSE3
-void ARGBToUVRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+// Requires AVX512VNNI and AVX512VBMI.
+// The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in
+// [0, 65535], so Y is byte 1 of each int32 and no saturation is needed.
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpsllw $15,%%zmm16,%%zmm5 \n"
+ "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
+ "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // kRGBToY (unsigned)
+ "vpmovzxwd 0x60(%[c]),%%zmm17 \n" // kAddY (32-bit)
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c)
+ "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c)
+ "vbroadcasti64x4 %[perm],%%zmm16 \n" // byte 1 of int32
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kARGBToV), // %5
- "m"(kARGBToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-#endif // HAS_ARGBTOUVROW_SSSE3
-
-#ifdef HAS_ARGBTOUVROW_AVX2
-// vpshufb for vphaddw + vpackuswb packed to shorts.
-static const lvec8 kShufARGBToUV_AVX = {
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15};
-void ARGBToUVRow_AVX2(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+ "1: \n"
+ "vpxorq (%[src_argb]),%%zmm5,%%zmm0 \n"
+ "vmovdqa64 %%zmm17,%%zmm18 \n"
+ "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n"
+ "vpxorq 0x40(%[src_argb]),%%zmm5,%%zmm1 \n"
+ "vmovdqa64 %%zmm17,%%zmm19 \n"
+ "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n"
+ "vpxorq 0x80(%[src_argb]),%%zmm5,%%zmm2 \n"
+ "vmovdqa64 %%zmm17,%%zmm20 \n"
+ "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n"
+ "vpxorq 0xc0(%[src_argb]),%%zmm5,%%zmm3 \n"
+ "vmovdqa64 %%zmm17,%%zmm21 \n"
+ "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vpermt2b %%zmm19,%%zmm16,%%zmm18 \n" // Y 0..31
+ "vpermt2b %%zmm21,%%zmm16,%%zmm20 \n" // Y 32..63
+ "vmovdqu8 %%ymm18,(%[dst_y]) \n"
+ "vmovdqu8 %%ymm20,0x20(%[dst_y]) \n"
+ "lea 0x40(%[dst_y]),%[dst_y] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n"
+ "vpxorq %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm5,%%zmm3,%%zmm3 \n"
+ "vmovdqa64 %%zmm17,%%zmm18 \n"
+ "vmovdqa64 %%zmm17,%%zmm19 \n"
+ "vmovdqa64 %%zmm17,%%zmm20 \n"
+ "vmovdqa64 %%zmm17,%%zmm21 \n"
+ "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n"
+ "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n"
+ "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n"
+ "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n"
+ "vpermt2b %%zmm19,%%zmm16,%%zmm18 \n"
+ "vpermt2b %%zmm21,%%zmm16,%%zmm20 \n"
+ "vmovdqu8 %%ymm18,(%[dst_y])%{%%k1%} \n"
+ "vmovdqu8 %%ymm20,0x20(%[dst_y])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermbDWordByte1_AVX512VNNIVBMI)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+
+#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
+ uintptr_t mask;
asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
+ "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
+ "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
+ "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
+ "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
+ "vbroadcasti64x4 %[shuf],%%zmm24 \n" // kShuffleMaskRGB24ToARGB
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpsllw $15,%%zmm16,%%zmm5 \n"
+ "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
+ "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1
+ "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // zmm4 = kRGBToY
+ "vbroadcasti32x4 0x60(%[c]),%%zmm17 \n" // zmm17 = kAddY
+ "vpmaddubsw %%zmm5,%%zmm4,%%zmm19 \n"
+ "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n"
+ "vpackssdw %%zmm19,%%zmm19,%%zmm19 \n"
+ "vpsubw %%zmm19,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias
+ "vmovdqu32 %[perm],%%zmm25 \n" // kPermdARGBToY_AVX512BW
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n"
-
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kAddUV128), // %5
- "m"(kARGBToV), // %6
- "m"(kARGBToU), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ "1: \n"
+ "vmovdqu8 (%[src_rgb]),%%zmm0 \n"
+ "vmovdqu8 64(%[src_rgb]),%%zmm1 \n"
+ "vmovdqu8 128(%[src_rgb]),%%zmm2 \n"
+ "lea 192(%[src_rgb]),%[src_rgb] \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vmovdqa64 %%zmm0,%%zmm19 \n"
+ "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n"
+ "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm24,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm24,%%zmm19,%%zmm19 \n"
+ "vpshufb %%zmm24,%%zmm1,%%zmm2 \n"
+ "vpshufb %%zmm24,%%zmm3,%%zmm3 \n"
+ "vpsubb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm5,%%zmm19,%%zmm19 \n"
+ "vpsubb %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsubb %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpmaddubsw %%zmm19,%%zmm4,%%zmm19 \n"
+ "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n"
+ "vpackssdw %%zmm19,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm25,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y]) \n"
+ "lea 0x40(%[dst_y]),%[dst_y] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k4 \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k2 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k3 \n"
+ "vmovdqu8 (%[src_rgb]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 64(%[src_rgb]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu8 128(%[src_rgb]),%%zmm2%{%%k3%}%{z%} \n"
+ "vpermq %%zmm2,%%zmm23,%%zmm3 \n"
+ "vmovdqa64 %%zmm0,%%zmm19 \n"
+ "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n"
+ "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n"
+ "vpermq %%zmm0,%%zmm20,%%zmm0 \n"
+ "vpshufb %%zmm24,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm24,%%zmm19,%%zmm19 \n"
+ "vpshufb %%zmm24,%%zmm1,%%zmm2 \n"
+ "vpshufb %%zmm24,%%zmm3,%%zmm3 \n"
+ "vpsubb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm5,%%zmm19,%%zmm19 \n"
+ "vpsubb %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsubb %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpmaddubsw %%zmm19,%%zmm4,%%zmm19 \n"
+ "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n"
+ "vpackssdw %%zmm19,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm25,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_y])%{%%k4%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_rgb] "+r"(src_rgb), [dst_y] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp), [mask] "=&r"(mask)
+ : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW),
+ [q0] "m"(kPermqRGB24ToARGB0_AVX512BW),
+ [q1] "m"(kPermqRGB24ToARGB1_AVX512BW),
+ [q2] "m"(kPermqRGB24ToARGB2_AVX512BW),
+ [q3] "m"(kPermqRGB24ToARGB3_AVX512BW),
+ [shuf] "m"(kShuffleMaskRGB24ToARGB)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25",
+ "k1", "k2", "k3", "k4");
}
-#endif // HAS_ARGBTOUVROW_AVX2
-
-#ifdef HAS_ABGRTOUVROW_AVX2
-void ABGRToUVRow_AVX2(const uint8_t* src_abgr0,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+#endif // HAS_RGBTOYMATRIXROW_AVX512BW
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3
+void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
+ "movdqu 0x80(%4),%%xmm5 \n" // kAddUV
+ "movdqu 0x20(%4),%%xmm3 \n" // kRGBToU
+ "movdqu 0x40(%4),%%xmm4 \n" // kRGBToV
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n"
+ "1: \n"
+#if defined(__x86_64__)
+ "movdqu (%0),%%xmm8 \n"
+ "movdqu 0x10(%0),%%xmm9 \n"
+ "movdqu 0x20(%0),%%xmm10 \n"
+ "movdqu 0x30(%0),%%xmm11 \n"
+ "movdqa %%xmm8,%%xmm0 \n"
+ "movdqa %%xmm9,%%xmm1 \n"
+ "movdqa %%xmm10,%%xmm2 \n"
+ "movdqa %%xmm11,%%xmm6 \n"
+#else
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm6 \n"
+#endif
+ "pmaddubsw %%xmm3,%%xmm0 \n"
+ "pmaddubsw %%xmm3,%%xmm1 \n"
+ "pmaddubsw %%xmm3,%%xmm2 \n"
+ "pmaddubsw %%xmm3,%%xmm6 \n"
+ "phaddw %%xmm1,%%xmm0 \n"
+ "phaddw %%xmm6,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm0,%%xmm1 \n"
+ "psubw %%xmm2,%%xmm6 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "psrlw $0x8,%%xmm6 \n"
+ "packuswb %%xmm6,%%xmm1 \n"
+ "movdqu %%xmm1,(%1) \n"
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_abgr0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kAddUV128), // %5
- "m"(kABGRToV), // %6
- "m"(kABGRToU), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+#if defined(__x86_64__)
+ "pmaddubsw %%xmm4,%%xmm8 \n"
+ "pmaddubsw %%xmm4,%%xmm9 \n"
+ "pmaddubsw %%xmm4,%%xmm10 \n"
+ "pmaddubsw %%xmm4,%%xmm11 \n"
+ "phaddw %%xmm9,%%xmm8 \n"
+ "phaddw %%xmm11,%%xmm10 \n"
+ "movdqa %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm8,%%xmm1 \n"
+ "psubw %%xmm10,%%xmm6 \n"
+#else
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm6 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm6 \n"
+ "phaddw %%xmm1,%%xmm0 \n"
+ "phaddw %%xmm6,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm0,%%xmm1 \n"
+ "psubw %%xmm2,%%xmm6 \n"
+#endif
+ "psrlw $0x8,%%xmm1 \n"
+ "psrlw $0x8,%%xmm6 \n"
+ "packuswb %%xmm6,%%xmm1 \n"
+ "movdqu %%xmm1,0x00(%1,%2,1) \n"
+
+ "lea 0x40(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "subl $0x10,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"(c) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10", "xmm11"
+#endif
+ );
}
-#endif // HAS_ABGRTOUVROW_AVX2
-
-#ifdef HAS_ARGBTOUVJROW_AVX2
-void ARGBToUVJRow_AVX2(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+#endif // HAS_ARGBTOUV444MATRIXROW_SSSE3
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX2
+void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
+ "vmovdqu 0x20(%4),%%ymm3 \n" // kRGBToU
+ "vmovdqu 0x40(%4),%%ymm4 \n" // kRGBToV
+ "vmovdqu 0x80(%4),%%ymm5 \n" // kAddUV
+ "vmovdqu %5,%%ymm7 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
+ "1: \n"
+#if defined(__x86_64__)
+ "vmovdqu (%0),%%ymm8 \n"
+ "vmovdqu 0x20(%0),%%ymm9 \n"
+ "vmovdqu 0x40(%0),%%ymm10 \n"
+ "vmovdqu 0x60(%0),%%ymm11 \n"
+ "vpmaddubsw %%ymm3,%%ymm8,%%ymm0 \n"
+ "vpmaddubsw %%ymm3,%%ymm9,%%ymm1 \n"
+ "vpmaddubsw %%ymm3,%%ymm10,%%ymm2 \n"
+ "vpmaddubsw %%ymm3,%%ymm11,%%ymm6 \n"
+#else
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm6 \n"
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm3,%%ymm6,%%ymm6 \n"
+#endif
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vphaddw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpsubw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpsubw %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // unmutate.
+ "vmovdqu %%ymm0,(%1) \n"
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kAddUVJ128), // %5
- "m"(kARGBToVJ), // %6
- "m"(kARGBToUJ), // %7
- "m"(kShufARGBToUV_AVX) // %8
+#if defined(__x86_64__)
+ "vpmaddubsw %%ymm4,%%ymm8,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm9,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm10,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm11,%%ymm6 \n"
+#else
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm6 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm6,%%ymm6 \n"
+#endif
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vphaddw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpsubw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpsubw %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // unmutate.
+ "vmovdqu %%ymm0,(%1,%2,1) \n"
+ "lea 0x80(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "subl $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"(c), // %4
+ "m"(kPermdARGBToY_AVX) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10", "xmm11"
+#endif
+ );
}
-#endif // HAS_ARGBTOUVJROW_AVX2
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX2
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU
+ "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1
+ "vbroadcasti32x4 0x80(%[c]),%%zmm5 \n" // kAddUV
+ "vmovdqu32 %[perm],%%zmm17 \n"
+ "sub %[dst_u],%[dst_v] \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
-#ifdef HAS_ARGBTOUVJROW_SSSE3
-void ARGBToUVJRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_argb]),%%zmm18 \n"
+ "vmovdqu8 0x40(%[src_argb]),%%zmm19 \n"
+ "vmovdqu8 0x80(%[src_argb]),%%zmm20 \n"
+ "vmovdqu8 0xc0(%[src_argb]),%%zmm21 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vpmaddubsw %%zmm3,%%zmm18,%%zmm0 \n"
+ "vpmaddubsw %%zmm3,%%zmm19,%%zmm1 \n"
+ "vpmaddubsw %%zmm3,%%zmm20,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm21,%%zmm6 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
+ "vmovdqu8 %%zmm0,(%[dst_u]) \n"
+
+ "vpmaddubsw %%zmm4,%%zmm18,%%zmm0 \n"
+ "vpmaddubsw %%zmm4,%%zmm19,%%zmm1 \n"
+ "vpmaddubsw %%zmm4,%%zmm20,%%zmm2 \n"
+ "vpmaddubsw %%zmm4,%%zmm21,%%zmm6 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
+ "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1) \n"
+ "lea 0x40(%[dst_u]),%[dst_u] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm18%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm19%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm20%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm21%{%%k4%}%{z%} \n"
+ "vpmaddubsw %%zmm3,%%zmm18,%%zmm0 \n"
+ "vpmaddubsw %%zmm3,%%zmm19,%%zmm1 \n"
+ "vpmaddubsw %%zmm3,%%zmm20,%%zmm2 \n"
+ "vpmaddubsw %%zmm3,%%zmm21,%%zmm6 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n"
+
+ "vpmaddubsw %%zmm4,%%zmm18,%%zmm0 \n"
+ "vpmaddubsw %%zmm4,%%zmm19,%%zmm1 \n"
+ "vpmaddubsw %%zmm4,%%zmm20,%%zmm2 \n"
+ "vpmaddubsw %%zmm4,%%zmm21,%%zmm6 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3",
+ "k4");
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
+ "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU (signed)
+ "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV (signed)
+ "vbroadcasti32x4 0x80(%[c]),%%zmm5 \n" // kAddUV
+ "vmovdqu32 %[perm],%%zmm17 \n"
+ "sub %[dst_u],%[dst_v] \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "paddw %%xmm5,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kARGBToVJ), // %5
- "m"(kARGBToUJ), // %6
- "m"(kAddUVJ128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-#endif // HAS_ARGBTOUVJROW_SSSE3
-
-#ifdef HAS_ARGBTOUV444ROW_SSSE3
-void ARGBToUV444Row_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+ "1: \n"
+ "vmovdqu8 (%[src_argb]),%%zmm18 \n"
+ "vmovdqu8 0x40(%[src_argb]),%%zmm19 \n"
+ "vmovdqu8 0x80(%[src_argb]),%%zmm20 \n"
+ "vmovdqu8 0xc0(%[src_argb]),%%zmm21 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm16,%%zmm16,%%zmm16 \n"
+ "vpdpbusd %%zmm3,%%zmm18,%%zmm0 \n"
+ "vpdpbusd %%zmm3,%%zmm19,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm20,%%zmm2 \n"
+ "vpdpbusd %%zmm3,%%zmm21,%%zmm16 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpsubw %%zmm2,%%zmm5,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
+ "vmovdqu8 %%zmm0,(%[dst_u]) \n"
+
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm16,%%zmm16,%%zmm16 \n"
+ "vpdpbusd %%zmm4,%%zmm18,%%zmm0 \n"
+ "vpdpbusd %%zmm4,%%zmm19,%%zmm1 \n"
+ "vpdpbusd %%zmm4,%%zmm20,%%zmm2 \n"
+ "vpdpbusd %%zmm4,%%zmm21,%%zmm16 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpsubw %%zmm2,%%zmm5,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
+ "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1) \n"
+ "lea 0x40(%[dst_u]),%[dst_u] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm18%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm19%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm20%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm21%{%%k4%}%{z%} \n"
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm16,%%zmm16,%%zmm16 \n"
+ "vpdpbusd %%zmm3,%%zmm18,%%zmm0 \n"
+ "vpdpbusd %%zmm3,%%zmm19,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm20,%%zmm2 \n"
+ "vpdpbusd %%zmm3,%%zmm21,%%zmm16 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpsubw %%zmm2,%%zmm5,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n"
+
+ "vpxorq %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpxorq %%zmm16,%%zmm16,%%zmm16 \n"
+ "vpdpbusd %%zmm4,%%zmm18,%%zmm0 \n"
+ "vpdpbusd %%zmm4,%%zmm19,%%zmm1 \n"
+ "vpdpbusd %%zmm4,%%zmm20,%%zmm2 \n"
+ "vpdpbusd %%zmm4,%%zmm21,%%zmm16 \n"
+ "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpsubw %%zmm2,%%zmm5,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm2,%%zmm2 \n"
+ "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+// Requires AVX512VNNI and AVX512VBMI.
+// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at
+// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each
+// int32. The inverse is folded into the vpternlogd that merges the halves.
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %4,%%xmm3 \n"
- "movdqa %5,%%xmm4 \n"
- "movdqa %6,%%xmm5 \n"
- "sub %1,%2 \n"
+ "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU (signed)
+ "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV (signed)
+ "vpmovzxwd 0x80(%[c]),%%zmm5 \n" // kAddUV (32-bit)
+ "vpternlogd $0x55,%%zmm5,%%zmm5,%%zmm5 \n" // ~kAddUV
+ "vbroadcasti64x4 %[perm],%%zmm16 \n" // byte 1 of int32
+ "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n"
+ "vpxord %%zmm7,%%zmm7,%%zmm7 \n"
+ "vinserti64x4 $1,%%ymm6,%%zmm7,%%zmm6 \n" // high 256 bits select
+ "sub %[dst_u],%[dst_v] \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm6 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm2 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm2 \n"
- "packsswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "pmaddubsw %%xmm3,%%xmm0 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm2 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm2 \n"
- "packsswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "lea 0x40(%0),%0 \n"
- "movdqu %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu8 (%[src_argb]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n"
+ "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n"
+ "vmovdqu8 0xc0(%[src_argb]),%%zmm7 \n"
+ "lea 0x100(%[src_argb]),%[src_argb]\n"
+ "vmovdqa64 %%zmm5,%%zmm17 \n"
+ "vmovdqa64 %%zmm5,%%zmm18 \n"
+ "vmovdqa64 %%zmm5,%%zmm19 \n"
+ "vmovdqa64 %%zmm5,%%zmm20 \n"
+ "vmovdqa64 %%zmm5,%%zmm21 \n"
+ "vmovdqa64 %%zmm5,%%zmm22 \n"
+ "vmovdqa64 %%zmm5,%%zmm23 \n"
+ "vmovdqa64 %%zmm5,%%zmm24 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm17 \n" // U
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm21 \n" // V
+ "vpdpbusd %%zmm3,%%zmm1,%%zmm18 \n"
+ "vpdpbusd %%zmm4,%%zmm1,%%zmm22 \n"
+ "vpdpbusd %%zmm3,%%zmm2,%%zmm19 \n"
+ "vpdpbusd %%zmm4,%%zmm2,%%zmm23 \n"
+ "vpdpbusd %%zmm3,%%zmm7,%%zmm20 \n"
+ "vpdpbusd %%zmm4,%%zmm7,%%zmm24 \n"
+ "vpermt2b %%zmm18,%%zmm16,%%zmm17 \n" // U 0..31
+ "vpermt2b %%zmm20,%%zmm16,%%zmm19 \n" // U 32..63
+ "vpermt2b %%zmm22,%%zmm16,%%zmm21 \n" // V 0..31
+ "vpermt2b %%zmm24,%%zmm16,%%zmm23 \n" // V 32..63
+ "vpternlogd $0x27,%%zmm6,%%zmm19,%%zmm17 \n" // ~(low 17, high 19)
+ "vpternlogd $0x27,%%zmm6,%%zmm23,%%zmm21 \n" // ~(low 21, high 23)
+ "vmovdqu8 %%zmm17,(%[dst_u]) \n"
+ "vmovdqu8 %%zmm21,(%[dst_u],%[dst_v],1) \n"
+ "lea 0x40(%[dst_u]),%[dst_u] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $16,%%k1,%%k2 \n"
+ "kshiftrq $32,%%k1,%%k3 \n"
+ "kshiftrq $48,%%k1,%%k4 \n"
+ "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqu32 0xc0(%[src_argb]),%%zmm7%{%%k4%}%{z%} \n"
+ "vmovdqa64 %%zmm5,%%zmm17 \n"
+ "vmovdqa64 %%zmm5,%%zmm18 \n"
+ "vmovdqa64 %%zmm5,%%zmm19 \n"
+ "vmovdqa64 %%zmm5,%%zmm20 \n"
+ "vmovdqa64 %%zmm5,%%zmm21 \n"
+ "vmovdqa64 %%zmm5,%%zmm22 \n"
+ "vmovdqa64 %%zmm5,%%zmm23 \n"
+ "vmovdqa64 %%zmm5,%%zmm24 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm17 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm21 \n"
+ "vpdpbusd %%zmm3,%%zmm1,%%zmm18 \n"
+ "vpdpbusd %%zmm4,%%zmm1,%%zmm22 \n"
+ "vpdpbusd %%zmm3,%%zmm2,%%zmm19 \n"
+ "vpdpbusd %%zmm4,%%zmm2,%%zmm23 \n"
+ "vpdpbusd %%zmm3,%%zmm7,%%zmm20 \n"
+ "vpdpbusd %%zmm4,%%zmm7,%%zmm24 \n"
+ "vpermt2b %%zmm18,%%zmm16,%%zmm17 \n"
+ "vpermt2b %%zmm20,%%zmm16,%%zmm19 \n"
+ "vpermt2b %%zmm22,%%zmm16,%%zmm21 \n"
+ "vpermt2b %%zmm24,%%zmm16,%%zmm23 \n"
+ "vpternlogd $0x27,%%zmm6,%%zmm19,%%zmm17 \n"
+ "vpternlogd $0x27,%%zmm6,%%zmm23,%%zmm21 \n"
+ "vmovdqu8 %%zmm17,(%[dst_u])%{%%k1%} \n"
+ "vmovdqu8 %%zmm21,(%[dst_u],%[dst_v],1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [c] "r"(c), [perm] "m"(kPermbDWordByte1_AVX512VNNIVBMI)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22",
+ "xmm23", "xmm24", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+
+#ifdef HAS_ARGBTOUVMATRIXROW_SSSE3
+
+// ARGBARGB to AARRGGBB shuffle
+static const lvec8 kShuffleAARRGGBB = {
+ 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15,
+ 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15,
+};
+
+// 8x2 -> 4x1 ARGB pixels converted to 4 U and 4 V
+// ARGBToUV does rounding average of 4 ARGB pixels
+void OMITFP ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+#if defined(__i386__)
+ uvec8 rgb_to_u;
+ uvec8 rgb_to_v;
+ uvec16 add_uv;
+ memcpy(&rgb_to_u, c->kRGBToU, sizeof(rgb_to_u));
+ memcpy(&rgb_to_v, c->kRGBToV, sizeof(rgb_to_v));
+ memcpy(&add_uv, c->kAddUV, sizeof(add_uv));
+#endif
+ asm volatile(
+#if defined(__i386__)
+ "movdqa %5,%%xmm4 \n" // RGBToU
+ "movdqa %6,%%xmm5 \n" // RGBToV
+ "pcmpeqb %%xmm6,%%xmm6 \n" // 0x0101
+ "pabsb %%xmm6,%%xmm6 \n"
+ "movdqa %8,%%xmm7 \n" // kShuffleAARRGGBB
+#else
+ "movdqu 0x20(%5),%%xmm4 \n" // RGBToU
+ "movdqu 0x40(%5),%%xmm5 \n" // RGBToV
+ "pcmpeqb %%xmm6,%%xmm6 \n" // 0x0101
+ "pabsb %%xmm6,%%xmm6 \n"
+ "movdqa %6,%%xmm7 \n" // kShuffleAARRGGBB
+#endif
+ "sub %1,%2 \n"
+
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // Read 8x2 ARGB Pixels
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%4,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%4,1),%%xmm3 \n"
+ "pshufb %%xmm7,%%xmm0 \n" // aarrggbb
+ "pshufb %%xmm7,%%xmm1 \n"
+ "pshufb %%xmm7,%%xmm2 \n"
+ "pshufb %%xmm7,%%xmm3 \n"
+ "pmaddubsw %%xmm6,%%xmm0 \n" // 8x2 -> 4x2
+ "pmaddubsw %%xmm6,%%xmm1 \n"
+ "pmaddubsw %%xmm6,%%xmm2 \n"
+ "pmaddubsw %%xmm6,%%xmm3 \n"
+ "paddw %%xmm2,%%xmm0 \n" // 4x2 -> 4x1
+ "paddw %%xmm3,%%xmm1 \n"
+ "pxor %%xmm2,%%xmm2 \n" // 0 for vpavgw
+ "psrlw $1,%%xmm0 \n"
+ "psrlw $1,%%xmm1 \n"
+ "pavgw %%xmm2,%%xmm0 \n"
+ "pavgw %%xmm2,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n" // mutates
+
+#if defined(__i386__)
+ "movdqa %7,%%xmm2 \n" // kAddUV
+#else
+ "movdqu 0x80(%5),%%xmm2 \n" // kAddUV
+#endif
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pmaddubsw %%xmm5,%%xmm1 \n" // 4 V
+ "pmaddubsw %%xmm4,%%xmm0 \n" // 4 U
+ "phaddw %%xmm1,%%xmm0 \n" // uuuuvvvv
+ "psubw %%xmm0,%%xmm2 \n"
+ "psrlw $0x8,%%xmm2 \n"
+ "packuswb %%xmm2,%%xmm2 \n"
+ "movd %%xmm2,(%1) \n" // Write 4 U's
+ "pshufd $0x55,%%xmm2,%%xmm2 \n" // Copy V to low 4 bytes
+ "movd %%xmm2,0x00(%1,%2,1) \n" // Write 4 V's
+
+ "lea 0x20(%0),%0 \n"
+ "lea 0x4(%1),%1 \n"
+ "subl $0x8,%3 \n"
+ "jg 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
- "+rm"(width) // %3
- : "m"(kARGBToV), // %4
- "m"(kARGBToU), // %5
- "m"(kAddUV128) // %6
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6");
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"((ptrdiff_t)(src_stride_argb)), // %4
+#if defined(__i386__)
+ "m"(rgb_to_u), // %5
+ "m"(rgb_to_v), // %6
+ "m"(add_uv), // %7
+ "m"(kShuffleAARRGGBB) // %8
+#else
+ "r"(c), // %5
+ "m"(kShuffleAARRGGBB) // %6
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-#endif // HAS_ARGBTOUV444ROW_SSSE3
-void BGRAToYRow_SSSE3(const uint8_t* src_bgra, uint8_t* dst_y, int width) {
- asm volatile(
- "movdqa %4,%%xmm5 \n"
- "movdqa %3,%%xmm4 \n"
+#endif // HAS_ARGBTOUVMATRIXROW_SSSE3
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm3,%%xmm2 \n"
- "psrlw $0x7,%%xmm0 \n"
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kBGRAToY), // %3
- "m"(kAddY16) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
-}
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
-void BGRAToUVRow_SSSE3(const uint8_t* src_bgra0,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+// 16x2 -> 8x1 ARGB pixels converted to 8 U and 8 V
+// ARGBToUV does rounding average of 4 ARGB pixels
+void OMITFP ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+#if defined(__i386__)
+ uvec8 rgb_to_u;
+ uvec8 rgb_to_v;
+ uvec16 add_uv;
+ memcpy(&rgb_to_u, c->kRGBToU, sizeof(rgb_to_u));
+ memcpy(&rgb_to_v, c->kRGBToV, sizeof(rgb_to_v));
+ memcpy(&add_uv, c->kAddUV, sizeof(add_uv));
+#endif
asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
+#if defined(__i386__)
+ "vbroadcasti128 %5,%%ymm4 \n" // RGBToU
+ "vbroadcasti128 %6,%%ymm5 \n" // RGBToV
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0x0101
+ "vpabsb %%ymm6,%%ymm6 \n"
+ "vmovdqa %8,%%ymm7 \n" // kShuffleAARRGGBB
+#else
+ "vbroadcasti128 0x20(%5),%%ymm4 \n" // RGBToU
+ "vbroadcasti128 0x40(%5),%%ymm5 \n" // RGBToV
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0x0101
+ "vpabsb %%ymm6,%%ymm6 \n"
+ "vmovdqa %6,%%ymm7 \n" // kShuffleAARRGGBB
+#endif
+ "sub %1,%2 \n"
+
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // Read 16x2 ARGB Pixels
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x00(%0,%4,1),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%4,1),%%ymm3 \n"
+ "vpshufb %%ymm7,%%ymm0,%%ymm0 \n" // aarrggbb
+ "vpshufb %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n" // 16x2 -> 8x2
+ "vpmaddubsw %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm0,%%ymm2,%%ymm0 \n" // 8x2 -> 8x1
+ "vpaddw %%ymm1,%%ymm3,%%ymm1 \n"
+ "vpxor %%ymm2,%%ymm2,%%ymm2 \n" // 0 for vpavgw
+ "vpsrlw $1,%%ymm0,%%ymm0 \n"
+ "vpsrlw $1,%%ymm1,%%ymm1 \n"
+ "vpavgw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpavgw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n" // 8 ARGB Pixels
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_bgra0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_bgra)), // %4
- "m"(kBGRAToV), // %5
- "m"(kBGRAToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
+#if defined(__i386__)
+ "vbroadcasti128 %7,%%ymm2 \n" // kAddUV
+#else
+ "vmovdqu 0x80(%5),%%ymm2 \n" // kAddUV
+#endif
+ "vpmaddubsw %%ymm5,%%ymm0,%%ymm1 \n" // 8 V
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" // 8 U
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // uuuuvvvv uuuuvvvv
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n" // uuuuuuuu vvvvvvvv
+ "vpsubw %%ymm0,%%ymm2,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" // mutates 8U8u- 8V8v
+ "vmovq %%xmm0,(%1) \n" // Write 8 U's
+ "vextractf128 $0x1,%%ymm0,%%xmm0 \n" // Copy V to low 8 bytes
+ "vmovq %%xmm0,0x00(%1,%2,1) \n" // Write 8 V's
+
+ "lea 0x40(%0),%0 \n"
+ "lea 0x8(%1),%1 \n"
+ "subl $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"((ptrdiff_t)(src_stride_argb)), // %4
+#if defined(__i386__)
+ "m"(rgb_to_u), // %5
+ "m"(rgb_to_v), // %6
+ "m"(add_uv), // %7
+ "m"(kShuffleAARRGGBB) // %8
+#else
+ "r"(c), // %5
+ "m"(kShuffleAARRGGBB) // %6
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-
-void ABGRToYRow_SSSE3(const uint8_t* src_abgr, uint8_t* dst_y, int width) {
+#endif // HAS_ARGBTOUVMATRIXROW_AVX2
+
+// RGB to BT601 coefficients
+// UB 0.875 coefficient = 112
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW
+
+// 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V
+// ARGBToUV does rounding average of 4 ARGB pixels
+
+LIBYUV_TARGET_AVX512BW
+void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %4,%%xmm5 \n"
- "movdqa %3,%%xmm4 \n"
+ "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU
+ "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpabsb %%zmm16,%%zmm20 \n" // 0x0101
+ "vbroadcasti32x4 0x80(%[c]),%%zmm17 \n" // kAddUV
+ "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB
+ "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW
+ "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW
+ "sub %1,%2 \n"
+ "subl $0x20,%3 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm3,%%xmm2 \n"
- "psrlw $0x7,%%xmm0 \n"
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kABGRToY), // %3
- "m"(kAddY16) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "vmovups 0x00(%0,%[stride],1),%%zmm2 \n"
+ "vmovups 0x40(%0,%[stride],1),%%zmm3 \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n" // unscramble pixels
+
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm1 \n" // 16 U
+ "vpmaddubsw %%zmm5,%%zmm0,%%zmm0 \n" // 16 V
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm0,%%zmm1,%%zmm0 \n" // mutates (U in lower, V
+ // in upper)
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n" // unmutate
+
+ "vmovdqu %%xmm0,(%1) \n" // Write 16 U's
+ "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,0x00(%1,%2,1) \n" // Write 16 V's
+
+ "lea 0x80(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "subl $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "addl $0x20,%3 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "knotw %%k1,%%k3 \n"
+ "knotw %%k2,%%k4 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n"
+ "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n"
+ "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n"
+
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm1 \n"
+ "vpmaddubsw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpackssdw %%zmm0,%%zmm1,%%zmm0 \n"
+ "vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n"
+
+ "addl $1,%3 \n"
+ "shrl $1,%3 \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu8 %%xmm0,(%1)%{%%k1%} \n"
+ "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n"
+ "vmovdqu8 %%xmm0,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride]
+ [c] "r"(c), // %[c]
+ [shuf] "m"(kShuffleAARRGGBB), // %[shuf]
+ [permy] "m"(kPermdARGBToY_AVX512BW), // %[permy]
+ [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW
-void RGBAToYRow_SSSE3(const uint8_t* src_rgba, uint8_t* dst_y, int width) {
- asm volatile(
- "movdqa %4,%%xmm5 \n"
- "movdqa %3,%%xmm4 \n"
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNI
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm3,%%xmm2 \n"
- "psrlw $0x7,%%xmm0 \n"
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_rgba), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kRGBAToY), // %3
- "m"(kAddY16) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
-}
+// 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V
+// ARGBToUV does rounding average of 4 ARGB pixels
-void ABGRToUVRow_SSSE3(const uint8_t* src_abgr0,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
+ "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU (signed)
+ "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV (signed)
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpabsb %%zmm16,%%zmm20 \n" // 0x0101
+ "vbroadcasti32x4 0x80(%[c]),%%zmm17 \n" // kAddUV
+ "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB
+ "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW
+ "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW
+ "sub %1,%2 \n"
+ "subl $0x20,%3 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_abgr0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kABGRToV), // %5
- "m"(kABGRToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-
-void RGBAToUVRow_SSSE3(const uint8_t* src_rgba0,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "vmovups 0x00(%0,%[stride],1),%%zmm2 \n"
+ "vmovups 0x40(%0,%[stride],1),%%zmm3 \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw and
+ // vpdpbusd
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n" // unscramble pixels
+
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" // 16 U (negated)
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm2 \n" // 16 V (negated)
+ "vpackssdw %%zmm2,%%zmm1,%%zmm0 \n" // mutates (U in lower, V
+ // in upper)
+ "vpsubw %%zmm0,%%zmm17,%%zmm0 \n" // 0x8000 - dot16
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n" // unmutate
+
+ "vmovdqu %%xmm0,(%1) \n" // Write 16 U's
+ "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,0x00(%1,%2,1) \n" // Write 16 V's
+
+ "lea 0x80(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "subl $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "addl $0x20,%3 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "knotw %%k1,%%k3 \n"
+ "knotw %%k2,%%k4 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n"
+ "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n"
+ "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n"
+
+ "vpxorq %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm2 \n"
+ "vpackssdw %%zmm2,%%zmm1,%%zmm0 \n"
+ "vpsubw %%zmm0,%%zmm17,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm18,%%zmm0 \n"
+
+ "addl $1,%3 \n"
+ "shrl $1,%3 \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu8 %%xmm0,(%1)%{%%k1%} \n"
+ "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n"
+ "vmovdqu8 %%xmm0,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride]
+ [c] "r"(c), // %[c]
+ [shuf] "m"(kShuffleAARRGGBB), // %[shuf]
+ [permy] "m"(kPermdARGBToY_AVX512BW), // %[permy]
+ [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNI
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
+
+// Requires AVX512VNNI and AVX512VBMI.
+// 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V
+// ARGBToUV does rounding average of 4 ARGB pixels
+
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ uintptr_t temp;
asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
+ "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU (signed)
+ "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV (signed)
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpabsb %%zmm16,%%zmm20 \n" // 0x0101
+ "vpmovzxwd 0x80(%[c]),%%zmm17 \n" // kAddUV (32-bit)
+ "vpternlogd $0x55,%%zmm17,%%zmm17,%%zmm17 \n" // ~kAddUV
+ "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB
+ "vbroadcasti64x4 %[permb],%%zmm18 \n" // byte 1 of int32
+ "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW
+ "sub %1,%2 \n"
+ "subl $0x20,%3 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_rgba0), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_rgba)), // %4
- "m"(kRGBAToV), // %5
- "m"(kRGBAToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "vmovups 0x00(%0,%[stride],1),%%zmm2 \n"
+ "vmovups 0x40(%0,%[stride],1),%%zmm3 \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw and
+ // vpdpbusd
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" // mutates
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n" // unscramble pixels
+
+ "vmovdqa64 %%zmm17,%%zmm1 \n" // ~kAddUV
+ "vmovdqa64 %%zmm17,%%zmm3 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" // ~(16 U)
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm3 \n" // ~(16 V)
+ "vpermt2b %%zmm3,%%zmm18,%%zmm1 \n" // byte 1 of U and V
+ "vpternlogd $0x55,%%ymm1,%%ymm1,%%ymm1 \n" // invert
+
+ "vmovdqu %%xmm1,(%1) \n" // Write 16 U's
+ "vextracti32x4 $0x1,%%ymm1,0x00(%1,%2,1) \n" // Write 16 V's
+
+ "lea 0x80(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "subl $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "addl $0x20,%3 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "knotw %%k1,%%k3 \n"
+ "knotw %%k2,%%k4 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n"
+ "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n"
+ "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n"
+ "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n"
+ "vpshufb %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
+ "vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpermd %%zmm0,%%zmm19,%%zmm0 \n"
+
+ "vmovdqa64 %%zmm17,%%zmm1 \n"
+ "vmovdqa64 %%zmm17,%%zmm3 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm3 \n"
+ "vpermt2b %%zmm3,%%zmm18,%%zmm1 \n"
+ "vpternlogd $0x55,%%ymm1,%%ymm1,%%ymm1 \n"
+
+ "addl $1,%3 \n"
+ "shrl $1,%3 \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu8 %%xmm1,(%1)%{%%k1%} \n"
+ "vextracti32x4 $0x1,%%ymm1,%%xmm1 \n"
+ "vmovdqu8 %%xmm1,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride]
+ [c] "r"(c), // %[c]
+ [shuf] "m"(kShuffleAARRGGBB), // %[shuf]
+ [permb] "m"(kPermbDWordByte1_AVX512VNNIVBMI), // %[permb]
+ [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
#if defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2)
// Read 8 UV from 444
#define READYUV444 \
- "movq (%[u_buf]),%%xmm0 \n" \
+ "movq (%[u_buf]),%%xmm3 \n" \
"movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x8(%[u_buf]),%[u_buf] \n" \
- "punpcklbw %%xmm1,%%xmm0 \n" \
+ "punpcklbw %%xmm1,%%xmm3 \n" \
"movq (%[y_buf]),%%xmm4 \n" \
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n"
// Read 4 UV from 422, upsample to 8 UV
#define READYUV422 \
- "movd (%[u_buf]),%%xmm0 \n" \
+ "movd (%[u_buf]),%%xmm3 \n" \
"movd 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x4(%[u_buf]),%[u_buf] \n" \
- "punpcklbw %%xmm1,%%xmm0 \n" \
- "punpcklwd %%xmm0,%%xmm0 \n" \
+ "punpcklbw %%xmm1,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
"movq (%[y_buf]),%%xmm4 \n" \
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n"
// Read 4 UV from 422 10 bit, upsample to 8 UV
-// TODO(fbarchard): Consider shufb to replace pack/unpack
-// TODO(fbarchard): Consider pmulhuw to replace psraw
-// TODO(fbarchard): Consider pmullw to replace psllw and allow different bits.
#define READYUV210 \
- "movq (%[u_buf]),%%xmm0 \n" \
+ "movq (%[u_buf]),%%xmm3 \n" \
"movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x8(%[u_buf]),%[u_buf] \n" \
- "punpcklwd %%xmm1,%%xmm0 \n" \
- "psraw $0x2,%%xmm0 \n" \
- "packuswb %%xmm0,%%xmm0 \n" \
- "punpcklwd %%xmm0,%%xmm0 \n" \
+ "punpcklwd %%xmm1,%%xmm3 \n" \
+ "psrlw $2,%%xmm3 \n" \
+ "packuswb %%xmm3,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm2 \n" \
+ "psllw $6,%%xmm4 \n" \
+ "psrlw $4,%%xmm2 \n" \
+ "por %%xmm2,%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+
+#define READYUVA210 \
+ "movq (%[u_buf]),%%xmm3 \n" \
+ "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x8(%[u_buf]),%[u_buf] \n" \
+ "punpcklwd %%xmm1,%%xmm3 \n" \
+ "psrlw $2,%%xmm3 \n" \
+ "packuswb %%xmm3,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm2 \n" \
+ "psllw $6,%%xmm4 \n" \
+ "psrlw $4,%%xmm2 \n" \
+ "por %%xmm2,%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n" \
+ "movdqu (%[a_buf]),%%xmm5 \n" \
+ "psrlw $2,%%xmm5 \n" \
+ "packuswb %%xmm5,%%xmm5 \n" \
+ "lea 0x10(%[a_buf]),%[a_buf] \n"
+
+// Read 8 UV from 444 10 bit
+#define READYUV410 \
+ "movdqu (%[u_buf]),%%xmm3 \n" \
+ "movdqu 0x00(%[u_buf],%[v_buf],1),%%xmm2 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "psrlw $2,%%xmm3 \n" \
+ "psrlw $2,%%xmm2 \n" \
+ "movdqa %%xmm3,%%xmm1 \n" \
+ "punpcklwd %%xmm2,%%xmm3 \n" \
+ "punpckhwd %%xmm2,%%xmm1 \n" \
+ "packuswb %%xmm1,%%xmm3 \n" \
"movdqu (%[y_buf]),%%xmm4 \n" \
- "psllw $0x6,%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm2 \n" \
+ "psllw $6,%%xmm4 \n" \
+ "psrlw $4,%%xmm2 \n" \
+ "por %%xmm2,%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+
+// Read 8 UV from 444 10 bit. With 8 Alpha.
+#define READYUVA410 \
+ "movdqu (%[u_buf]),%%xmm3 \n" \
+ "movdqu 0x00(%[u_buf],%[v_buf],1),%%xmm2 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "psrlw $2,%%xmm3 \n" \
+ "psrlw $2,%%xmm2 \n" \
+ "movdqa %%xmm3,%%xmm1 \n" \
+ "punpcklwd %%xmm2,%%xmm3 \n" \
+ "punpckhwd %%xmm2,%%xmm1 \n" \
+ "packuswb %%xmm1,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm2 \n" \
+ "psllw $6,%%xmm4 \n" \
+ "psrlw $4,%%xmm2 \n" \
+ "por %%xmm2,%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n" \
+ "movdqu (%[a_buf]),%%xmm5 \n" \
+ "psrlw $2,%%xmm5 \n" \
+ "packuswb %%xmm5,%%xmm5 \n" \
+ "lea 0x10(%[a_buf]),%[a_buf] \n"
+
+// Read 4 UV from 422 12 bit, upsample to 8 UV
+#define READYUV212 \
+ "movq (%[u_buf]),%%xmm3 \n" \
+ "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x8(%[u_buf]),%[u_buf] \n" \
+ "punpcklwd %%xmm1,%%xmm3 \n" \
+ "psrlw $0x4,%%xmm3 \n" \
+ "packuswb %%xmm3,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm2 \n" \
+ "psllw $4,%%xmm4 \n" \
+ "psrlw $8,%%xmm2 \n" \
+ "por %%xmm2,%%xmm4 \n" \
"lea 0x10(%[y_buf]),%[y_buf] \n"
// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
#define READYUVA422 \
- "movd (%[u_buf]),%%xmm0 \n" \
+ "movd (%[u_buf]),%%xmm3 \n" \
"movd 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x4(%[u_buf]),%[u_buf] \n" \
- "punpcklbw %%xmm1,%%xmm0 \n" \
- "punpcklwd %%xmm0,%%xmm0 \n" \
+ "punpcklbw %%xmm1,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
+ "movq (%[y_buf]),%%xmm4 \n" \
+ "punpcklbw %%xmm4,%%xmm4 \n" \
+ "lea 0x8(%[y_buf]),%[y_buf] \n" \
+ "movq (%[a_buf]),%%xmm5 \n" \
+ "lea 0x8(%[a_buf]),%[a_buf] \n"
+
+// Read 8 UV from 444. With 8 Alpha.
+#define READYUVA444 \
+ "movq (%[u_buf]),%%xmm3 \n" \
+ "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x8(%[u_buf]),%[u_buf] \n" \
+ "punpcklbw %%xmm1,%%xmm3 \n" \
"movq (%[y_buf]),%%xmm4 \n" \
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n" \
@@ -1966,90 +3995,152 @@ void RGBAToUVRow_SSSE3(const uint8_t* src_rgba0,
// Read 4 UV from NV12, upsample to 8 UV
#define READNV12 \
- "movq (%[uv_buf]),%%xmm0 \n" \
+ "movq (%[uv_buf]),%%xmm3 \n" \
"lea 0x8(%[uv_buf]),%[uv_buf] \n" \
- "punpcklwd %%xmm0,%%xmm0 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
"movq (%[y_buf]),%%xmm4 \n" \
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n"
// Read 4 VU from NV21, upsample to 8 UV
#define READNV21 \
- "movq (%[vu_buf]),%%xmm0 \n" \
+ "movq (%[vu_buf]),%%xmm3 \n" \
"lea 0x8(%[vu_buf]),%[vu_buf] \n" \
- "pshufb %[kShuffleNV21], %%xmm0 \n" \
+ "pshufb %[kShuffleNV21], %%xmm3 \n" \
"movq (%[y_buf]),%%xmm4 \n" \
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n"
-// Read 4 YUY2 with 8 Y and update 4 UV to 8 UV.
+// Read 4 YUY2 with 8 Y and upsample 4 UV to 8 UV.
+// xmm6 kShuffleYUY2Y,
+// xmm7 kShuffleYUY2UV
#define READYUY2 \
"movdqu (%[yuy2_buf]),%%xmm4 \n" \
- "pshufb %[kShuffleYUY2Y], %%xmm4 \n" \
- "movdqu (%[yuy2_buf]),%%xmm0 \n" \
- "pshufb %[kShuffleYUY2UV], %%xmm0 \n" \
- "lea 0x10(%[yuy2_buf]),%[yuy2_buf] \n"
-
-// Read 4 UYVY with 8 Y and update 4 UV to 8 UV.
+ "lea 0x10(%[yuy2_buf]),%[yuy2_buf] \n" \
+ "movdqa %%xmm4,%%xmm3 \n" \
+ "pshufb %%xmm6,%%xmm4 \n" \
+ "pshufb %%xmm7,%%xmm3 \n"
+
+// Read 4 UYVY with 8 Y and upsample 4 UV to 8 UV.
+// xmm6 kShuffleUYVYY,
+// xmm7 kShuffleUYVYUV
#define READUYVY \
"movdqu (%[uyvy_buf]),%%xmm4 \n" \
- "pshufb %[kShuffleUYVYY], %%xmm4 \n" \
- "movdqu (%[uyvy_buf]),%%xmm0 \n" \
- "pshufb %[kShuffleUYVYUV], %%xmm0 \n" \
- "lea 0x10(%[uyvy_buf]),%[uyvy_buf] \n"
+ "lea 0x10(%[uyvy_buf]),%[uyvy_buf] \n" \
+ "movdqa %%xmm4,%%xmm3 \n" \
+ "pshufb %%xmm6,%%xmm4 \n" \
+ "pshufb %%xmm7,%%xmm3 \n"
+
+// Read 4 UV from P210, upsample to 8 UV
+#define READP210 \
+ "movdqu (%[uv_buf]),%%xmm3 \n" \
+ "lea 0x10(%[uv_buf]),%[uv_buf] \n" \
+ "psrlw $0x8,%%xmm3 \n" \
+ "packuswb %%xmm3,%%xmm3 \n" \
+ "punpcklwd %%xmm3,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+
+// Read 8 UV from P410
+#define READP410 \
+ "movdqu (%[uv_buf]),%%xmm3 \n" \
+ "movdqu 0x10(%[uv_buf]),%%xmm1 \n" \
+ "lea 0x20(%[uv_buf]),%[uv_buf] \n" \
+ "psrlw $0x8,%%xmm3 \n" \
+ "psrlw $0x8,%%xmm1 \n" \
+ "packuswb %%xmm1,%%xmm3 \n" \
+ "movdqu (%[y_buf]),%%xmm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
#if defined(__x86_64__)
#define YUVTORGB_SETUP(yuvconstants) \
- "movdqa (%[yuvconstants]),%%xmm8 \n" \
- "movdqa 32(%[yuvconstants]),%%xmm9 \n" \
- "movdqa 64(%[yuvconstants]),%%xmm10 \n" \
- "movdqa 96(%[yuvconstants]),%%xmm11 \n" \
- "movdqa 128(%[yuvconstants]),%%xmm12 \n" \
- "movdqa 160(%[yuvconstants]),%%xmm13 \n" \
- "movdqa 192(%[yuvconstants]),%%xmm14 \n"
+ "pcmpeqb %%xmm13,%%xmm13 \n" \
+ "movdqu (%[yuvconstants]),%%xmm8 \n" \
+ "pxor %%xmm12,%%xmm12 \n" \
+ "movdqu 32(%[yuvconstants]),%%xmm9 \n" \
+ "pavgb %%xmm12,%%xmm13 \n" \
+ "movdqu 64(%[yuvconstants]),%%xmm10 \n" \
+ "movdqu 96(%[yuvconstants]),%%xmm11 \n" \
+ "movdqu 128(%[yuvconstants]),%%xmm12 \n"
+
+#define YUVTORGB_SETUP_AR30(yuvconstants) \
+ YUVTORGB_SETUP(yuvconstants) \
+ "pcmpeqb %%xmm0,%%xmm0 \n" \
+ "psrlw $14,%%xmm0 \n" \
+ "psllw $3,%%xmm0 \n" \
+ "psubw %%xmm0,%%xmm12 \n"
+
// Convert 8 pixels: 8 UV and 8 Y
#define YUVTORGB16(yuvconstants) \
- "movdqa %%xmm0,%%xmm1 \n" \
- "movdqa %%xmm0,%%xmm2 \n" \
- "movdqa %%xmm0,%%xmm3 \n" \
- "movdqa %%xmm11,%%xmm0 \n" \
- "pmaddubsw %%xmm8,%%xmm1 \n" \
- "psubw %%xmm1,%%xmm0 \n" \
- "movdqa %%xmm12,%%xmm1 \n" \
- "pmaddubsw %%xmm9,%%xmm2 \n" \
- "psubw %%xmm2,%%xmm1 \n" \
- "movdqa %%xmm13,%%xmm2 \n" \
- "pmaddubsw %%xmm10,%%xmm3 \n" \
- "psubw %%xmm3,%%xmm2 \n" \
- "pmulhuw %%xmm14,%%xmm4 \n" \
+ "psubb %%xmm13,%%xmm3 \n" \
+ "pmulhuw %%xmm11,%%xmm4 \n" \
+ "movdqa %%xmm8,%%xmm0 \n" \
+ "movdqa %%xmm9,%%xmm1 \n" \
+ "movdqa %%xmm10,%%xmm2 \n" \
+ "paddw %%xmm12,%%xmm4 \n" \
+ "pmaddubsw %%xmm3,%%xmm0 \n" \
+ "pmaddubsw %%xmm3,%%xmm1 \n" \
+ "pmaddubsw %%xmm3,%%xmm2 \n" \
"paddsw %%xmm4,%%xmm0 \n" \
- "paddsw %%xmm4,%%xmm1 \n" \
- "paddsw %%xmm4,%%xmm2 \n"
-#define YUVTORGB_REGS \
- "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
+ "paddsw %%xmm4,%%xmm2 \n" \
+ "psubsw %%xmm1,%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm1 \n"
+
+#define YUVTORGB16_AR30(yuvconstants) YUVTORGB16(yuvconstants)
+
+#define YUVTORGB_REGS "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",
#else
#define YUVTORGB_SETUP(yuvconstants)
+#define YUVTORGB_SETUP_AR30(yuvconstants)
+
// Convert 8 pixels: 8 UV and 8 Y
#define YUVTORGB16(yuvconstants) \
- "movdqa %%xmm0,%%xmm1 \n" \
- "movdqa %%xmm0,%%xmm2 \n" \
- "movdqa %%xmm0,%%xmm3 \n" \
- "movdqa 96(%[yuvconstants]),%%xmm0 \n" \
- "pmaddubsw (%[yuvconstants]),%%xmm1 \n" \
- "psubw %%xmm1,%%xmm0 \n" \
- "movdqa 128(%[yuvconstants]),%%xmm1 \n" \
- "pmaddubsw 32(%[yuvconstants]),%%xmm2 \n" \
- "psubw %%xmm2,%%xmm1 \n" \
- "movdqa 160(%[yuvconstants]),%%xmm2 \n" \
- "pmaddubsw 64(%[yuvconstants]),%%xmm3 \n" \
- "psubw %%xmm3,%%xmm2 \n" \
- "pmulhuw 192(%[yuvconstants]),%%xmm4 \n" \
+ "movdqu 96(%[yuvconstants]),%%xmm0 \n" \
+ "pmulhuw %%xmm0,%%xmm4 \n" \
+ "pcmpeqb %%xmm0,%%xmm0 \n" \
+ "pabsb %%xmm0,%%xmm0 \n" \
+ "psllw $7,%%xmm0 \n" \
+ "psubb %%xmm0,%%xmm3 \n" \
+ "movdqu (%[yuvconstants]),%%xmm0 \n" \
+ "movdqu 32(%[yuvconstants]),%%xmm1 \n" \
+ "movdqu 64(%[yuvconstants]),%%xmm2 \n" \
+ "pmaddubsw %%xmm3,%%xmm0 \n" \
+ "pmaddubsw %%xmm3,%%xmm1 \n" \
+ "pmaddubsw %%xmm3,%%xmm2 \n" \
+ "movdqu 128(%[yuvconstants]),%%xmm3 \n" \
+ "paddw %%xmm3,%%xmm4 \n" \
"paddsw %%xmm4,%%xmm0 \n" \
- "paddsw %%xmm4,%%xmm1 \n" \
- "paddsw %%xmm4,%%xmm2 \n"
+ "paddsw %%xmm4,%%xmm2 \n" \
+ "psubsw %%xmm1,%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm1 \n"
+
+#define YUVTORGB16_AR30(yuvconstants) \
+ "movdqu 96(%[yuvconstants]),%%xmm0 \n" \
+ "pmulhuw %%xmm0,%%xmm4 \n" \
+ "pcmpeqb %%xmm0,%%xmm0 \n" \
+ "pabsb %%xmm0,%%xmm0 \n" \
+ "psllw $7,%%xmm0 \n" \
+ "psubb %%xmm0,%%xmm3 \n" \
+ "movdqu (%[yuvconstants]),%%xmm0 \n" \
+ "movdqu 32(%[yuvconstants]),%%xmm1 \n" \
+ "movdqu 64(%[yuvconstants]),%%xmm2 \n" \
+ "pmaddubsw %%xmm3,%%xmm0 \n" \
+ "pmaddubsw %%xmm3,%%xmm1 \n" \
+ "pmaddubsw %%xmm3,%%xmm2 \n" \
+ "movdqu 128(%[yuvconstants]),%%xmm3 \n" \
+ "paddw %%xmm3,%%xmm4 \n" \
+ "pcmpeqb %%xmm3,%%xmm3 \n" \
+ "psrlw $14,%%xmm3 \n" \
+ "psllw $3,%%xmm3 \n" \
+ "psubw %%xmm3,%%xmm4 \n" \
+ "paddsw %%xmm4,%%xmm0 \n" \
+ "paddsw %%xmm4,%%xmm2 \n" \
+ "psubsw %%xmm1,%%xmm4 \n" \
+ "movdqa %%xmm4,%%xmm1 \n"
+
#define YUVTORGB_REGS
-#endif
+#endif // defined(__x86_64__)
#define YUVTORGB(yuvconstants) \
YUVTORGB16(yuvconstants) \
@@ -2083,28 +4174,40 @@ void RGBAToUVRow_SSSE3(const uint8_t* src_rgba0,
"movdqu %%xmm0,0x10(%[dst_rgba]) \n" \
"lea 0x20(%[dst_rgba]),%[dst_rgba] \n"
+// Store 8 RGB24 values.
+#define STORERGB24 \
+ "punpcklbw %%xmm1,%%xmm0 \n" \
+ "punpcklbw %%xmm2,%%xmm2 \n" \
+ "movdqa %%xmm0,%%xmm1 \n" \
+ "punpcklwd %%xmm2,%%xmm0 \n" \
+ "punpckhwd %%xmm2,%%xmm1 \n" \
+ "pshufb %%xmm5,%%xmm0 \n" \
+ "pshufb %%xmm6,%%xmm1 \n" \
+ "palignr $0xc,%%xmm0,%%xmm1 \n" \
+ "movq %%xmm0,(%[dst_rgb24]) \n" \
+ "movdqu %%xmm1,0x8(%[dst_rgb24]) \n" \
+ "lea 0x18(%[dst_rgb24]),%[dst_rgb24] \n"
+
// Store 8 AR30 values.
#define STOREAR30 \
- "psraw $0x4,%%xmm0 \n" \
- "psraw $0x4,%%xmm1 \n" \
- "psraw $0x4,%%xmm2 \n" \
"pminsw %%xmm7,%%xmm0 \n" \
"pminsw %%xmm7,%%xmm1 \n" \
"pminsw %%xmm7,%%xmm2 \n" \
"pmaxsw %%xmm6,%%xmm0 \n" \
"pmaxsw %%xmm6,%%xmm1 \n" \
"pmaxsw %%xmm6,%%xmm2 \n" \
- "psllw $0x4,%%xmm2 \n" \
+ "movdqa %%xmm1,%%xmm3 \n" \
+ "pand %%xmm7,%%xmm3 \n" \
+ "pand %%xmm7,%%xmm2 \n" \
+ "psrlw $4,%%xmm0 \n" \
+ "psrlw $10,%%xmm1 \n" \
+ "psllw $6,%%xmm3 \n" \
+ "por %%xmm3,%%xmm0 \n" \
+ "por %%xmm5,%%xmm2 \n" \
+ "por %%xmm1,%%xmm2 \n" \
"movdqa %%xmm0,%%xmm3 \n" \
"punpcklwd %%xmm2,%%xmm0 \n" \
"punpckhwd %%xmm2,%%xmm3 \n" \
- "movdqa %%xmm1,%%xmm2 \n" \
- "punpcklwd %%xmm5,%%xmm1 \n" \
- "punpckhwd %%xmm5,%%xmm2 \n" \
- "pslld $0xa,%%xmm1 \n" \
- "pslld $0xa,%%xmm2 \n" \
- "por %%xmm1,%%xmm0 \n" \
- "por %%xmm2,%%xmm3 \n" \
"movdqu %%xmm0,(%[dst_ar30]) \n" \
"movdqu %%xmm3,0x10(%[dst_ar30]) \n" \
"lea 0x20(%[dst_ar30]), %[dst_ar30] \n"
@@ -2117,16 +4220,16 @@ void OMITFP I444ToARGBRow_SSSE3(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV444
YUVTORGB(yuvconstants)
STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2138,35 +4241,146 @@ void OMITFP I444ToARGBRow_SSSE3(const uint8_t* y_buf,
);
}
+#ifdef HAS_I444ALPHATOARGBROW_SSSE3
+void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "movd %[temp],%%xmm7 \n"
+ "movd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movd %%xmm7,%[temp] \n"
+ "movq (%[u_buf]),%%xmm3 \n"
+ "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x8(%[u_buf]),%[u_buf] \n"
+ "punpcklbw %%xmm1,%%xmm3 \n"
+ "movq (%[y_buf]),%%xmm4 \n"
+ "punpcklbw %%xmm4,%%xmm4 \n"
+ "lea 0x8(%[y_buf]),%[y_buf] \n"
+ "movd %%xmm6,%[temp] \n"
+ "movq (%[temp]),%%xmm5 \n"
+ "lea 0x8(%[temp]),%[temp] \n"
+ "movd %[temp],%%xmm6 \n"
+ YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA444 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_I444ALPHATOARGBROW_SSSE3
+
+#ifdef HAS_I422TORGB24ROW_SSSE3
void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
+#if defined(__i386__)
+ uint32_t shuf0[4];
+ uint32_t shuf1[4];
+ memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0));
+ memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1));
+#endif
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n"
- "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n"
- "sub %[u_buf],%[v_buf] \n"
+ "movdqu %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n"
+ "movdqu %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n"
+ "sub %[u_buf],%[v_buf] \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
- "punpcklbw %%xmm1,%%xmm0 \n"
- "punpcklbw %%xmm2,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklwd %%xmm2,%%xmm0 \n"
- "punpckhwd %%xmm2,%%xmm1 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "pshufb %%xmm6,%%xmm1 \n"
- "palignr $0xc,%%xmm0,%%xmm1 \n"
- "movq %%xmm0,(%[dst_rgb24]) \n"
- "movdqu %%xmm1,0x8(%[dst_rgb24]) \n"
- "lea 0x18(%[dst_rgb24]),%[dst_rgb24] \n"
- "subl $0x8,%[width] \n"
- "jg 1b \n"
+ STORERGB24
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
+#if defined(__i386__)
+ [width]"+m"(width) // %[width]
+#else
+ [width]"+rm"(width) // %[width]
+#endif
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+#if defined(__i386__)
+ [kShuffleMaskARGBToRGB24_0]"m"(shuf0),
+ [kShuffleMaskARGBToRGB24_1]"m"(shuf1)
+#else
+ [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]),
+ [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1])
+#endif
+ : "memory", "cc", YUVTORGB_REGS
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
+ );
+}
+#endif // HAS_I422TORGB24ROW_SSSE3
+
+#ifdef HAS_I444TORGB24ROW_SSSE3
+void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uint32_t shuf0[4];
+ uint32_t shuf1[4];
+ memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0));
+ memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1));
+#endif
+ asm volatile (
+ YUVTORGB_SETUP(yuvconstants)
+ "movdqu %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n"
+ "movdqu %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n"
+ "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV444
+ YUVTORGB(yuvconstants)
+ STORERGB24
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2177,12 +4391,18 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
[width]"+rm"(width) // %[width]
#endif
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0),
- [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)
+#if defined(__i386__)
+ [kShuffleMaskARGBToRGB24_0]"m"(shuf0),
+ [kShuffleMaskARGBToRGB24_1]"m"(shuf1)
+#else
+ [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]),
+ [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1])
+#endif
: "memory", "cc", YUVTORGB_REGS
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
);
}
+#endif // HAS_I444TORGB24ROW_SSSE3
void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
@@ -2192,16 +4412,16 @@ void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2213,6 +4433,7 @@ void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf,
);
}
+#ifdef HAS_I422TOAR30ROW_SSSE3
void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -2220,32 +4441,37 @@ void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n" // AR30 constants
- "psrlw $14,%%xmm5 \n"
- "psllw $4,%%xmm5 \n" // 2 alpha bits
- "pxor %%xmm6,%%xmm6 \n"
- "pcmpeqb %%xmm7,%%xmm7 \n" // 0 for min
- "psrlw $6,%%xmm7 \n" // 1023 for max
+ YUVTORGB_SETUP_AR30(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
- YUVTORGB16(yuvconstants)
+ YUVTORGB16_AR30(yuvconstants)
STOREAR30
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+#if defined(__i386__)
+ [width]"+m"(width) // %[width]
+#else
[width]"+rm"(width) // %[width]
+#endif
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
: "memory", "cc", YUVTORGB_REGS
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
}
+#endif // HAS_I422TOAR30ROW_SSSE3
// 10 bit YUV to ARGB
void OMITFP I210ToARGBRow_SSSE3(const uint16_t* y_buf,
@@ -2256,16 +4482,46 @@ void OMITFP I210ToARGBRow_SSSE3(const uint16_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210
YUVTORGB(yuvconstants)
STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ );
+}
+
+// 12 bit YUV to ARGB
+void OMITFP I212ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
+ YUVTORGB_SETUP(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV212
+ YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2286,21 +4542,21 @@ void OMITFP I210ToAR30Row_SSSE3(const uint16_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $14,%%xmm5 \n"
- "psllw $4,%%xmm5 \n" // 2 alpha bits
- "pxor %%xmm6,%%xmm6 \n"
- "pcmpeqb %%xmm7,%%xmm7 \n" // 0 for min
- "psrlw $6,%%xmm7 \n" // 1023 for max
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210
YUVTORGB16(yuvconstants)
STOREAR30
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2312,155 +4568,508 @@ void OMITFP I210ToAR30Row_SSSE3(const uint16_t* y_buf,
);
}
-#ifdef HAS_I422ALPHATOARGBROW_SSSE3
-void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- // clang-format off
+// 12 bit YUV to AR30
+void OMITFP I212ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
- READYUVA422
- YUVTORGB(yuvconstants)
- STOREARGB
- "subl $0x8,%[width] \n"
- "jg 1b \n"
+ "1: \n"
+ READYUV212
+ YUVTORGB16(yuvconstants)
+ STOREAR30
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
-#if defined(__i386__)
- [width]"+m"(width) // %[width]
-#else
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
[width]"+rm"(width) // %[width]
-#endif
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
: "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
}
-#endif // HAS_I422ALPHATOARGBROW_SSSE3
-void OMITFP NV12ToARGBRow_SSSE3(const uint8_t* y_buf,
- const uint8_t* uv_buf,
+// 10 bit YUV to ARGB
+void OMITFP I410ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
- READNV12
+ "1: \n"
+ READYUV410
YUVTORGB(yuvconstants)
STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
[dst_argb]"+r"(dst_argb), // %[dst_argb]
[width]"+rm"(width) // %[width]
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
);
- // clang-format on
}
-void OMITFP NV21ToARGBRow_SSSE3(const uint8_t* y_buf,
- const uint8_t* vu_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- // clang-format off
- asm volatile (
+#ifdef HAS_I210ALPHATOARGBROW_SSSE3
+// 10 bit YUVA to ARGB
+void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "movd %[temp],%%xmm7 \n"
+ "movd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movd %%xmm7,%[temp] \n"
+ "movq (%[u_buf]),%%xmm3 \n"
+ "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x8(%[u_buf]),%[u_buf] \n"
+ "punpcklwd %%xmm1,%%xmm3 \n"
+ "psraw $2,%%xmm3 \n"
+ "packuswb %%xmm3,%%xmm3 \n"
+ "punpcklwd %%xmm3,%%xmm3 \n"
+ "movdqu (%[y_buf]),%%xmm4 \n"
+ "movdqa %%xmm4,%%xmm2 \n"
+ "psllw $6,%%xmm4 \n"
+ "psrlw $4,%%xmm2 \n"
+ "paddw %%xmm2,%%xmm4 \n"
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+ "movd %%xmm6,%[temp] \n"
+ "movdqu (%[temp]),%%xmm5 \n"
+ "psraw $2,%%xmm5 \n"
+ "packuswb %%xmm5,%%xmm5 \n"
+ "lea 0x10(%[temp]),%[temp] \n"
+ "movd %[temp],%%xmm6 \n"
+ YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA210 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_I210ALPHATOARGBROW_SSSE3
+
+#ifdef HAS_I410ALPHATOARGBROW_SSSE3
+// 10 bit YUVA to ARGB
+void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "movd %[temp],%%xmm7 \n"
+ "movd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movd %%xmm7,%[temp] \n"
+ "movdqu (%[u_buf]),%%xmm3 \n"
+ "movdqu 0x00(%[u_buf],%[temp],1),%%xmm2 \n"
+ "lea 0x10(%[u_buf]),%[u_buf] \n"
+ "psraw $2,%%xmm3 \n"
+ "psraw $2,%%xmm2 \n"
+ "movdqa %%xmm3,%%xmm1 \n"
+ "punpcklwd %%xmm2,%%xmm3 \n"
+ "punpckhwd %%xmm2,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm3 \n"
+ "movdqu (%[y_buf]),%%xmm4 \n"
+ "movdqa %%xmm4,%%xmm2 \n"
+ "psllw $6,%%xmm4 \n"
+ "psrlw $4,%%xmm2 \n"
+ "paddw %%xmm2,%%xmm4 \n"
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+ "movd %%xmm6,%[temp] \n"
+ "movdqu (%[temp]),%%xmm5 \n"
+ "psraw $2,%%xmm5 \n"
+ "packuswb %%xmm5,%%xmm5 \n"
+ "lea 0x10(%[temp]),%[temp] \n"
+ "movd %[temp],%%xmm6 \n"
+ YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA410 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_I410ALPHATOARGBROW_SSSE3
+
+// 10 bit YUV to AR30
+void OMITFP I410ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
- READNV21
- YUVTORGB(yuvconstants)
- STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "1: \n"
+ READYUV410
+ YUVTORGB16(yuvconstants)
+ STOREAR30
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
- [vu_buf]"+r"(vu_buf), // %[vu_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
[width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleNV21]"m"(kShuffleNV21)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
+}
+
+#ifdef HAS_I422ALPHATOARGBROW_SSSE3
+void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "movd %[temp],%%xmm7 \n"
+ "movd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movd %%xmm7,%[temp] \n"
+ "movd (%[u_buf]),%%xmm3 \n"
+ "movd 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x4(%[u_buf]),%[u_buf] \n"
+ "punpcklbw %%xmm1,%%xmm3 \n"
+ "punpcklwd %%xmm3,%%xmm3 \n"
+ "movq (%[y_buf]),%%xmm4 \n"
+ "punpcklbw %%xmm4,%%xmm4 \n"
+ "lea 0x8(%[y_buf]),%[y_buf] \n"
+ "movd %%xmm6,%[temp] \n"
+ "movq (%[temp]),%%xmm5 \n"
+ "lea 0x8(%[temp]),%[temp] \n"
+ "movd %[temp],%%xmm6 \n"
+ YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA422 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_I422ALPHATOARGBROW_SSSE3
+
+void OMITFP NV12ToARGBRow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READNV12 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+}
+
+void OMITFP NV21ToARGBRow_SSSE3(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READNV21 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [vu_buf] "+r"(vu_buf), // %[vu_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleNV21] "m"(kShuffleNV21)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
void OMITFP YUY2ToARGBRow_SSSE3(const uint8_t* yuy2_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
+ asm volatile(
+ "movdqa %[kShuffleYUY2Y],%%xmm6 \n"
+ "movdqa %[kShuffleYUY2UV],%%xmm7 \n" YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READYUY2 YUVTORGB(yuvconstants) STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [yuy2_buf] "+r"(yuy2_buf), // %[yuy2_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleYUY2Y] "m"(kShuffleYUY2Y), [kShuffleYUY2UV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
+ "xmm5", "xmm6", "xmm7");
+}
+
+void OMITFP UYVYToARGBRow_SSSE3(const uint8_t* uyvy_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "movdqa %[kShuffleUYVYY],%%xmm6 \n"
+ "movdqa %[kShuffleUYVYUV],%%xmm7 \n" YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READUYVY YUVTORGB(yuvconstants) STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [uyvy_buf] "+r"(uyvy_buf), // %[uyvy_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleUYVYY] "m"(kShuffleUYVYY), [kShuffleUYVYUV] "m"(kShuffleUYVYUV)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
+ "xmm5", "xmm6", "xmm7");
+}
+
+void OMITFP P210ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READP210 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[u_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+}
+
+void OMITFP P410ToARGBRow_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+
+ LABELALIGN "1: \n" READP410 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[u_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
+}
+
+void OMITFP P210ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
- READYUY2
- YUVTORGB(yuvconstants)
- STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [yuy2_buf]"+r"(yuy2_buf), // %[yuy2_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleYUY2Y]"m"(kShuffleYUY2Y),
- [kShuffleYUY2UV]"m"(kShuffleYUY2UV)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ "1: \n"
+ READP210
+ YUVTORGB16(yuvconstants)
+ STOREAR30
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [uv_buf]"+r"(uv_buf), // %[uv_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
}
-void OMITFP UYVYToARGBRow_SSSE3(const uint8_t* uyvy_buf,
- uint8_t* dst_argb,
+void OMITFP P410ToAR30Row_SSSE3(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n" // all 1s
+ "movdqa %%xmm5,%%xmm7 \n"
+ "psrlw $6,%%xmm7 \n"
+ "psllw $4,%%xmm7 \n" // 1023 * 16 for max
+ "psllw $14,%%xmm5 \n" // 0xc000 for 2 alpha bits
+ "pxor %%xmm6,%%xmm6 \n" // 0 for min
LABELALIGN
- "1: \n"
- READUYVY
- YUVTORGB(yuvconstants)
- STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [uyvy_buf]"+r"(uyvy_buf), // %[uyvy_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleUYVYY]"m"(kShuffleUYVYY),
- [kShuffleUYVYUV]"m"(kShuffleUYVYUV)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ "1: \n"
+ READP410
+ YUVTORGB16(yuvconstants)
+ STOREAR30
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [uv_buf]"+r"(uv_buf), // %[uv_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
}
void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
@@ -2471,16 +5080,16 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
STORERGBA
- "sub $0x8,%[width] \n"
- "jg 1b \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2492,16 +5101,16 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
);
}
-#endif // HAS_I422TOARGBROW_SSSE3
+#endif // defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2)
// Read 16 UV from 444
#define READYUV444_AVX2 \
- "vmovdqu (%[u_buf]),%%xmm0 \n" \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
"vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x10(%[u_buf]),%[u_buf] \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
"vpermq $0xd8,%%ymm1,%%ymm1 \n" \
- "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
"vmovdqu (%[y_buf]),%%xmm4 \n" \
"vpermq $0xd8,%%ymm4,%%ymm4 \n" \
"vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
@@ -2509,42 +5118,170 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
// Read 8 UV from 422, upsample to 16 UV.
#define READYUV422_AVX2 \
- "vmovq (%[u_buf]),%%xmm0 \n" \
+ "vmovq (%[u_buf]),%%xmm3 \n" \
"vmovq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x8(%[u_buf]),%[u_buf] \n" \
- "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
- "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vmovdqu (%[y_buf]),%%xmm4 \n" \
"vpermq $0xd8,%%ymm4,%%ymm4 \n" \
"vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
"lea 0x10(%[y_buf]),%[y_buf] \n"
-// Read 8 UV from 210 10 bit, upsample to 16 UV
-// TODO(fbarchard): Consider vshufb to replace pack/unpack
+#define READYUV422_AVX512BW \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" \
+ "vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" \
+ "vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" \
+ "vmovdqa64 %%zmm16,%%zmm1 \n" \
+ "vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+#define READYUV422_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%} \n" \
+ "vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%} \n" \
+ "vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" \
+ "vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" \
+ "vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" \
+ "vmovdqa64 %%zmm16,%%zmm1 \n" \
+ "vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+// Read 8 UV from 210, upsample to 16 UV
+// TODO(fbarchard): Consider vpshufb to replace pack/unpack
// TODO(fbarchard): Consider vunpcklpd to combine the 2 registers into 1.
#define READYUV210_AVX2 \
- "vmovdqu (%[u_buf]),%%xmm0 \n" \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \
+ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "vpsllw $6,%%ymm4,%%ymm2 \n" \
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" \
+ "vpor %%ymm2,%%ymm4,%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+
+// Read 8 UV from 210, upsample to 16 UV. With 16 Alpha.
+#define READYUVA210_AVX2 \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \
+ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "vpsllw $6,%%ymm4,%%ymm2 \n" \
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" \
+ "vpor %%ymm2,%%ymm4,%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vmovdqu (%[a_buf]),%%ymm5 \n" \
+ "vpsrlw $2,%%ymm5,%%ymm5 \n" \
+ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" \
+ "lea 0x20(%[a_buf]),%[a_buf] \n"
+
+// Read 16 UV from 410
+#define READYUV410_AVX2 \
+ "vmovdqu (%[u_buf]),%%ymm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%ymm2 \n" \
+ "lea 0x20(%[u_buf]),%[u_buf] \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm2,%%ymm2 \n" \
+ "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" \
+ "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "vpsllw $6,%%ymm4,%%ymm2 \n" \
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" \
+ "vpor %%ymm2,%%ymm4,%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+
+// Read 8 UV from 212 12 bit, upsample to 16 UV
+#define READYUV212_AVX2 \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
"vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x10(%[u_buf]),%[u_buf] \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
"vpermq $0xd8,%%ymm1,%%ymm1 \n" \
- "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n" \
- "vpsraw $0x2,%%ymm0,%%ymm0 \n" \
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" \
- "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $0x4,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vmovdqu (%[y_buf]),%%ymm4 \n" \
- "vpsllw $0x6,%%ymm4,%%ymm4 \n" \
+ "vpsllw $4,%%ymm4,%%ymm2 \n" \
+ "vpsrlw $8,%%ymm4,%%ymm4 \n" \
+ "vpor %%ymm2,%%ymm4,%%ymm4 \n" \
"lea 0x20(%[y_buf]),%[y_buf] \n"
+// Read 16 UV from 410. With 16 Alpha.
+#define READYUVA410_AVX2 \
+ "vmovdqu (%[u_buf]),%%ymm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%ymm2 \n" \
+ "lea 0x20(%[u_buf]),%[u_buf] \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm2,%%ymm2 \n" \
+ "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" \
+ "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "vpsllw $6,%%ymm4,%%ymm2 \n" \
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" \
+ "vpor %%ymm2,%%ymm4,%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vmovdqu (%[a_buf]),%%ymm5 \n" \
+ "vpsrlw $2,%%ymm5,%%ymm5 \n" \
+ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" \
+ "lea 0x20(%[a_buf]),%[a_buf] \n"
+
+// Read 16 UV from 444. With 16 Alpha.
+#define READYUVA444_AVX2 \
+ "vmovdqu (%[u_buf]),%%xmm3 \n" \
+ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
+ "lea 0x10(%[u_buf]),%[u_buf] \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%xmm4 \n" \
+ "vpermq $0xd8,%%ymm4,%%ymm4 \n" \
+ "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
+ "lea 0x10(%[y_buf]),%[y_buf] \n" \
+ "vmovdqu (%[a_buf]),%%xmm5 \n" \
+ "vpermq $0xd8,%%ymm5,%%ymm5 \n" \
+ "lea 0x10(%[a_buf]),%[a_buf] \n"
+
// Read 8 UV from 422, upsample to 16 UV. With 16 Alpha.
#define READYUVA422_AVX2 \
- "vmovq (%[u_buf]),%%xmm0 \n" \
+ "vmovq (%[u_buf]),%%xmm3 \n" \
"vmovq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \
"lea 0x8(%[u_buf]),%[u_buf] \n" \
- "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
- "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vmovdqu (%[y_buf]),%%xmm4 \n" \
"vpermq $0xd8,%%ymm4,%%ymm4 \n" \
"vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
@@ -2555,10 +5292,10 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
// Read 8 UV from NV12, upsample to 16 UV.
#define READNV12_AVX2 \
- "vmovdqu (%[uv_buf]),%%xmm0 \n" \
+ "vmovdqu (%[uv_buf]),%%xmm3 \n" \
"lea 0x10(%[uv_buf]),%[uv_buf] \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
- "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vmovdqu (%[y_buf]),%%xmm4 \n" \
"vpermq $0xd8,%%ymm4,%%ymm4 \n" \
"vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
@@ -2566,75 +5303,167 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
// Read 8 VU from NV21, upsample to 16 UV.
#define READNV21_AVX2 \
- "vmovdqu (%[vu_buf]),%%xmm0 \n" \
+ "vmovdqu (%[vu_buf]),%%xmm3 \n" \
"lea 0x10(%[vu_buf]),%[vu_buf] \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
- "vpshufb %[kShuffleNV21], %%ymm0, %%ymm0 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vpshufb %[kShuffleNV21], %%ymm3, %%ymm3 \n" \
"vmovdqu (%[y_buf]),%%xmm4 \n" \
"vpermq $0xd8,%%ymm4,%%ymm4 \n" \
"vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" \
"lea 0x10(%[y_buf]),%[y_buf] \n"
+// Read 4 UV from P210, upsample to 8 UV
+#define READP210_AVX2 \
+ "vmovdqu (%[uv_buf]),%%ymm3 \n" \
+ "lea 0x20(%[uv_buf]),%[uv_buf] \n" \
+ "vpsrlw $0x8,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+
+// Read 8 UV from P410
+#define READP410_AVX2 \
+ "vmovdqu (%[uv_buf]),%%ymm3 \n" \
+ "vmovdqu 0x20(%[uv_buf]),%%ymm1 \n" \
+ "lea 0x40(%[uv_buf]),%[uv_buf] \n" \
+ "vpsrlw $0x8,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \
+ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n" \
+ "vmovdqu (%[y_buf]),%%ymm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+
// Read 8 YUY2 with 16 Y and upsample 8 UV to 16 UV.
+// ymm6 kShuffleYUY2Y,
+// ymm7 kShuffleYUY2UV
#define READYUY2_AVX2 \
- "vmovdqu (%[yuy2_buf]),%%ymm4 \n" \
- "vpshufb %[kShuffleYUY2Y], %%ymm4, %%ymm4 \n" \
- "vmovdqu (%[yuy2_buf]),%%ymm0 \n" \
- "vpshufb %[kShuffleYUY2UV], %%ymm0, %%ymm0 \n" \
+ "vmovdqu (%[yuy2_buf]),%%ymm1 \n" \
+ "vpshufb %%ymm6,%%ymm1,%%ymm4 \n" \
+ "vpshufb %%ymm7,%%ymm1,%%ymm3 \n" \
"lea 0x20(%[yuy2_buf]),%[yuy2_buf] \n"
// Read 8 UYVY with 16 Y and upsample 8 UV to 16 UV.
+// ymm6 kShuffleUYVYY,
+// ymm7 kShuffleUYVYUV
#define READUYVY_AVX2 \
- "vmovdqu (%[uyvy_buf]),%%ymm4 \n" \
- "vpshufb %[kShuffleUYVYY], %%ymm4, %%ymm4 \n" \
- "vmovdqu (%[uyvy_buf]),%%ymm0 \n" \
- "vpshufb %[kShuffleUYVYUV], %%ymm0, %%ymm0 \n" \
+ "vmovdqu (%[uyvy_buf]),%%ymm1 \n" \
+ "vpshufb %%ymm6,%%ymm1,%%ymm4 \n" \
+ "vpshufb %%ymm7,%%ymm1,%%ymm3 \n" \
"lea 0x20(%[uyvy_buf]),%[uyvy_buf] \n"
#if defined(__x86_64__)
-#define YUVTORGB_SETUP_AVX2(yuvconstants) \
- "vmovdqa (%[yuvconstants]),%%ymm8 \n" \
- "vmovdqa 32(%[yuvconstants]),%%ymm9 \n" \
- "vmovdqa 64(%[yuvconstants]),%%ymm10 \n" \
- "vmovdqa 96(%[yuvconstants]),%%ymm11 \n" \
- "vmovdqa 128(%[yuvconstants]),%%ymm12 \n" \
- "vmovdqa 160(%[yuvconstants]),%%ymm13 \n" \
- "vmovdqa 192(%[yuvconstants]),%%ymm14 \n"
+#define YUVTORGB_SETUP_AVX2(yuvconstants) \
+ "vpcmpeqb %%ymm13,%%ymm13,%%ymm13 \n" \
+ "vmovdqu (%[yuvconstants]),%%ymm8 \n" \
+ "vpabsb %%ymm13,%%ymm13 \n" \
+ "vmovdqu 32(%[yuvconstants]),%%ymm9 \n" \
+ "vpsllw $7,%%ymm13,%%ymm13 \n" \
+ "vmovdqu 64(%[yuvconstants]),%%ymm10 \n" \
+ "vmovdqu 96(%[yuvconstants]),%%ymm11 \n" \
+ "vmovdqu 128(%[yuvconstants]),%%ymm12 \n"
+
+#define YUVTORGB_SETUP_AVX512BW(yuvconstants) \
+ "vpternlogd $0xff,%%zmm13,%%zmm13,%%zmm13 \n" \
+ "vpbroadcastq (%[yuvconstants]),%%zmm8 \n" \
+ "vpabsb %%zmm13,%%zmm13 \n" \
+ "vpsllw $7,%%zmm13,%%zmm13 \n" \
+ "vpbroadcastq 32(%[yuvconstants]),%%zmm9 \n" \
+ "vpbroadcastq 64(%[yuvconstants]),%%zmm10 \n" \
+ "vpbroadcastq 96(%[yuvconstants]),%%zmm11 \n" \
+ "vpbroadcastq 128(%[yuvconstants]),%%zmm12 \n" \
+ "vmovups (%[quadsplitperm]),%%zmm16 \n" \
+ "vmovups (%[unperm]),%%zmm18 \n"
+
+#define YUVTORGB_SETUP_AR30_AVX2(yuvconstants) \
+ YUVTORGB_SETUP_AVX2(yuvconstants) \
+ "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpsrlw $14,%%ymm0,%%ymm0 \n" \
+ "vpsllw $3,%%ymm0,%%ymm0 \n" \
+ "vpsubw %%ymm0,%%ymm12,%%ymm12 \n"
+
+#define YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants) \
+ YUVTORGB_SETUP_AVX512BW(yuvconstants) \
+ "vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $14,%%zmm0,%%zmm0 \n" \
+ "vpsllw $3,%%zmm0,%%zmm0 \n" \
+ "vpsubw %%zmm0,%%zmm12,%%zmm12 \n"
#define YUVTORGB16_AVX2(yuvconstants) \
- "vpmaddubsw %%ymm10,%%ymm0,%%ymm2 \n" \
- "vpmaddubsw %%ymm9,%%ymm0,%%ymm1 \n" \
- "vpmaddubsw %%ymm8,%%ymm0,%%ymm0 \n" \
- "vpsubw %%ymm2,%%ymm13,%%ymm2 \n" \
- "vpsubw %%ymm1,%%ymm12,%%ymm1 \n" \
- "vpsubw %%ymm0,%%ymm11,%%ymm0 \n" \
- "vpmulhuw %%ymm14,%%ymm4,%%ymm4 \n" \
+ "vpsubb %%ymm13,%%ymm3,%%ymm3 \n" \
+ "vpmulhuw %%ymm11,%%ymm4,%%ymm4 \n" \
+ "vpmaddubsw %%ymm3,%%ymm8,%%ymm0 \n" \
+ "vpmaddubsw %%ymm3,%%ymm9,%%ymm1 \n" \
+ "vpmaddubsw %%ymm3,%%ymm10,%%ymm2 \n" \
+ "vpaddw %%ymm4,%%ymm12,%%ymm4 \n" \
"vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \
- "vpaddsw %%ymm4,%%ymm1,%%ymm1 \n" \
+ "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \
"vpaddsw %%ymm4,%%ymm2,%%ymm2 \n"
-#define YUVTORGB_REGS_AVX2 \
- "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
+#define YUVTORGB16_AR30_AVX2(yuvconstants) YUVTORGB16_AVX2(yuvconstants)
+
+#define YUVTORGB16_AVX512BW(yuvconstants) \
+ "vpsubb %%zmm13,%%zmm3,%%zmm3 \n" \
+ "vpmulhuw %%zmm11,%%zmm4,%%zmm4 \n" \
+ "vpmaddubsw %%zmm3,%%zmm8,%%zmm0 \n" \
+ "vpmaddubsw %%zmm3,%%zmm9,%%zmm1 \n" \
+ "vpmaddubsw %%zmm3,%%zmm10,%%zmm2 \n" \
+ "vpaddw %%zmm4,%%zmm12,%%zmm4 \n" \
+ "vpaddsw %%zmm4,%%zmm0,%%zmm0 \n" \
+ "vpsubsw %%zmm1,%%zmm4,%%zmm1 \n" \
+ "vpaddsw %%zmm4,%%zmm2,%%zmm2 \n"
+
+#define YUVTORGB_REGS_AVX2 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",
+#define YUVTORGB_REGS_AVX512BW \
+ "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm16", "xmm18",
#else // Convert 16 pixels: 16 UV and 16 Y.
#define YUVTORGB_SETUP_AVX2(yuvconstants)
+#define YUVTORGB_SETUP_AR30_AVX2(yuvconstants)
+#define YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants)
#define YUVTORGB16_AVX2(yuvconstants) \
- "vpmaddubsw 64(%[yuvconstants]),%%ymm0,%%ymm2 \n" \
- "vpmaddubsw 32(%[yuvconstants]),%%ymm0,%%ymm1 \n" \
- "vpmaddubsw (%[yuvconstants]),%%ymm0,%%ymm0 \n" \
- "vmovdqu 160(%[yuvconstants]),%%ymm3 \n" \
- "vpsubw %%ymm2,%%ymm3,%%ymm2 \n" \
+ "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpabsb %%ymm0,%%ymm0 \n" \
+ "vpsllw $7,%%ymm0,%%ymm0 \n" \
+ "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \
+ "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \
+ "vmovdqu (%[yuvconstants]),%%ymm0 \n" \
+ "vmovdqu 32(%[yuvconstants]),%%ymm1 \n" \
+ "vmovdqu 64(%[yuvconstants]),%%ymm2 \n" \
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \
+ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \
"vmovdqu 128(%[yuvconstants]),%%ymm3 \n" \
- "vpsubw %%ymm1,%%ymm3,%%ymm1 \n" \
- "vmovdqu 96(%[yuvconstants]),%%ymm3 \n" \
- "vpsubw %%ymm0,%%ymm3,%%ymm0 \n" \
- "vpmulhuw 192(%[yuvconstants]),%%ymm4,%%ymm4 \n" \
+ "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \
"vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \
- "vpaddsw %%ymm4,%%ymm1,%%ymm1 \n" \
+ "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \
"vpaddsw %%ymm4,%%ymm2,%%ymm2 \n"
+
+#define YUVTORGB16_AR30_AVX2(yuvconstants) \
+ "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \
+ "vpabsb %%ymm0,%%ymm0 \n" \
+ "vpsllw $7,%%ymm0,%%ymm0 \n" \
+ "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \
+ "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \
+ "vmovdqu (%[yuvconstants]),%%ymm0 \n" \
+ "vmovdqu 32(%[yuvconstants]),%%ymm1 \n" \
+ "vmovdqu 64(%[yuvconstants]),%%ymm2 \n" \
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \
+ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \
+ "vmovdqu 128(%[yuvconstants]),%%ymm3 \n" \
+ "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $14,%%ymm3,%%ymm3 \n" \
+ "vpsllw $3,%%ymm3,%%ymm3 \n" \
+ "vpsubw %%ymm3,%%ymm4,%%ymm4 \n" \
+ "vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \
+ "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \
+ "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n"
+
#define YUVTORGB_REGS_AVX2
-#endif
+#endif // defined(__x86_64__)
#define YUVTORGB_AVX2(yuvconstants) \
YUVTORGB16_AVX2(yuvconstants) \
@@ -2645,6 +5474,15 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
"vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \
"vpackuswb %%ymm2,%%ymm2,%%ymm2 \n"
+#define YUVTORGB_AVX512BW(yuvconstants) \
+ YUVTORGB16_AVX512BW(yuvconstants) \
+ "vpsraw $0x6,%%zmm0,%%zmm0 \n" \
+ "vpsraw $0x6,%%zmm1,%%zmm1 \n" \
+ "vpsraw $0x6,%%zmm2,%%zmm2 \n" \
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" \
+ "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \
+ "vpackuswb %%zmm2,%%zmm2,%%zmm2 \n"
+
// Store 16 ARGB values.
#define STOREARGB_AVX2 \
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
@@ -2655,34 +5493,186 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
"vpunpckhwd %%ymm2,%%ymm0,%%ymm0 \n" \
"vmovdqu %%ymm1,(%[dst_argb]) \n" \
"vmovdqu %%ymm0,0x20(%[dst_argb]) \n" \
- "lea 0x40(%[dst_argb]), %[dst_argb] \n"
+ "lea 0x40(%[dst_argb]), %[dst_argb] \n"
+
+// Store 32 ARGB values.
+#define STOREARGB_AVX512BW \
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpermq %%zmm0,%%zmm18,%%zmm0 \n" \
+ "vpunpcklbw %%zmm5,%%zmm2,%%zmm2 \n" \
+ "vpermq %%zmm2,%%zmm18,%%zmm2 \n" \
+ "vpunpcklwd %%zmm2,%%zmm0,%%zmm1 \n" \
+ "vpunpckhwd %%zmm2,%%zmm0,%%zmm0 \n" \
+ "vmovups %%zmm1,(%[dst_argb]) \n" \
+ "vmovups %%zmm0,0x40(%[dst_argb]) \n" \
+ "lea 0x80(%[dst_argb]), %[dst_argb] \n"
+
+// Store masked ARGB values with AVX512BW.
+#define STOREARGB_MASK_AVX512BW \
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpermq %%zmm0,%%zmm18,%%zmm0 \n" \
+ "vpunpcklbw %%zmm5,%%zmm2,%%zmm2 \n" \
+ "vpermq %%zmm2,%%zmm18,%%zmm2 \n" \
+ "vpunpcklwd %%zmm2,%%zmm0,%%zmm1 \n" \
+ "vpunpckhwd %%zmm2,%%zmm0,%%zmm0 \n" \
+ "kshiftrd $16,%%k1,%%k2 \n" \
+ "vmovdqu32 %%zmm1,(%[dst_argb])%{%%k1%} \n" \
+ "vmovdqu32 %%zmm0,0x40(%[dst_argb])%{%%k2%} \n"
+
+// Store 16 RGB24 values.
+#define STORERGB24_AVX2 \
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
+ "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" \
+ "vmovdqa %%ymm0,%%ymm1 \n" \
+ "vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n" \
+ "vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n" \
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" \
+ "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" \
+ "vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n" \
+ "vextracti128 $1,%%ymm0,%%xmm2 \n" \
+ "vextracti128 $1,%%ymm1,%%xmm3 \n" \
+ "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" \
+ "vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n" \
+ "vmovdqu %%xmm0,(%[dst_rgb24]) \n" \
+ "vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n" \
+ "vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n" \
+ "lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n"
+
+// Store 32 RGB24 values with VBMI.
+#define STORERGB24_AVX512VBMI \
+ "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \
+ "vmovdqa64 %%zmm0,%%zmm3 \n" \
+ "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \
+ "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \
+ "vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \
+ "vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" \
+ "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
+
+// Store masked RGB24 values with VBMI.
+#define STORERGB24_MASK_AVX512VBMI \
+ "lea (%q[width],%q[width],2),%k[width] \n" \
+ "bzhi %q[width],%[temp],%[y_buf] \n" \
+ "kmovq %[y_buf],%%k3 \n" \
+ "xor %k[y_buf],%k[y_buf] \n" \
+ "sub $0x40,%k[width] \n" \
+ "cmovg %[temp],%[y_buf] \n" \
+ "bzhi %q[width],%[y_buf],%[y_buf] \n" \
+ "kmovd %k[y_buf],%%k4 \n" \
+ "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \
+ "vmovdqa64 %%zmm0,%%zmm3 \n" \
+ "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \
+ "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \
+ "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \
+ "vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%} \n"
+
+// Store 32 RGB24 values with AVX512BW.
+#define STORERGB24_AVX512BW \
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \
+ "vmovdqa64 %%zmm0,%%zmm1 \n" \
+ "vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \
+ "vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \
+ "vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \
+ "vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \
+ "vmovdqa64 %%zmm20,%%zmm3 \n" \
+ "vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \
+ "vmovdqa64 %%zmm21,%%zmm4 \n" \
+ "vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \
+ "vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \
+ "vmovdqu8 %%ymm4,0x40(%[dst_rgb24]) \n" \
+ "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
+
+// Store masked RGB24 values with AVX512BW.
+#define STORERGB24_MASK_AVX512BW \
+ "lea (%q[width],%q[width],2),%k[width] \n" \
+ "bzhi %q[width],%[temp],%[y_buf] \n" \
+ "kmovq %[y_buf],%%k3 \n" \
+ "xor %k[y_buf],%k[y_buf] \n" \
+ "sub $0x40,%k[width] \n" \
+ "cmovg %[temp],%[y_buf] \n" \
+ "bzhi %q[width],%[y_buf],%[y_buf] \n" \
+ "kmovd %k[y_buf],%%k4 \n" \
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
+ "vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \
+ "vmovdqa64 %%zmm0,%%zmm1 \n" \
+ "vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \
+ "vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \
+ "vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \
+ "vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \
+ "vmovdqa64 %%zmm20,%%zmm3 \n" \
+ "vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \
+ "vmovdqa64 %%zmm21,%%zmm4 \n" \
+ "vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \
+ "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \
+ "vmovdqu8 %%ymm4,0x40(%[dst_rgb24])%{%%k4%} \n"
+
+// Store 32 AR30 values.
+#define STOREAR30_AVX512BW \
+ "vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \
+ "vpminsw %%zmm7,%%zmm1,%%zmm1 \n" \
+ "vpminsw %%zmm7,%%zmm2,%%zmm2 \n" \
+ "vpmaxsw %%zmm6,%%zmm0,%%zmm0 \n" \
+ "vpmaxsw %%zmm6,%%zmm1,%%zmm1 \n" \
+ "vpmaxsw %%zmm6,%%zmm2,%%zmm2 \n" \
+ "vpsrlw $4,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $10,%%zmm1,%%zmm3 \n" \
+ "vpsllw $6,%%zmm1,%%zmm1 \n" \
+ "vpternlogd $0xf8,%%zmm19,%%zmm1,%%zmm0 \n" \
+ "vpternlogd $0xe4,%%zmm7,%%zmm3,%%zmm2 \n" \
+ "vpord %%zmm5,%%zmm2,%%zmm2 \n" \
+ "vmovdqa64 %%zmm0,%%zmm3 \n" \
+ "vpermt2w %%zmm2,%%zmm20,%%zmm0 \n" \
+ "vpermt2w %%zmm2,%%zmm21,%%zmm3 \n" \
+ "vmovdqu32 %%zmm0,(%[dst_ar30]) \n" \
+ "vmovdqu32 %%zmm3,0x40(%[dst_ar30]) \n" \
+ "lea 0x80(%[dst_ar30]), %[dst_ar30] \n"
+
+// Store masked AR30 values with AVX512BW.
+#define STOREAR30_MASK_AVX512BW \
+ "vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \
+ "vpminsw %%zmm7,%%zmm1,%%zmm1 \n" \
+ "vpminsw %%zmm7,%%zmm2,%%zmm2 \n" \
+ "vpmaxsw %%zmm6,%%zmm0,%%zmm0 \n" \
+ "vpmaxsw %%zmm6,%%zmm1,%%zmm1 \n" \
+ "vpmaxsw %%zmm6,%%zmm2,%%zmm2 \n" \
+ "vpsrlw $4,%%zmm0,%%zmm0 \n" \
+ "vpsrlw $10,%%zmm1,%%zmm3 \n" \
+ "vpsllw $6,%%zmm1,%%zmm1 \n" \
+ "vpternlogd $0xf8,%%zmm19,%%zmm1,%%zmm0 \n" \
+ "vpternlogd $0xe4,%%zmm7,%%zmm3,%%zmm2 \n" \
+ "vpord %%zmm5,%%zmm2,%%zmm2 \n" \
+ "vmovdqa64 %%zmm0,%%zmm3 \n" \
+ "vpermt2w %%zmm2,%%zmm20,%%zmm0 \n" \
+ "vpermt2w %%zmm2,%%zmm21,%%zmm3 \n" \
+ "kshiftrd $16,%%k1,%%k2 \n" \
+ "vmovdqu32 %%zmm0,(%[dst_ar30])%{%%k1%} \n" \
+ "vmovdqu32 %%zmm3,0x40(%[dst_ar30])%{%%k2%} \n"
// Store 16 AR30 values.
#define STOREAR30_AVX2 \
- "vpsraw $0x4,%%ymm0,%%ymm0 \n" \
- "vpsraw $0x4,%%ymm1,%%ymm1 \n" \
- "vpsraw $0x4,%%ymm2,%%ymm2 \n" \
- "vpminsw %%ymm7,%%ymm0,%%ymm0 \n" \
- "vpminsw %%ymm7,%%ymm1,%%ymm1 \n" \
- "vpminsw %%ymm7,%%ymm2,%%ymm2 \n" \
- "vpmaxsw %%ymm6,%%ymm0,%%ymm0 \n" \
- "vpmaxsw %%ymm6,%%ymm1,%%ymm1 \n" \
- "vpmaxsw %%ymm6,%%ymm2,%%ymm2 \n" \
- "vpsllw $0x4,%%ymm2,%%ymm2 \n" \
- "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
- "vpermq $0xd8,%%ymm1,%%ymm1 \n" \
- "vpermq $0xd8,%%ymm2,%%ymm2 \n" \
- "vpunpckhwd %%ymm2,%%ymm0,%%ymm3 \n" \
- "vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n" \
- "vpunpckhwd %%ymm5,%%ymm1,%%ymm2 \n" \
- "vpunpcklwd %%ymm5,%%ymm1,%%ymm1 \n" \
- "vpslld $0xa,%%ymm1,%%ymm1 \n" \
- "vpslld $0xa,%%ymm2,%%ymm2 \n" \
- "vpor %%ymm1,%%ymm0,%%ymm0 \n" \
- "vpor %%ymm2,%%ymm3,%%ymm3 \n" \
- "vmovdqu %%ymm0,(%[dst_ar30]) \n" \
- "vmovdqu %%ymm3,0x20(%[dst_ar30]) \n" \
- "lea 0x40(%[dst_ar30]), %[dst_ar30] \n"
+ "vpminsw %%ymm7,%%ymm0,%%ymm0 \n" \
+ "vpminsw %%ymm7,%%ymm1,%%ymm1 \n" \
+ "vpminsw %%ymm7,%%ymm2,%%ymm2 \n" \
+ "vpmaxsw %%ymm6,%%ymm0,%%ymm0 \n" \
+ "vpmaxsw %%ymm6,%%ymm1,%%ymm1 \n" \
+ "vpmaxsw %%ymm6,%%ymm2,%%ymm2 \n" \
+ "vpand %%ymm7,%%ymm1,%%ymm3 \n" \
+ "vpand %%ymm7,%%ymm2,%%ymm2 \n" \
+ "vpsrlw $4,%%ymm0,%%ymm0 \n" \
+ "vpsrlw $10,%%ymm1,%%ymm1 \n" \
+ "vpsllw $6,%%ymm3,%%ymm3 \n" \
+ "vpor %%ymm3,%%ymm0,%%ymm0 \n" \
+ "vpor %%ymm5,%%ymm2,%%ymm2 \n" \
+ "vpor %%ymm1,%%ymm2,%%ymm2 \n" \
+ "vpermq $0xd8,%%ymm2,%%ymm2 \n" \
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n" \
+ "vpunpcklwd %%ymm2,%%ymm0,%%ymm1 \n" \
+ "vpunpckhwd %%ymm2,%%ymm0,%%ymm3 \n" \
+ "vmovdqu %%ymm1,(%[dst_ar30]) \n" \
+ "vmovdqu %%ymm3,0x20(%[dst_ar30]) \n" \
+ "lea 0x40(%[dst_ar30]), %[dst_ar30] \n"
#ifdef HAS_I444TOARGBROW_AVX2
// 16 pixels
@@ -2695,17 +5685,17 @@ void OMITFP I444ToARGBRow_AVX2(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV444_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2729,18 +5719,18 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2751,7 +5741,364 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
);
}
-#endif // HAS_I422TOARGBROW_AVX2
+#endif // defined(HAS_I422TOARGBROW_AVX2)
+
+#if defined(HAS_I422TORGB24ROW_AVX2)
+// 16 pixels
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGB24 (48
+// bytes).
+void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uint32_t shuf0[4];
+ uint32_t shuf1[4];
+ memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0));
+ memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1));
+#endif
+ asm volatile (
+ YUVTORGB_SETUP_AVX2(yuvconstants)
+ "vbroadcasti128 %[kShuffleMaskARGBToRGB24_1],%%ymm5 \n"
+ "vbroadcasti128 %[kShuffleMaskARGBToRGB24_0],%%ymm6 \n"
+ "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX2
+ YUVTORGB_AVX2(yuvconstants)
+ STORERGB24_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
+#if defined(__i386__)
+ [width]"+m"(width) // %[width]
+#else
+ [width]"+rm"(width) // %[width]
+#endif
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+#if defined(__i386__)
+ [kShuffleMaskARGBToRGB24_0]"m"(shuf0),
+ [kShuffleMaskARGBToRGB24_1]"m"(shuf1)
+#else
+ [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]),
+ [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1])
+#endif
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
+ );
+}
+#endif // defined(HAS_I422TORGB24ROW_AVX2)
+
+#if defined(HAS_I422TOARGBROW_AVX512BW) || \
+ defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I422TORGB24ROW_AVX512BW) || \
+ defined(HAS_I422TOAR30ROW_AVX512BW) || \
+ defined(HAS_I444TOARGBROW_AVX512BW) || \
+ defined(HAS_I444ALPHATOARGBROW_AVX512BW) || \
+ defined(HAS_I444TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I422TORGBAROW_AVX512BW) || \
+ defined(HAS_YUY2TOARGBROW_AVX512BW) || \
+ defined(HAS_UYVYTOARGBROW_AVX512BW) || \
+ defined(HAS_I210TOARGBROW_AVX512BW) || \
+ defined(HAS_I210TOAR30ROW_AVX512BW) || \
+ defined(HAS_I210ALPHATOARGBROW_AVX512BW) || \
+ defined(HAS_I212TOARGBROW_AVX512BW) || \
+ defined(HAS_I212TOAR30ROW_AVX512BW) || \
+ defined(HAS_I410TOARGBROW_AVX512BW) || \
+ defined(HAS_I410TOAR30ROW_AVX512BW) || \
+ defined(HAS_I410ALPHATOARGBROW_AVX512BW) || \
+ defined(HAS_P210TOARGBROW_AVX512BW) || \
+ defined(HAS_P210TOAR30ROW_AVX512BW) || \
+ defined(HAS_P410TOARGBROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW)
+static const uint16_t kSplitQuadWords[32] = {
+ 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+ 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7};
+#endif
+
+#if defined(HAS_I422TOAR30ROW_AVX512BW) || \
+ defined(HAS_I210TOAR30ROW_AVX512BW) || \
+ defined(HAS_I212TOAR30ROW_AVX512BW) || \
+ defined(HAS_I410TOAR30ROW_AVX512BW) || \
+ defined(HAS_P210TOAR30ROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW)
+static const uint16_t kPermAR30_0[32] = {
+ 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
+ 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
+static const uint16_t kPermAR30_1[32] = {
+ 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
+ 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
+#endif
+
+#if defined(HAS_I422TOARGBROW_AVX512BW)
+// 32 pixels
+// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 ARGB (128
+// bytes).
+LIBYUV_TARGET_AVX512BW
+void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile (
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ [width]"+r"(width), // %[width]
+ [temp]"=&r"(temp) // %[temp]
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+ [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
+ [unperm]"r"(kUnpermuteAVX512) // %[unperm]
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "k1", "k2"
+ );
+}
+#endif // defined(HAS_I422TOARGBROW_AVX512BW)
+
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I444TORGB24ROW_AVX512VBMI)
+static const uint8_t kMaskBG[64] = {
+ 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
+ 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
+ 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
+ 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
+ 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
+ 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
+static const uint8_t kMaskDST0[64] = {
+ 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
+ 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
+ 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
+ 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
+ 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
+ 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
+static const uint8_t kMaskDST1[64] = {
+ 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
+ 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
+ 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) ||
+ // defined(HAS_I444TORGB24ROW_AVX512VBMI)
+
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
+// 32 pixels
+// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96
+// bytes).
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile (
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vmovdqu32 %[kMaskBG],%%zmm20 \n"
+ "vmovdqu32 %[kMaskDST0],%%zmm21 \n"
+ "vmovdqu32 %[kMaskDST1],%%zmm22 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_AVX512VBMI
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_MASK_AVX512VBMI
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
+ [width]"+r"(width), // %[width]
+ [temp]"=&r"(temp) // %[temp]
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+ [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
+ [unperm]"r"(kUnpermuteAVX512), // %[unperm]
+ [kMaskBG]"m"(kMaskBG), // %[kMaskBG]
+ [kMaskDST0]"m"(kMaskDST0), // %[kMaskDST0]
+ [kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1]
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm20", "xmm21", "xmm22",
+ "k1", "k2", "k3", "k4"
+ );
+}
+#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI)
+
+#if defined(HAS_I422TORGB24ROW_AVX512BW)
+static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
+static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
+
+// 32 pixels
+// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96
+// bytes).
+LIBYUV_TARGET_AVX512BW
+void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile (
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_1],%%zmm5 \n"
+ "vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_0],%%zmm6 \n"
+ "vmovdqu64 %[kStitchRGB24_0],%%zmm20 \n"
+ "vmovdqu64 %[kStitchRGB24_1],%%zmm21 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
+ [width]"+r"(width), // %[width]
+ [temp]"=&r"(temp) // %[temp]
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+ [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
+ [unperm]"r"(kUnpermuteAVX512), // %[unperm]
+ [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]),
+ [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]),
+ [kStitchRGB24_0]"m"(kStitchRGB24_0), // %[kStitchRGB24_0]
+ [kStitchRGB24_1]"m"(kStitchRGB24_1) // %[kStitchRGB24_1]
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm20", "xmm21",
+ "k1", "k2", "k3", "k4"
+ );
+}
+#endif // defined(HAS_I422TORGB24ROW_AVX512BW)
+
+#if defined(HAS_I422TOAR30ROW_AVX512BW)
+// 32 pixels
+// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 AR30 (128
+// bytes).
+LIBYUV_TARGET_AVX512BW
+void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile (
+ YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" // all 1s
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n" // 1023 * 16 for max
+ "vpsllw $10,%%zmm5,%%zmm19 \n" // 0xfc00 for G mask
+ "vpsllw $14,%%zmm5,%%zmm5 \n" // 0xc000 for 2 alpha bits
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n" // 0 for min
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+r"(width), // %[width]
+ [temp]"=&r"(temp) // %[temp]
+ : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
+ [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
+ [unperm]"r"(kUnpermuteAVX512), // %[unperm]
+ [permar30_0]"r"(kPermAR30_0), // %[permar30_0]
+ [permar30_1]"r"(kPermAR30_1) // %[permar30_1]
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
+ "xmm19", "xmm20", "xmm21", "k1", "k2"
+ );
+}
+#endif // defined(HAS_I422TOAR30ROW_AVX512BW)
#if defined(HAS_I422TOAR30ROW_AVX2)
// 16 pixels
@@ -2763,35 +6110,38 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // AR30 constants
- "vpsrlw $14,%%ymm5,%%ymm5 \n"
- "vpsllw $4,%%ymm5,%%ymm5 \n" // 2 alpha bits
- "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
- "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" // 1023 for max
- "vpsrlw $6,%%ymm7,%%ymm7 \n"
+ YUVTORGB_SETUP_AR30_AVX2(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
- YUVTORGB16_AVX2(yuvconstants)
+ YUVTORGB16_AR30_AVX2(yuvconstants)
STOREAR30_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+#if defined(__i386__)
+ [width]"+m"(width) // %[width]
+#else
[width]"+rm"(width) // %[width]
+#endif
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
: "memory", "cc", YUVTORGB_REGS_AVX2
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
}
-#endif // HAS_I422TOAR30ROW_AVX2
+#endif // defined(HAS_I422TOAR30ROW_AVX2)
#if defined(HAS_I210TOARGBROW_AVX2)
// 16 pixels
@@ -2804,18 +6154,18 @@ void OMITFP I210ToARGBRow_AVX2(const uint16_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2826,11 +6176,46 @@ void OMITFP I210ToARGBRow_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
);
}
-#endif // HAS_I210TOARGBROW_AVX2
+#endif // defined(HAS_I210TOARGBROW_AVX2)
-#if defined(HAS_I210TOAR30ROW_AVX2)
+#if defined(HAS_I212TOARGBROW_AVX2)
// 16 pixels
-// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 AR30 (64 bytes).
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
+void OMITFP I212ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
+ YUVTORGB_SETUP_AVX2(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV212_AVX2
+ YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ );
+}
+#endif // defined(HAS_I212TOARGBROW_AVX2)
+
+#if defined(HAS_I210TOAR30ROW_AVX2)
+// 16 pixels
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 AR30 (64 bytes).
void OMITFP I210ToAR30Row_AVX2(const uint16_t* y_buf,
const uint16_t* u_buf,
const uint16_t* v_buf,
@@ -2839,23 +6224,22 @@ void OMITFP I210ToAR30Row_AVX2(const uint16_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // AR30 constants
- "vpsrlw $14,%%ymm5,%%ymm5 \n"
- "vpsllw $4,%%ymm5,%%ymm5 \n" // 2 alpha bits
- "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
- "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" // 1023 for max
- "vpsrlw $6,%%ymm7,%%ymm7 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -2863,51 +6247,426 @@ void OMITFP I210ToAR30Row_AVX2(const uint16_t* y_buf,
[width]"+rm"(width) // %[width]
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
: "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
}
-#endif // HAS_I210TOAR30ROW_AVX2
+#endif // defined(HAS_I210TOAR30ROW_AVX2)
-#if defined(HAS_I422ALPHATOARGBROW_AVX2)
+#if defined(HAS_I212TOAR30ROW_AVX2)
// 16 pixels
-// 8 UV values upsampled to 16 UV, mixed with 16 Y and 16 A producing 16 ARGB.
-void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- // clang-format off
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 AR30 (64 bytes).
+void OMITFP I212ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
+ YUVTORGB_SETUP_AVX2(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
+
+ LABELALIGN
+ "1: \n"
+ READYUV212_AVX2
+ YUVTORGB16_AVX2(yuvconstants)
+ STOREAR30_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
+ );
+}
+#endif // defined(HAS_I212TOAR30ROW_AVX2)
+
+#if defined(HAS_I410TOARGBROW_AVX2)
+// 16 pixels
+// 16 UV values with 16 Y producing 16 ARGB (64 bytes).
+void OMITFP I410ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- READYUVA422_AVX2
+ "1: \n"
+ READYUV410_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
- "subl $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
[dst_argb]"+r"(dst_argb), // %[dst_argb]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ );
+}
+#endif // defined(HAS_I410TOARGBROW_AVX2)
+
+#if defined(HAS_I210ALPHATOARGBROW_AVX2)
+// 16 pixels
+// 8 UV, 16 Y and 16 A producing 16 ARGB (64 bytes).
+void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
#if defined(__i386__)
- [width]"+m"(width) // %[width]
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "vmovd %[temp],%%xmm7 \n"
+ "vmovd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovd %%xmm7,%[temp] \n"
+ "vmovdqu (%[u_buf]),%%xmm3 \n"
+ "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x10(%[u_buf]),%[u_buf] \n"
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpsraw $2,%%ymm3,%%ymm3 \n"
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n"
+ "vmovdqu (%[y_buf]),%%ymm4 \n"
+ "vpsllw $6,%%ymm4,%%ymm2 \n"
+ "vpsrlw $4,%%ymm4,%%ymm4 \n"
+ "vpaddw %%ymm2,%%ymm4,%%ymm4 \n"
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+ "vmovd %%xmm6,%[temp] \n"
+ "vmovdqu (%[temp]),%%ymm5 \n"
+ "vpsraw $2,%%ymm5,%%ymm5 \n"
+ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n"
+ "lea 0x20(%[temp]),%[temp] \n"
+ "vmovd %[temp],%%xmm6 \n"
+ YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
#else
- [width]"+rm"(width) // %[width]
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA210_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
#endif
+}
+#endif // defined(HAS_I210ALPHATOARGBROW_AVX2)
+
+#if defined(HAS_I410ALPHATOARGBROW_AVX2)
+// 16 pixels
+// 16 UV, 16 Y and 16 A producing 16 ARGB (64 bytes).
+void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "vmovd %[temp],%%xmm7 \n"
+ "vmovd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovd %%xmm7,%[temp] \n"
+ "vmovdqu (%[u_buf]),%%ymm3 \n"
+ "vmovdqu 0x00(%[u_buf],%[temp],1),%%ymm2 \n"
+ "lea 0x20(%[u_buf]),%[u_buf] \n"
+ "vpsraw $2,%%ymm3,%%ymm3 \n"
+ "vpsraw $2,%%ymm2,%%ymm2 \n"
+ "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n"
+ "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n"
+ "vmovdqu (%[y_buf]),%%ymm4 \n"
+ "vpsllw $6,%%ymm4,%%ymm2 \n"
+ "vpsrlw $4,%%ymm4,%%ymm4 \n"
+ "vpaddw %%ymm2,%%ymm4,%%ymm4 \n"
+ "lea 0x20(%[y_buf]),%[y_buf] \n"
+ "vmovd %%xmm6,%[temp] \n"
+ "vmovdqu (%[temp]),%%ymm5 \n"
+ "vpsraw $2,%%ymm5,%%ymm5 \n"
+ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n"
+ "lea 0x20(%[temp]),%[temp] \n"
+ "vmovd %[temp],%%xmm6 \n"
+ YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA410_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
+#endif
+}
+#endif // defined(HAS_I410ALPHATOARGBROW_AVX2)
+
+#if defined(HAS_I410TOAR30ROW_AVX2)
+// 16 pixels
+// 16 UV values with 16 Y producing 16 AR30 (64 bytes).
+void OMITFP I410ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
+ YUVTORGB_SETUP_AVX2(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
+
+ LABELALIGN
+ "1: \n"
+ READYUV410_AVX2
+ YUVTORGB16_AVX2(yuvconstants)
+ STOREAR30_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [u_buf]"+r"(u_buf), // %[u_buf]
+ [v_buf]"+r"(v_buf), // %[v_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+rm"(width) // %[width]
: [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
: "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
}
-#endif // HAS_I422ALPHATOARGBROW_AVX2
+#endif // defined(HAS_I410TOAR30ROW_AVX2)
+
+#if defined(HAS_I444ALPHATOARGBROW_AVX2)
+// 16 pixels
+// 16 UV values with 16 Y and 16 A producing 16 ARGB.
+void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "vmovd %[temp],%%xmm7 \n"
+ "vmovd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovd %%xmm7,%[temp] \n"
+ "vmovdqu (%[u_buf]),%%xmm3 \n"
+ "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x10(%[u_buf]),%[u_buf] \n"
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n"
+ "vmovdqu (%[y_buf]),%%xmm4 \n"
+ "vpermq $0xd8,%%ymm4,%%ymm4 \n"
+ "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n"
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+ "vmovd %%xmm6,%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm5 \n"
+ "vpermq $0xd8,%%ymm5,%%ymm5 \n"
+ "lea 0x10(%[temp]),%[temp] \n"
+ "vmovd %[temp],%%xmm6 \n"
+ YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA444_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
+#endif
+}
+#endif // defined(HAS_I444ALPHATOARGBROW_AVX2)
+
+#if defined(HAS_I422ALPHATOARGBROW_AVX2)
+// 16 pixels
+// 8 UV values upsampled to 16 UV, mixed with 16 Y and 16 A producing 16 ARGB.
+void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+#if defined(__i386__)
+ uintptr_t temp;
+ asm volatile(
+ "mov %[v_buf],%[temp] \n"
+ "sub %[u_buf],%[temp] \n"
+ "vmovd %[temp],%%xmm7 \n"
+ "vmovd %[a_buf],%%xmm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovd %%xmm7,%[temp] \n"
+ "vmovq (%[u_buf]),%%xmm3 \n"
+ "vmovq 0x00(%[u_buf],%[temp],1),%%xmm1 \n"
+ "lea 0x8(%[u_buf]),%[u_buf] \n"
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n"
+ "vmovdqu (%[y_buf]),%%xmm4 \n"
+ "vpermq $0xd8,%%ymm4,%%ymm4 \n"
+ "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n"
+ "lea 0x10(%[y_buf]),%[y_buf] \n"
+ "vmovd %%xmm6,%[temp] \n"
+ "vmovdqu (%[temp]),%%xmm5 \n"
+ "vpermq $0xd8,%%ymm5,%%ymm5 \n"
+ "lea 0x10(%[temp]),%[temp] \n"
+ "vmovd %[temp],%%xmm6 \n"
+ YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [dst_argb] "+r"(dst_argb),
+ [temp] "=&r"(temp),
+ [width] "+m"(width)
+ : [v_buf] "m"(v_buf),
+ [a_buf] "m"(a_buf),
+ [yuvconstants] "r"(yuvconstants)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+#else
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
+
+ LABELALIGN "1: \n" READYUVA422_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
+#endif
+}
+#endif // defined(HAS_I422ALPHATOARGBROW_AVX2)
#if defined(HAS_I422TORGBAROW_AVX2)
// 16 pixels
@@ -2920,11 +6679,11 @@ void OMITFP I422ToRGBARow_AVX2(const uint8_t* y_buf,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
YUVTORGB_AVX2(yuvconstants)
@@ -2951,7 +6710,7 @@ void OMITFP I422ToRGBARow_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
);
}
-#endif // HAS_I422TORGBAROW_AVX2
+#endif // defined(HAS_I422TORGBAROW_AVX2)
#if defined(HAS_NV12TOARGBROW_AVX2)
// 16 pixels.
@@ -2961,30 +6720,23 @@ void OMITFP NV12ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- READNV12_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN "1: \n" READNV12_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
-#endif // HAS_NV12TOARGBROW_AVX2
+#endif // defined(HAS_NV12TOARGBROW_AVX2)
#if defined(HAS_NV21TOARGBROW_AVX2)
// 16 pixels.
@@ -2994,31 +6746,24 @@ void OMITFP NV21ToARGBRow_AVX2(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- READNV21_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [vu_buf]"+r"(vu_buf), // %[vu_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleNV21]"m"(kShuffleNV21)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN "1: \n" READNV21_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [vu_buf] "+r"(vu_buf), // %[vu_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleNV21] "m"(kShuffleNV21)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
-#endif // HAS_NV21TOARGBROW_AVX2
+#endif // defined(HAS_NV21TOARGBROW_AVX2)
#if defined(HAS_YUY2TOARGBROW_AVX2)
// 16 pixels.
@@ -3027,31 +6772,25 @@ void OMITFP YUY2ToARGBRow_AVX2(const uint8_t* yuy2_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(
+ "vbroadcasti128 %[kShuffleYUY2Y],%%ymm6 \n"
+ "vbroadcasti128 %[kShuffleYUY2UV],%%ymm7 \n" YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READYUY2_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [yuy2_buf]"+r"(yuy2_buf), // %[yuy2_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleYUY2Y]"m"(kShuffleYUY2Y),
- [kShuffleYUY2UV]"m"(kShuffleYUY2UV)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READYUY2_AVX2 YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [yuy2_buf] "+r"(yuy2_buf), // %[yuy2_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleYUY2Y] "m"(kShuffleYUY2Y), [kShuffleYUY2UV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5", "xmm6", "xmm7");
}
-#endif // HAS_YUY2TOARGBROW_AVX2
+#endif // defined(HAS_YUY2TOARGBROW_AVX2)
#if defined(HAS_UYVYTOARGBROW_AVX2)
// 16 pixels.
@@ -3060,54 +6799,170 @@ void OMITFP UYVYToARGBRow_AVX2(const uint8_t* uyvy_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
+ asm volatile(
+ "vbroadcasti128 %[kShuffleUYVYY],%%ymm6 \n"
+ "vbroadcasti128 %[kShuffleUYVYUV],%%ymm7 \n" YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN "1: \n" READUYVY_AVX2 YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [uyvy_buf] "+r"(uyvy_buf), // %[uyvy_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleUYVYY] "m"(kShuffleUYVYY), [kShuffleUYVYUV] "m"(kShuffleUYVYUV)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7");
+}
+#endif // defined(HAS_UYVYTOARGBROW_AVX2)
+
+#if defined(HAS_P210TOARGBROW_AVX2)
+// 16 pixels.
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
+void OMITFP P210ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN "1: \n" READP210_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // defined(HAS_P210TOARGBROW_AVX2)
+
+#if defined(HAS_P410TOARGBROW_AVX2)
+// 16 pixels.
+// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
+void OMITFP P410ToARGBRow_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN "1: \n" READP410_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // defined(HAS_P410TOARGBROW_AVX2)
+
+#if defined(HAS_P210TOAR30ROW_AVX2)
+// 16 pixels
+// 16 UV values with 16 Y producing 16 AR30 (64 bytes).
+void OMITFP P210ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
LABELALIGN
- "1: \n"
- READUYVY_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [uyvy_buf]"+r"(uyvy_buf), // %[uyvy_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ "1: \n"
+ READP210_AVX2
+ YUVTORGB16_AVX2(yuvconstants)
+ STOREAR30_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [uv_buf]"+r"(uv_buf), // %[uv_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
[width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleUYVYY]"m"(kShuffleUYVYY),
- [kShuffleUYVYUV]"m"(kShuffleUYVYUV)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
);
- // clang-format on
}
-#endif // HAS_UYVYTOARGBROW_AVX2
+#endif // defined(HAS_P210TOAR30ROW_AVX2)
-#ifdef HAS_I400TOARGBROW_SSE2
-void I400ToARGBRow_SSE2(const uint8_t* y_buf, uint8_t* dst_argb, int width) {
- asm volatile(
- "mov $0x4a354a35,%%eax \n" // 4a35 = 18997 = 1.164
- "movd %%eax,%%xmm2 \n"
- "pshufd $0x0,%%xmm2,%%xmm2 \n"
- "mov $0x04880488,%%eax \n" // 0488 = 1160 = 1.164 *
- // 16
- "movd %%eax,%%xmm3 \n"
- "pshufd $0x0,%%xmm3,%%xmm3 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "pslld $0x18,%%xmm4 \n"
+#if defined(HAS_P410TOAR30ROW_AVX2)
+// 16 pixels
+// 16 UV values with 16 Y producing 16 AR30 (64 bytes).
+void OMITFP P410ToAR30Row_AVX2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile (
+ YUVTORGB_SETUP_AVX2(yuvconstants)
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // all 1s
+ "vpsrlw $6,%%ymm5,%%ymm7 \n"
+ "vpsllw $4,%%ymm7,%%ymm7 \n" // 1023 * 16 for max
+ "vpsllw $14,%%ymm5,%%ymm5 \n" // 0xc000 for 2 alpha bits
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n" // 0 for min
- LABELALIGN
- "1: \n"
+ LABELALIGN
+ "1: \n"
+ READP410_AVX2
+ YUVTORGB16_AVX2(yuvconstants)
+ STOREAR30_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : [y_buf]"+r"(y_buf), // %[y_buf]
+ [uv_buf]"+r"(uv_buf), // %[uv_buf]
+ [dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
+ [width]"+rm"(width) // %[width]
+ : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2
+ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
+ );
+}
+#endif // defined(HAS_P410TOAR30ROW_AVX2)
+
+#ifdef HAS_I400TOARGBROW_SSE2
+void I400ToARGBRow_SSE2(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "movdqu 96(%3),%%xmm2 \n" // yg = 18997 = 1.164
+ "movdqu 128(%3),%%xmm3 \n" // ygb = 1160 = 1.164 * 16
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0xff000000
+ "pslld $0x18,%%xmm4 \n"
+
+ LABELALIGN
+ "1: \n"
// Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164
"movq (%0),%%xmm0 \n"
"lea 0x8(%0),%0 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
- "psubusw %%xmm3,%%xmm0 \n"
- "psrlw $6, %%xmm0 \n"
+ "paddsw %%xmm3,%%xmm0 \n"
+ "psraw $6, %%xmm0 \n"
"packuswb %%xmm0,%%xmm0 \n"
// Step 2: Weave into ARGB
@@ -3121,41 +6976,39 @@ void I400ToARGBRow_SSE2(const uint8_t* y_buf, uint8_t* dst_argb, int width) {
"movdqu %%xmm1,0x10(%1) \n"
"lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
+ "subl $0x8,%2 \n"
"jg 1b \n"
- : "+r"(y_buf), // %0
- "+r"(dst_argb), // %1
- "+rm"(width) // %2
- :
- : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+ : "+r"(y_buf), // %0
+ "+r"(dst_argb), // %1
+ "+rm"(width) // %2
+ : "r"(yuvconstants) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
#endif // HAS_I400TOARGBROW_SSE2
#ifdef HAS_I400TOARGBROW_AVX2
// 16 pixels of Y converted to 16 pixels of ARGB (64 bytes).
// note: vpunpcklbw mutates and vpackuswb unmutates.
-void I400ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, int width) {
+void I400ToARGBRow_AVX2(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile(
- "mov $0x4a354a35,%%eax \n" // 0488 = 1160 = 1.164 *
- // 16
- "vmovd %%eax,%%xmm2 \n"
- "vbroadcastss %%xmm2,%%ymm2 \n"
- "mov $0x4880488,%%eax \n" // 4a35 = 18997 = 1.164
- "vmovd %%eax,%%xmm3 \n"
- "vbroadcastss %%xmm3,%%ymm3 \n"
- "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpslld $0x18,%%ymm4,%%ymm4 \n"
+ "vmovdqu 96(%3),%%ymm2 \n" // yg = 18997 = 1.164
+ "vmovdqu 128(%3),%%ymm3 \n" // ygb = -1160 = 1.164*16
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" // 0xff000000
+ "vpslld $0x18,%%ymm4,%%ymm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
// Step 1: Scale Y contribution to 16 G values. G = (y - 16) * 1.164
"vmovdqu (%0),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
"vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsubusw %%ymm3,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x6,%%ymm0,%%ymm0 \n"
+ "vpaddsw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpsraw $0x6,%%ymm0,%%ymm0 \n"
"vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
"vpunpcklbw %%ymm0,%%ymm0,%%ymm1 \n"
"vpermq $0xd8,%%ymm1,%%ymm1 \n"
@@ -3165,15 +7018,15 @@ void I400ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, int width) {
"vpor %%ymm4,%%ymm1,%%ymm1 \n"
"vmovdqu %%ymm0,(%1) \n"
"vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
+ "lea 0x40(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
"vzeroupper \n"
- : "+r"(y_buf), // %0
- "+r"(dst_argb), // %1
- "+rm"(width) // %2
- :
- : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+ : "+r"(y_buf), // %0
+ "+r"(dst_argb), // %1
+ "+rm"(width) // %2
+ : "r"(yuvconstants) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
#endif // HAS_I400TOARGBROW_AVX2
@@ -3183,107 +7036,582 @@ static const uvec8 kShuffleMirror = {15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u,
7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
- intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("movdqa %3,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu -0x10(%0,%2,1),%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirror) // %3
+ : "memory", "cc", "xmm0", "xmm5");
+}
+#endif // HAS_MIRRORROW_SSSE3
- "movdqa %3,%%xmm5 \n"
+#if defined(HAS_MIRRORROW_AVX512BW) || \
+ defined(HAS_MIRRORSPLITUVROW_AVX512BW) || \
+ defined(HAS_MIRRORUVROW_AVX512BW)
+static const uint16_t kMirrorUVPermute_AVX512[32] = {
+ 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u,
+ 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u,
+ 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
+#endif // defined(HAS_MIRRORROW_AVX512BW) ||
+ // defined(HAS_MIRRORSPLITUVROW_AVX512BW) ||
+ // defined(HAS_MIRRORUVROW_AVX512BW)
+
+#ifdef HAS_MIRRORROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 %[shuf],%%zmm5 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu -0x10(%0,%2,1),%%xmm0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirror) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ "1: \n"
+ "vmovdqu8 (%[src],%[width],1),%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vshufi64x2 $0x1b,%%zmm0,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm1 \n"
+ "vpsllw $0x8,%%zmm0,%%zmm0 \n"
+ "vporq %%zmm1,%%zmm0,%%zmm0 \n"
+ "mov $0x40,%k[temp] \n"
+ "sub %k[width],%k[temp] \n"
+ "shr $0x1,%k[temp] \n"
+ "vpbroadcastw %k[temp],%%zmm1 \n"
+ "vmovdqu16 %[rev],%%zmm2 \n"
+ "vpsubw %%zmm1,%%zmm2,%%zmm1 \n"
+ "vpermw %%zmm0,%%zmm1,%%zmm0 \n"
+ "test $0x1,%b[width] \n"
+ "je 3f \n"
+ "valignq $0x2,%%zmm0,%%zmm0,%%zmm1 \n"
+ "vpalignr $0x1,%%zmm0,%%zmm1,%%zmm0 \n"
+ "3: \n"
+ "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst), [width] "+r"(temp_width), [temp] "=&r"(temp)
+ : [src] "r"(src), [shuf] "m"(kShuffleMirror),
+ [rev] "m"(kMirrorUVPermute_AVX512)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5", "k1");
}
-#endif // HAS_MIRRORROW_SSSE3
+#endif // HAS_MIRRORROW_AVX512BW
#ifdef HAS_MIRRORROW_AVX2
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
- intptr_t temp_width = (intptr_t)(width);
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("vbroadcasti128 %3,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu -0x20(%0,%2,1),%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0x4e,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirror) // %3
+ : "memory", "cc", "xmm0", "xmm5");
+}
+#endif // HAS_MIRRORROW_AVX2
+
+#if defined(HAS_MIRRORSPLITUVROW_SSSE3) || \
+ defined(HAS_MIRRORSPLITUVROW_AVX2) || \
+ defined(HAS_MIRRORSPLITUVROW_AVX512BW)
+// Shuffle table for reversing the bytes of UV channels.
+static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u,
+ 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u};
+#endif // defined(HAS_MIRRORSPLITUVROW_SSSE3) || ...
+
+#ifdef HAS_MIRRORSPLITUVROW_SSSE3
+void MirrorSplitUVRow_SSSE3(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
asm volatile(
+ "movdqa %4,%%xmm1 \n"
+ "sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu -0x10(%0,%3,2),%%xmm0 \n"
+ "pshufb %%xmm1,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "movhps %%xmm0,0x00(%1,%2,1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(temp_width) // %3
+ : "m"(kShuffleMirrorSplitUV) // %4
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_MIRRORSPLITUVROW_SSSE3
- "vbroadcastf128 %3,%%ymm5 \n"
+#ifdef HAS_MIRRORSPLITUVROW_AVX512BW
+static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1};
+
+LIBYUV_TARGET_AVX512BW
+void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 %[shuf],%%zmm1 \n"
+ "sub %1,%2 \n"
+ "vmovdqu64 %[perm],%%zmm3 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu -0x20(%0,%2,1),%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpermq $0x4e,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirror) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ "1: \n"
+ "vmovdqu8 (%0,%3,2),%%zmm0 \n"
+ "vpshufb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpermq %%zmm0,%%zmm3,%%zmm0 \n"
+ "vextracti64x4 $0x1,%%zmm0,%%ymm2 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm2,0x00(%1,%2,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q3,%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "mov $0x20,%k[temp] \n"
+ "sub %k3,%k[temp] \n"
+ "vpbroadcastw %k[temp],%%zmm2 \n"
+ "vmovdqu16 %[mirror_uv],%%zmm4 \n"
+ "vpsubw %%zmm2,%%zmm4,%%zmm2 \n"
+ "vpermw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(temp_width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [shuf] "m"(kShuffleMirrorSplitUV), // %[shuf]
+ [perm] "m"(kMirrorSplitUVPermute), // %[perm]
+ [mirror_uv] "m"(kMirrorUVPermute_AVX512) // %[mirror_uv]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1");
}
-#endif // HAS_MIRRORROW_AVX2
+#endif // HAS_MIRRORSPLITUVROW_AVX512BW
-#ifdef HAS_MIRRORUVROW_SSSE3
-// Shuffle table for reversing the bytes of UV channels.
-static const uvec8 kShuffleMirrorUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u,
- 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u};
-void MirrorUVRow_SSSE3(const uint8_t* src,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- intptr_t temp_width = (intptr_t)(width);
+#ifdef HAS_MIRRORSPLITUVROW_AVX2
+void MirrorSplitUVRow_AVX2(const uint8_t* src,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
asm volatile(
- "movdqa %4,%%xmm1 \n"
- "lea -0x10(%0,%3,2),%0 \n"
- "sub %1,%2 \n"
+ "vbroadcasti128 %4,%%ymm1 \n"
+ "lea -0x20(%0,%3,2),%0 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea -0x10(%0),%0 \n"
- "pshufb %%xmm1,%%xmm0 \n"
- "movlpd %%xmm0,(%1) \n"
- "movhpd %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $8,%3 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(temp_width) // %3
- : "m"(kShuffleMirrorUV) // %4
- : "memory", "cc", "xmm0", "xmm1");
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "lea -0x20(%0),%0 \n"
+ "vpshufb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0x72,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm2 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "vmovdqu %%xmm2,0x00(%1,%2,1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(temp_width) // %3
+ : "m"(kShuffleMirrorSplitUV) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_MIRRORSPLITUVROW_AVX2
+
+#ifdef HAS_MIRRORUVROW_SSSE3
+// Shuffle table for reversing the UV.
+static const uvec8 kShuffleMirrorUV = {14u, 15u, 12u, 13u, 10u, 11u, 8u, 9u,
+ 6u, 7u, 4u, 5u, 2u, 3u, 0u, 1u};
+
+void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("movdqa %3,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu -0x10(%0,%2,2),%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorUV) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORUVROW_SSSE3
-#ifdef HAS_ARGBMIRRORROW_SSE2
+#ifdef HAS_MIRRORUVROW_AVX2
+void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("vbroadcasti128 %3,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu -0x20(%0,%2,2),%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0x4e,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorUV) // %3
+ : "memory", "cc", "xmm0", "xmm5");
+}
+#endif // HAS_MIRRORUVROW_AVX2
-void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
- intptr_t temp_width = (intptr_t)(width);
+#ifdef HAS_MIRRORUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ uintptr_t temp;
asm volatile(
+ "vmovdqu16 %[rev],%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpermw (%[src_uv],%[width],2),%%zmm5,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst_uv]) \n"
+ "lea 0x40(%[dst_uv]),%[dst_uv] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k[width],%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%[src_uv]),%%zmm0%{%%k1%}%{z%}\n"
+ "mov $0x20,%k[temp] \n"
+ "sub %k[width],%k[temp] \n"
+ "vpbroadcastw %k[temp],%%zmm1 \n"
+ "vpsubw %%zmm1,%%zmm5,%%zmm1 \n"
+ "vpermw %%zmm0,%%zmm1,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst_uv])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst_uv] "+r"(dst_uv), [width] "+r"(temp_width), [temp] "=&r"(temp)
+ : [src_uv] "r"(src_uv), [rev] "m"(kMirrorUVPermute_AVX512)
+ : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1");
+}
+#endif // HAS_MIRRORUVROW_AVX512BW
+
+#ifdef HAS_RGB24MIRRORROW_SSSE3
+
+// Shuffle first 5 pixels to last 5 mirrored. first byte zero
+static const uvec8 kShuffleMirrorRGB0 = {128u, 12u, 13u, 14u, 9u, 10u, 11u, 6u,
+ 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
- "lea -0x10(%0,%2,4),%0 \n"
+// Shuffle last 5 pixels to first 5 mirrored. last byte zero
+static const uvec8 kShuffleMirrorRGB1 = {
+ 13u, 14u, 15u, 10u, 11u, 12u, 7u, 8u, 9u, 4u, 5u, 6u, 1u, 2u, 3u, 128u};
+
+// Shuffle 5 pixels at a time (15 bytes)
+void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ src_rgb24 += temp_width * 3 - 48;
+ asm volatile(
+ "movdqa %3,%%xmm4 \n"
+ "movdqa %4,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "pshufd $0x1b,%%xmm0,%%xmm0 \n"
- "lea -0x10(%0),%0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- :
- : "memory", "cc", "xmm0");
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // first 5
+ "movdqu 15(%0),%%xmm1 \n" // next 5
+ "movdqu 30(%0),%%xmm2 \n" // next 5
+ "movdqu 32(%0),%%xmm3 \n" // last 1 special
+ "pshufb %%xmm4,%%xmm0 \n"
+ "pshufb %%xmm4,%%xmm1 \n"
+ "pshufb %%xmm4,%%xmm2 \n"
+ "pshufb %%xmm5,%%xmm3 \n"
+ "lea -0x30(%0),%0 \n"
+ "movdqu %%xmm0,32(%1) \n" // last 5
+ "movdqu %%xmm1,17(%1) \n" // next 5
+ "movdqu %%xmm2,2(%1) \n" // next 5
+ "movlpd %%xmm3,0(%1) \n" // first 1
+ "lea 0x30(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorRGB0), // %3
+ "m"(kShuffleMirrorRGB1) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_RGB24MIRRORROW_SSSE3
+
+#ifdef HAS_RGB24MIRRORROW_AVX2
+// Shuffle first 10 pixels to last 10 mirrored. first byte zero
+static const uvec8 kShuffleMirrorRGB0_AVX = {
+ 128u, 12u, 13u, 14u, 9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
+
+// Shuffle last 2 pixels to first 2 mirrored. last byte zero
+static const uvec8 kShuffleMirrorRGB1_AVX = {
+ 13u, 14u, 15u, 10u, 11u, 12u, 7u, 8u, 9u, 4u, 5u, 6u, 1u, 2u, 3u, 128u};
+
+void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ src_rgb24 += temp_width * 3 - 96;
+ asm volatile(
+ "vbroadcasti128 %3,%%ymm4 \n"
+ "vmovdqa %4,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // first 10
+ "vinserti128 $1,15(%0),%%ymm0,%%ymm0 \n"
+ "vmovdqu 30(%0),%%xmm1 \n" // next 10
+ "vinserti128 $1,45(%0),%%ymm1,%%ymm1 \n"
+ "vmovdqu 60(%0),%%xmm2 \n" // next 10
+ "vinserti128 $1,75(%0),%%ymm2,%%ymm2 \n"
+ "vmovdqu 80(%0),%%xmm3 \n" // last 2 special
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpshufb %%xmm5,%%xmm3,%%xmm3 \n"
+ "lea -0x60(%0),%0 \n"
+ "vmovdqu %%xmm0,80(%1) \n"
+ "vextracti128 $1,%%ymm0,65(%1) \n"
+ "vmovdqu %%xmm1,50(%1) \n"
+ "vextracti128 $1,%%ymm1,35(%1) \n"
+ "vmovdqu %%xmm2,20(%1) \n"
+ "vextracti128 $1,%%ymm2,5(%1) \n"
+ "vmovq %%xmm3,0(%1) \n"
+ "lea 0x60(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorRGB0_AVX), // %3
+ "m"(kShuffleMirrorRGB1_AVX) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_RGB24MIRRORROW_AVX2
+
+#ifdef HAS_RGB24MIRRORROW_AVX512VBMI
+// Mirror 64 RGB24 pixels (192 bytes = 3 zmm) per loop. Loads and stores are
+// 3 full contiguous zmm vectors. Output vector 0 is a 2 source byte permute
+// of input vectors 1 and 2, and output vectors 1 and 2 are 2 source byte
+// permutes of input vectors 0 and 1. Output byte 65 (byte 1 of output
+// vector 1) is the only byte that comes from a third vector: byte 0 of input
+// vector 2, which is merged in with a masked broadcast.
+static const uint8_t kPermMirrorRGB24_0[64] = {
+ 125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u,
+ 118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u,
+ 105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u,
+ 92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u,
+ 85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u,
+ 72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u};
+static const uint8_t kPermMirrorRGB24_1[64] = {
+ 127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u,
+ 114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u,
+ 107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u,
+ 94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u,
+ 81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u,
+ 74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u};
+static const uint8_t kPermMirrorRGB24_2[64] = {
+ 65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u,
+ 48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u,
+ 37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u,
+ 26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u,
+ 9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
+
+// Byte mask with the low n bits set, clamped to [0, 64].
+static inline uint64_t RGB24MirrorMask(int n) {
+ return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull;
+}
+
+LIBYUV_TARGET_AVX512VBMI
+void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ int r = width & 63;
+ ptrdiff_t temp_width = (ptrdiff_t)(width - r);
+ if (temp_width > 0) {
+ const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192;
+ asm volatile(
+ "vmovdqu8 %3,%%zmm20 \n"
+ "vmovdqu8 %4,%%zmm21 \n"
+ "vmovdqu8 %5,%%zmm22 \n"
+ "kmovq %6,%%k1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n" // last 64 pixels
+ "vmovdqu8 0x40(%0),%%zmm1 \n"
+ "vmovdqu8 0x80(%0),%%zmm2 \n"
+ "lea -0xc0(%0),%0 \n"
+ "vmovdqa64 %%zmm20,%%zmm3 \n"
+ "vmovdqa64 %%zmm21,%%zmm4 \n"
+ "vmovdqa64 %%zmm22,%%zmm5 \n"
+ "vpermi2b %%zmm2,%%zmm1,%%zmm3 \n" // out0 from in1:in2
+ "vpermi2b %%zmm1,%%zmm0,%%zmm4 \n" // out1 from in0:in1
+ "vpermi2b %%zmm1,%%zmm0,%%zmm5 \n" // out2 from in0:in1
+ "vpbroadcastb %%xmm2,%%zmm4%{%%k1%} \n" // out1 byte 1 = in2[0]
+ "vmovdqu8 %%zmm3,(%1) \n"
+ "vmovdqu8 %%zmm4,0x40(%1) \n"
+ "vmovdqu8 %%zmm5,0x80(%1) \n"
+ "lea 0xc0(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(temp_width) // %2
+ : "m"(kPermMirrorRGB24_0), // %3
+ "m"(kPermMirrorRGB24_1), // %4
+ "m"(kPermMirrorRGB24_2), // %5
+ "r"((uint64_t)2) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm20", "xmm21", "xmm22", "k1");
+ }
+ // Remaining 1 to 63 pixels are the first r pixels of the source. Masked
+ // loads and stores access only the 3 * r valid bytes. The tables, as
+ // global byte offsets into a 192 byte block, are adjusted by the 192 - 3 * r
+ // missing bytes. Bytes from the third vector (index >= 128) are selected
+ // with a mask from the sign bit of the index.
+ if (r) {
+ int nb = r * 3;
+ int off = 192 - nb;
+ asm volatile(
+ "kmovq %2,%%k1 \n"
+ "kmovq %3,%%k2 \n"
+ "kmovq %4,%%k3 \n"
+ "vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 0x40(%0),%%zmm1%{%%k2%}%{z%}\n"
+ "vmovdqu8 0x80(%0),%%zmm2%{%%k3%}%{z%}\n"
+ "vpbroadcastb %5,%%zmm6 \n" // off
+ "vpbroadcastb %6,%%zmm7 \n" // off - 64
+ "vmovdqu8 %7,%%zmm3 \n"
+ "vmovdqu8 %8,%%zmm4 \n"
+ "vmovdqu8 %9,%%zmm5 \n"
+ "vpsubb %%zmm7,%%zmm3,%%zmm3 \n"
+ "vpsubb %%zmm6,%%zmm4,%%zmm4 \n"
+ "vpsubb %%zmm6,%%zmm5,%%zmm5 \n"
+
+ "vpmovb2m %%zmm3,%%k4 \n" // out0
+ "vmovdqa64 %%zmm3,%%zmm6 \n"
+ "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
+ "vpermb %%zmm2,%%zmm3,%%zmm6%{%%k4%}\n"
+ "vmovdqu8 %%zmm6,(%1)%{%%k1%} \n"
+ "vpmovb2m %%zmm4,%%k4 \n" // out1
+ "vmovdqa64 %%zmm4,%%zmm6 \n"
+ "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
+ "vpermb %%zmm2,%%zmm4,%%zmm6%{%%k4%}\n"
+ "vmovdqu8 %%zmm6,0x40(%1)%{%%k2%} \n"
+ "vpmovb2m %%zmm5,%%k4 \n" // out2
+ "vmovdqa64 %%zmm5,%%zmm6 \n"
+ "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
+ "vpermb %%zmm2,%%zmm5,%%zmm6%{%%k4%}\n"
+ "vmovdqu8 %%zmm6,0x80(%1)%{%%k3%} \n"
+ "vzeroupper \n"
+ :
+ : "r"(src_rgb24), // %0
+ "r"(dst_rgb24), // %1
+ "r"(RGB24MirrorMask(nb)), // %2
+ "r"(RGB24MirrorMask(nb - 64)), // %3
+ "r"(RGB24MirrorMask(nb - 128)), // %4
+ "r"(off), // %5
+ "r"(off - 64), // %6
+ "m"(kPermMirrorRGB24_0), // %7
+ "m"(kPermMirrorRGB24_1), // %8
+ "m"(kPermMirrorRGB24_2) // %9
+ : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "k1", "k2", "k3", "k4");
+ }
+}
+#endif // HAS_RGB24MIRRORROW_AVX512VBMI
+
+#ifdef HAS_ARGBMIRRORROW_SSE2
+
+void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("lea -0x10(%0,%2,4),%0 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "pshufd $0x1b,%%xmm0,%%xmm0 \n"
+ "lea -0x10(%0),%0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ :
+ : "memory", "cc", "xmm0");
}
#endif // HAS_ARGBMIRRORROW_SSE2
@@ -3291,26 +7619,73 @@ void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
// Shuffle table for reversing the bytes.
static const ulvec32 kARGBShuffleMirror_AVX2 = {7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
- intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ asm volatile("vmovdqu %3,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpermd -0x20(%0,%2,4),%%ymm5,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kARGBShuffleMirror_AVX2) // %3
+ : "memory", "cc", "xmm0", "xmm5");
+}
+#endif // HAS_ARGBMIRRORROW_AVX2
- "vmovdqu %3,%%ymm5 \n"
+#ifdef HAS_ARGBMIRRORROW_AVX512BW
+static const uint32_t kARGBShuffleMirror_AVX512[16] = {
+ 15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
+
+// Mirror 16 ARGB pixels per loop. The remaining 1 to 15 pixels are at the
+// start of src. They are loaded with a k mask, permuted with the reverse
+// index minus (16 - remaining), and stored with the same k mask.
+LIBYUV_TARGET_AVX512BW
+void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ptrdiff_t temp_width = (ptrdiff_t)(width);
+ uintptr_t temp;
+ asm volatile(
+ "vmovdqu32 %4,%%zmm5 \n"
+ "sub $0x10,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vpermd -0x20(%0,%2,4),%%ymm5,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kARGBShuffleMirror_AVX2) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ "1: \n"
+ "vpermd (%0,%2,4),%%zmm5,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovw %k3,%%k1 \n"
+ "mov $0x10,%k3 \n"
+ "sub %k2,%k3 \n"
+ "vpbroadcastd %k3,%%zmm1 \n"
+ "vpsubd %%zmm1,%%zmm5,%%zmm1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpermd %%zmm0,%%zmm1,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width), // %2
+ "=&r"(temp) // %3
+ : "m"(kARGBShuffleMirror_AVX512) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1");
}
-#endif // HAS_ARGBMIRRORROW_AVX2
+#endif // HAS_ARGBMIRRORROW_AVX512BW
#ifdef HAS_SPLITUVROW_AVX2
void SplitUVRow_AVX2(const uint8_t* src_uv,
@@ -3318,29 +7693,29 @@ void SplitUVRow_AVX2(const uint8_t* src_uv,
uint8_t* dst_v,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
- "sub %1,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm2 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm3 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm3,%%ymm2,%%ymm2 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm2,%%ymm2 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm2,0x00(%1,%2,1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm2 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm3 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm2,0x00(%1,%2,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -3350,34 +7725,100 @@ void SplitUVRow_AVX2(const uint8_t* src_uv,
}
#endif // HAS_SPLITUVROW_AVX2
+#ifdef HAS_SPLITUVROW_AVX512BW
+static const uint64_t kSplitUVPermute[8] = {0, 2, 4, 6, 1, 3, 5, 7};
+
+LIBYUV_TARGET_AVX512BW
+void SplitUVRow_AVX512BW(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $0x8,%%zmm5,%%zmm5 \n"
+ "vmovdqu64 %[perm],%%zmm4 \n"
+ "sub %[dst_u],%[dst_v] \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_uv]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src_uv]),%%zmm1 \n"
+ "lea 0x80(%[src_uv]),%[src_uv] \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm1,%%zmm3 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpermq %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpermq %%zmm2,%%zmm4,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u]) \n"
+ "vmovdqu8 %%zmm2,(%[dst_u],%[dst_v],1) \n"
+ "lea 0x40(%[dst_u]),%[dst_u] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "kshiftrq $32,%%k1,%%k2 \n"
+ "vmovdqu16 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm2 \n"
+ "vpsrlw $0x8,%%zmm1,%%zmm3 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpermq %%zmm0,%%zmm4,%%zmm0 \n"
+ "vpermq %%zmm2,%%zmm4,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n"
+ "vmovdqu8 %%zmm2,(%[dst_u],%[dst_v],1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_uv] "+r"(src_uv), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [perm] "m"(kSplitUVPermute)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1",
+ "k2");
+}
+#endif // HAS_SPLITUVROW_AVX512BW
+
#ifdef HAS_SPLITUVROW_SSE2
void SplitUVRow_SSE2(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
- "sub %1,%2 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "movdqa %%xmm1,%%xmm3 \n"
- "pand %%xmm5,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "psrlw $0x8,%%xmm2 \n"
- "psrlw $0x8,%%xmm3 \n"
- "packuswb %%xmm3,%%xmm2 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm2,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "psrlw $0x8,%%xmm2 \n"
+ "psrlw $0x8,%%xmm3 \n"
+ "packuswb %%xmm3,%%xmm2 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm2,0x00(%1,%2,1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -3387,36 +7828,287 @@ void SplitUVRow_SSE2(const uint8_t* src_uv,
}
#endif // HAS_SPLITUVROW_SSE2
-#ifdef HAS_MERGEUVROW_AVX2
-void MergeUVRow_AVX2(const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uv,
- int width) {
+#ifdef HAS_DETILEROW_AVX2
+void DetileRow_AVX2(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width) {
asm volatile(
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vinserti128 $0x1,(%0,%3,1),%%ymm0,%%ymm0 \n"
+ "lea (%0,%3,2),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride) // %3
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_DETILEROW_AVX2
- "sub %0,%1 \n"
+#ifdef HAS_DETILEROW_16_AVX2
+void DetileRow_16_AVX2(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width) {
+ asm volatile(
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x00(%0,%1,1),%%ymm1 \n"
- "lea 0x20(%0),%0 \n"
- "vpunpcklbw %%ymm1,%%ymm0,%%ymm2 \n"
- "vpunpckhbw %%ymm1,%%ymm0,%%ymm0 \n"
- "vextractf128 $0x0,%%ymm2,(%2) \n"
- "vextractf128 $0x0,%%ymm0,0x10(%2) \n"
- "vextractf128 $0x1,%%ymm2,0x20(%2) \n"
- "vextractf128 $0x1,%%ymm0,0x30(%2) \n"
- "lea 0x40(%2),%2 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_u), // %0
- "+r"(src_v), // %1
- "+r"(dst_uv), // %2
- "+r"(width) // %3
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu (%0,%3,2),%%ymm1 \n"
+ "lea (%0,%3,4),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride) // %3
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_DETILEROW_16_AVX2
+
+#ifdef HAS_DETILETOYUY2_AVX2
+// Read 32 Y, 16 UV, and write 16 YUYV (64 bytes).
+void DetileToYUY2_AVX2(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vinserti128 $0x1,(%0,%4,1),%%ymm0,%%ymm0 \n"
+ "lea (%0,%4,2),%0 \n"
+ "vmovdqu (%1),%%xmm1 \n"
+ "vinserti128 $0x1,(%1,%5,1),%%ymm1,%%ymm1 \n"
+ "lea (%1,%5,2),%1 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm3 \n"
+ "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "vmovdqu %%ymm1,0x20(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "subl $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_uv), // %1
+ "+r"(dst_yuy2), // %2
+ "+rm"(width) // %3
+ : "r"(src_y_tile_stride), // %4
+ "r"(src_uv_tile_stride) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_DETILETOYUY2_AVX2
+
+#ifdef HAS_UNPACKMT2T_AVX2
+void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) {
+ static const uint32_t kMaskC0 = 0xc0c0c0c0u;
+ if (size >= 80) {
+ asm volatile(
+ "vpbroadcastd %[c0],%%ymm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%xmm0 \n"
+ "vpsllw $2,%%xmm0,%%xmm1 \n"
+ "vinserti128 $0x1,%%xmm0,%%ymm1,%%ymm1 \n"
+ "vpsllw $4,%%ymm1,%%ymm0 \n"
+ "vpand %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x10(%[src]),%%ymm2 \n"
+ "vmovdqu 0x30(%[src]),%%ymm3 \n"
+ "lea 0x50(%[src]),%[src] \n"
+ "vpsrlw $2,%%ymm2,%%ymm4 \n"
+ "vpsrlw $2,%%ymm3,%%ymm5 \n"
+ "vpandn %%ymm4,%%ymm6,%%ymm4 \n"
+ "vpandn %%ymm5,%%ymm6,%%ymm5 \n"
+ "vpor %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm5,%%ymm1,%%ymm1 \n"
+
+ "vpunpcklbw %%ymm2,%%ymm0,%%ymm4 \n"
+ "vpunpckhbw %%ymm2,%%ymm0,%%ymm5 \n"
+ "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "vmovdqu %%ymm2,0x20(%[dst]) \n"
+
+ "vpunpcklbw %%ymm3,%%ymm1,%%ymm4 \n"
+ "vpunpckhbw %%ymm3,%%ymm1,%%ymm5 \n"
+ "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm1 \n"
+ "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm3 \n"
+ "vmovdqu %%ymm1,0x40(%[dst]) \n"
+ "vmovdqu %%ymm3,0x60(%[dst]) \n"
+
+ "lea 0x80(%[dst]),%[dst] \n"
+ "sub $0x50,%[size] \n"
+ "cmp $0x50,%[size] \n"
+ "jae 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [size] "+r"(size)
+ : [c0] "m"(kMaskC0)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
+ }
+}
+#endif // HAS_UNPACKMT2T_AVX2
+
+#ifdef HAS_DETILESPLITUVROW_AVX2
+static const uvec8 kDeinterlaceUV = {0, 2, 4, 6, 8, 10, 12, 14,
+ 1, 3, 5, 7, 9, 11, 13, 15};
+
+void DetileSplitUVRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 %4,%%ymm1 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vinserti128 $0x1,(%0,%5,1),%%ymm0,%%ymm0 \n"
+ "lea (%0,%5,2),%0 \n"
+ "vpshufb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "vextracti128 $0x1,%%ymm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vpshufb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovq %%xmm0,(%1) \n"
+ "vmovhps %%xmm0,(%2) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "m"(kDeinterlaceUV), // %4
+ "r"(src_tile_stride) // %5
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_DETILESPLITUVROW_AVX2
+
+#ifdef HAS_MERGEUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MergeUVRow_AVX512BW(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%zmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n"
+ "vporq %%zmm0,%%zmm1,%%zmm2 \n"
+ "vmovdqu16 %%zmm2,(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k3,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%zmm1%{%%k1%}%{z%} \n"
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n"
+ "vporq %%zmm0,%%zmm1,%%zmm2 \n"
+ "vmovdqu16 %%zmm2,(%2)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
:
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1");
+}
+#endif // HAS_MERGEUVROW_AVX512BW
+
+#ifdef HAS_MERGEUVROW_AVX2
+void MergeUVRow_AVX2(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile("sub %0,%1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%ymm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm0,%%ymm1,%%ymm2 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_MERGEUVROW_AVX2
@@ -3425,78 +8117,185 @@ void MergeUVRow_SSE2(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
- asm volatile(
-
- "sub %0,%1 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%1,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "punpcklbw %%xmm1,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm2 \n"
- "movdqu %%xmm0,(%2) \n"
- "movdqu %%xmm2,0x10(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_u), // %0
- "+r"(src_v), // %1
- "+r"(dst_uv), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ asm volatile("sub %0,%1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x00(%0,%1,1),%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklbw %%xmm1,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm2 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "movdqu %%xmm2,0x10(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_MERGEUVROW_SSE2
-// Use scale to convert lsb formats to msb, depending how many bits there are:
-// 128 = 9 bits
-// 64 = 10 bits
-// 16 = 12 bits
-// 1 = 16 bits
#ifdef HAS_MERGEUVROW_16_AVX2
void MergeUVRow_16_AVX2(const uint16_t* src_u,
const uint16_t* src_v,
uint16_t* dst_uv,
- int scale,
+ int depth,
int width) {
- // clang-format off
- asm volatile (
- "vmovd %4,%%xmm3 \n"
- "vpunpcklwd %%xmm3,%%xmm3,%%xmm3 \n"
- "vbroadcastss %%xmm3,%%ymm3 \n"
- "sub %0,%1 \n"
+ asm volatile(
+ "vmovd %4,%%xmm3 \n"
+ "vmovd %5,%%xmm4 \n"
- // 16 pixels per loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu (%0,%1,1),%%ymm1 \n"
- "add $0x20,%0 \n"
-
- "vpmullw %%ymm3,%%ymm0,%%ymm0 \n"
- "vpmullw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpunpcklwd %%ymm1,%%ymm0,%%ymm2 \n" // mutates
- "vpunpckhwd %%ymm1,%%ymm0,%%ymm0 \n"
- "vextractf128 $0x0,%%ymm2,(%2) \n"
- "vextractf128 $0x0,%%ymm0,0x10(%2) \n"
- "vextractf128 $0x1,%%ymm2,0x20(%2) \n"
- "vextractf128 $0x1,%%ymm0,0x30(%2) \n"
- "add $0x40,%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_u), // %0
- "+r"(src_v), // %1
- "+r"(dst_uv), // %2
- "+r"(width) // %3
- : "r"(scale) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
- // clang-format on
+ "sub %0,%1 \n"
+ // 8 pixels per loop.
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%0),%%ymm0 \n"
+ "vpmovzxwd 0x00(%0,%1,1),%%ymm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpsllw %%xmm3,%%ymm0,%%ymm0 \n"
+ "vpslld %%xmm4,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm0,%%ymm1,%%ymm2 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(16 - depth), // %4
+ "r"(32 - depth) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
-#endif // HAS_MERGEUVROW_AVX2
+#endif // HAS_MERGEUVROW_16_AVX2
+
+#ifdef HAS_MERGEUVROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MergeUVRow_16_AVX512BW(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vmovd %[shift_u],%%xmm3 \n"
+ "vmovd %[shift_v],%%xmm4 \n"
+ "sub %0,%1 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%0),%%zmm0 \n"
+ "vpmovzxwd 0x20(%0),%%zmm2 \n"
+ "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n"
+ "vpmovzxwd 0x20(%0,%1,1),%%zmm5 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsllw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vpsllw %%xmm3,%%zmm2,%%zmm2 \n"
+ "vpslld %%xmm4,%%zmm1,%%zmm1 \n"
+ "vpslld %%xmm4,%%zmm5,%%zmm5 \n"
+ "vpord %%zmm0,%%zmm1,%%zmm0 \n"
+ "vpord %%zmm2,%%zmm5,%%zmm2 \n"
+ "vmovdqu32 %%zmm0,(%2) \n"
+ "vmovdqu32 %%zmm2,0x40(%2) \n"
+ "lea 0x80(%2),%2 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "cmp $0x10,%3 \n"
+ "jl 3f \n"
+ "vpmovzxwd (%0),%%zmm0 \n"
+ "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpsllw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vpslld %%xmm4,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm0,%%zmm1,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k3,%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vpmovzxwd (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmovzxwd 0x00(%0,%1,1),%%zmm1%{%%k1%}%{z%} \n"
+ "vpsllw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vpslld %%xmm4,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm0,%%zmm1,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%2)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [shift_u] "r"(16 - depth), // %[shift_u]
+ [shift_v] "r"(32 - depth) // %[shift_v]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1");
+}
+#endif // HAS_MERGEUVROW_16_AVX512BW
+
+#ifdef HAS_SPLITUVROW_16_AVX2
+const uvec8 kSplitUVShuffle16 = {0, 1, 4, 5, 8, 9, 12, 13,
+ 2, 3, 6, 7, 10, 11, 14, 15};
+void SplitUVRow_16_AVX2(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ depth = 16 - depth;
+ asm volatile(
+ "vmovd %4,%%xmm3 \n"
+ "vbroadcasti128 %5,%%ymm4 \n"
+ "sub %1,%2 \n"
+
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "add $0x40,%0 \n"
+
+ "vpsrlw %%xmm3,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm3,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vextractf128 $0x0,%%ymm0,(%1) \n"
+ "vextractf128 $0x0,%%ymm1,0x10(%1) \n"
+ "vextractf128 $0x1,%%ymm0,(%1,%2) \n"
+ "vextractf128 $0x1,%%ymm1,0x10(%1,%2) \n"
+ "add $0x20,%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(depth), // %4
+ "m"(kSplitUVShuffle16) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SPLITUVROW_16_AVX2
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 128 = 9 bits
@@ -3508,212 +8307,903 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
- "vmovd %3,%%xmm3 \n"
- "vpunpcklwd %%xmm3,%%xmm3,%%xmm3 \n"
- "vbroadcastss %%xmm3,%%ymm3 \n"
- "sub %0,%1 \n"
+ asm volatile(
+ "vmovd %3,%%xmm3 \n"
+ "vpbroadcastw %%xmm3,%%ymm3 \n"
+ "sub %0,%1 \n"
- // 16 pixels per loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vpmullw %%ymm3,%%ymm0,%%ymm0 \n"
- "vpmullw %%ymm3,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%0,%1) \n"
- "vmovdqu %%ymm1,0x20(%0,%1) \n"
- "add $0x40,%0 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm3");
- // clang-format on
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vpmullw (%0),%%ymm3,%%ymm0 \n"
+ "vpmullw 0x20(%0),%%ymm3,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%0,%1) \n"
+ "vmovdqu %%ymm1,0x20(%0,%1) \n"
+ "add $0x40,%0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm3");
}
#endif // HAS_MULTIPLYROW_16_AVX2
-// Use scale to convert lsb formats to msb, depending how many bits there are:
-// 32768 = 9 bits
-// 16384 = 10 bits
-// 4096 = 12 bits
-// 256 = 16 bits
-void Convert16To8Row_SSSE3(const uint16_t* src_y,
- uint8_t* dst_y,
- int scale,
- int width) {
- // clang-format off
- asm volatile (
- "movd %3,%%xmm2 \n"
- "punpcklwd %%xmm2,%%xmm2 \n"
- "pshufd $0x0,%%xmm2,%%xmm2 \n"
+#ifdef HAS_MULTIPLYROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[scale],%%zmm2 \n"
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "add $0x20,%0 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "add $0x10,%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vpmullw (%0),%%zmm2,%%zmm0 \n"
+ "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "vmovdqu16 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vpmullw (%0),%%zmm2,%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [scale] "r"(scale) // %[scale]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1");
}
+#endif // HAS_MULTIPLYROW_16_AVX512BW
-#ifdef HAS_CONVERT16TO8ROW_AVX2
+// Use scale to convert msb formats to lsb, depending how many bits there are:
+// 512 = 9 bits
+// 1024 = 10 bits
+// 4096 = 12 bits
+// 65536 = 16 bits
+#ifdef HAS_DIVIDEROW_16_AVX2
+void DivideRow_16_AVX2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ asm volatile(
+ "vmovd %3,%%xmm3 \n"
+ "vpbroadcastw %%xmm3,%%ymm3 \n"
+ "sub %0,%1 \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpmulhuw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%0,%1) \n"
+ "vmovdqu %%ymm1,0x20(%0,%1) \n"
+ "add $0x40,%0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ "+r"(scale) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm3");
+}
+#endif // HAS_DIVIDEROW_16_AVX2
+
+#ifdef HAS_DIVIDEROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void DivideRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[scale],%%zmm2 \n"
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vpmulhuw (%0),%%zmm2,%%zmm0 \n"
+ "vpmulhuw 0x40(%0),%%zmm2,%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "vmovdqu16 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vpmulhuw (%0),%%zmm2,%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [scale] "r"(scale) // %[scale]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1");
+}
+#endif // HAS_DIVIDEROW_16_AVX512BW
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits
+// 16384 = 10 bits
+// 4096 = 12 bits
+// 256 = 16 bits
+#ifdef HAS_CONVERT16TO8ROW_SSSE3
+void Convert16To8Row_SSSE3(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ asm volatile(
+ "movd %3,%%xmm2 \n"
+ "punpcklwd %%xmm2,%%xmm2 \n"
+ "pshufd $0x0,%%xmm2,%%xmm2 \n"
+
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "add $0x20,%0 \n"
+ "pmulhuw %%xmm2,%%xmm0 \n"
+ "pmulhuw %%xmm2,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "add $0x10,%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_CONVERT16TO8ROW_SSSE3
+
+#ifdef HAS_CONVERT16TO8ROW_AVX2
void Convert16To8Row_AVX2(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
- "vmovd %3,%%xmm2 \n"
- "vpunpcklwd %%xmm2,%%xmm2,%%xmm2 \n"
- "vbroadcastss %%xmm2,%%ymm2 \n"
+ asm volatile(
+ "vmovd %3,%%xmm2 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "add $0x40,%0 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // mutates
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "add $0x20,%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "add $0x40,%0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "add $0x20,%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_CONVERT16TO8ROW_AVX2
+#ifdef HAS_CONVERT16TO8ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert16To8Row_AVX512BW(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[scale],%%zmm2 \n"
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "vmovdqu16 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpmovuswb %%zmm0,%%ymm0 \n"
+ "vpmovuswb %%zmm1,%%ymm1 \n"
+ "vmovdqu8 %%ymm0,(%1) \n"
+ "vmovdqu8 %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmovuswb %%zmm0,%%ymm0 \n"
+ "vmovdqu8 %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmovuswb %%zmm0,%%ymm0 \n"
+ "vmovdqu8 %%ymm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [scale] "r"(scale) // %[scale]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1");
+}
+#endif // HAS_CONVERT16TO8ROW_AVX512BW
+
+#ifdef HAS_HALFROW_16TO8_SSSE3
+void HalfRow_16To8_SSSE3(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ asm volatile(
+ "movd %4,%%xmm2 \n"
+ "punpcklwd %%xmm2,%%xmm2 \n"
+ "pshufd $0x0,%%xmm2,%%xmm2 \n"
+
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu (%0,%3,2),%%xmm3 \n"
+ "movdqu 0x10(%0,%3,2),%%xmm4 \n"
+ "add $0x20,%0 \n"
+ "pavgw %%xmm3,%%xmm0 \n"
+ "pavgw %%xmm4,%%xmm1 \n"
+ "pmulhuw %%xmm2,%%xmm0 \n"
+ "pmulhuw %%xmm2,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "add $0x10,%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride), // %3
+ "r"(scale) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_HALFROW_16TO8_SSSE3
+
+#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3
+static const uvec8 kShuffleMaskHalfWidth16To8 = {
+ 0u, 1u, 4u, 5u, 8u, 9u, 12u, 13u,
+ 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+
+void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ asm volatile(
+ "movd %4,%%xmm2 \n"
+ "punpcklwd %%xmm2,%%xmm2 \n"
+ "pshufd $0x0,%%xmm2,%%xmm2 \n"
+ "pcmpeqd %%xmm5,%%xmm5 \n"
+ "pcmpeqw %%xmm6,%%xmm6 \n"
+ "psrld $16,%%xmm5 \n"
+ "pmaddwd %%xmm6,%%xmm6 \n"
+ "movdqa %5,%%xmm7 \n"
+
+ // 8 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu (%0,%3,2),%%xmm3 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "movdqa %%xmm3,%%xmm4 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "psrld $16,%%xmm1 \n"
+ "pand %%xmm5,%%xmm3 \n"
+ "psrld $16,%%xmm4 \n"
+ "paddd %%xmm1,%%xmm0 \n"
+ "paddd %%xmm4,%%xmm3 \n"
+ "paddd %%xmm3,%%xmm0 \n"
+ "paddd %%xmm6,%%xmm0 \n"
+ "psrld $2,%%xmm0 \n"
+ "pshufb %%xmm7,%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x10(%0,%3,2),%%xmm3 \n"
+ "add $0x20,%0 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "psrld $16,%%xmm4 \n"
+ "paddd %%xmm4,%%xmm1 \n"
+ "movdqa %%xmm3,%%xmm4 \n"
+ "pand %%xmm5,%%xmm3 \n"
+ "psrld $16,%%xmm4 \n"
+ "paddd %%xmm4,%%xmm3 \n"
+ "paddd %%xmm3,%%xmm1 \n"
+ "paddd %%xmm6,%%xmm1 \n"
+ "psrld $2,%%xmm1 \n"
+ "pshufb %%xmm7,%%xmm1 \n"
+ "punpcklqdq %%xmm1,%%xmm0 \n"
+ "pmulhuw %%xmm2,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "add $0x08,%1 \n"
+ "sub $0x08,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride), // %3
+ "rm"(scale), // %4
+ "m"(kShuffleMaskHalfWidth16To8) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_HALFWIDTHROW_16TO8_SSSE3
+
+#ifdef HAS_HALFROW_16TO8_AVX2
+void HalfRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ asm volatile(
+ "vmovd %4,%%xmm2 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n"
+ "vpavgw 0x20(%0,%3,2),%%ymm1,%%ymm1 \n"
+ "add $0x40,%0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "add $0x20,%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride), // %3
+ "rm"(scale) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_HALFROW_16TO8_AVX2
+
+#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
+void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ asm volatile(
+ "vmovd %4,%%xmm2 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
+ "vpcmpeqw %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $15,%%ymm5,%%ymm5 \n"
+ "vpsllw $1,%%ymm5,%%ymm6 \n"
+ "vpsllw $15,%%ymm5,%%ymm7 \n"
+
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu (%0,%3,2),%%ymm3 \n"
+ "vmovdqu 0x20(%0,%3,2),%%ymm4 \n"
+ "add $0x40,%0 \n"
+ "vpxor %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpxor %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpxor %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpxor %%ymm7,%%ymm4,%%ymm4 \n"
+ "vpmaddwd %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpmaddwd %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpmaddwd %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpmaddwd %%ymm5,%%ymm4,%%ymm4 \n"
+ "vpaddd %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpsrld $2,%%ymm0,%%ymm0 \n"
+ "vpsrld $2,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "add $0x10,%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride), // %3
+ "rm"(scale) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_HALFWIDTHROW_16TO8_AVX2
+
+#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ int n = (width - 1) & ~15;
+ int r = (width - 1) & 15;
+ SIMD_ALIGNED(uint16_t pair[64]);
+ SIMD_ALIGNED(uint8_t out[16]);
+ if (n > 0) {
+ HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n);
+ src_uv += n * 2;
+ dst_uv += n;
+ }
+ if (r > 0) {
+ HalfWidthRow_16To8_Any_AVX2(src_uv, src_uv_stride, dst_uv, scale, r);
+ src_uv += r * 2;
+ dst_uv += r;
+ }
+ memset(pair, 0, sizeof(pair));
+ pair[0] = src_uv[0];
+ pair[1] = src_uv[0];
+ pair[32] = src_uv[src_uv_stride];
+ pair[33] = src_uv[src_uv_stride];
+ HalfWidthRow_16To8_AVX2(pair, 32, out, scale, 16);
+ dst_uv[0] = out[0];
+}
+#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2
+
+#ifdef HAS_HALFROW_AVX2
+void HalfRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpavgb (%0,%3,1),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride) // %3
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_HALFROW_AVX2
+
+#ifdef HAS_HALFROW_16_AVX2
+void HalfRow_16_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint16_t* dst_uv,
+ int width) {
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(src_uv_stride) // %3
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_HALFROW_16_AVX2
+
+#ifdef HAS_HALFROW_16TO8_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[scale],%%zmm2 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu64 (%[src0]),%%zmm0 \n"
+ "vmovdqu64 0x40(%[src0]),%%zmm1 \n"
+ "vpavgw (%[src1]),%%zmm0,%%zmm0 \n"
+ "vpavgw 0x40(%[src1]),%%zmm1,%%zmm1 \n"
+ "add $0x80,%[src0] \n"
+ "add $0x80,%[src1] \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpmovuswb %%zmm0,(%[dst]) \n"
+ "vpmovuswb %%zmm1,0x20(%[dst]) \n"
+ "add $0x40,%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%q[temp] \n"
+ "bzhi %q[width],%q[temp],%q[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "shr $32,%q[temp] \n"
+ "kmovd %k[temp],%%k2 \n"
+ "vmovdqu16 (%[src0]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src0]),%%zmm1%{%%k2%}%{z%}\n"
+ "vmovdqu16 (%[src1]),%%zmm3%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src1]),%%zmm4%{%%k2%}%{z%}\n"
+ "vpavgw %%zmm3,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpmovuswb %%zmm0,(%[dst])%{%%k1%} \n"
+ "vpmovuswb %%zmm1,0x20(%[dst])%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [scale] "r"(scale)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1", "k2");
+}
+#endif // HAS_HALFROW_16TO8_AVX512BW
+
+#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[scale],%%zmm2 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $15,%%zmm5,%%zmm5 \n"
+ "vpsllw $1,%%zmm5,%%zmm6 \n"
+ "vpsllw $15,%%zmm5,%%zmm7 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu64 (%[src0]),%%zmm0 \n"
+ "vmovdqu64 0x40(%[src0]),%%zmm1 \n"
+ "vmovdqu64 (%[src1]),%%zmm3 \n"
+ "vmovdqu64 0x40(%[src1]),%%zmm4 \n"
+ "add $0x80,%[src0] \n"
+ "add $0x80,%[src1] \n"
+ "vpxord %%zmm7,%%zmm0,%%zmm0 \n"
+ "vpxord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm3,%%zmm3 \n"
+ "vpxord %%zmm7,%%zmm4,%%zmm4 \n"
+ "vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddwd %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm3,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm6,%%zmm1,%%zmm1 \n"
+ "vpsrld $2,%%zmm0,%%zmm0 \n"
+ "vpsrld $2,%%zmm1,%%zmm1 \n"
+ "vpmovusdw %%zmm0,%%ymm0 \n"
+ "vpmovusdw %%zmm1,%%ymm1 \n"
+ "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmovuswb %%zmm0,(%[dst]) \n"
+ "add $0x20,%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k[width],%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "kmovd %k[temp],%%k3 \n"
+ "shr $16,%k[temp] \n"
+ "kmovw %k[temp],%%k2 \n"
+ "vmovdqu32 (%[src0]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src0]),%%zmm1%{%%k2%}%{z%}\n"
+ "vmovdqu32 (%[src1]),%%zmm3%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src1]),%%zmm4%{%%k2%}%{z%}\n"
+ "vpxord %%zmm7,%%zmm0,%%zmm0 \n"
+ "vpxord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm3,%%zmm3 \n"
+ "vpxord %%zmm7,%%zmm4,%%zmm4 \n"
+ "vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n"
+ "vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmaddwd %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm3,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm6,%%zmm1,%%zmm1 \n"
+ "vpsrld $2,%%zmm0,%%zmm0 \n"
+ "vpsrld $2,%%zmm1,%%zmm1 \n"
+ "vpmovusdw %%zmm0,%%ymm0 \n"
+ "vpmovusdw %%zmm1,%%ymm1 \n"
+ "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmovuswb %%zmm0,(%[dst])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [scale] "r"(scale)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "k1", "k2", "k3");
+}
+#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW
+
// Use scale to convert to lsb formats depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
-// TODO(fbarchard): reduce to SSE2
+// 65536 = 16 bits
+#ifdef HAS_CONVERT8TO16ROW_SSE2
void Convert8To16Row_SSE2(const uint8_t* src_y,
uint16_t* dst_y,
- int scale,
+ int bits,
int width) {
- // clang-format off
- asm volatile (
- "movd %3,%%xmm2 \n"
- "punpcklwd %%xmm2,%%xmm2 \n"
- "pshufd $0x0,%%xmm2,%%xmm2 \n"
-
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "add $0x10,%0 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "add $0x20,%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ const int shift = 16 - bits;
+ asm volatile("movd %3,%%xmm2 \n"
+
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm0,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm1 \n"
+ "add $0x10,%0 \n"
+ "psrlw %%xmm2,%%xmm0 \n"
+ "psrlw %%xmm2,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "add $0x20,%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(shift) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
+#endif // HAS_CONVERT8TO16ROW_SSE2
#ifdef HAS_CONVERT8TO16ROW_AVX2
void Convert8To16Row_AVX2(const uint8_t* src_y,
uint16_t* dst_y,
- int scale,
+ int bits,
int width) {
- // clang-format off
- asm volatile (
- "vmovd %3,%%xmm2 \n"
- "vpunpcklwd %%xmm2,%%xmm2,%%xmm2 \n"
- "vbroadcastss %%xmm2,%%ymm2 \n"
-
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "add $0x20,%0 \n"
- "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n"
- "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "add $0x40,%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ const int shift = 16 - bits;
+ asm volatile("vmovd %3,%%xmm2 \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "add $0x20,%0 \n"
+ "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm2,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm2,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "add $0x40,%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(shift) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_CONVERT8TO16ROW_AVX2
-#ifdef HAS_SPLITRGBROW_SSSE3
+#ifdef HAS_CONVERT8TO16ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert8To16Row_AVX512BW(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ const int shift = 16 - bits;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %[c0101],%%zmm2 \n"
+ "vmovd %[shift],%%xmm3 \n"
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "vpmovzxbw 0x20(%0),%%zmm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmullw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "vmovdqu16 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [shift] "r"(shift), // %[shift]
+ [c0101] "r"(0x0101) // %[c0101]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "k1");
+}
+#endif // HAS_CONVERT8TO16ROW_AVX512BW
+
+#ifdef HAS_SPLITRGBROW_SSSE3
// Shuffle table for converting RGB to Planar.
-static const uvec8 kShuffleMaskRGBToR0 = {0u, 3u, 6u, 9u, 12u, 15u,
- 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToR1 = {128u, 128u, 128u, 128u, 128u, 128u,
- 2u, 5u, 8u, 11u, 14u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToR2 = {128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 128u, 1u,
- 4u, 7u, 10u, 13u};
-
-static const uvec8 kShuffleMaskRGBToG0 = {1u, 4u, 7u, 10u, 13u, 128u,
- 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToG1 = {128u, 128u, 128u, 128u, 128u, 0u,
- 3u, 6u, 9u, 12u, 15u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToG2 = {128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 128u, 2u,
- 5u, 8u, 11u, 14u};
-
-static const uvec8 kShuffleMaskRGBToB0 = {2u, 5u, 8u, 11u, 14u, 128u,
- 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToB1 = {128u, 128u, 128u, 128u, 128u, 1u,
- 4u, 7u, 10u, 13u, 128u, 128u,
- 128u, 128u, 128u, 128u};
-static const uvec8 kShuffleMaskRGBToB2 = {128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 0u, 3u,
- 6u, 9u, 12u, 15u};
+static const uvec8 kSplitRGBShuffle[9] = {
+ {0u, 3u, 6u, 9u, 12u, 15u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 128u, 2u, 5u, 8u, 11u, 14u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 1u, 4u,
+ 7u, 10u, 13u},
+ {1u, 4u, 7u, 10u, 13u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 0u, 3u, 6u, 9u, 12u, 15u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 2u, 5u,
+ 8u, 11u, 14u},
+ {2u, 5u, 8u, 11u, 14u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 1u, 4u, 7u, 10u, 13u, 128u, 128u, 128u, 128u,
+ 128u, 128u},
+ {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, 0u, 3u, 6u, 9u,
+ 12u, 15u}};
void SplitRGBRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_r,
@@ -3721,262 +9211,1249 @@ void SplitRGBRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_b,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "pshufb %5, %%xmm0 \n"
- "pshufb %6, %%xmm1 \n"
- "pshufb %7, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
-
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "pshufb %8, %%xmm0 \n"
- "pshufb %9, %%xmm1 \n"
- "pshufb %10, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
-
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "pshufb %11, %%xmm0 \n"
- "pshufb %12, %%xmm1 \n"
- "pshufb %13, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%3) \n"
- "lea 0x10(%3),%3 \n"
- "lea 0x30(%0),%0 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "pshufb 0(%5), %%xmm0 \n"
+ "pshufb 16(%5), %%xmm1 \n"
+ "pshufb 32(%5), %%xmm2 \n"
+ "por %%xmm1,%%xmm0 \n"
+ "por %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "pshufb 48(%5),%%xmm0 \n"
+ "pshufb 64(%5),%%xmm1 \n"
+ "pshufb 80(%5), %%xmm2 \n"
+ "por %%xmm1,%%xmm0 \n"
+ "por %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "pshufb 96(%5), %%xmm0 \n"
+ "pshufb 112(%5), %%xmm1 \n"
+ "pshufb 128(%5), %%xmm2 \n"
+ "por %%xmm1,%%xmm0 \n"
+ "por %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%3) \n"
+ "lea 0x10(%3),%3 \n"
+ "lea 0x30(%0),%0 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
"+r"(dst_b), // %3
"+r"(width) // %4
- : "m"(kShuffleMaskRGBToR0), // %5
- "m"(kShuffleMaskRGBToR1), // %6
- "m"(kShuffleMaskRGBToR2), // %7
- "m"(kShuffleMaskRGBToG0), // %8
- "m"(kShuffleMaskRGBToG1), // %9
- "m"(kShuffleMaskRGBToG2), // %10
- "m"(kShuffleMaskRGBToB0), // %11
- "m"(kShuffleMaskRGBToB1), // %12
- "m"(kShuffleMaskRGBToB2) // %13
+ : "r"(&kSplitRGBShuffle[0]) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_SPLITRGBROW_SSSE3
-#ifdef HAS_MERGERGBROW_SSSE3
+#ifdef HAS_SPLITRGBROW_SSE41
+// Shuffle table for converting RGB to Planar, SSE4.1. Note: these are used for
+// the AVX2 implementation as well.
+static const uvec8 kSplitRGBShuffleSSE41[5] = {
+ {0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u},
+ {1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u},
+ {2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u},
+ {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u},
+ {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u},
+};
-// Shuffle table for converting RGB to Planar.
-static const uvec8 kShuffleMaskRToRGB0 = {0u, 128u, 128u, 1u, 128u, 128u,
- 2u, 128u, 128u, 3u, 128u, 128u,
- 4u, 128u, 128u, 5u};
-static const uvec8 kShuffleMaskGToRGB0 = {128u, 0u, 128u, 128u, 1u, 128u,
- 128u, 2u, 128u, 128u, 3u, 128u,
- 128u, 4u, 128u, 128u};
-static const uvec8 kShuffleMaskBToRGB0 = {128u, 128u, 0u, 128u, 128u, 1u,
- 128u, 128u, 2u, 128u, 128u, 3u,
- 128u, 128u, 4u, 128u};
-
-static const uvec8 kShuffleMaskGToRGB1 = {5u, 128u, 128u, 6u, 128u, 128u,
- 7u, 128u, 128u, 8u, 128u, 128u,
- 9u, 128u, 128u, 10u};
-static const uvec8 kShuffleMaskBToRGB1 = {128u, 5u, 128u, 128u, 6u, 128u,
- 128u, 7u, 128u, 128u, 8u, 128u,
- 128u, 9u, 128u, 128u};
-static const uvec8 kShuffleMaskRToRGB1 = {128u, 128u, 6u, 128u, 128u, 7u,
- 128u, 128u, 8u, 128u, 128u, 9u,
- 128u, 128u, 10u, 128u};
-
-static const uvec8 kShuffleMaskBToRGB2 = {10u, 128u, 128u, 11u, 128u, 128u,
- 12u, 128u, 128u, 13u, 128u, 128u,
- 14u, 128u, 128u, 15u};
-static const uvec8 kShuffleMaskRToRGB2 = {128u, 11u, 128u, 128u, 12u, 128u,
- 128u, 13u, 128u, 128u, 14u, 128u,
- 128u, 15u, 128u, 128u};
-static const uvec8 kShuffleMaskGToRGB2 = {128u, 128u, 11u, 128u, 128u, 12u,
- 128u, 128u, 13u, 128u, 128u, 14u,
- 128u, 128u, 15u, 128u};
-
-void MergeRGBRow_SSSE3(const uint8_t* src_r,
- const uint8_t* src_g,
- const uint8_t* src_b,
- uint8_t* dst_rgb,
+void SplitRGBRow_SSE41(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
int width) {
asm volatile(
+ "movdqa 48(%5), %%xmm0 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm1 \n"
+ "movdqu 0x10(%0),%%xmm2 \n"
+ "movdqu 0x20(%0),%%xmm3 \n"
+ "lea 0x30(%0),%0 \n"
+ "movdqa %%xmm1, %%xmm4 \n"
+ "pblendvb %%xmm3, %%xmm1 \n"
+ "pblendvb %%xmm2, %%xmm3 \n"
+ "pblendvb %%xmm4, %%xmm2 \n"
+ "palignr $0xF, %%xmm0, %%xmm0 \n"
+ "pblendvb %%xmm2, %%xmm1 \n"
+ "pblendvb %%xmm3, %%xmm2 \n"
+ "pblendvb %%xmm4, %%xmm3 \n"
+ "palignr $0x1, %%xmm0, %%xmm0 \n"
+ "pshufb 0(%5), %%xmm1 \n"
+ "pshufb 16(%5), %%xmm2 \n"
+ "pshufb 32(%5), %%xmm3 \n"
+ "movdqu %%xmm1,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "movdqu %%xmm2,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "movdqu %%xmm3,(%3) \n"
+ "lea 0x10(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "r"(&kSplitRGBShuffleSSE41[0]) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SPLITRGBROW_SSE41
+
+#ifdef HAS_SPLITRGBROW_AVX2
+void SplitRGBRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 48(%5), %%ymm0 \n"
+ "vbroadcasti128 64(%5), %%ymm7 \n"
+#if defined(__x86_64__)
+ "vbroadcasti128 0(%5), %%ymm8 \n"
+ "vbroadcasti128 16(%5), %%ymm9 \n"
+ "vbroadcasti128 32(%5), %%ymm10 \n"
+#endif
+ "1: \n"
+ "vmovdqu (%0),%%ymm4 \n"
+ "vmovdqu 0x20(%0),%%ymm5 \n"
+ "vmovdqu 0x40(%0),%%ymm6 \n"
+ "lea 0x60(%0),%0 \n"
+ "vpblendd $240, %%ymm5, %%ymm4, %%ymm1 \n"
+ "vperm2i128 $33, %%ymm6, %%ymm4, %%ymm2 \n"
+ "vpblendd $240, %%ymm6, %%ymm5, %%ymm3 \n"
+ "vpblendvb %%ymm0, %%ymm3, %%ymm1, %%ymm4 \n"
+ "vpblendvb %%ymm0, %%ymm1, %%ymm2, %%ymm5 \n"
+ "vpblendvb %%ymm0, %%ymm2, %%ymm3, %%ymm6 \n"
+ "vpblendvb %%ymm7, %%ymm5, %%ymm4, %%ymm1 \n"
+ "vpblendvb %%ymm7, %%ymm6, %%ymm5, %%ymm2 \n"
+ "vpblendvb %%ymm7, %%ymm4, %%ymm6, %%ymm3 \n"
+#if defined(__x86_64__)
+ "vpshufb %%ymm8, %%ymm1, %%ymm1 \n"
+ "vpshufb %%ymm9, %%ymm2, %%ymm2 \n"
+ "vpshufb %%ymm10, %%ymm3, %%ymm3 \n"
+#else
+ "vbroadcasti128 0(%5), %%ymm4 \n"
+ "vbroadcasti128 16(%5), %%ymm5 \n"
+ "vbroadcasti128 32(%5), %%ymm6 \n"
+ "vpshufb %%ymm4, %%ymm1, %%ymm1 \n"
+ "vpshufb %%ymm5, %%ymm2, %%ymm2 \n"
+ "vpshufb %%ymm6, %%ymm3, %%ymm3 \n"
+#endif
+ "vmovdqu %%ymm1,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "vmovdqu %%ymm3,(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "r"(&kSplitRGBShuffleSSE41[0]) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10"
+#endif
+ );
+}
+#endif // HAS_SPLITRGBROW_AVX2
+
+#ifdef HAS_MERGERGBROW_AVX2
+// Shuffle table for converting Planar to RGB.
+static const uvec8 kMergeRGBShuffle[9] = {
+ {0u, 128u, 128u, 1u, 128u, 128u, 2u, 128u, 128u, 3u, 128u, 128u, 4u, 128u,
+ 128u, 5u},
+ {128u, 0u, 128u, 128u, 1u, 128u, 128u, 2u, 128u, 128u, 3u, 128u, 128u, 4u,
+ 128u, 128u},
+ {128u, 128u, 0u, 128u, 128u, 1u, 128u, 128u, 2u, 128u, 128u, 3u, 128u, 128u,
+ 4u, 128u},
+ {128u, 128u, 6u, 128u, 128u, 7u, 128u, 128u, 8u, 128u, 128u, 9u, 128u, 128u,
+ 10u, 128u},
+ {5u, 128u, 128u, 6u, 128u, 128u, 7u, 128u, 128u, 8u, 128u, 128u, 9u, 128u,
+ 128u, 10u},
+ {128u, 5u, 128u, 128u, 6u, 128u, 128u, 7u, 128u, 128u, 8u, 128u, 128u, 9u,
+ 128u, 128u},
+ {128u, 11u, 128u, 128u, 12u, 128u, 128u, 13u, 128u, 128u, 14u, 128u, 128u,
+ 15u, 128u, 128u},
+ {128u, 128u, 11u, 128u, 128u, 12u, 128u, 128u, 13u, 128u, 128u, 14u, 128u,
+ 128u, 15u, 128u},
+ {10u, 128u, 128u, 11u, 128u, 128u, 12u, 128u, 128u, 13u, 128u, 128u, 14u,
+ 128u, 128u, 15u}};
+
+void MergeRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+#if defined(__x86_64__)
+ "vbroadcasti128 (%5),%%ymm8 \n"
+ "vbroadcasti128 16(%5),%%ymm9 \n"
+ "vbroadcasti128 32(%5),%%ymm10 \n"
+ "vbroadcasti128 48(%5),%%ymm11 \n"
+ "vbroadcasti128 64(%5),%%ymm12 \n"
+ "vbroadcasti128 80(%5),%%ymm13 \n"
+ "vbroadcasti128 96(%5),%%ymm14 \n"
+ "vbroadcasti128 112(%5),%%ymm15 \n"
+#endif
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu (%1),%%xmm1 \n"
- "movdqu (%2),%%xmm2 \n"
- "pshufb %5, %%xmm0 \n"
- "pshufb %6, %%xmm1 \n"
- "pshufb %7, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%3) \n"
-
- "movdqu (%0),%%xmm0 \n"
- "movdqu (%1),%%xmm1 \n"
- "movdqu (%2),%%xmm2 \n"
- "pshufb %8, %%xmm0 \n"
- "pshufb %9, %%xmm1 \n"
- "pshufb %10, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,16(%3) \n"
-
- "movdqu (%0),%%xmm0 \n"
- "movdqu (%1),%%xmm1 \n"
- "movdqu (%2),%%xmm2 \n"
- "pshufb %11, %%xmm0 \n"
- "pshufb %12, %%xmm1 \n"
- "pshufb %13, %%xmm2 \n"
- "por %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,32(%3) \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm3 \n"
+ "vmovdqu (%0,%1,1),%%ymm4 \n"
+ "vmovdqu (%0,%2,1),%%ymm5 \n"
+ "lea 0x20(%0),%0 \n"
- "lea 0x10(%0),%0 \n"
- "lea 0x10(%1),%1 \n"
- "lea 0x10(%2),%2 \n"
- "lea 0x30(%3),%3 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
+#if defined(__x86_64__)
+ "vpshufb %%ymm8,%%ymm3,%%ymm0 \n"
+ "vpshufb %%ymm9,%%ymm4,%%ymm6 \n"
+ "vpshufb %%ymm10,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+
+ "vpshufb %%ymm11,%%ymm3,%%ymm1 \n"
+ "vpshufb %%ymm12,%%ymm4,%%ymm6 \n"
+ "vpshufb %%ymm13,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm7,%%ymm1,%%ymm1 \n"
+
+ "vpshufb %%ymm14,%%ymm3,%%ymm2 \n"
+ "vpshufb %%ymm15,%%ymm4,%%ymm6 \n"
+ "vbroadcasti128 128(%5),%%ymm7 \n"
+ "vpor %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm7,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm7,%%ymm2,%%ymm2 \n"
+#else
+ "vbroadcasti128 (%5),%%ymm6 \n"
+ "vbroadcasti128 16(%5),%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm0 \n"
+ "vpshufb %%ymm7,%%ymm4,%%ymm6 \n"
+ "vbroadcasti128 32(%5),%%ymm7 \n"
+ "vpor %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm7,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+
+ "vbroadcasti128 48(%5),%%ymm6 \n"
+ "vbroadcasti128 64(%5),%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm1 \n"
+ "vpshufb %%ymm7,%%ymm4,%%ymm6 \n"
+ "vbroadcasti128 80(%5),%%ymm7 \n"
+ "vpor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm7,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm7,%%ymm1,%%ymm1 \n"
+
+ "vbroadcasti128 96(%5),%%ymm6 \n"
+ "vbroadcasti128 112(%5),%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm2 \n"
+ "vpshufb %%ymm7,%%ymm4,%%ymm6 \n"
+ "vbroadcasti128 128(%5),%%ymm7 \n"
+ "vpor %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm7,%%ymm5,%%ymm7 \n"
+ "vpor %%ymm7,%%ymm2,%%ymm2 \n"
+#endif
+
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm6 \n"
+ "vperm2i128 $0x30,%%ymm0,%%ymm2,%%ymm7 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm6,(%3) \n"
+ "vmovdqu %%ymm7,0x20(%3) \n"
+ "vmovdqu %%ymm1,0x40(%3) \n"
+ "lea 0x60(%3),%3 \n"
+ "subl $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
"+r"(dst_rgb), // %3
- "+r"(width) // %4
- : "m"(kShuffleMaskRToRGB0), // %5
- "m"(kShuffleMaskGToRGB0), // %6
- "m"(kShuffleMaskBToRGB0), // %7
- "m"(kShuffleMaskRToRGB1), // %8
- "m"(kShuffleMaskGToRGB1), // %9
- "m"(kShuffleMaskBToRGB1), // %10
- "m"(kShuffleMaskRToRGB2), // %11
- "m"(kShuffleMaskGToRGB2), // %12
- "m"(kShuffleMaskBToRGB2) // %13
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ "+rm"(width) // %4
+ : "r"(&kMergeRGBShuffle[0]) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
+#endif
+ );
}
-#endif // HAS_MERGERGBROW_SSSE3
+#endif // HAS_MERGERGBROW_AVX2
-#ifdef HAS_COPYROW_SSE2
-void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_MERGEARGBROW_SSE2
+void MergeARGBRow_SSE2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "test $0xf,%0 \n"
- "jne 2f \n"
- "test $0xf,%1 \n"
- "jne 2f \n"
-
- LABELALIGN
- "1: \n"
- "movdqa (%0),%%xmm0 \n"
- "movdqa 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "movdqa %%xmm0,(%1) \n"
- "movdqa %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "jmp 9f \n"
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "sub %0,%3 \n"
LABELALIGN
- "2: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 2b \n"
-
- LABELALIGN "9: \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
+ "1: \n"
+
+ "movq (%0,%2),%%xmm0 \n" // B
+ "movq (%0),%%xmm1 \n" // R
+ "movq (%0,%1),%%xmm2 \n" // G
+ "punpcklbw %%xmm1,%%xmm0 \n" // BR
+ "movq (%0,%3),%%xmm1 \n" // A
+ "punpcklbw %%xmm1,%%xmm2 \n" // GA
+ "movdqa %%xmm0,%%xmm1 \n" // BR
+ "punpckhbw %%xmm2,%%xmm1 \n" // BGRA (hi)
+ "punpcklbw %%xmm2,%%xmm0 \n" // BGRA (lo)
+ "movdqu %%xmm0,(%4) \n"
+ "movdqu %%xmm1,16(%4) \n"
+
+ "lea 8(%0),%0 \n"
+ "lea 32(%4),%4 \n"
+ "sub $0x8,%5 \n"
+ "jg 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
:
- : "memory", "cc", "xmm0", "xmm1");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
-#endif // HAS_COPYROW_SSE2
+#endif // HAS_MERGEARGBROW_SSE2
-#ifdef HAS_COPYROW_AVX
-void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
+#ifdef HAS_MERGEXRGBROW_SSE2
+void MergeXRGBRow_SSE2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x40,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
+ "1: \n"
+
+ "movq (%2),%%xmm0 \n" // B
+ "movq (%0),%%xmm1 \n" // R
+ "movq (%1),%%xmm2 \n" // G
+ "punpcklbw %%xmm1,%%xmm0 \n" // BR
+ "pcmpeqd %%xmm1,%%xmm1 \n" // A(255)
+ "punpcklbw %%xmm1,%%xmm2 \n" // GA
+ "movdqa %%xmm0,%%xmm1 \n" // BR
+ "punpckhbw %%xmm2,%%xmm1 \n" // BGRA (hi)
+ "punpcklbw %%xmm2,%%xmm0 \n" // BGRA (lo)
+ "movdqu %%xmm0,(%3) \n"
+ "movdqu %%xmm1,16(%3) \n"
+
+ "lea 8(%0),%0 \n"
+ "lea 8(%1),%1 \n"
+ "lea 8(%2),%2 \n"
+ "lea 32(%3),%3 \n"
+ "sub $0x8,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+r"(width) // %4
:
- : "memory", "cc", "xmm0", "xmm1");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
-#endif // HAS_COPYROW_AVX
+#endif // HAS_MERGEXRGBROW_SSE2
-#ifdef HAS_COPYROW_ERMS
-// Multiple of 1.
-void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width) {
- size_t width_tmp = (size_t)(width);
+#ifdef HAS_MERGEARGBROW_AVX2
+void MergeARGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "sub %0,%3 \n"
- "rep movsb \n"
- : "+S"(src), // %0
- "+D"(dst), // %1
- "+c"(width_tmp) // %2
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0,%2),%%xmm0 \n" // B
+ "vmovdqu (%0,%1),%%xmm1 \n" // R
+ "vinserti128 $1,(%0),%%ymm0,%%ymm0 \n" // G
+ "vinserti128 $1,(%0,%3),%%ymm1,%%ymm1 \n" // A
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm0,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm2,%%ymm0,%%ymm0 \n"
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm0,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%4) \n" // First 8
+ "vmovdqu %%ymm1,32(%4) \n" // Next 8
+
+ "lea 16(%0),%0 \n"
+ "lea 64(%4),%4 \n"
+ "sub $0x10,%5 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
:
- : "memory", "cc");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
-#endif // HAS_COPYROW_ERMS
-
-#ifdef HAS_ARGBCOPYALPHAROW_SSE2
-// width in pixels
-void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
- "pcmpeqb %%xmm0,%%xmm0 \n"
- "pslld $0x18,%%xmm0 \n"
- "pcmpeqb %%xmm1,%%xmm1 \n"
- "psrld $0x8,%%xmm1 \n"
+#endif // HAS_MERGEARGBROW_AVX2
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm2 \n"
- "movdqu 0x10(%0),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "movdqu (%1),%%xmm4 \n"
- "movdqu 0x10(%1),%%xmm5 \n"
- "pand %%xmm0,%%xmm2 \n"
- "pand %%xmm0,%%xmm3 \n"
- "pand %%xmm1,%%xmm4 \n"
- "pand %%xmm1,%%xmm5 \n"
- "por %%xmm4,%%xmm2 \n"
- "por %%xmm5,%%xmm3 \n"
- "movdqu %%xmm2,(%1) \n"
- "movdqu %%xmm3,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+#ifdef HAS_MERGEXRGBROW_AVX2
+void MergeXRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+
+ "vmovdqu (%2),%%xmm0 \n" // B
+ "vpcmpeqb %%ymm1,%%ymm1,%%ymm1 \n" // A(255)
+ "vinserti128 $0,(%1),%%ymm1,%%ymm1 \n" // R
+ "vinserti128 $1,(%0),%%ymm0,%%ymm0 \n" // G
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm0,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm2,%%ymm0,%%ymm0 \n"
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm0,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%3) \n" // First 8
+ "vmovdqu %%ymm1,32(%3) \n" // Next 8
+
+ "lea 16(%0),%0 \n"
+ "lea 16(%1),%1 \n"
+ "lea 16(%2),%2 \n"
+ "lea 64(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+rm"(width) // %4
+ ::"memory",
+ "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_MERGEXRGBROW_AVX2
+
+#ifdef HAS_SPLITARGBROW_SSE2
+void SplitARGBRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ asm volatile(
+ "sub %1,%2 \n"
+ "sub %1,%3 \n"
+ "sub %1,%4 \n"
+
+ LABELALIGN
+ "1: \n"
+
+ "movdqu (%0),%%xmm0 \n" // 00-0F
+ "movdqu 16(%0),%%xmm1 \n" // 10-1F
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklqdq %%xmm1,%%xmm0 \n" // 00-07 10-17
+ "punpckhqdq %%xmm1,%%xmm2 \n" // 08-0F 18-1F
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n" // 08192A3B4C5D6E7F (lo)
+ "punpckhbw %%xmm2,%%xmm1 \n" // 08192A3B4C5D6E7F (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklqdq %%xmm1,%%xmm0 \n" // 08192A3B08192A3B
+ "punpckhqdq %%xmm1,%%xmm2 \n" // 4C5D6E7F4C5D6E7F
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n" // 048C159D26AE37BF (lo)
+ "punpckhbw %%xmm2,%%xmm1 \n" // 048C159D26AE37BF (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckldq %%xmm1,%%xmm0 \n" // 048C048C159D159D (BG)
+ "punpckhdq %%xmm1,%%xmm2 \n" // 26AE26AE37BF37BF (RA)
+ "movlps %%xmm0,(%1,%3) \n" // B
+ "movhps %%xmm0,(%1,%2) \n" // G
+ "movlps %%xmm2,(%1) \n" // R
+ "movhps %%xmm2,(%1,%4) \n" // A
+
+ "lea 32(%0),%0 \n"
+ "lea 8(%1),%1 \n"
+ "sub $0x8,%5 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(dst_a), // %4
+ "+rm"(width) // %5
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_SPLITARGBROW_SSE2
+
+#ifdef HAS_SPLITXRGBROW_SSE2
+void SplitXRGBRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "1: \n"
+
+ "movdqu (%0),%%xmm0 \n" // 00-0F
+ "movdqu 16(%0),%%xmm1 \n" // 10-1F
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklqdq %%xmm1,%%xmm0 \n" // 00-07 10-17
+ "punpckhqdq %%xmm1,%%xmm2 \n" // 08-0F 18-1F
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n" // 08192A3B4C5D6E7F (lo)
+ "punpckhbw %%xmm2,%%xmm1 \n" // 08192A3B4C5D6E7F (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklqdq %%xmm1,%%xmm0 \n" // 08192A3B08192A3B
+ "punpckhqdq %%xmm1,%%xmm2 \n" // 4C5D6E7F4C5D6E7F
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n" // 048C159D26AE37BF (lo)
+ "punpckhbw %%xmm2,%%xmm1 \n" // 048C159D26AE37BF (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckldq %%xmm1,%%xmm0 \n" // 048C048C159D159D (BG)
+ "punpckhdq %%xmm1,%%xmm2 \n" // 26AE26AE37BF37BF (RA)
+ "movlps %%xmm0,(%3) \n" // B
+ "movhps %%xmm0,(%2) \n" // G
+ "movlps %%xmm2,(%1) \n" // R
+
+ "lea 32(%0),%0 \n"
+ "lea 8(%1),%1 \n"
+ "lea 8(%2),%2 \n"
+ "lea 8(%3),%3 \n"
+ "sub $0x8,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_SPLITXRGBROW_SSE2
+
+static const uvec8 kShuffleMaskARGBSplit = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+#ifdef HAS_SPLITARGBROW_SSSE3
+void SplitARGBRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+#if defined(__i386__)
+ uint32_t shuf[4];
+ memcpy(shuf, &kShuffleMaskARGBSplit, sizeof(shuf));
+#endif
+ asm volatile(
+ "movdqu %6,%%xmm3 \n"
+ "sub %1,%2 \n"
+ "sub %1,%3 \n"
+ "sub %1,%4 \n"
+
+ LABELALIGN
+ "1: \n"
+
+ "movdqu (%0),%%xmm0 \n" // 00-0F
+ "movdqu 16(%0),%%xmm1 \n" // 10-1F
+ "pshufb %%xmm3,%%xmm0 \n" // 048C159D26AE37BF (lo)
+ "pshufb %%xmm3,%%xmm1 \n" // 048C159D26AE37BF (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckldq %%xmm1,%%xmm0 \n" // 048C048C159D159D (BG)
+ "punpckhdq %%xmm1,%%xmm2 \n" // 26AE26AE37BF37BF (RA)
+ "movlps %%xmm0,(%1,%3) \n" // B
+ "movhps %%xmm0,(%1,%2) \n" // G
+ "movlps %%xmm2,(%1) \n" // R
+ "movhps %%xmm2,(%1,%4) \n" // A
+
+ "lea 32(%0),%0 \n"
+ "lea 8(%1),%1 \n"
+ "subl $0x8,%5 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(dst_a), // %4
+#if defined(__i386__)
+ "+m"(width) // %5
+ : "m"(shuf) // %6
+#else
+ "+rm"(width) // %5
+ : "m"(kShuffleMaskARGBSplit) // %6
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_SPLITARGBROW_SSSE3
+
+#ifdef HAS_SPLITXRGBROW_SSSE3
+void SplitXRGBRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "movdqa %5,%%xmm3 \n"
+
+ LABELALIGN
+ "1: \n"
+
+ "movdqu (%0),%%xmm0 \n" // 00-0F
+ "movdqu 16(%0),%%xmm1 \n" // 10-1F
+ "pshufb %%xmm3,%%xmm0 \n" // 048C159D26AE37BF (lo)
+ "pshufb %%xmm3,%%xmm1 \n" // 048C159D26AE37BF (hi)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckldq %%xmm1,%%xmm0 \n" // 048C048C159D159D (BG)
+ "punpckhdq %%xmm1,%%xmm2 \n" // 26AE26AE37BF37BF (RA)
+ "movlps %%xmm0,(%3) \n" // B
+ "movhps %%xmm0,(%2) \n" // G
+ "movlps %%xmm2,(%1) \n" // R
+
+ "lea 32(%0),%0 \n"
+ "lea 8(%1),%1 \n"
+ "lea 8(%2),%2 \n"
+ "lea 8(%3),%3 \n"
+ "sub $0x8,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "m"(kShuffleMaskARGBSplit) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_SPLITXRGBROW_SSSE3
+
+#ifdef HAS_SPLITARGBROW_AVX2
+static const ulvec32 kShuffleMaskARGBPermute = {0, 4, 1, 5, 2, 6, 3, 7};
+void SplitARGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+#if defined(__i386__)
+ uint32_t shuf[4];
+ uint32_t perm[8];
+ memcpy(shuf, &kShuffleMaskARGBSplit, sizeof(shuf));
+ memcpy(perm, &kShuffleMaskARGBPermute, sizeof(perm));
+#endif
+ asm volatile(
+ "sub %1,%2 \n"
+ "sub %1,%3 \n"
+ "sub %1,%4 \n"
+ "vmovdqu %7,%%ymm3 \n"
+ "vbroadcasti128 %6,%%ymm4 \n"
+
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0),%%xmm0 \n" // 00-0F
+ "vmovdqu 16(%0),%%xmm1 \n" // 10-1F
+ "vinserti128 $1,32(%0),%%ymm0,%%ymm0 \n" // 00-0F 20-2F
+ "vinserti128 $1,48(%0),%%ymm1,%%ymm1 \n" // 10-1F 30-3F
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpermd %%ymm0,%%ymm3,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm3,%%ymm1 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm2 \n" // GA
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm0 \n" // BR
+ "vmovdqu %%xmm0,(%1,%3) \n" // B
+ "vextracti128 $1,%%ymm0,(%1) \n" // R
+ "vmovdqu %%xmm2,(%1,%2) \n" // G
+ "vextracti128 $1,%%ymm2,(%1,%4) \n" // A
+ "lea 64(%0),%0 \n"
+ "lea 16(%1),%1 \n"
+ "subl $0x10,%5 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(dst_a), // %4
+#if defined(__i386__)
+ "+m"(width) // %5
+ : "m"(shuf), // %6
+ "m"(perm) // %7
+#else
+ "+rm"(width) // %5
+ : "m"(kShuffleMaskARGBSplit), // %6
+ "m"(kShuffleMaskARGBPermute) // %7
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SPLITARGBROW_AVX2
+
+#ifdef HAS_SPLITXRGBROW_AVX2
+void SplitXRGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "vmovdqa %6,%%ymm3 \n"
+ "vbroadcasti128 %5,%%ymm4 \n"
+
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0),%%xmm0 \n" // 00-0F
+ "vmovdqu 16(%0),%%xmm1 \n" // 10-1F
+ "vinserti128 $1,32(%0),%%ymm0,%%ymm0 \n" // 00-0F 20-2F
+ "vinserti128 $1,48(%0),%%ymm1,%%ymm1 \n" // 10-1F 30-3F
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpermd %%ymm0,%%ymm3,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm3,%%ymm1 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm2 \n" // GA
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm0 \n" // BR
+ "vmovdqu %%xmm0,(%3) \n" // B
+ "vextracti128 $1,%%ymm0,(%1) \n" // R
+ "vmovdqu %%xmm2,(%2) \n" // G
+
+ "lea 64(%0),%0 \n"
+ "lea 16(%1),%1 \n"
+ "lea 16(%2),%2 \n"
+ "lea 16(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "m"(kShuffleMaskARGBSplit), // %5
+ "m"(kShuffleMaskARGBPermute) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SPLITXRGBROW_AVX2
+
+#ifdef HAS_MERGEXR30ROW_AVX2
+void MergeXR30Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ int shift = depth - 10;
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // AR30 constants
+ "vpsrlw $14,%%ymm5,%%ymm5 \n"
+ "vpsllw $4,%%ymm5,%%ymm5 \n" // 2 alpha bits
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $6,%%ymm6,%%ymm6 \n"
+ "vmovd %5,%%xmm4 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu (%0,%1),%%ymm1 \n"
+ "vmovdqu (%0,%2),%%ymm2 \n"
+ "vpsrlw %%xmm4,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm4,%%ymm1,%%ymm1 \n"
+ "vpsrlw %%xmm4,%%ymm2,%%ymm2 \n"
+ "vpminuw %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpminuw %%ymm1,%%ymm6,%%ymm1 \n"
+ "vpminuw %%ymm2,%%ymm6,%%ymm2 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm2,%%ymm2 \n"
+ "vpsllw $0x4,%%ymm0,%%ymm0 \n" // Shift R to target bit
+ "vpunpckhwd %%ymm0,%%ymm2,%%ymm3 \n" // RB
+ "vpunpcklwd %%ymm0,%%ymm2,%%ymm0 \n"
+ "vpunpckhwd %%ymm5,%%ymm1,%%ymm2 \n" // AG
+ "vpunpcklwd %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpslld $0xa,%%ymm1,%%ymm1 \n" // Shift AG to target bit
+ "vpslld $0xa,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n" // Combine
+ "vpor %%ymm2,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm0,(%3) \n"
+ "vmovdqu %%ymm3,0x20(%3) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar30), // %3
+ "+r"(width) // %4
+#if defined(__i386__)
+ : "m"(shift) // %5
+#else
+ : "rm"(shift) // %5
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_MERGEXR30ROW_AVX2
+
+#ifdef HAS_MERGEAR64ROW_AVX2
+static const lvec32 MergeAR64Permute = {0, 4, 2, 6, 1, 5, 3, 7};
+void MergeAR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ uint32_t mask16 = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
+ uint32_t mask = (mask16 << 16) | mask16;
+#if defined(__i386__)
+ uint32_t perm[8];
+ memcpy(perm, &MergeAR64Permute, sizeof(perm));
+#endif
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "sub %0,%3 \n"
+ "vmovdqu %8,%%ymm5 \n"
+ "vmovd %6,%%xmm6 \n"
+ "vbroadcastss %7,%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // R
+ "vmovdqu (%0,%1),%%ymm1 \n" // G
+ "vmovdqu (%0,%2),%%ymm2 \n" // B
+ "vmovdqu (%0,%3),%%ymm3 \n" // A
+ "vpminuw %%ymm0,%%ymm7,%%ymm0 \n"
+ "vpminuw %%ymm1,%%ymm7,%%ymm1 \n"
+ "vpminuw %%ymm2,%%ymm7,%%ymm2 \n"
+ "vpminuw %%ymm3,%%ymm7,%%ymm3 \n"
+ "vpsllw %%xmm6,%%ymm0,%%ymm0 \n"
+ "vpsllw %%xmm6,%%ymm1,%%ymm1 \n"
+ "vpsllw %%xmm6,%%ymm2,%%ymm2 \n"
+ "vpsllw %%xmm6,%%ymm3,%%ymm3 \n"
+ "vpermd %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm5,%%ymm1 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpermd %%ymm3,%%ymm5,%%ymm3 \n"
+ "vpunpcklwd %%ymm1,%%ymm2,%%ymm4 \n" // BG(low)
+ "vpunpckhwd %%ymm1,%%ymm2,%%ymm1 \n" // BG(hi)
+ "vpunpcklwd %%ymm3,%%ymm0,%%ymm2 \n" // RA(low)
+ "vpunpckhwd %%ymm3,%%ymm0,%%ymm0 \n" // RA(hi)
+ "vpunpckldq %%ymm2,%%ymm4,%%ymm3 \n" // BGRA(1)
+ "vpunpckhdq %%ymm2,%%ymm4,%%ymm4 \n" // BGRA(3)
+ "vpunpckldq %%ymm0,%%ymm1,%%ymm2 \n" // BGRA(2)
+ "vpunpckhdq %%ymm0,%%ymm1,%%ymm1 \n" // BGRA(4)
+ "vmovdqu %%ymm3,(%4) \n"
+ "vmovdqu %%ymm2,0x20(%4) \n"
+ "vmovdqu %%ymm4,0x40(%4) \n"
+ "vmovdqu %%ymm1,0x60(%4) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x80(%4),%4 \n"
+ "subl $0x10,%5 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_ar64), // %4
+#if defined(__i386__)
+ "+m"(width) // %5
+ : "m"(shift), // %6
+ "m"(mask), // %7
+ "m"(perm) // %8
+#else
+ "+rm"(width) // %5
+ : "m"(shift), // %6
+ "m"(mask), // %7
+ "m"(MergeAR64Permute) // %8
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_MERGEAR64ROW_AVX2
+
+#ifdef HAS_MERGEXR64ROW_AVX2
+void MergeXR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ uint32_t mask16 = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
+ uint32_t mask = (mask16 << 16) | mask16;
+#if defined(__i386__)
+ uint32_t perm[8];
+ memcpy(perm, &MergeAR64Permute, sizeof(perm));
+#endif
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "vmovdqu %7,%%ymm5 \n"
+ "vmovd %5,%%xmm6 \n"
+ "vbroadcastss %6,%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // R
+ "vmovdqu (%0,%1),%%ymm1 \n" // G
+ "vmovdqu (%0,%2),%%ymm2 \n" // B
+ "vpminuw %%ymm0,%%ymm7,%%ymm0 \n"
+ "vpminuw %%ymm1,%%ymm7,%%ymm1 \n"
+ "vpminuw %%ymm2,%%ymm7,%%ymm2 \n"
+ "vpsllw %%xmm6,%%ymm0,%%ymm0 \n"
+ "vpsllw %%xmm6,%%ymm1,%%ymm1 \n"
+ "vpsllw %%xmm6,%%ymm2,%%ymm2 \n"
+ "vpermd %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm5,%%ymm1 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n" // A (0xffff)
+ "vpunpcklwd %%ymm1,%%ymm2,%%ymm4 \n" // BG(low)
+ "vpunpckhwd %%ymm1,%%ymm2,%%ymm1 \n" // BG(hi)
+ "vpunpcklwd %%ymm3,%%ymm0,%%ymm2 \n" // RA(low)
+ "vpunpckhwd %%ymm3,%%ymm0,%%ymm0 \n" // RA(hi)
+ "vpunpckldq %%ymm2,%%ymm4,%%ymm3 \n" // BGRA(1)
+ "vpunpckhdq %%ymm2,%%ymm4,%%ymm4 \n" // BGRA(3)
+ "vpunpckldq %%ymm0,%%ymm1,%%ymm2 \n" // BGRA(2)
+ "vpunpckhdq %%ymm0,%%ymm1,%%ymm1 \n" // BGRA(4)
+ "vmovdqu %%ymm3,(%3) \n"
+ "vmovdqu %%ymm2,0x20(%3) \n"
+ "vmovdqu %%ymm4,0x40(%3) \n"
+ "vmovdqu %%ymm1,0x60(%3) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x80(%3),%3 \n"
+ "subl $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar64), // %3
+ "+rm"(width) // %4
+ : "m"(shift), // %5
+ "m"(mask), // %6
+#if defined(__i386__)
+ "m"(perm) // %7
+#else
+ "m"(MergeAR64Permute) // %7
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_MERGEXR64ROW_AVX2
+
+#ifdef HAS_MERGEARGB16TO8ROW_AVX2
+static const uvec8 MergeARGB16To8Shuffle = {0, 8, 1, 9, 2, 10, 3, 11,
+ 4, 12, 5, 13, 6, 14, 7, 15};
+void MergeARGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ int shift = depth - 8;
+#if defined(__i386__)
+ uint32_t shuf[4];
+ memcpy(shuf, &MergeARGB16To8Shuffle, sizeof(shuf));
+#endif
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "sub %0,%3 \n"
+ "vbroadcasti128 %7,%%ymm5 \n"
+ "vmovd %6,%%xmm6 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsrlw $8,%%ymm7,%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // R
+ "vmovdqu (%0,%1),%%ymm1 \n" // G
+ "vmovdqu (%0,%2),%%ymm2 \n" // B
+ "vmovdqu (%0,%3),%%ymm3 \n" // A
+ "vpsrlw %%xmm6,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm6,%%ymm1,%%ymm1 \n"
+ "vpsrlw %%xmm6,%%ymm2,%%ymm2 \n"
+ "vpsrlw %%xmm6,%%ymm3,%%ymm3 \n"
+ "vpminuw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpminuw %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpminuw %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpminuw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpackuswb %%ymm1,%%ymm2,%%ymm1 \n" // BG (planar)
+ "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n" // RA (planar)
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" // BG (interleave)
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" // RA (interleave)
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm0,%%ymm1,%%ymm2 \n" // BGRA (low)
+ "vpunpckhwd %%ymm0,%%ymm1,%%ymm0 \n" // BGRA (hi)
+ "vmovdqu %%ymm2,(%4) \n"
+ "vmovdqu %%ymm0,0x20(%4) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%4),%4 \n"
+ "subl $0x10,%5 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+#if defined(__i386__)
+ "+m"(width) // %5
+ : "m"(shift), // %6
+ "m"(shuf) // %7
+#else
+ "+rm"(width) // %5
+ : "m"(shift), // %6
+ "m"(MergeARGB16To8Shuffle) // %7
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7");
+}
+#endif // HAS_MERGEARGB16TO8ROW_AVX2
+
+#ifdef HAS_MERGEXRGB16TO8ROW_AVX2
+void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ int shift = depth - 8;
+#if defined(__i386__)
+ uint32_t shuf[4];
+ memcpy(shuf, &MergeARGB16To8Shuffle, sizeof(shuf));
+#endif
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "vbroadcasti128 %6,%%ymm5 \n"
+ "vmovd %5,%%xmm6 \n"
+ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpsrlw $8,%%ymm3,%%ymm3 \n" // A (0xff)
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // R
+ "vmovdqu (%0,%1),%%ymm1 \n" // G
+ "vmovdqu (%0,%2),%%ymm2 \n" // B
+ "vpsrlw %%xmm6,%%ymm0,%%ymm0 \n"
+ "vpsrlw %%xmm6,%%ymm1,%%ymm1 \n"
+ "vpsrlw %%xmm6,%%ymm2,%%ymm2 \n"
+ "vpminuw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpminuw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpminuw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm1,%%ymm2,%%ymm1 \n" // BG (planar)
+ "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n" // RA (planar)
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" // BG (interleave)
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" // RA (interleave)
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm0,%%ymm1,%%ymm2 \n" // BGRA (low)
+ "vpunpckhwd %%ymm0,%%ymm1,%%ymm0 \n" // BGRA (hi)
+ "vmovdqu %%ymm2,(%3) \n"
+ "vmovdqu %%ymm0,0x20(%3) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%3),%3 \n"
+ "subl $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+rm"(width) // %4
+ : "m"(shift), // %5
+#if defined(__i386__)
+ "m"(shuf) // %6
+#else
+ "m"(MergeARGB16To8Shuffle) // %6
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6");
+}
+#endif // HAS_MERGEXRGB16TO8ROW_AVX2
+
+#ifdef HAS_COPYROW_SSE2
+void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "test $0xf,%0 \n"
+ "jne 2f \n"
+ "test $0xf,%1 \n"
+ "jne 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqa (%0),%%xmm0 \n"
+ "movdqa 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "movdqa %%xmm0,(%1) \n"
+ "movdqa %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "jmp 9f \n"
+
+ LABELALIGN
+ "2: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 2b \n"
+
+ LABELALIGN "9: \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_COPYROW_SSE2
+
+#ifdef HAS_COPYROW_AVX
+void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_COPYROW_AVX
+
+#ifdef HAS_COPYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x80,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n"
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vmovups %%zmm0,(%1) \n"
+ "vmovups %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x80,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x80,%2 \n"
+ "jle 99f \n"
+ "cmp $0x40,%2 \n"
+ "jl 3f \n"
+ "vmovups (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vmovups %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q2,%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ :
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_COPYROW_AVX512BW
+
+#ifdef HAS_COPYROW_ERMS
+// Multiple of 1.
+void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width) {
+ size_t width_tmp = (size_t)(width);
+ asm volatile("rep movsb \n"
+ : "+S"(src), // %0
+ "+D"(dst), // %1
+ "+c"(width_tmp) // %2
+ :
+ : "memory", "cc");
+}
+#endif // HAS_COPYROW_ERMS
+
+#ifdef HAS_ARGBCOPYALPHAROW_SSE2
+// width in pixels
+void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "pcmpeqb %%xmm0,%%xmm0 \n"
+ "pslld $0x18,%%xmm0 \n"
+ "pcmpeqb %%xmm1,%%xmm1 \n"
+ "psrld $0x8,%%xmm1 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm2 \n"
+ "movdqu 0x10(%0),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "movdqu (%1),%%xmm4 \n"
+ "movdqu 0x10(%1),%%xmm5 \n"
+ "pand %%xmm0,%%xmm2 \n"
+ "pand %%xmm0,%%xmm3 \n"
+ "pand %%xmm1,%%xmm4 \n"
+ "pand %%xmm1,%%xmm5 \n"
+ "por %%xmm4,%%xmm2 \n"
+ "por %%xmm5,%%xmm3 \n"
+ "movdqu %%xmm2,(%1) \n"
+ "movdqu %%xmm3,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -3989,22 +10466,22 @@ void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
// width in pixels
void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpsrld $0x8,%%ymm0,%%ymm0 \n"
+ "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x8,%%ymm0,%%ymm0 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm1 \n"
- "vmovdqu 0x20(%0),%%ymm2 \n"
- "lea 0x40(%0),%0 \n"
- "vpblendvb %%ymm0,(%1),%%ymm1,%%ymm1 \n"
- "vpblendvb %%ymm0,0x20(%1),%%ymm2,%%ymm2 \n"
- "vmovdqu %%ymm1,(%1) \n"
- "vmovdqu %%ymm2,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm1 \n"
+ "vmovdqu 0x20(%0),%%ymm2 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpblendvb %%ymm0,(%1),%%ymm1,%%ymm1 \n"
+ "vpblendvb %%ymm0,0x20(%1),%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm1,(%1) \n"
+ "vmovdqu %%ymm2,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -4019,20 +10496,18 @@ void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
uint8_t* dst_a,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
- "movdqu (%0), %%xmm0 \n"
- "movdqu 0x10(%0), %%xmm1 \n"
- "lea 0x20(%0), %0 \n"
- "psrld $0x18, %%xmm0 \n"
- "psrld $0x18, %%xmm1 \n"
- "packssdw %%xmm1, %%xmm0 \n"
- "packuswb %%xmm0, %%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1), %1 \n"
- "sub $0x8, %2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0), %%xmm0 \n"
+ "movdqu 0x10(%0), %%xmm1 \n"
+ "lea 0x20(%0), %0 \n"
+ "psrld $0x18, %%xmm0 \n"
+ "psrld $0x18, %%xmm1 \n"
+ "packssdw %%xmm1, %%xmm0 \n"
+ "packuswb %%xmm0, %%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "lea 0x8(%1), %1 \n"
+ "sub $0x8, %2 \n"
+ "jg 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_a), // %1
"+rm"(width) // %2
@@ -4050,29 +10525,29 @@ void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_a,
int width) {
asm volatile(
- "vmovdqa %3,%%ymm4 \n"
- "vbroadcastf128 %4,%%ymm5 \n"
+ "vmovdqa %3,%%ymm4 \n"
+ "vbroadcasti128 %4,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0), %%ymm0 \n"
- "vmovdqu 0x20(%0), %%ymm1 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" // vpsrld $0x18, %%ymm0
- "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
- "vmovdqu 0x40(%0), %%ymm2 \n"
- "vmovdqu 0x60(%0), %%ymm3 \n"
- "lea 0x80(%0), %0 \n"
- "vpackssdw %%ymm1, %%ymm0, %%ymm0 \n" // mutates
- "vpshufb %%ymm5,%%ymm2,%%ymm2 \n"
- "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
- "vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // mutates
- "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates.
- "vpermd %%ymm0,%%ymm4,%%ymm0 \n" // unmutate.
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20, %2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0), %%ymm0 \n"
+ "vmovdqu 0x20(%0), %%ymm1 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" // vpsrld $0x18, %%ymm0
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x40(%0), %%ymm2 \n"
+ "vmovdqu 0x60(%0), %%ymm3 \n"
+ "lea 0x80(%0), %0 \n"
+ "vpackssdw %%ymm1, %%ymm0, %%ymm0 \n" // mutates
+ "vpshufb %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // mutates
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates.
+ "vpermd %%ymm0,%%ymm4,%%ymm0 \n" // unmutate.
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20, %2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_a), // %1
"+rm"(width) // %2
@@ -4082,35 +10557,91 @@ void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
}
#endif // HAS_ARGBEXTRACTALPHAROW_AVX2
+#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%0),%%zmm0 \n"
+ "vmovdqu32 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpsrld $0x18,%%zmm0,%%zmm0 \n"
+ "vpsrld $0x18,%%zmm1,%%zmm1 \n"
+ "vpmovdb %%zmm0,(%1) \n"
+ "vpmovdb %%zmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "cmp $0x10,%2 \n"
+ "jl 3f \n"
+ "vmovdqu32 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x18,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jz 99f \n"
+
+ "3: \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovw %k3,%%k1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrld $0x18,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_a), // %1
+ "+r"(width), // %2
+ "=&r"(temp) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW
+
#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2
// width in pixels
void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "pcmpeqb %%xmm0,%%xmm0 \n"
- "pslld $0x18,%%xmm0 \n"
- "pcmpeqb %%xmm1,%%xmm1 \n"
- "psrld $0x8,%%xmm1 \n"
+ "pcmpeqb %%xmm0,%%xmm0 \n"
+ "pslld $0x18,%%xmm0 \n"
+ "pcmpeqb %%xmm1,%%xmm1 \n"
+ "psrld $0x8,%%xmm1 \n"
LABELALIGN
- "1: \n"
- "movq (%0),%%xmm2 \n"
- "lea 0x8(%0),%0 \n"
- "punpcklbw %%xmm2,%%xmm2 \n"
- "punpckhwd %%xmm2,%%xmm3 \n"
- "punpcklwd %%xmm2,%%xmm2 \n"
- "movdqu (%1),%%xmm4 \n"
- "movdqu 0x10(%1),%%xmm5 \n"
- "pand %%xmm0,%%xmm2 \n"
- "pand %%xmm0,%%xmm3 \n"
- "pand %%xmm1,%%xmm4 \n"
- "pand %%xmm1,%%xmm5 \n"
- "por %%xmm4,%%xmm2 \n"
- "por %%xmm5,%%xmm3 \n"
- "movdqu %%xmm2,(%1) \n"
- "movdqu %%xmm3,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movq (%0),%%xmm2 \n"
+ "lea 0x8(%0),%0 \n"
+ "punpcklbw %%xmm2,%%xmm2 \n"
+ "movdqa %%xmm2,%%xmm3 \n"
+ "punpckhwd %%xmm3,%%xmm3 \n"
+ "punpcklwd %%xmm2,%%xmm2 \n"
+ "movdqu (%1),%%xmm4 \n"
+ "movdqu 0x10(%1),%%xmm5 \n"
+ "pand %%xmm0,%%xmm2 \n"
+ "pand %%xmm0,%%xmm3 \n"
+ "pand %%xmm1,%%xmm4 \n"
+ "pand %%xmm1,%%xmm5 \n"
+ "por %%xmm4,%%xmm2 \n"
+ "por %%xmm5,%%xmm3 \n"
+ "movdqu %%xmm2,(%1) \n"
+ "movdqu %%xmm3,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -4123,24 +10654,24 @@ void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
// width in pixels
void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpsrld $0x8,%%ymm0,%%ymm0 \n"
+ "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x8,%%ymm0,%%ymm0 \n"
LABELALIGN
- "1: \n"
- "vpmovzxbd (%0),%%ymm1 \n"
- "vpmovzxbd 0x8(%0),%%ymm2 \n"
- "lea 0x10(%0),%0 \n"
- "vpslld $0x18,%%ymm1,%%ymm1 \n"
- "vpslld $0x18,%%ymm2,%%ymm2 \n"
- "vpblendvb %%ymm0,(%1),%%ymm1,%%ymm1 \n"
- "vpblendvb %%ymm0,0x20(%1),%%ymm2,%%ymm2 \n"
- "vmovdqu %%ymm1,(%1) \n"
- "vmovdqu %%ymm2,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vpmovzxbd (%0),%%ymm1 \n"
+ "vpmovzxbd 0x8(%0),%%ymm2 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpslld $0x18,%%ymm1,%%ymm1 \n"
+ "vpslld $0x18,%%ymm2,%%ymm2 \n"
+ "vpblendvb %%ymm0,(%1),%%ymm1,%%ymm1 \n"
+ "vpblendvb %%ymm0,0x20(%1),%%ymm2,%%ymm2 \n"
+ "vmovdqu %%ymm1,(%1) \n"
+ "vmovdqu %%ymm2,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -4149,60 +10680,202 @@ void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
}
#endif // HAS_ARGBCOPYYTOALPHAROW_AVX2
-#ifdef HAS_SETROW_X86
-void SetRow_X86(uint8_t* dst, uint8_t v8, int width) {
- size_t width_tmp = (size_t)(width >> 2);
- const uint32_t v32 = v8 * 0x01010101u; // Duplicate byte to all bytes.
+#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ uintptr_t temp;
asm volatile(
+ "kmovq %4,%%k1 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
- "rep stosl \n"
- : "+D"(dst), // %0
- "+c"(width_tmp) // %1
- : "a"(v32) // %2
- : "memory", "cc");
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbd (%0),%%zmm1 \n"
+ "vpmovzxbd 0x10(%0),%%zmm2 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpslld $0x18,%%zmm1,%%zmm1 \n"
+ "vpslld $0x18,%%zmm2,%%zmm2 \n"
+ "vmovdqu8 %%zmm1,(%1)%{%%k1%} \n"
+ "vmovdqu8 %%zmm2,0x40(%1)%{%%k1%} \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "cmp $0x10,%2 \n"
+ "jl 3f \n"
+ "vpmovzxbd (%0),%%zmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpslld $0x18,%%zmm1,%%zmm1 \n"
+ "vmovdqu8 %%zmm1,(%1)%{%%k1%} \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jz 99f \n"
+
+ "3: \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovw %k3,%%k1 \n"
+ "vpmovzxbd (%0),%%zmm1%{%%k1%}%{z%} \n"
+ "vpslld $0x18,%%zmm1,%%zmm1 \n"
+ "shl $0x2,%k2 \n"
+ "bzhi %q2,%4,%3 \n"
+ "kmovq %3,%%k1 \n"
+ "vmovdqu8 %%zmm1,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width), // %2
+ "=&r"(temp) // %3
+ : "r"(0x8888888888888888ull) // %4
+ : "memory", "cc", "xmm1", "xmm2", "k1");
}
+#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+#ifdef HAS_SETROW_ERMS
void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) {
size_t width_tmp = (size_t)(width);
- asm volatile(
-
- "rep stosb \n"
- : "+D"(dst), // %0
- "+c"(width_tmp) // %1
- : "a"(v8) // %2
- : "memory", "cc");
+ asm volatile("rep stosb \n"
+ : "+D"(dst), // %0
+ "+c"(width_tmp) // %1
+ : "a"(v8) // %2
+ : "memory", "cc");
}
+#endif // HAS_SETROW_ERMS
-void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width) {
- size_t width_tmp = (size_t)(width);
+#ifdef HAS_SETROW_AVX2
+void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) {
asm volatile(
+ "vmovd %2,%%xmm0 \n"
+ "vpbroadcastb %%xmm0,%%ymm0 \n"
+ "sub $0x40,%1 \n"
+ "jl 2f \n"
- "rep stosl \n"
- : "+D"(dst_argb), // %0
- "+c"(width_tmp) // %1
- : "a"(v32) // %2
- : "memory", "cc");
+ LABELALIGN
+ "1: \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "vmovdqu %%ymm0,0x20(%0) \n"
+ "lea 0x40(%0),%0 \n"
+ "sub $0x40,%1 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%1 \n"
+ "jle 99f \n"
+ "vmovdqu %%ymm0,(%0) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst), // %0
+ "+r"(width) // %1
+ : "r"((int)v8) // %2
+ : "memory", "cc", "xmm0");
}
-#endif // HAS_SETROW_X86
+#endif // HAS_SETROW_AVX2
-#ifdef HAS_YUY2TOYROW_SSE2
-void YUY2ToYRow_SSE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+#ifdef HAS_ARGBSETROW_AVX2
+void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
+ "vmovd %2,%%xmm0 \n"
+ "vpbroadcastd %%xmm0,%%ymm0 \n"
+ "sub $0x20,%1 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pand %%xmm5,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "vmovdqu %%ymm0,0x20(%0) \n"
+ "vmovdqu %%ymm0,0x40(%0) \n"
+ "vmovdqu %%ymm0,0x60(%0) \n"
+ "lea 0x80(%0),%0 \n"
+ "sub $0x20,%1 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%1 \n"
+ "jle 99f \n"
+ "cmp $0x10,%1 \n"
+ "jl 3f \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "vmovdqu %%ymm0,0x20(%0) \n"
+ "lea 0x40(%0),%0 \n"
+ "sub $0x10,%1 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "vmovdqu %%ymm0,(%0) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width) // %1
+ : "r"(v32) // %2
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_ARGBSETROW_AVX2
+
+#ifdef HAS_COPYROW_16_AVX2
+void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) {
+ asm volatile(
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm3 \n"
+ "lea 0x80(%0),%0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "vmovdqu %%ymm3,0x60(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_COPYROW_16_AVX2
+
+#ifdef HAS_YUY2TOYROW_SSE2
+void YUY2ToYRow_SSE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ asm volatile(
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -4210,43 +10883,70 @@ void YUY2ToYRow_SSE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm5");
}
+void YUY2ToNVUVRow_SSE2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%3,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%3,1),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "pavgb %%xmm2,%%xmm0 \n"
+ "pavgb %%xmm3,%%xmm1 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"((ptrdiff_t)(stride_yuy2)) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+
void YUY2ToUVRow_SSE2(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
- "sub %1,%2 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x00(%0,%4,1),%%xmm2 \n"
- "movdqu 0x10(%0,%4,1),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "pavgb %%xmm3,%%xmm1 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pand %%xmm5,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_yuy2), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"((intptr_t)(stride_yuy2)) // %4
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%4,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%4,1),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "pavgb %%xmm2,%%xmm0 \n"
+ "pavgb %%xmm3,%%xmm1 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm1 \n"
+ "movq %%xmm0,(%1) \n"
+ "movq %%xmm1,0x00(%1,%2,1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(stride_yuy2)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
}
@@ -4255,28 +10955,28 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2,
uint8_t* dst_v,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
- "sub %1,%2 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pand %%xmm5,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm1 \n"
+ "movq %%xmm0,(%1) \n"
+ "movq %%xmm1,0x00(%1,%2,1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -4287,19 +10987,17 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2,
void UYVYToYRow_SSE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -4313,37 +11011,37 @@ void UYVYToUVRow_SSE2(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
- "sub %1,%2 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x00(%0,%4,1),%%xmm2 \n"
- "movdqu 0x10(%0,%4,1),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "pavgb %%xmm3,%%xmm1 \n"
- "pand %%xmm5,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pand %%xmm5,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_uyvy), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"((intptr_t)(stride_uyvy)) // %4
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%4,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%4,1),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "pavgb %%xmm2,%%xmm0 \n"
+ "pavgb %%xmm3,%%xmm1 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm1 \n"
+ "movq %%xmm0,(%1) \n"
+ "movq %%xmm1,0x00(%1,%2,1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_uyvy), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(stride_uyvy)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
}
@@ -4352,28 +11050,28 @@ void UYVYToUV422Row_SSE2(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrlw $0x8,%%xmm5 \n"
- "sub %1,%2 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrlw $0x8,%%xmm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pand %%xmm5,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pand %%xmm5,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm1 \n"
+ "movq %%xmm0,(%1) \n"
+ "movq %%xmm1,0x00(%1,%2,1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -4386,23 +11084,23 @@ void UYVYToUV422Row_SSE2(const uint8_t* src_uyvy,
#ifdef HAS_YUY2TOYROW_AVX2
void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -4410,44 +11108,71 @@ void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm5");
}
+void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpavgb 0x00(%0,%3,1),%%ymm0,%%ymm0 \n"
+ "vpavgb 0x20(%0,%3,1),%%ymm1,%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"((ptrdiff_t)(stride_yuy2)) // %3
+ : "memory", "cc", "xmm0", "xmm1");
+}
+
void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
- "sub %1,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm1 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm1,%%ymm1 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
+ "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vextractf128 $0x0,%%ymm1,(%1) \n"
"vextractf128 $0x0,%%ymm0,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_yuy2), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"((intptr_t)(stride_yuy2)) // %4
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(stride_yuy2)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm5");
}
@@ -4456,31 +11181,31 @@ void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
uint8_t* dst_v,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
- "sub %1,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm1 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm1,%%ymm1 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vextractf128 $0x0,%%ymm1,(%1) \n"
"vextractf128 $0x0,%%ymm0,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -4491,21 +11216,19 @@ void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -4518,38 +11241,38 @@ void UYVYToUVRow_AVX2(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
- "sub %1,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm1 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm1,%%ymm1 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
+ "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vextractf128 $0x0,%%ymm1,(%1) \n"
"vextractf128 $0x0,%%ymm0,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_uyvy), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"((intptr_t)(stride_uyvy)) // %4
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uyvy), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(stride_uyvy)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm5");
}
@@ -4558,31 +11281,31 @@ void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
- "sub %1,%2 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrlw $0x8,%%ymm5,%%ymm5 \n"
+ "sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm1 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm1,%%ymm1 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vextractf128 $0x0,%%ymm1,(%1) \n"
"vextractf128 $0x0,%%ymm0,0x00(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -4592,92 +11315,347 @@ void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy,
}
#endif // HAS_YUY2TOYROW_AVX2
-#ifdef HAS_ARGBBLENDROW_SSSE3
-// Shuffle table for isolating alpha.
-static const uvec8 kShuffleAlpha = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80,
- 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80};
+#ifdef HAS_YUY2TOYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
-// Blend 8 pixels at a time
-void ARGBBlendRow_SSSE3(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
+ LABELALIGN
+ "1: \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "vmovdqu16 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "vpmovwb %%zmm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ :
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_YUY2TOYROW_AVX512BW
+
+#ifdef HAS_YUY2TOUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "pcmpeqb %%xmm7,%%xmm7 \n"
- "psrlw $0xf,%%xmm7 \n"
- "pcmpeqb %%xmm6,%%xmm6 \n"
- "psrlw $0x8,%%xmm6 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psllw $0x8,%%xmm5 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "pslld $0x18,%%xmm4 \n"
- "sub $0x4,%3 \n"
- "jl 49f \n"
+ "sub %1,%2 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
- // 4 pixel loop.
LABELALIGN
- "40: \n"
- "movdqu (%0),%%xmm3 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm3,%%xmm0 \n"
- "pxor %%xmm4,%%xmm3 \n"
- "movdqu (%1),%%xmm2 \n"
- "pshufb %4,%%xmm3 \n"
- "pand %%xmm6,%%xmm2 \n"
- "paddw %%xmm7,%%xmm3 \n"
- "pmullw %%xmm3,%%xmm2 \n"
- "movdqu (%1),%%xmm1 \n"
- "lea 0x10(%1),%1 \n"
- "psrlw $0x8,%%xmm1 \n"
- "por %%xmm4,%%xmm0 \n"
- "pmullw %%xmm3,%%xmm1 \n"
- "psrlw $0x8,%%xmm2 \n"
- "paddusb %%xmm2,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jge 40b \n"
-
- "49: \n"
- "add $0x3,%3 \n"
- "jl 99f \n"
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x18,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm1,(%1) \n"
+ "vpmovdb %%zmm0,0x00(%1,%2,1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "add $0x1,%k3 \n"
+ "shr $0x1,%k3 \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k3,%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n"
+ "vpavgb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpsrld $0x8,%%zmm0,%%zmm1 \n"
+ "vpsrld $0x18,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm1,(%1)%{%%k1%} \n"
+ "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [stride] "r"((ptrdiff_t)(stride_yuy2)) // %[stride]
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_YUY2TOUVROW_AVX512BW
+
+#ifdef HAS_UYVYTOYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "vmovdqu16 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm1,%%zmm1 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "vpmovwb %%zmm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "cmp $0x20,%2 \n"
+ "jl 3f \n"
+ "vmovdqu16 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jle 99f \n"
+
+ "3: \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k2,%k[temp],%k[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_uyvy), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ :
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_UYVYTOYROW_AVX512BW
+
+#ifdef HAS_UYVYTOUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
+ int stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub %1,%2 \n"
+ "sub $0x20,%3 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmovdb %%zmm0,(%1) \n"
+ "vpsrld $0x10,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm0,0x00(%1,%2,1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%3 \n"
+ "jle 99f \n"
+ "add $0x1,%k3 \n"
+ "shr $0x1,%k3 \n"
+ "mov $-1,%k[temp] \n"
+ "bzhi %k3,%k[temp],%k[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n"
+ "vpavgb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm0,(%1)%{%%k1%} \n"
+ "vpsrld $0x10,%%zmm0,%%zmm0 \n"
+ "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_uyvy), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width), // %3
+ [temp] "=&r"(temp) // %[temp]
+ : [stride] "r"((ptrdiff_t)(stride_uyvy)) // %[stride]
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_UYVYTOUVROW_AVX512BW
+
+#if defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW)
+// Shuffle table for isolating alpha.
+SIMD_ALIGNED(static const uvec8 kShuffleAlpha[2]) = {
+ {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80,
+ 15u, 0x80},
+ {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80,
+ 15u, 0x80}};
+#endif // defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW)
+
+#ifdef HAS_ARGBBLENDROW_AVX2
+// Blend 8 pixels at a time, with 4-pixel and 1-pixel tails.
+void ARGBBlendRow_AVX2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsrlw $0xf,%%ymm7,%%ymm7 \n"
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $0x8,%%ymm6,%%ymm6 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsllw $0x8,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpslld $0x18,%%ymm4,%%ymm4 \n"
+#if defined(__x86_64__)
+ "vbroadcasti128 %4,%%ymm8 \n"
+#endif
+ "sub $0x8,%3 \n"
+ "jl 2f \n"
+
+ // 8 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm3 \n"
+ "vmovdqu (%1),%%ymm2 \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "vpxor %%ymm4,%%ymm3,%%ymm0 \n"
+#if defined(__x86_64__)
+ "vpshufb %%ymm8,%%ymm0,%%ymm0 \n"
+#else
+ "vpshufb %4,%%ymm0,%%ymm0 \n"
+#endif
+ "vpaddw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpmullw %%ymm0,%%ymm2,%%ymm2 \n"
+ "vpmullw %%ymm0,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpaddusb %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddusb %%ymm1,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm3,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x4,%3 \n"
+ "jl 3f \n"
+ "vmovdqu (%0),%%xmm3 \n"
+ "vmovdqu (%1),%%xmm2 \n"
+ "lea 0x10(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "vpxor %%xmm4,%%xmm3,%%xmm0 \n"
+#if defined(__x86_64__)
+ "vpshufb %%xmm8,%%xmm0,%%xmm0 \n"
+#else
+ "vpshufb %4,%%xmm0,%%xmm0 \n"
+#endif
+ "vpaddw %%xmm7,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x8,%%xmm2,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm2,%%xmm2 \n"
+ "vpmullw %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpmullw %%xmm0,%%xmm1,%%xmm1 \n"
+ "vpor %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpsrlw $0x8,%%xmm2,%%xmm2 \n"
+ "vpand %%xmm5,%%xmm1,%%xmm1 \n"
+ "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n"
+ "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n"
+ "vmovdqu %%xmm3,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x4,%3 \n"
+
+ "3: \n"
+ "add $0x3,%3 \n"
+ "jl 99f \n"
// 1 pixel loop.
- "91: \n"
- "movd (%0),%%xmm3 \n"
- "lea 0x4(%0),%0 \n"
- "movdqa %%xmm3,%%xmm0 \n"
- "pxor %%xmm4,%%xmm3 \n"
- "movd (%1),%%xmm2 \n"
- "pshufb %4,%%xmm3 \n"
- "pand %%xmm6,%%xmm2 \n"
- "paddw %%xmm7,%%xmm3 \n"
- "pmullw %%xmm3,%%xmm2 \n"
- "movd (%1),%%xmm1 \n"
- "lea 0x4(%1),%1 \n"
- "psrlw $0x8,%%xmm1 \n"
- "por %%xmm4,%%xmm0 \n"
- "pmullw %%xmm3,%%xmm1 \n"
- "psrlw $0x8,%%xmm2 \n"
- "paddusb %%xmm2,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movd %%xmm0,(%2) \n"
- "lea 0x4(%2),%2 \n"
- "sub $0x1,%3 \n"
- "jge 91b \n"
- "99: \n"
- : "+r"(src_argb0), // %0
- "+r"(src_argb1), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- : "m"(kShuffleAlpha) // %4
+ LABELALIGN
+ "4: \n"
+ "vmovd (%0),%%xmm3 \n"
+ "vmovd (%1),%%xmm2 \n"
+ "lea 0x4(%0),%0 \n"
+ "lea 0x4(%1),%1 \n"
+ "vpxor %%xmm4,%%xmm3,%%xmm0 \n"
+#if defined(__x86_64__)
+ "vpshufb %%xmm8,%%xmm0,%%xmm0 \n"
+#else
+ "vpshufb %4,%%xmm0,%%xmm0 \n"
+#endif
+ "vpaddw %%xmm7,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x8,%%xmm2,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm2,%%xmm2 \n"
+ "vpmullw %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpmullw %%xmm0,%%xmm1,%%xmm1 \n"
+ "vpor %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpsrlw $0x8,%%xmm2,%%xmm2 \n"
+ "vpand %%xmm5,%%xmm1,%%xmm1 \n"
+ "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n"
+ "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n"
+ "vmovd %%xmm3,(%2) \n"
+ "lea 0x4(%2),%2 \n"
+ "sub $0x1,%3 \n"
+ "jge 4b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ : "m"(kShuffleAlpha[0]) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8"
+#endif
+ );
}
-#endif // HAS_ARGBBLENDROW_SSSE3
+#endif // HAS_ARGBBLENDROW_AVX2
#ifdef HAS_BLENDPLANEROW_SSSE3
// Blend 8 pixels at a time.
@@ -4691,36 +11669,36 @@ void BlendPlaneRow_SSSE3(const uint8_t* src0,
uint8_t* dst,
int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psllw $0x8,%%xmm5 \n"
- "mov $0x80808080,%%eax \n"
- "movd %%eax,%%xmm6 \n"
- "pshufd $0x0,%%xmm6,%%xmm6 \n"
- "mov $0x807f807f,%%eax \n"
- "movd %%eax,%%xmm7 \n"
- "pshufd $0x0,%%xmm7,%%xmm7 \n"
- "sub %2,%0 \n"
- "sub %2,%1 \n"
- "sub %2,%3 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psllw $0x8,%%xmm5 \n"
+ "pcmpeqb %%xmm6,%%xmm6 \n"
+ "pabsb %%xmm6,%%xmm6 \n"
+ "psllw $7,%%xmm6 \n"
+ "mov $0x807f807f,%%eax \n"
+ "movd %%eax,%%xmm7 \n"
+ "pshufd $0x0,%%xmm7,%%xmm7 \n"
+ "sub %2,%0 \n"
+ "sub %2,%1 \n"
+ "sub %2,%3 \n"
// 8 pixel loop.
LABELALIGN
- "1: \n"
- "movq (%2),%%xmm0 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "pxor %%xmm5,%%xmm0 \n"
- "movq (%0,%2,1),%%xmm1 \n"
- "movq (%1,%2,1),%%xmm2 \n"
- "punpcklbw %%xmm2,%%xmm1 \n"
- "psubb %%xmm6,%%xmm1 \n"
- "pmaddubsw %%xmm1,%%xmm0 \n"
- "paddw %%xmm7,%%xmm0 \n"
- "psrlw $0x8,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%3,%2,1) \n"
- "lea 0x8(%2),%2 \n"
- "sub $0x8,%4 \n"
- "jg 1b \n"
+ "1: \n"
+ "movq (%2),%%xmm0 \n"
+ "punpcklbw %%xmm0,%%xmm0 \n"
+ "pxor %%xmm5,%%xmm0 \n"
+ "movq (%0,%2,1),%%xmm1 \n"
+ "movq (%1,%2,1),%%xmm2 \n"
+ "punpcklbw %%xmm2,%%xmm1 \n"
+ "psubb %%xmm6,%%xmm1 \n"
+ "pmaddubsw %%xmm1,%%xmm0 \n"
+ "paddw %%xmm7,%%xmm0 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%3,%2,1) \n"
+ "lea 0x8(%2),%2 \n"
+ "sub $0x8,%4 \n"
+ "jg 1b \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(alpha), // %2
@@ -4743,44 +11721,44 @@ void BlendPlaneRow_AVX2(const uint8_t* src0,
uint8_t* dst,
int width) {
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsllw $0x8,%%ymm5,%%ymm5 \n"
- "mov $0x80808080,%%eax \n"
- "vmovd %%eax,%%xmm6 \n"
- "vbroadcastss %%xmm6,%%ymm6 \n"
- "mov $0x807f807f,%%eax \n"
- "vmovd %%eax,%%xmm7 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsllw $0x8,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpabsb %%ymm6,%%ymm6 \n"
+ "vpsllw $7,%%ymm6,%%ymm6 \n"
+ "mov $0x807f807f,%%eax \n"
+ "vmovd %%eax,%%xmm7 \n"
"vbroadcastss %%xmm7,%%ymm7 \n"
- "sub %2,%0 \n"
- "sub %2,%1 \n"
- "sub %2,%3 \n"
+ "sub %2,%0 \n"
+ "sub %2,%1 \n"
+ "sub %2,%3 \n"
// 32 pixel loop.
LABELALIGN
- "1: \n"
- "vmovdqu (%2),%%ymm0 \n"
- "vpunpckhbw %%ymm0,%%ymm0,%%ymm3 \n"
- "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
- "vpxor %%ymm5,%%ymm3,%%ymm3 \n"
- "vpxor %%ymm5,%%ymm0,%%ymm0 \n"
- "vmovdqu (%0,%2,1),%%ymm1 \n"
- "vmovdqu (%1,%2,1),%%ymm2 \n"
- "vpunpckhbw %%ymm2,%%ymm1,%%ymm4 \n"
- "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n"
- "vpsubb %%ymm6,%%ymm4,%%ymm4 \n"
- "vpsubb %%ymm6,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "vpmaddubsw %%ymm1,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm7,%%ymm3,%%ymm3 \n"
- "vpaddw %%ymm7,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm3,%%ymm3 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%3,%2,1) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x20,%4 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%2),%%ymm0 \n"
+ "vpunpckhbw %%ymm0,%%ymm0,%%ymm3 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpxor %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpxor %%ymm5,%%ymm0,%%ymm0 \n"
+ "vmovdqu (%0,%2,1),%%ymm1 \n"
+ "vmovdqu (%1,%2,1),%%ymm2 \n"
+ "vpunpckhbw %%ymm2,%%ymm1,%%ymm4 \n"
+ "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpsubb %%ymm6,%%ymm4,%%ymm4 \n"
+ "vpsubb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpmaddubsw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm3,%%ymm3 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%3,%2,1) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(alpha), // %2
@@ -4792,95 +11770,201 @@ void BlendPlaneRow_AVX2(const uint8_t* src0,
}
#endif // HAS_BLENDPLANEROW_AVX2
+#ifdef HAS_BLENDPLANEROW_AVX512BW
+// Blend 64 pixels at a time.
+// unsigned version of math
+// =((A2*C2)+(B2*(255-C2))+255)/256
+// signed version of math
+// =(((A2-128)*C2)+((B2-128)*(255-C2))+32768+127)/256
+LIBYUV_TARGET_AVX512BW
+void BlendPlaneRow_AVX512BW(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpsllw $0x8,%%zmm17,%%zmm16 \n" // 0xff00
+ "vpabsb %%zmm17,%%zmm17 \n" // 0x0101
+ "vpsllw $7,%%zmm17,%%zmm17 \n" // 0x8080
+ "mov $0x807f807f,%k[temp] \n"
+ "vmovd %k[temp],%%xmm18 \n"
+ "vbroadcastss %%xmm18,%%zmm18 \n" // 0x807f
+ "sub %2,%0 \n"
+ "sub %2,%1 \n"
+ "sub %2,%3 \n"
+ "sub $64,%4 \n"
+ "jl 2f \n"
+
+ // 64 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%2),%%zmm0 \n"
+ "vpunpckhbw %%zmm0,%%zmm0,%%zmm3 \n"
+ "vpunpcklbw %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpxorq %%zmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%0,%2,1),%%zmm1 \n"
+ "vmovdqu8 (%1,%2,1),%%zmm2 \n"
+ "vpunpckhbw %%zmm2,%%zmm1,%%zmm4 \n"
+ "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm17,%%zmm4,%%zmm4 \n"
+ "vpsubb %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm3,%%zmm3 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%3,%2,1) \n"
+ "add $64,%2 \n"
+ "sub $64,%4 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $64,%4 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q4,%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "vmovdqu8 (%2),%%zmm0%{%%k1%}%{z%} \n"
+ "vpunpckhbw %%zmm0,%%zmm0,%%zmm3 \n"
+ "vpunpcklbw %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpxorq %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpxorq %%zmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%0,%2,1),%%zmm1%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%1,%2,1),%%zmm2%{%%k1%}%{z%} \n"
+ "vpunpckhbw %%zmm2,%%zmm1,%%zmm4 \n"
+ "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm17,%%zmm4,%%zmm4 \n"
+ "vpsubb %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpmaddubsw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm3,%%zmm3 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%3,%2,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src0), // %0
+ "+r"(src1), // %1
+ "+r"(alpha), // %2
+ "+r"(dst), // %3
+ "+r"(width), // %4
+ [temp] "=&r"(temp) // %[temp]
+ ::"memory",
+ "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", "xmm18",
+ "k1");
+}
+#endif // HAS_BLENDPLANEROW_AVX512BW
+
#ifdef HAS_ARGBATTENUATEROW_SSSE3
-// Shuffle table duplicating alpha
-static const uvec8 kShuffleAlpha0 = {3u, 3u, 3u, 3u, 3u, 3u, 128u, 128u,
- 7u, 7u, 7u, 7u, 7u, 7u, 128u, 128u};
-static const uvec8 kShuffleAlpha1 = {11u, 11u, 11u, 11u, 11u, 11u, 128u, 128u,
- 15u, 15u, 15u, 15u, 15u, 15u, 128u, 128u};
+// Shuffle table duplicating alpha.
+static const vec8 kAttenuateShuffle = {6, -128, 6, -128, 6, -128,
+ -128, -128, 14, -128, 14, -128,
+ 14, -128, -128, -128};
+
// Attenuate 4 pixels at a time.
void ARGBAttenuateRow_SSSE3(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
asm volatile(
- "pcmpeqb %%xmm3,%%xmm3 \n"
- "pslld $0x18,%%xmm3 \n"
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
+ "movdqa %3,%%xmm4 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "pslld $0x18,%%xmm5 \n"
+ "pxor %%xmm6,%%xmm6 \n"
+ "pcmpeqb %%xmm7,%%xmm7 \n"
+ "punpcklbw %%xmm6,%%xmm7 \n"
+ "sub %0,%1 \n"
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "movdqu (%0),%%xmm1 \n"
- "punpcklbw %%xmm1,%%xmm1 \n"
- "pmulhuw %%xmm1,%%xmm0 \n"
- "movdqu (%0),%%xmm1 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "movdqu (%0),%%xmm2 \n"
- "punpckhbw %%xmm2,%%xmm2 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "movdqu (%0),%%xmm2 \n"
- "lea 0x10(%0),%0 \n"
- "pand %%xmm3,%%xmm2 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "por %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleAlpha0), // %3
- "m"(kShuffleAlpha1) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "1: \n"
+ "movdqu (%0),%%xmm6 \n"
+ "movdqa %%xmm6,%%xmm0 \n"
+ "movdqa %%xmm6,%%xmm1 \n"
+ "punpcklbw %%xmm5,%%xmm0 \n"
+ "punpckhbw %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "pshufb %%xmm4,%%xmm2 \n" // a,a,a,0
+ "pshufb %%xmm4,%%xmm3 \n"
+ "pmullw %%xmm2,%%xmm0 \n" // rgb * alpha
+ "pmullw %%xmm3,%%xmm1 \n"
+ "paddw %%xmm7,%%xmm0 \n" // + 255
+ "paddw %%xmm7,%%xmm1 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "pand %%xmm5,%%xmm6 \n"
+ "por %%xmm6,%%xmm0 \n"
+ "movdqu %%xmm0,(%0,%1) \n"
+ "lea 0x10(%0),%0 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kAttenuateShuffle) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_ARGBATTENUATEROW_SSSE3
#ifdef HAS_ARGBATTENUATEROW_AVX2
+
// Shuffle table duplicating alpha.
-static const uvec8 kShuffleAlpha_AVX2 = {6u, 7u, 6u, 7u, 6u, 7u,
- 128u, 128u, 14u, 15u, 14u, 15u,
- 14u, 15u, 128u, 128u};
+static const lvec8 kAttenuateShuffle_AVX2 = {
+ 6, -128, 6, -128, 6, -128, -128, -128, 14, -128, 14,
+ -128, 14, -128, -128, -128, 22, -128, 22, -128, 22, -128,
+ -128, -128, 30, -128, 30, -128, 30, -128, -128, -128};
+
// Attenuate 8 pixels at a time.
void ARGBAttenuateRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vbroadcastf128 %3,%%ymm4 \n"
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpslld $0x18,%%ymm5,%%ymm5 \n"
- "sub %0,%1 \n"
+ "vmovdqa %3,%%ymm4 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpslld $0x18,%%ymm5,%%ymm5 \n"
+ "vpxor %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpunpcklbw %%ymm6,%%ymm7,%%ymm7 \n"
+ "sub %0,%1 \n"
// 8 pixel loop.
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm6 \n"
- "vpunpcklbw %%ymm6,%%ymm6,%%ymm0 \n"
- "vpunpckhbw %%ymm6,%%ymm6,%%ymm1 \n"
- "vpshufb %%ymm4,%%ymm0,%%ymm2 \n"
- "vpshufb %%ymm4,%%ymm1,%%ymm3 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpand %%ymm5,%%ymm6,%%ymm6 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpor %%ymm6,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,0x00(%0,%1,1) \n"
- "lea 0x20(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleAlpha_AVX2) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "1: \n"
+ "vmovdqu (%0),%%ymm6 \n"
+ "vpunpcklbw %%ymm5,%%ymm6,%%ymm0 \n"
+ "vpunpckhbw %%ymm5,%%ymm6,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm2 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm3 \n"
+ "vpmullw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmullw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpaddw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm6,%%ymm1 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%0,%1,1) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kAttenuateShuffle_AVX2) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_ARGBATTENUATEROW_AVX2
@@ -4893,33 +11977,33 @@ void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb,
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movzb 0x03(%0),%3 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "movd 0x00(%4,%3,4),%%xmm2 \n"
- "movzb 0x07(%0),%3 \n"
- "movd 0x00(%4,%3,4),%%xmm3 \n"
- "pshuflw $0x40,%%xmm2,%%xmm2 \n"
- "pshuflw $0x40,%%xmm3,%%xmm3 \n"
- "movlhps %%xmm3,%%xmm2 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "movdqu (%0),%%xmm1 \n"
- "movzb 0x0b(%0),%3 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "movd 0x00(%4,%3,4),%%xmm2 \n"
- "movzb 0x0f(%0),%3 \n"
- "movd 0x00(%4,%3,4),%%xmm3 \n"
- "pshuflw $0x40,%%xmm2,%%xmm2 \n"
- "pshuflw $0x40,%%xmm3,%%xmm3 \n"
- "movlhps %%xmm3,%%xmm2 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movzb 0x03(%0),%3 \n"
+ "punpcklbw %%xmm0,%%xmm0 \n"
+ "movd 0x00(%4,%3,4),%%xmm2 \n"
+ "movzb 0x07(%0),%3 \n"
+ "movd 0x00(%4,%3,4),%%xmm3 \n"
+ "pshuflw $0x40,%%xmm2,%%xmm2 \n"
+ "pshuflw $0x40,%%xmm3,%%xmm3 \n"
+ "movlhps %%xmm3,%%xmm2 \n"
+ "pmulhuw %%xmm2,%%xmm0 \n"
+ "movdqu (%0),%%xmm1 \n"
+ "movzb 0x0b(%0),%3 \n"
+ "punpckhbw %%xmm1,%%xmm1 \n"
+ "movd 0x00(%4,%3,4),%%xmm2 \n"
+ "movzb 0x0f(%0),%3 \n"
+ "movd 0x00(%4,%3,4),%%xmm3 \n"
+ "pshuflw $0x40,%%xmm2,%%xmm2 \n"
+ "pshuflw $0x40,%%xmm3,%%xmm3 \n"
+ "movlhps %%xmm3,%%xmm2 \n"
+ "pmulhuw %%xmm2,%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width), // %2
@@ -4939,53 +12023,53 @@ void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
int width) {
uintptr_t alpha;
asm volatile(
- "sub %0,%1 \n"
- "vbroadcastf128 %5,%%ymm5 \n"
+ "sub %0,%1 \n"
+ "vbroadcasti128 %5,%%ymm5 \n"
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
// replace VPGATHER
- "movzb 0x03(%0),%3 \n"
- "vmovd 0x00(%4,%3,4),%%xmm0 \n"
- "movzb 0x07(%0),%3 \n"
- "vmovd 0x00(%4,%3,4),%%xmm1 \n"
- "movzb 0x0b(%0),%3 \n"
- "vpunpckldq %%xmm1,%%xmm0,%%xmm6 \n"
- "vmovd 0x00(%4,%3,4),%%xmm2 \n"
- "movzb 0x0f(%0),%3 \n"
- "vmovd 0x00(%4,%3,4),%%xmm3 \n"
- "movzb 0x13(%0),%3 \n"
- "vpunpckldq %%xmm3,%%xmm2,%%xmm7 \n"
- "vmovd 0x00(%4,%3,4),%%xmm0 \n"
- "movzb 0x17(%0),%3 \n"
- "vmovd 0x00(%4,%3,4),%%xmm1 \n"
- "movzb 0x1b(%0),%3 \n"
- "vpunpckldq %%xmm1,%%xmm0,%%xmm0 \n"
- "vmovd 0x00(%4,%3,4),%%xmm2 \n"
- "movzb 0x1f(%0),%3 \n"
- "vmovd 0x00(%4,%3,4),%%xmm3 \n"
- "vpunpckldq %%xmm3,%%xmm2,%%xmm2 \n"
+ "movzb 0x03(%0),%3 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm0 \n"
+ "movzb 0x07(%0),%3 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm1 \n"
+ "movzb 0x0b(%0),%3 \n"
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm6 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm2 \n"
+ "movzb 0x0f(%0),%3 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm3 \n"
+ "movzb 0x13(%0),%3 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm7 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm0 \n"
+ "movzb 0x17(%0),%3 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm1 \n"
+ "movzb 0x1b(%0),%3 \n"
+ "vpunpckldq %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm2 \n"
+ "movzb 0x1f(%0),%3 \n"
+ "vmovd 0x00(%4,%3,4),%%xmm3 \n"
+ "vpunpckldq %%xmm3,%%xmm2,%%xmm2 \n"
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm3 \n"
"vpunpcklqdq %%xmm2,%%xmm0,%%xmm0 \n"
"vinserti128 $0x1,%%xmm0,%%ymm3,%%ymm3 \n"
// end of VPGATHER
- "vmovdqu (%0),%%ymm6 \n"
- "vpunpcklbw %%ymm6,%%ymm6,%%ymm0 \n"
- "vpunpckhbw %%ymm6,%%ymm6,%%ymm1 \n"
- "vpunpcklwd %%ymm3,%%ymm3,%%ymm2 \n"
- "vpunpckhwd %%ymm3,%%ymm3,%%ymm3 \n"
- "vpshufb %%ymm5,%%ymm2,%%ymm2 \n"
- "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,0x00(%0,%1,1) \n"
- "lea 0x20(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "vmovdqu (%0),%%ymm6 \n"
+ "vpunpcklbw %%ymm6,%%ymm6,%%ymm0 \n"
+ "vpunpckhbw %%ymm6,%%ymm6,%%ymm1 \n"
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm2 \n"
+ "vpunpckhwd %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpshufb %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%0,%1,1) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width), // %2
@@ -4997,52 +12081,79 @@ void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
}
#endif // HAS_ARGBUNATTENUATEROW_AVX2
-#ifdef HAS_ARGBGRAYROW_SSSE3
-// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels
-void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+#if defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) || \
+ defined(HAS_ARGBGRAYROW_AVX512VNNI)
+static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+#endif // defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) ||
+ // defined(HAS_ARGBGRAYROW_AVX512VNNI)
+
+#ifdef HAS_ARGBGRAYROW_AVX2
+void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0}
+ static const uint32_t kOne16 = 0x00010001u;
+ static const uint32_t kBias = 32896u;
asm volatile(
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
+ "vpbroadcastd %[coeff],%%ymm2 \n"
+ "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsllw $7,%%ymm7,%%ymm3 \n"
+ "vpbroadcastb %%xmm3,%%ymm3 \n" // 0x80808080
+ "vpslld $24,%%ymm7,%%ymm7 \n" // 0xff000000
+ "vpbroadcastd %[one16],%%ymm4 \n"
+ "vpbroadcastd %[bias],%%ymm5 \n"
+ "vbroadcasti128 %[shuf],%%ymm6 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "psrlw $0x7,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movdqu (%0),%%xmm2 \n"
- "movdqu 0x10(%0),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "psrld $0x18,%%xmm2 \n"
- "psrld $0x18,%%xmm3 \n"
- "packuswb %%xmm3,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm3 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpcklbw %%xmm2,%%xmm3 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklwd %%xmm3,%%xmm0 \n"
- "punpckhwd %%xmm3,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kARGBToYJ), // %3
- "m"(kAddYJ64) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpxor %%ymm3,%%ymm0,%%ymm1 \n"
+ "vpmaddubsw %%ymm1,%%ymm2,%%ymm1 \n"
+ "vpmaddwd %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpsrld $8,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovd (%[src]),%%xmm0 \n"
+ "lea 0x4(%[src]),%[src] \n"
+ "vpxor %%xmm3,%%xmm0,%%xmm1 \n"
+ "vpmaddubsw %%xmm1,%%xmm2,%%xmm1 \n"
+ "vpmaddwd %%xmm4,%%xmm1,%%xmm1 \n"
+ "vpaddd %%xmm5,%%xmm1,%%xmm1 \n"
+ "vpsrld $8,%%xmm1,%%xmm1 \n"
+ "vpshufb %%xmm6,%%xmm1,%%xmm1 \n"
+ "vpand %%xmm7,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%[dst]) \n"
+ "lea 0x4(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [coeff] "m"(kCoeff), [one16] "m"(kOne16), [bias] "m"(kBias),
+ [shuf] "m"(kShufGrayDword)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-#endif // HAS_ARGBGRAYROW_SSSE3
+#endif // HAS_ARGBGRAYROW_AVX2
-#ifdef HAS_ARGBSEPIAROW_SSSE3
+#if defined(HAS_ARGBSEPIAROW_AVX2) || defined(HAS_ARGBSEPIAROW_AVX512VNNI)
// b = (r * 35 + g * 68 + b * 17) >> 7
// g = (r * 45 + g * 88 + b * 22) >> 7
// r = (r * 50 + g * 98 + b * 24) >> 7
@@ -5055,318 +12166,564 @@ static const vec8 kARGBToSepiaG = {22, 88, 45, 0, 22, 88, 45, 0,
static const vec8 kARGBToSepiaR = {24, 98, 50, 0, 24, 98, 50, 0,
24, 98, 50, 0, 24, 98, 50, 0};
+#endif
-// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels.
-void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width) {
+#ifdef HAS_ARGBSEPIAROW_AVX2
+// Convert ARGB pixels to Sepia ARGB pixels.
+void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) {
asm volatile(
- "movdqa %2,%%xmm2 \n"
- "movdqa %3,%%xmm3 \n"
- "movdqa %4,%%xmm4 \n"
+ "vbroadcasti128 %2,%%ymm2 \n"
+ "vbroadcasti128 %3,%%ymm3 \n"
+ "vbroadcasti128 %4,%%ymm4 \n"
+ "sub $0x8,%1 \n"
+ "jl 2f \n"
// 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm6 \n"
- "pmaddubsw %%xmm2,%%xmm0 \n"
- "pmaddubsw %%xmm2,%%xmm6 \n"
- "phaddw %%xmm6,%%xmm0 \n"
- "psrlw $0x7,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movdqu (%0),%%xmm5 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm5 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "phaddw %%xmm1,%%xmm5 \n"
- "psrlw $0x7,%%xmm5 \n"
- "packuswb %%xmm5,%%xmm5 \n"
- "punpcklbw %%xmm5,%%xmm0 \n"
- "movdqu (%0),%%xmm5 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm5 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "phaddw %%xmm1,%%xmm5 \n"
- "psrlw $0x7,%%xmm5 \n"
- "packuswb %%xmm5,%%xmm5 \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "psrld $0x18,%%xmm6 \n"
- "psrld $0x18,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "punpcklbw %%xmm6,%%xmm5 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklwd %%xmm5,%%xmm0 \n"
- "punpckhwd %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%0) \n"
- "movdqu %%xmm1,0x10(%0) \n"
- "lea 0x20(%0),%0 \n"
- "sub $0x8,%1 \n"
- "jg 1b \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm5 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n"
+ "vphaddw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vphaddw %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $0x7,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x7,%%ymm6,%%ymm6 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpshufd $0x01,%%ymm1,%%ymm5 \n"
+ "vpunpcklbw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpsrld $0x18,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpunpcklbw %%ymm0,%%ymm6,%%ymm6 \n"
+ "vpunpcklwd %%ymm6,%%ymm1,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x8,%1 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%1 \n"
+ "jle 99f \n"
+
+ // 1 pixel remainder loop.
+ LABELALIGN
+ "3: \n"
+ "vmovd (%0),%%xmm0 \n"
+ "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n"
+ "vpmaddubsw %%xmm3,%%xmm0,%%xmm5 \n"
+ "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n"
+ "vphaddw %%xmm5,%%xmm1,%%xmm1 \n"
+ "vphaddw %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpsrlw $0x7,%%xmm1,%%xmm1 \n"
+ "vpsrlw $0x7,%%xmm6,%%xmm6 \n"
+ "vpackuswb %%xmm1,%%xmm1,%%xmm1 \n"
+ "vpackuswb %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpshufd $0x01,%%xmm1,%%xmm5 \n"
+ "vpunpcklbw %%xmm5,%%xmm1,%%xmm1 \n"
+ "vpsrld $0x18,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vpunpcklbw %%xmm0,%%xmm6,%%xmm6 \n"
+ "vpunpcklwd %%xmm6,%%xmm1,%%xmm0 \n"
+ "vmovd %%xmm0,(%0) \n"
+ "lea 0x04(%0),%0 \n"
+ "sub $0x1,%1 \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
: "m"(kARGBToSepiaB), // %2
"m"(kARGBToSepiaG), // %3
"m"(kARGBToSepiaR) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
-#endif // HAS_ARGBSEPIAROW_SSSE3
+#endif // HAS_ARGBSEPIAROW_AVX2
-#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3
-// Tranform 8 ARGB pixels (32 bytes) with color matrix.
-// Same as Sepia except matrix is provided.
-void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width) {
+#ifdef HAS_ARGBSEPIAROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) {
+ uintptr_t temp;
asm volatile(
- "movdqu (%3),%%xmm5 \n"
- "pshufd $0x00,%%xmm5,%%xmm2 \n"
- "pshufd $0x55,%%xmm5,%%xmm3 \n"
- "pshufd $0xaa,%%xmm5,%%xmm4 \n"
- "pshufd $0xff,%%xmm5,%%xmm5 \n"
+ "vpbroadcastd %[kSepiaB],%%zmm2 \n"
+ "vpbroadcastd %[kSepiaG],%%zmm3 \n"
+ "vpbroadcastd %[kSepiaR],%%zmm4 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrld $24,%%zmm5,%%zmm6 \n" // 0x000000ff
+ "vpslld $24,%%zmm5,%%zmm5 \n" // 0xff000000
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm7 \n"
- "pmaddubsw %%xmm2,%%xmm0 \n"
- "pmaddubsw %%xmm2,%%xmm7 \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "phaddsw %%xmm7,%%xmm0 \n"
- "phaddsw %%xmm1,%%xmm6 \n"
- "psraw $0x6,%%xmm0 \n"
- "psraw $0x6,%%xmm6 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "punpcklbw %%xmm6,%%xmm0 \n"
- "movdqu (%0),%%xmm1 \n"
- "movdqu 0x10(%0),%%xmm7 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm7 \n"
- "phaddsw %%xmm7,%%xmm1 \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu 0x10(%0),%%xmm7 \n"
- "pmaddubsw %%xmm5,%%xmm6 \n"
- "pmaddubsw %%xmm5,%%xmm7 \n"
- "phaddsw %%xmm7,%%xmm6 \n"
- "psraw $0x6,%%xmm1 \n"
- "psraw $0x6,%%xmm6 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "punpcklbw %%xmm6,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm6 \n"
- "punpcklwd %%xmm1,%%xmm0 \n"
- "punpckhwd %%xmm1,%%xmm6 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm6,0x10(%1) \n"
- "lea 0x20(%0),%0 \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(matrix_argb) // %3
+ "vmovdqu32 (%[dst]),%%zmm0 \n"
+ "vpxord %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm7,%%zmm7 \n"
+ "vpxord %%zmm8,%%zmm8,%%zmm8 \n"
+ "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpsrld $7,%%zmm1,%%zmm1 \n"
+ "vpsrld $7,%%zmm7,%%zmm7 \n"
+ "vpsrld $7,%%zmm8,%%zmm8 \n"
+ "vpminud %%zmm6,%%zmm7,%%zmm7 \n"
+ "vpminud %%zmm6,%%zmm8,%%zmm8 \n"
+ "vpslld $8,%%zmm7,%%zmm7 \n"
+ "vpslld $16,%%zmm8,%%zmm8 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm8,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[dst]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpxord %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm7,%%zmm7 \n"
+ "vpxord %%zmm8,%%zmm8,%%zmm8 \n"
+ "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpsrld $7,%%zmm1,%%zmm1 \n"
+ "vpsrld $7,%%zmm7,%%zmm7 \n"
+ "vpsrld $7,%%zmm8,%%zmm8 \n"
+ "vpminud %%zmm6,%%zmm7,%%zmm7 \n"
+ "vpminud %%zmm6,%%zmm8,%%zmm8 \n"
+ "vpslld $8,%%zmm7,%%zmm7 \n"
+ "vpslld $16,%%zmm8,%%zmm8 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm8,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [width] "+r"(width), [temp] "=&r"(temp)
+ : [kSepiaB] "m"(kARGBToSepiaB), [kSepiaG] "m"(kARGBToSepiaG),
+ [kSepiaR] "m"(kARGBToSepiaR)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ "xmm7", "xmm8", "k1");
+}
+#endif // HAS_ARGBSEPIAROW_AVX512VNNI
+
+#ifdef HAS_ARGBCOLORMATRIXROW_AVX2
+static const vec8 kShufTranspose4x4 = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+// Transform ARGB pixels with color matrix.
+void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 (%3),%%ymm5 \n"
+ "vbroadcasti128 %4,%%ymm7 \n"
+ "vpshufd $0x00,%%ymm5,%%ymm2 \n"
+ "vpshufd $0x55,%%ymm5,%%ymm3 \n"
+ "vpshufd $0xaa,%%ymm5,%%ymm4 \n"
+ "vpshufd $0xff,%%ymm5,%%ymm5 \n"
+ "sub $0x8,%2 \n"
+ "jl 2f \n"
+
+ // 8 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm6 \n"
+ "vphaddsw %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n"
+ "vpmaddubsw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vphaddsw %%ymm0,%%ymm6,%%ymm6 \n"
+ "vpsraw $0x6,%%ymm1,%%ymm1 \n"
+ "vpsraw $0x6,%%ymm6,%%ymm6 \n"
+ "vpackuswb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm7,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm1,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%2 \n"
+ "jle 99f \n"
+
+ // 1 pixel remainder loop.
+ LABELALIGN
+ "3: \n"
+ "vmovd (%0),%%xmm0 \n"
+ "lea 0x04(%0),%0 \n"
+ "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n"
+ "vpmaddubsw %%xmm3,%%xmm0,%%xmm6 \n"
+ "vphaddsw %%xmm6,%%xmm1,%%xmm1 \n"
+ "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n"
+ "vpmaddubsw %%xmm5,%%xmm0,%%xmm0 \n"
+ "vphaddsw %%xmm0,%%xmm6,%%xmm6 \n"
+ "vpsraw $0x6,%%xmm1,%%xmm1 \n"
+ "vpsraw $0x6,%%xmm6,%%xmm6 \n"
+ "vpackuswb %%xmm6,%%xmm1,%%xmm1 \n"
+ "vpshufb %%xmm7,%%xmm1,%%xmm1 \n"
+ "vmovd %%xmm1,(%1) \n"
+ "lea 0x04(%1),%1 \n"
+ "sub $0x1,%2 \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(matrix_argb), // %3
+ "m"(kShufTranspose4x4) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
}
-#endif // HAS_ARGBCOLORMATRIXROW_SSSE3
+#endif // HAS_ARGBCOLORMATRIXROW_AVX2
+
+#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd (%[mat]),%%zmm2 \n"
+ "vpbroadcastd 0x04(%[mat]),%%zmm3 \n"
+ "vpbroadcastd 0x08(%[mat]),%%zmm4 \n"
+ "vpbroadcastd 0x0c(%[mat]),%%zmm5 \n"
+ "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n"
+ "vpsrld $24,%%zmm6,%%zmm6 \n" // 255
+ "vpxord %%zmm9,%%zmm9,%%zmm9 \n" // 0
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpxord %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm7,%%zmm7 \n"
+ "vpxord %%zmm8,%%zmm8,%%zmm8 \n"
+ "vpxord %%zmm10,%%zmm10,%%zmm10 \n"
+ "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n"
+ "vpsrad $6,%%zmm1,%%zmm1 \n"
+ "vpsrad $6,%%zmm7,%%zmm7 \n"
+ "vpsrad $6,%%zmm8,%%zmm8 \n"
+ "vpsrad $6,%%zmm10,%%zmm10 \n"
+ "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n"
+ "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n"
+ "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n"
+ "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n"
+ "vpminud %%zmm6,%%zmm1,%%zmm1 \n"
+ "vpminud %%zmm6,%%zmm7,%%zmm7 \n"
+ "vpminud %%zmm6,%%zmm8,%%zmm8 \n"
+ "vpminud %%zmm6,%%zmm10,%%zmm10 \n"
+ "vpslld $8,%%zmm7,%%zmm7 \n"
+ "vpslld $16,%%zmm8,%%zmm8 \n"
+ "vpslld $24,%%zmm10,%%zmm10 \n"
+ "vpord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm10,%%zmm8,%%zmm8 \n"
+ "vpord %%zmm8,%%zmm1,%%zmm1 \n"
+ "vmovdqu32 %%zmm1,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpxord %%zmm1,%%zmm1,%%zmm1 \n"
+ "vpxord %%zmm7,%%zmm7,%%zmm7 \n"
+ "vpxord %%zmm8,%%zmm8,%%zmm8 \n"
+ "vpxord %%zmm10,%%zmm10,%%zmm10 \n"
+ "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n"
+ "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n"
+ "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n"
+ "vpsrad $6,%%zmm1,%%zmm1 \n"
+ "vpsrad $6,%%zmm7,%%zmm7 \n"
+ "vpsrad $6,%%zmm8,%%zmm8 \n"
+ "vpsrad $6,%%zmm10,%%zmm10 \n"
+ "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n"
+ "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n"
+ "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n"
+ "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n"
+ "vpminud %%zmm6,%%zmm1,%%zmm1 \n"
+ "vpminud %%zmm6,%%zmm7,%%zmm7 \n"
+ "vpminud %%zmm6,%%zmm8,%%zmm8 \n"
+ "vpminud %%zmm6,%%zmm10,%%zmm10 \n"
+ "vpslld $8,%%zmm7,%%zmm7 \n"
+ "vpslld $16,%%zmm8,%%zmm8 \n"
+ "vpslld $24,%%zmm10,%%zmm10 \n"
+ "vpord %%zmm7,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm10,%%zmm8,%%zmm8 \n"
+ "vpord %%zmm8,%%zmm1,%%zmm1 \n"
+ "vmovdqu32 %%zmm1,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [mat] "r"(matrix_argb)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "xmm8", "xmm9", "xmm10", "k1");
+}
+#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI
-#ifdef HAS_ARGBQUANTIZEROW_SSE2
-// Quantize 4 ARGB pixels (16 bytes).
-void ARGBQuantizeRow_SSE2(uint8_t* dst_argb,
+#ifdef HAS_ARGBQUANTIZEROW_AVX2
+// Quantize ARGB pixels.
+void ARGBQuantizeRow_AVX2(uint8_t* dst_argb,
int scale,
int interval_size,
int interval_offset,
int width) {
asm volatile(
- "movd %2,%%xmm2 \n"
- "movd %3,%%xmm3 \n"
- "movd %4,%%xmm4 \n"
- "pshuflw $0x40,%%xmm2,%%xmm2 \n"
- "pshufd $0x44,%%xmm2,%%xmm2 \n"
- "pshuflw $0x40,%%xmm3,%%xmm3 \n"
- "pshufd $0x44,%%xmm3,%%xmm3 \n"
- "pshuflw $0x40,%%xmm4,%%xmm4 \n"
- "pshufd $0x44,%%xmm4,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
- "pcmpeqb %%xmm6,%%xmm6 \n"
- "pslld $0x18,%%xmm6 \n"
+ "vpbroadcastw %2,%%ymm2 \n"
+ "vpbroadcastw %3,%%ymm3 \n"
+ "vpbroadcastw %4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpslld $0x18,%%ymm6,%%ymm6 \n"
+ "sub $0x8,%1 \n"
+ "jl 2f \n"
- // 4 pixel loop.
+ // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "punpcklbw %%xmm5,%%xmm0 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "movdqu (%0),%%xmm1 \n"
- "punpckhbw %%xmm5,%%xmm1 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "pmullw %%xmm3,%%xmm0 \n"
- "movdqu (%0),%%xmm7 \n"
- "pmullw %%xmm3,%%xmm1 \n"
- "pand %%xmm6,%%xmm7 \n"
- "paddw %%xmm4,%%xmm0 \n"
- "paddw %%xmm4,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "por %%xmm7,%%xmm0 \n"
- "movdqu %%xmm0,(%0) \n"
- "lea 0x10(%0),%0 \n"
- "sub $0x4,%1 \n"
- "jg 1b \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm0,%%ymm7 \n"
+ "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpmullw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpmullw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpaddw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpandn %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpor %%ymm7,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x8,%1 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%1 \n"
+ "jle 99f \n"
+
+ // 1 pixel remainder loop.
+ LABELALIGN
+ "3: \n"
+ "vmovd (%0),%%xmm0 \n"
+ "vpand %%xmm6,%%xmm0,%%xmm7 \n"
+ "vpunpcklbw %%xmm5,%%xmm0,%%xmm0 \n"
+ "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n"
+ "vpmullw %%xmm3,%%xmm0,%%xmm0 \n"
+ "vpaddw %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vpandn %%xmm0,%%xmm6,%%xmm0 \n"
+ "vpor %%xmm7,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%0) \n"
+ "lea 0x04(%0),%0 \n"
+ "sub $0x1,%1 \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
- : "r"(scale), // %2
- "r"(interval_size), // %3
- "r"(interval_offset) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ : "m"(scale), // %2
+ "m"(interval_size), // %3
+ "m"(interval_offset) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
}
-#endif // HAS_ARGBQUANTIZEROW_SSE2
+#endif // HAS_ARGBQUANTIZEROW_AVX2
-#ifdef HAS_ARGBSHADEROW_SSE2
-// Shade 4 pixels at a time by specified value.
-void ARGBShadeRow_SSE2(const uint8_t* src_argb,
+#ifdef HAS_ARGBSHADEROW_AVX2
+// Shade ARGB pixels by specified value.
+void ARGBShadeRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value) {
asm volatile(
- "movd %3,%%xmm2 \n"
- "punpcklbw %%xmm2,%%xmm2 \n"
- "punpcklqdq %%xmm2,%%xmm2 \n"
+ "vpbroadcastd %3,%%ymm2 \n"
+ "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n"
+ "sub $0x8,%2 \n"
+ "jl 2f \n"
- // 4 pixel loop.
+ // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "pmulhuw %%xmm2,%%xmm1 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%2 \n"
+ "jle 99f \n"
+
+ // 1 pixel remainder loop.
+ LABELALIGN
+ "3: \n"
+ "vmovd (%0),%%xmm0 \n"
+ "lea 0x04(%0),%0 \n"
+ "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n"
+ "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x8,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%1) \n"
+ "lea 0x04(%1),%1 \n"
+ "sub $0x1,%2 \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
- : "r"(value) // %3
+ : "m"(value) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
-#endif // HAS_ARGBSHADEROW_SSE2
+#endif // HAS_ARGBSHADEROW_AVX2
#ifdef HAS_ARGBMULTIPLYROW_SSE2
// Multiply 2 rows of ARGB pixels together, 4 pixels at a time.
-void ARGBMultiplyRow_SSE2(const uint8_t* src_argb0,
+void ARGBMultiplyRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
-
- "pxor %%xmm5,%%xmm5 \n"
+ "pxor %%xmm5,%%xmm5 \n"
+ "pcmpeqw %%xmm4,%%xmm4 \n"
+ "psrlw $0xf,%%xmm4 \n"
+ "psllw $0x7,%%xmm4 \n"
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movdqu (%1),%%xmm2 \n"
- "lea 0x10(%1),%1 \n"
- "movdqu %%xmm0,%%xmm1 \n"
- "movdqu %%xmm2,%%xmm3 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpckhbw %%xmm5,%%xmm3 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "pmulhuw %%xmm3,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqu (%1),%%xmm2 \n"
+ "lea 0x10(%1),%1 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "movdqa %%xmm2,%%xmm3 \n"
+ "punpcklbw %%xmm5,%%xmm0 \n"
+ "punpckhbw %%xmm5,%%xmm1 \n"
+ "punpcklbw %%xmm5,%%xmm2 \n"
+ "punpckhbw %%xmm5,%%xmm3 \n"
+ "pmullw %%xmm2,%%xmm0 \n"
+ "pmullw %%xmm3,%%xmm1 \n"
+ "paddw %%xmm4,%%xmm0 \n"
+ "paddw %%xmm4,%%xmm1 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x4,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
:
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_ARGBMULTIPLYROW_SSE2
#ifdef HAS_ARGBMULTIPLYROW_AVX2
// Multiply 2 rows of ARGB pixels together, 8 pixels at a time.
-void ARGBMultiplyRow_AVX2(const uint8_t* src_argb0,
+void ARGBMultiplyRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
- asm volatile(
-
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
-
- // 4 pixel loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm1 \n"
- "lea 0x20(%0),%0 \n"
- "vmovdqu (%1),%%ymm3 \n"
- "lea 0x20(%1),%1 \n"
- "vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n"
- "vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n"
- "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
- "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x8,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb0), // %0
- "+r"(src_argb1), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- :
- : "memory", "cc"
-#if defined(__AVX2__)
- ,
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"
-#endif
- );
+ asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+
+ // 8 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu (%1),%%ymm3 \n"
+ "lea 0x20(%1),%1 \n"
+ "vpunpcklbw %%ymm1,%%ymm5,%%ymm0 \n"
+ "vpunpckhbw %%ymm1,%%ymm5,%%ymm1 \n"
+ "vpsrlw $0x1,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x1,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
+ "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpmulhrsw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhrsw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
}
#endif // HAS_ARGBMULTIPLYROW_AVX2
#ifdef HAS_ARGBADDROW_SSE2
// Add 2 rows of ARGB pixels together, 4 pixels at a time.
-void ARGBAddRow_SSE2(const uint8_t* src_argb0,
+void ARGBAddRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movdqu (%1),%%xmm1 \n"
- "lea 0x10(%1),%1 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqu (%1),%%xmm1 \n"
+ "lea 0x10(%1),%1 \n"
+ "paddusb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x4,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -5377,24 +12734,24 @@ void ARGBAddRow_SSE2(const uint8_t* src_argb0,
#ifdef HAS_ARGBADDROW_AVX2
// Add 2 rows of ARGB pixels together, 4 pixels at a time.
-void ARGBAddRow_AVX2(const uint8_t* src_argb0,
+void ARGBAddRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "lea 0x20(%0),%0 \n"
- "vpaddusb (%1),%%ymm0,%%ymm0 \n"
- "lea 0x20(%1),%1 \n"
- "vmovdqu %%ymm0,(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x8,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpaddusb (%1),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%1),%1 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -5405,24 +12762,24 @@ void ARGBAddRow_AVX2(const uint8_t* src_argb0,
#ifdef HAS_ARGBSUBTRACTROW_SSE2
// Subtract 2 rows of ARGB pixels, 4 pixels at a time.
-void ARGBSubtractRow_SSE2(const uint8_t* src_argb0,
+void ARGBSubtractRow_SSE2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movdqu (%1),%%xmm1 \n"
- "lea 0x10(%1),%1 \n"
- "psubusb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqu (%1),%%xmm1 \n"
+ "lea 0x10(%1),%1 \n"
+ "psubusb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x4,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -5433,24 +12790,24 @@ void ARGBSubtractRow_SSE2(const uint8_t* src_argb0,
#ifdef HAS_ARGBSUBTRACTROW_AVX2
// Subtract 2 rows of ARGB pixels, 8 pixels at a time.
-void ARGBSubtractRow_AVX2(const uint8_t* src_argb0,
+void ARGBSubtractRow_AVX2(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "lea 0x20(%0),%0 \n"
- "vpsubusb (%1),%%ymm0,%%ymm0 \n"
- "lea 0x20(%1),%1 \n"
- "vmovdqu %%ymm0,(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x8,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpsubusb (%1),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%1),%1 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -5459,1502 +12816,6303 @@ void ARGBSubtractRow_AVX2(const uint8_t* src_argb0,
}
#endif // HAS_ARGBSUBTRACTROW_AVX2
-#ifdef HAS_SOBELXROW_SSE2
+#ifdef HAS_SOBELXROW_AVX2
// SobelX as a matrix is
// -1 0 1
// -2 0 2
// -1 0 1
-void SobelXRow_SSE2(const uint8_t* src_y0,
+void SobelXRow_AVX2(const uint8_t* src_y0,
const uint8_t* src_y1,
const uint8_t* src_y2,
uint8_t* dst_sobelx,
int width) {
asm volatile(
- "sub %0,%1 \n"
- "sub %0,%2 \n"
- "sub %0,%3 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+ "sub %0,%3 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%ymm0 \n"
+ "vpmovzxbw 0x02(%0),%%ymm1 \n"
+ "vpsubw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n"
+ "vpmovzxbw 0x02(%0,%1,1),%%ymm2 \n"
+ "vpsubw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpmovzxbw 0x00(%0,%2,1),%%ymm2 \n"
+ "vpmovzxbw 0x02(%0,%2,1),%%ymm3 \n"
+ "vpsubw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpabsw %%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,0x00(%0,%3,1) \n"
+ "lea 0x10(%0),%0 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y0), // %0
+ "+r"(src_y1), // %1
+ "+r"(src_y2), // %2
+ "+r"(dst_sobelx), // %3
+ "+r"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_SOBELXROW_AVX2
+
+#ifdef HAS_SOBELYROW_AVX2
+// SobelY as a matrix is
+// -1 -2 -1
+// 0 0 0
+// 1 2 1
+void SobelYRow_AVX2(const uint8_t* src_y0,
+ const uint8_t* src_y1,
+ uint8_t* dst_sobely,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "sub %0,%2 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%ymm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n"
+ "vpsubw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpmovzxbw 0x01(%0),%%ymm1 \n"
+ "vpmovzxbw 0x01(%0,%1,1),%%ymm2 \n"
+ "vpsubw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpmovzxbw 0x02(%0),%%ymm2 \n"
+ "vpmovzxbw 0x02(%0,%1,1),%%ymm3 \n"
+ "vpsubw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpabsw %%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,0x00(%0,%2,1) \n"
+ "lea 0x10(%0),%0 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y0), // %0
+ "+r"(src_y1), // %1
+ "+r"(dst_sobely), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_SOBELYROW_AVX2
+
+#ifdef HAS_SOBELROW_AVX2
+// Adds Sobel X and Sobel Y and stores Sobel into ARGB.
+// A = 255
+// R = Sobel
+// G = Sobel
+// B = Sobel
+void SobelRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpslld $0x18,%%ymm5,%%ymm5 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vpaddusb 0x00(%0,%1,1),%%xmm0,%%xmm0 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpshufd $0xd8,%%xmm0,%%xmm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm0,%%ymm0,%%ymm1 \n"
+ "vpunpckhwd %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm5,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm1,(%2) \n"
+ "vmovdqu %%ymm0,0x20(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_sobelx), // %0
+ "+r"(src_sobely), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
+}
+#endif // HAS_SOBELROW_AVX2
+
+#ifdef HAS_SOBELTOPLANEROW_AVX2
+// Adds Sobel X and Sobel Y and stores Sobel into a plane.
+void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile("sub %0,%1 \n"
+
+ // 32 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpaddusb 0x00(%0,%1,1),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_sobelx), // %0
+ "+r"(src_sobely), // %1
+ "+r"(dst_y), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_SOBELTOPLANEROW_AVX2
+
+#ifdef HAS_SOBELXYROW_AVX2
+// Mixes Sobel X, Sobel Y and Sobel into ARGB.
+// A = 255
+// R = Sobel X
+// G = Sobel
+// B = Sobel Y
+void SobelXYRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vmovdqu 0x00(%0,%1,1),%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpaddusb %%xmm1,%%xmm0,%%xmm2 \n"
+ "vpshufd $0xd8,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xd8,%%xmm1,%%xmm1 \n"
+ "vpshufd $0xd8,%%xmm2,%%xmm2 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm2,%%ymm2 \n"
+ "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm0,%%ymm1,%%ymm2 \n"
+ "vpunpckhwd %%ymm0,%%ymm1,%%ymm0 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "vmovdqu %%ymm0,0x20(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_sobelx), // %0
+ "+r"(src_sobely), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5");
+}
+#endif // HAS_SOBELXYROW_AVX2
+
+#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2
+static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+static const vec8 kCumSumMaddLo[2] = {
+ {1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0},
+ {1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0}};
+static const vec8 kCumSumMaddHi[2] = {
+ {1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0},
+ {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}};
+
+void ComputeCumulativeSumRow_AVX2(const uint8_t* row,
+ int32_t* cumsum,
+ const int32_t* previous_cumsum,
+ int width) {
+ asm volatile(
+ "vpxor %%xmm0,%%xmm0,%%xmm0 \n"
+ "vbroadcasti128 %4,%%ymm5 \n"
+ "vmovdqu %5,%%ymm6 \n"
+ "vmovdqu %6,%%ymm7 \n"
+ "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0xf,%%ymm1,%%ymm1 \n"
+ "sub $0x4,%3 \n"
+ "jl 49f \n"
+
+ // 4 pixel loop.
+ LABELALIGN
+ "40: \n"
+ "vbroadcasti128 (%0),%%ymm2 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpshufb %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n"
+ "vpmaddwd %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpmaddwd %%ymm1,%%ymm4,%%ymm4 \n"
+ "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n"
+ "vpaddd (%2),%%ymm3,%%ymm3 \n"
+ "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n"
+ "lea 0x40(%2),%2 \n"
+ "vpaddd %%ymm0,%%ymm3,%%ymm3 \n"
+ "vpaddd %%ymm0,%%ymm4,%%ymm4 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm3,(%1) \n"
+ "vmovdqu %%ymm4,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x4,%3 \n"
+ "jge 40b \n"
+
+ "49: \n"
+ "add $0x3,%3 \n"
+ "jl 19f \n"
+
+ // 1 pixel loop.
+ LABELALIGN
+ "10: \n"
+ "vpmovzxbd (%0),%%xmm2 \n"
+ "lea 0x4(%0),%0 \n"
+ "vpaddd %%xmm2,%%xmm0,%%xmm0 \n"
+ "vpaddd (%2),%%xmm0,%%xmm2 \n"
+ "lea 0x10(%2),%2 \n"
+ "vmovdqu %%xmm2,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x1,%3 \n"
+ "jge 10b \n"
+
+ "19: \n"
+ "vzeroupper \n"
+ : "+r"(row), // %0
+ "+r"(cumsum), // %1
+ "+r"(previous_cumsum), // %2
+ "+r"(width) // %3
+ : "m"(kCumSumTranspose), // %4
+ "m"(kCumSumMaddLo), // %5
+ "m"(kCumSumMaddHi) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2
+
+#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+void CumulativeSumToAverageRow_AVX2(const int32_t* topleft,
+ const int32_t* botleft,
+ int width,
+ int area,
+ uint8_t* dst,
+ int count) {
+ asm volatile(
+ "vpxor %%xmm5,%%xmm5,%%xmm5 \n"
+ "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n"
+ "vpcmpeqd %%xmm4,%%xmm4,%%xmm4 \n"
+ "vpslld $25,%%xmm4,%%xmm4 \n"
+ "vpsrld $2,%%xmm4,%%xmm4 \n"
+ "vdivss %%xmm5,%%xmm4,%%xmm4 \n"
+ "vpshufd $0x0,%%xmm4,%%xmm4 \n"
+ "subl $0x4,%3 \n"
+ "jl 19f \n"
+ "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n"
+ "cmpl $0x80,%5 \n"
+ "ja 10f \n"
+ "cmpl $0x1,%5 \n"
+ "jle 10f \n"
+
+ "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpsrld $0x10,%%xmm6,%%xmm6 \n"
+ "vcvtdq2ps %%xmm6,%%xmm6 \n"
+ "vaddss %%xmm6,%%xmm5,%%xmm6 \n"
+ "vmulss %%xmm4,%%xmm6,%%xmm6 \n"
+ "vcvtps2dq %%xmm6,%%xmm6 \n"
+ "vpackusdw %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpbroadcastw %%xmm6,%%ymm6 \n"
+
+ // 4 pixel small loop.
+ LABELALIGN
+ "4: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n"
+ "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n"
+ "lea 0x40(%0),%0 \n"
+ "vpsubd (%1),%%ymm0,%%ymm0 \n"
+ "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n"
+ "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n"
+ "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n"
+ "lea 0x40(%1),%1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm1 \n"
+ "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xd8,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "subl $0x4,%3 \n"
+ "jge 4b \n"
+ "vzeroupper \n"
+ "jmp 19f \n"
+
+ // 4 pixel loop.
+ LABELALIGN
+ "10: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n"
+ "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n"
+ "lea 0x40(%0),%0 \n"
+ "vpsubd (%1),%%ymm0,%%ymm0 \n"
+ "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n"
+ "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n"
+ "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n"
+ "lea 0x40(%1),%1 \n"
+ "vcvtdq2ps %%ymm0,%%ymm0 \n"
+ "vcvtdq2ps %%ymm1,%%ymm1 \n"
+ "vmulps %%ymm4,%%ymm0,%%ymm0 \n"
+ "vmulps %%ymm4,%%ymm1,%%ymm1 \n"
+ "vcvttps2dq %%ymm0,%%ymm0 \n"
+ "vcvttps2dq %%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm1 \n"
+ "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpshufd $0xd8,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "subl $0x4,%3 \n"
+ "jge 10b \n"
+ "vzeroupper \n"
+
+ "19: \n"
+ "addl $0x3,%3 \n"
+ "jl 99f \n"
+
+ // 1 pixel loop.
+ LABELALIGN
+ "20: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vpsubd 0x00(%0,%4,4),%%xmm0,%%xmm0\n"
+ "lea 0x10(%0),%0 \n"
+ "vmovdqu 0x00(%1,%4,4),%%xmm1 \n"
+ "vpsubd (%1),%%xmm1,%%xmm1 \n"
+ "lea 0x10(%1),%1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vcvtdq2ps %%xmm0,%%xmm0 \n"
+ "vmulps %%xmm4,%%xmm0,%%xmm0 \n"
+ "vcvttps2dq %%xmm0,%%xmm0 \n"
+ "vpackusdw %%xmm0,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%2) \n"
+ "lea 0x4(%2),%2 \n"
+ "subl $0x1,%3 \n"
+ "jge 20b \n"
+ "99: \n"
+ : "+r"(topleft), // %0
+ "+r"(botleft), // %1
+ "+r"(dst), // %2
+ "+rm"(count) // %3
+ : "r"((ptrdiff_t)(width)), // %4
+ "rm"(area) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+
+#ifdef HAS_ARGBAFFINEROW_AVX2
+static const float kAffineRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f,
+ 4.0f, 5.0f, 6.0f, 7.0f};
+static const float kAffineStep8 = 8.0f;
+// Copy ARGB pixels from source image with slope to a row of destination.
+LIBYUV_API
+void ARGBAffineRow_AVX2(const uint8_t* src_argb,
+ int src_argb_stride,
+ uint8_t* dst_argb,
+ const float* src_dudv,
+ int width) {
+ asm volatile(
+ "vbroadcastss (%[uv]),%%ymm2 \n"
+ "vbroadcastss 0x04(%[uv]),%%ymm3 \n"
+ "vbroadcastss 0x08(%[uv]),%%ymm4 \n"
+ "vbroadcastss 0x0c(%[uv]),%%ymm5 \n"
+ "vmovups %[ramp],%%ymm0 \n"
+ "vmulps %%ymm0,%%ymm4,%%ymm1 \n"
+ "vaddps %%ymm1,%%ymm2,%%ymm2 \n"
+ "vmulps %%ymm0,%%ymm5,%%ymm1 \n"
+ "vaddps %%ymm1,%%ymm3,%%ymm3 \n"
+ "vbroadcastss %[step8],%%ymm0 \n"
+ "vmulps %%ymm0,%%ymm4,%%ymm4 \n"
+ "vmulps %%ymm0,%%ymm5,%%ymm5 \n"
+ "vpbroadcastd %[stride],%%ymm6 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vcvttps2dq %%ymm2,%%ymm0 \n"
+ "vcvttps2dq %%ymm3,%%ymm1 \n"
+ "vpslld $2,%%ymm0,%%ymm0 \n"
+ "vpmulld %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[src],%%ymm0,1),%%ymm1\n"
+ "vaddps %%ymm4,%%ymm2,%%ymm2 \n"
+ "vaddps %%ymm5,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm1,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+ "vmovss 0x08(%[uv]),%%xmm4 \n"
+ "vmovss 0x0c(%[uv]),%%xmm5 \n"
+
+ // 1 pixel remainder loop.
+ LABELALIGN
+ "3: \n"
+ "vcvttps2dq %%xmm2,%%xmm0 \n"
+ "vcvttps2dq %%xmm3,%%xmm1 \n"
+ "vpslld $2,%%xmm0,%%xmm0 \n"
+ "vpmulld %%xmm6,%%xmm1,%%xmm1 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpcmpeqd %%xmm7,%%xmm7,%%xmm7 \n"
+ "vpsrldq $12,%%xmm7,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[src],%%xmm0,1),%%xmm1\n"
+ "vaddss %%xmm4,%%xmm2,%%xmm2 \n"
+ "vaddss %%xmm5,%%xmm3,%%xmm3 \n"
+ "vmovd %%xmm1,(%[dst]) \n"
+ "lea 0x04(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [src] "r"(src_argb), [uv] "r"(src_dudv), [stride] "m"(src_argb_stride),
+ [ramp] "m"(kAffineRamp), [step8] "m"(kAffineStep8)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#endif // HAS_ARGBAFFINEROW_AVX2
+
+#ifdef HAS_INTERPOLATEROW_AVX2
+// Bilinear filter 32x2 -> 32x1
+void InterpolateRow_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ asm volatile(
+ "sub %1,%0 \n"
+ "cmp $0x0,%3 \n"
+ "je 100f \n"
+ "cmp $0x80,%3 \n"
+ "je 50f \n"
+
+ "vmovd %3,%%xmm0 \n"
+ "neg %3 \n"
+ "add $0x100,%3 \n"
+ "vmovd %3,%%xmm5 \n"
+ "vpunpcklbw %%xmm0,%%xmm5,%%xmm5 \n"
+ "vpunpcklwd %%xmm5,%%xmm5,%%xmm5 \n"
+ "vbroadcastss %%xmm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
+ "vpsllw $7,%%ymm4,%%ymm4 \n"
+
+ // General purpose row blend.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vmovdqu 0x00(%1,%4,1),%%ymm2 \n"
+ "vpunpckhbw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpsubb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpsubb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm1,%%ymm5,%%ymm1 \n"
+ "vpmaddubsw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpaddw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpaddw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "jmp 99f \n"
+
+ // Blend 50 / 50.
+ LABELALIGN
+ "50: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vpavgb 0x00(%1,%4,1),%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 50b \n"
+ "jmp 99f \n"
+
+ // Blend 100 / 0 - Copy row unchanged.
+ LABELALIGN
+ "100: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 100b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(width), // %2
+ "+r"(source_y_fraction) // %3
+ : "r"(src_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5");
+}
+#endif // HAS_INTERPOLATEROW_AVX2
+
+#ifdef HAS_INTERPOLATEROW_AVX512BW
+// Bilinear filter 64x2 -> 64x1 with masked tail for any width.
+LIBYUV_TARGET_AVX512BW
+void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ asm volatile(
+ "sub %1,%0 \n"
+ "cmp $0x0,%3 \n"
+ "je 100f \n"
+ "cmp $0x80,%3 \n"
+ "je 50f \n"
+
+ "imul $0xff,%3,%3 \n"
+ "add $0x100,%3 \n"
+ "vpbroadcastw %3,%%zmm5 \n"
+ "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
+ "vpabsb %%zmm4,%%zmm4 \n"
+ "vpsllw $7,%%zmm4,%%zmm4 \n"
+
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ // General purpose row blend.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu64 (%1),%%zmm0 \n"
+ "vmovdqu64 0x00(%1,%4,1),%%zmm2 \n"
+ "vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n"
+ "vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpaddw %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpaddw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm1,%%zmm1 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%q3 \n"
+ "bzhi %q2,%q3,%q3 \n"
+ "kmovq %q3,%%k1 \n"
+ "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n"
+ "vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpsubb %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n"
+ "vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n"
+ "vpaddw %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpaddw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x8,%%zmm1,%%zmm1 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Blend 50 / 50.
+ LABELALIGN
+ "50: \n"
+ "sub $0x40,%2 \n"
+ "jl 52f \n"
+ LABELALIGN
+ "51: \n"
+ "vmovdqu64 (%1),%%zmm0 \n"
+ "vpavgb 0x00(%1,%4,1),%%zmm0,%%zmm0 \n"
+ "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 51b \n"
+
+ "52: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%q3 \n"
+ "bzhi %q2,%q3,%q3 \n"
+ "kmovq %q3,%%k1 \n"
+ "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n"
+ "vpavgb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Blend 100 / 0 - Copy row unchanged.
+ LABELALIGN
+ "100: \n"
+ "sub $0x40,%2 \n"
+ "jl 102f \n"
+ LABELALIGN
+ "101: \n"
+ "vmovdqu64 (%1),%%zmm0 \n"
+ "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 101b \n"
+
+ "102: \n"
+ "add $0x40,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%q3 \n"
+ "bzhi %q2,%q3,%q3 \n"
+ "kmovq %q3,%%k1 \n"
+ "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(width), // %2
+ "+r"(source_y_fraction) // %3
+ : "r"(src_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5", "k1");
+}
+#endif // HAS_INTERPOLATEROW_AVX512BW
+
+#ifdef HAS_INTERPOLATEROW_16_AVX2
+// Bilinear filter 16x2 -> 16x1
+void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ asm volatile(
+ "sub %1,%0 \n"
+ "cmp $0x0,%3 \n"
+ "je 100f \n"
+ "cmp $0x80,%3 \n"
+ "je 50f \n"
+
+ "vmovd %3,%%xmm0 \n"
+ "neg %3 \n"
+ "add $0x100,%3 \n"
+ "vmovd %3,%%xmm5 \n"
+ "vpunpcklwd %%xmm0,%%xmm5,%%xmm5 \n"
+ "vpbroadcastd %%xmm5,%%ymm5 \n"
+ "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsllw $15,%%ymm4,%%ymm4 \n" // 0x8000 unsigned bias
+ "vpsrlw $8,%%ymm4,%%ymm3 \n" // 0x00800080 = 8388736
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vmovdqu (%1,%4,2),%%ymm1 \n"
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpsubw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpsubw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddwd %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpmaddwd %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpsrad $0x8,%%ymm2,%%ymm2 \n"
+ "vpsrad $0x8,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "jmp 99f \n"
+
+ "50: \n" LABELALIGN
+ "2: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vpavgw (%1,%4,2),%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 2b \n"
+ "jmp 99f \n"
+
+ "100: \n" LABELALIGN
+ "3: \n"
+ "vmovdqu (%1),%%ymm0 \n"
+ "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(width), // %2
+ "+r"(source_y_fraction) // %3
+ : "r"(src_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_INTERPOLATEROW_16_AVX2
+
+#ifdef HAS_ARGBSHUFFLEROW_SSSE3
+// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
+void ARGBShuffleRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ asm volatile("movdqu (%3),%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(shuffler) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
+}
+#endif // HAS_ARGBSHUFFLEROW_SSSE3
+
+#ifdef HAS_ARGBSHUFFLEROW_AVX2
+// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
+void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ asm volatile("vbroadcasti128 (%3),%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(shuffler) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
+}
+#endif // HAS_ARGBSHUFFLEROW_AVX2
+
+#ifdef HAS_ARGBSHUFFLEROW_AVX512BW
+// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
+LIBYUV_TARGET_AVX512BW
+void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 (%[shuffler]),%%zmm5 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "vmovdqu8 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vmovdqu8 %%zmm0,(%1) \n"
+ "vmovdqu8 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q2,%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
+ "vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
+ "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
+ "vmovdqu32 %%zmm1,0x40(%1)%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ [temp] "=&r"(temp) // %[temp]
+ : [shuffler] "r"(shuffler) // %[shuffler]
+ : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1", "k2");
+}
+#endif // HAS_ARGBSHUFFLEROW_AVX512BW
+
+#ifdef HAS_I422TOYUY2ROW_SSE2
+void I422ToYUY2Row_SSE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile("sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%1),%%xmm2 \n"
+ "movq 0x00(%1,%2,1),%%xmm1 \n"
+ "add $0x8,%1 \n"
+ "punpcklbw %%xmm1,%%xmm2 \n"
+ "movdqu (%0),%%xmm0 \n"
+ "add $0x10,%0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n"
+ "punpckhbw %%xmm2,%%xmm1 \n"
+ "movdqu %%xmm0,(%3) \n"
+ "movdqu %%xmm1,0x10(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_yuy2), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_I422TOYUY2ROW_SSE2
+
+#ifdef HAS_I422TOUYVYROW_SSE2
+void I422ToUYVYRow_SSE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width) {
+ asm volatile("sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%1),%%xmm2 \n"
+ "movq 0x00(%1,%2,1),%%xmm1 \n"
+ "add $0x8,%1 \n"
+ "punpcklbw %%xmm1,%%xmm2 \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqa %%xmm2,%%xmm1 \n"
+ "add $0x10,%0 \n"
+ "punpcklbw %%xmm0,%%xmm1 \n"
+ "punpckhbw %%xmm0,%%xmm2 \n"
+ "movdqu %%xmm1,(%3) \n"
+ "movdqu %%xmm2,0x10(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_uyvy), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_I422TOUYVYROW_SSE2
+
+#ifdef HAS_I422TOYUY2ROW_AVX2
+void I422ToYUY2Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile("sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%1),%%ymm1 \n"
+ "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
+ "add $0x10,%1 \n"
+ "vpsllw $0x8,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm2,%%ymm2 \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "add $0x20,%0 \n"
+ "vpunpcklbw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpunpckhbw %%ymm2,%%ymm0,%%ymm2 \n"
+ "vextractf128 $0x0,%%ymm1,(%3) \n"
+ "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
+ "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
+ "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
+ "lea 0x40(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_yuy2), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_I422TOYUY2ROW_AVX2
+
+#ifdef HAS_I422TOUYVYROW_AVX2
+void I422ToUYVYRow_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width) {
+ asm volatile("sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%1),%%ymm1 \n"
+ "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
+ "add $0x10,%1 \n"
+ "vpsllw $0x8,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm2,%%ymm2 \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "add $0x20,%0 \n"
+ "vpunpcklbw %%ymm0,%%ymm2,%%ymm1 \n"
+ "vpunpckhbw %%ymm0,%%ymm2,%%ymm2 \n"
+ "vextractf128 $0x0,%%ymm1,(%3) \n"
+ "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
+ "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
+ "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
+ "lea 0x40(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_uyvy), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_I422TOUYVYROW_AVX2
+
+#ifdef HAS_ARGBPOLYNOMIALROW_SSE2
+void ARGBPolynomialRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width) {
+ asm volatile("pxor %%xmm3,%%xmm3 \n"
+
+ // 2 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n"
+ "lea 0x8(%0),%0 \n"
+ "punpcklbw %%xmm3,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm4 \n"
+ "punpcklwd %%xmm3,%%xmm0 \n"
+ "punpckhwd %%xmm3,%%xmm4 \n"
+ "cvtdq2ps %%xmm0,%%xmm0 \n"
+ "cvtdq2ps %%xmm4,%%xmm4 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "movdqa %%xmm4,%%xmm5 \n"
+ "movups 0x10(%3),%%xmm7 \n"
+ "mulps %%xmm7,%%xmm0 \n"
+ "mulps %%xmm7,%%xmm4 \n"
+ "movups (%3),%%xmm7 \n"
+ "addps %%xmm7,%%xmm0 \n"
+ "addps %%xmm7,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "mulps %%xmm1,%%xmm2 \n"
+ "mulps %%xmm5,%%xmm6 \n"
+ "mulps %%xmm2,%%xmm1 \n"
+ "mulps %%xmm6,%%xmm5 \n"
+ "movups 0x20(%3),%%xmm7 \n"
+ "mulps %%xmm7,%%xmm2 \n"
+ "mulps %%xmm7,%%xmm6 \n"
+ "movups 0x30(%3),%%xmm7 \n"
+ "mulps %%xmm7,%%xmm1 \n"
+ "mulps %%xmm7,%%xmm5 \n"
+ "addps %%xmm2,%%xmm0 \n"
+ "addps %%xmm6,%%xmm4 \n"
+ "addps %%xmm1,%%xmm0 \n"
+ "addps %%xmm5,%%xmm4 \n"
+ "cvttps2dq %%xmm0,%%xmm0 \n"
+ "cvttps2dq %%xmm4,%%xmm4 \n"
+ "packssdw %%xmm4,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x2,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(poly) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#endif // HAS_ARGBPOLYNOMIALROW_SSE2
+
+#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
+void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 (%3),%%ymm4 \n"
+ "vbroadcasti128 0x10(%3),%%ymm5 \n"
+ "vbroadcasti128 0x20(%3),%%ymm6 \n"
+ "vbroadcasti128 0x30(%3),%%ymm7 \n"
+
+ // 2 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbd (%0),%%ymm0 \n" // 2 ARGB pixels
+ "lea 0x8(%0),%0 \n"
+ "vcvtdq2ps %%ymm0,%%ymm0 \n" // X 8 floats
+ "vmulps %%ymm0,%%ymm0,%%ymm2 \n" // X * X
+ "vmulps %%ymm7,%%ymm0,%%ymm3 \n" // C3 * X
+ "vfmadd132ps %%ymm5,%%ymm4,%%ymm0 \n" // result = C0 + C1 * X
+ "vfmadd231ps %%ymm6,%%ymm2,%%ymm0 \n" // result += C2 * X * X
+ "vfmadd231ps %%ymm3,%%ymm2,%%ymm0 \n" // result += C3 * X * X *
+ // X
+ "vcvttps2dq %%ymm0,%%ymm0 \n"
+ "vpackssdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovq %%xmm0,(%1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x2,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(poly) // %3
+ : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7");
+}
+#endif // HAS_ARGBPOLYNOMIALROW_AVX2
+
+#if defined(HAS_HALFFLOATROW_AVX2)
+// 2^-112 moves the float exponent into half float position. Applied as a
+// separate multiply after scale so no denormal constant is formed when
+// scale < 2^-14 (see HalfFloatRow_C).
+static const float kScaleBias = 1.9259299444e-34f;
+#endif
+
+#ifdef HAS_HALFFLOATROW_AVX2
+void HalfFloatRow_AVX2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ asm volatile(
+ "vbroadcastss %3, %%ymm4 \n"
+ "vbroadcastss %4, %%ymm6 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "sub %0,%1 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm2 \n" // 16 shorts
+ "add $0x20,%0 \n"
+ "vpunpckhwd %%ymm5,%%ymm2,%%ymm3 \n" // mutates
+ "vpunpcklwd %%ymm5,%%ymm2,%%ymm2 \n"
+ "vcvtdq2ps %%ymm3,%%ymm3 \n"
+ "vcvtdq2ps %%ymm2,%%ymm2 \n"
+ "vmulps %%ymm3,%%ymm4,%%ymm3 \n" // * scale
+ "vmulps %%ymm2,%%ymm4,%%ymm2 \n"
+ "vmulps %%ymm3,%%ymm6,%%ymm3 \n" // * 2^-112
+ "vmulps %%ymm2,%%ymm6,%%ymm2 \n"
+ "vpsrld $0xd,%%ymm3,%%ymm3 \n"
+ "vpsrld $0xd,%%ymm2,%%ymm2 \n"
+ "vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // unmutates
+ "vmovdqu %%ymm2,-0x20(%0,%1,1) \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+#if defined(__x86_64__)
+ : "x"(scale), // %3
+ "x"(kScaleBias) // %4
+#else
+ : "m"(scale), // %3
+ "m"(kScaleBias) // %4
+#endif
+ : "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_HALFFLOATROW_AVX2
+
+#ifdef HAS_HALFFLOATROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfFloatRow_AVX512BW(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcastss %[scale],%%zmm4 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%[src]),%%zmm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vcvtdq2ps %%zmm0,%%zmm0 \n"
+ "vmulps %%zmm4,%%zmm0,%%zmm0 \n"
+ "vcvtps2ph $3,%%zmm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vpmovzxwd (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vcvtdq2ps %%zmm0,%%zmm0 \n"
+ "vmulps %%zmm4,%%zmm0,%%zmm0 \n"
+ "vcvtps2ph $3,%%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ :
+ [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp)
+ : [scale] "m"(scale)
+ : "memory", "cc", "xmm0", "xmm4", "k1");
+}
+#endif // HAS_HALFFLOATROW_AVX512BW
+
+#ifdef HAS_HALFFLOATROW_F16C
+void HalfFloatRow_F16C(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ asm volatile(
+ "vbroadcastss %3, %%ymm4 \n"
+ "sub %0,%1 \n"
+
+ // 16 pixel loop (pure AVX1 + F16C).
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%0),%%xmm2 \n" // 16 shorts -> 16 ints
+ "vpmovzxwd 0x08(%0),%%xmm0 \n"
+ "vinsertf128 $1,%%xmm0,%%ymm2,%%ymm2 \n"
+ "vpmovzxwd 0x10(%0),%%xmm3 \n"
+ "vpmovzxwd 0x18(%0),%%xmm0 \n"
+ "vinsertf128 $1,%%xmm0,%%ymm3,%%ymm3 \n"
+ "vcvtdq2ps %%ymm2,%%ymm2 \n"
+ "vcvtdq2ps %%ymm3,%%ymm3 \n"
+ "vmulps %%ymm2,%%ymm4,%%ymm2 \n"
+ "vmulps %%ymm3,%%ymm4,%%ymm3 \n"
+ "vcvtps2ph $3, %%ymm2, %%xmm2 \n"
+ "vcvtps2ph $3, %%ymm3, %%xmm3 \n"
+ "vmovdqu %%xmm2,0x00(%0,%1,1) \n"
+ "vmovdqu %%xmm3,0x10(%0,%1,1) \n"
+ "add $0x20,%0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_HALFFLOATROW_F16C
+
+#ifdef HAS_HALFFLOATROW_F16C
+void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ // 16 pixel loop (pure AVX1 + F16C).
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%0),%%xmm2 \n" // 16 shorts -> 16 ints
+ "vpmovzxwd 0x08(%0),%%xmm0 \n"
+ "vinsertf128 $1,%%xmm0,%%ymm2,%%ymm2 \n"
+ "vpmovzxwd 0x10(%0),%%xmm3 \n"
+ "vpmovzxwd 0x18(%0),%%xmm0 \n"
+ "vinsertf128 $1,%%xmm0,%%ymm3,%%ymm3 \n"
+ "vcvtdq2ps %%ymm2,%%ymm2 \n"
+ "vcvtdq2ps %%ymm3,%%ymm3 \n"
+ "vcvtps2ph $3, %%ymm2, %%xmm2 \n"
+ "vcvtps2ph $3, %%ymm3, %%xmm3 \n"
+ "vmovdqu %%xmm2,0x00(%0,%1,1) \n"
+ "vmovdqu %%xmm3,0x10(%0,%1,1) \n"
+ "add $0x20,%0 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm2", "xmm3");
+}
+#endif // HAS_HALFFLOATROW_F16C
+
+#ifdef HAS_BYTETOFLOATROW_AVX2
+void ByteToFloatRow_AVX2(const uint8_t* src,
+ float* dst,
+ float scale,
+ int width) {
+ asm volatile("vbroadcastss %3,%%ymm2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbd (%0),%%ymm0 \n"
+ "vpmovzxbd 0x08(%0),%%ymm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vcvtdq2ps %%ymm0,%%ymm0 \n"
+ "vcvtdq2ps %%ymm1,%%ymm1 \n"
+ "vmulps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmulps %%ymm2,%%ymm1,%%ymm1 \n"
+ "vmovups %%ymm0,(%1) \n"
+ "vmovups %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+#if defined(__x86_64__)
+ : "x"(scale) // %3
+#else
+ : "m"(scale) // %3
+#endif
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_BYTETOFLOATROW_AVX2
+
+#ifdef HAS_GAUSSROW_AVX2
+static const int32_t kGaussRound128 = 128;
+static const int8_t kShufGauss32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13,
+ -1, -1, -1, -1, -1, -1, -1, -1};
+void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) {
+ asm volatile(
+ "vpbroadcastd %[k128],%%ymm4 \n"
+ "vbroadcasti128 %[kShuf],%%ymm5 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "vmovdqu 0x04(%[src]),%%ymm1 \n"
+ "vmovdqu 0x08(%[src]),%%ymm2 \n"
+ "vpaddd 0x10(%[src]),%%ymm0,%%ymm0 \n"
+ "vpaddd 0x0c(%[src]),%%ymm1,%%ymm1 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpslld $2,%%ymm1,%%ymm1 \n"
+ "vpslld $1,%%ymm2,%%ymm3 \n"
+ "vpslld $2,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpsrld $8,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovd (%[src]),%%xmm0 \n"
+ "vmovd 0x04(%[src]),%%xmm1 \n"
+ "vmovd 0x08(%[src]),%%xmm2 \n"
+ "vmovd 0x0c(%[src]),%%xmm3 \n"
+ "vpaddd %%xmm3,%%xmm1,%%xmm1 \n"
+ "vmovd 0x10(%[src]),%%xmm3 \n"
+ "vpaddd %%xmm3,%%xmm0,%%xmm0 \n"
+ "lea 0x04(%[src]),%[src] \n"
+ "vpslld $2,%%xmm1,%%xmm1 \n"
+ "vpslld $1,%%xmm2,%%xmm3 \n"
+ "vpslld $2,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm3,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpaddd %%xmm4,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm2,%%xmm0,%%xmm0 \n"
+ "vpsrld $8,%%xmm0,%%xmm0 \n"
+ "vpextrw $0,%%xmm0,(%[dst]) \n"
+ "lea 0x02(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width)
+ : [k128] "m"(kGaussRound128), [kShuf] "m"(kShufGauss32To16)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_GAUSSROW_AVX2
+
+#ifdef HAS_GAUSSCOL_AVX2
+void GaussCol_AVX2(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width) {
+#if defined(__x86_64__)
+ ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0);
+ ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0);
+ ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0);
+ ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0);
+ asm volatile(
+ "vpxor %%xmm5,%%xmm5,%%xmm5 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%[p0]),%%ymm0 \n"
+ "vpmovzxwd (%[p0],%[off4],1),%%ymm4 \n"
+ "vpmovzxwd (%[p0],%[off1],1),%%ymm1 \n"
+ "vpmovzxwd (%[p0],%[off3],1),%%ymm3 \n"
+ "vpmovzxwd (%[p0],%[off2],1),%%ymm2 \n"
+ "lea 0x10(%[p0]),%[p0] \n"
+ "vpaddd %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpslld $2,%%ymm1,%%ymm1 \n"
+ "vpslld $1,%%ymm2,%%ymm3 \n"
+ "vpslld $2,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dp]) \n"
+ "lea 0x20(%[dp]),%[dp] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vpinsrw $0,(%[p0]),%%xmm5,%%xmm0 \n"
+ "vpinsrw $0,(%[p0],%[off4],1),%%xmm5,%%xmm4\n"
+ "vpinsrw $0,(%[p0],%[off1],1),%%xmm5,%%xmm1\n"
+ "vpinsrw $0,(%[p0],%[off3],1),%%xmm5,%%xmm3\n"
+ "vpinsrw $0,(%[p0],%[off2],1),%%xmm5,%%xmm2\n"
+ "lea 0x02(%[p0]),%[p0] \n"
+ "vpaddd %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpaddd %%xmm3,%%xmm1,%%xmm1 \n"
+ "vpslld $2,%%xmm1,%%xmm1 \n"
+ "vpslld $1,%%xmm2,%%xmm3 \n"
+ "vpslld $2,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm3,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpaddd %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%[dp]) \n"
+ "lea 0x04(%[dp]),%[dp] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width)
+ : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3),
+ [off4] "r"(off4)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+#else
+ int n = width & ~7;
+ ptrdiff_t idx = 0;
+ const uint16_t* tmp;
+ asm volatile(
+ "vpxor %%xmm5,%%xmm5,%%xmm5 \n"
+ "cmpl $0,%[n] \n"
+ "jle 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "mov %[s0],%[tmp] \n"
+ "vpmovzxwd (%[tmp],%[idx],2),%%ymm0 \n"
+ "mov %[s4],%[tmp] \n"
+ "vpmovzxwd (%[tmp],%[idx],2),%%ymm4 \n"
+ "mov %[s1],%[tmp] \n"
+ "vpmovzxwd (%[tmp],%[idx],2),%%ymm1 \n"
+ "mov %[s3],%[tmp] \n"
+ "vpmovzxwd (%[tmp],%[idx],2),%%ymm3 \n"
+ "mov %[s2],%[tmp] \n"
+ "vpmovzxwd (%[tmp],%[idx],2),%%ymm2 \n"
+ "add $0x8,%[idx] \n"
+ "vpaddd %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpslld $2,%%ymm1,%%ymm1 \n"
+ "vpslld $1,%%ymm2,%%ymm3 \n"
+ "vpslld $2,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dp]) \n"
+ "lea 0x20(%[dp]),%[dp] \n"
+ "cmpl %[n],%[idx] \n"
+ "jl 1b \n"
+
+ "2: \n"
+ "cmpl %[width],%[idx] \n"
+ "jge 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "mov %[s0],%[tmp] \n"
+ "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm0\n"
+ "mov %[s4],%[tmp] \n"
+ "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm4\n"
+ "mov %[s1],%[tmp] \n"
+ "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm1\n"
+ "mov %[s3],%[tmp] \n"
+ "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm3\n"
+ "mov %[s2],%[tmp] \n"
+ "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm2\n"
+ "add $0x1,%[idx] \n"
+ "vpaddd %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpaddd %%xmm3,%%xmm1,%%xmm1 \n"
+ "vpslld $2,%%xmm1,%%xmm1 \n"
+ "vpslld $1,%%xmm2,%%xmm3 \n"
+ "vpslld $2,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm3,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpaddd %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,(%[dp]) \n"
+ "lea 0x04(%[dp]),%[dp] \n"
+ "cmpl %[width],%[idx] \n"
+ "jl 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst)
+ : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3),
+ [s4] "m"(src4), [n] "m"(n), [width] "m"(width)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_GAUSSCOL_AVX2
+
+#if defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2)
+static const float kGaussF32_4 = 4.0f;
+static const float kGaussF32_6 = 6.0f;
+#endif // defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2)
+
+#ifdef HAS_GAUSSROW_F32_AVX2
+static const float kGaussF32_Scale = 1.0f / 256.0f;
+void GaussRow_F32_AVX2(const float* src, float* dst, int width) {
+ asm volatile(
+ "vbroadcastss %[k4],%%ymm4 \n"
+ "vbroadcastss %[k6],%%ymm5 \n"
+ "vbroadcastss %[kScale],%%ymm6 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%[src]),%%ymm0 \n"
+ "vmulps 0x04(%[src]),%%ymm4,%%ymm1 \n"
+ "vmulps 0x08(%[src]),%%ymm5,%%ymm2 \n"
+ "vmulps 0x0c(%[src]),%%ymm4,%%ymm3 \n"
+ "vaddps %%ymm1,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm3,%%ymm0,%%ymm0 \n"
+ "vaddps 0x10(%[src]),%%ymm0,%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vmulps %%ymm6,%%ymm0,%%ymm0 \n"
+ "vmovups %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovss (%[src]),%%xmm0 \n"
+ "vmulss 0x04(%[src]),%%xmm4,%%xmm1 \n"
+ "vmulss 0x08(%[src]),%%xmm5,%%xmm2 \n"
+ "vmulss 0x0c(%[src]),%%xmm4,%%xmm3 \n"
+ "vaddss %%xmm1,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm3,%%xmm0,%%xmm0 \n"
+ "vaddss 0x10(%[src]),%%xmm0,%%xmm0 \n"
+ "lea 0x04(%[src]),%[src] \n"
+ "vmulss %%xmm6,%%xmm0,%%xmm0 \n"
+ "vmovss %%xmm0,(%[dst]) \n"
+ "lea 0x04(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width)
+ : [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6),
+ [kScale] "m"(kGaussF32_Scale)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
+}
+#endif // HAS_GAUSSROW_F32_AVX2
+
+#ifdef HAS_GAUSSCOL_F32_AVX2
+void GaussCol_F32_AVX2(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width) {
+#if defined(__x86_64__)
+ ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0);
+ ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0);
+ ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0);
+ ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0);
+ asm volatile(
+ "vbroadcastss %[k4],%%ymm4 \n"
+ "vbroadcastss %[k6],%%ymm5 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%[p0]),%%ymm0 \n"
+ "vmulps (%[p0],%[off1],1),%%ymm4,%%ymm1\n"
+ "vmulps (%[p0],%[off2],1),%%ymm5,%%ymm2\n"
+ "vmulps (%[p0],%[off3],1),%%ymm4,%%ymm3\n"
+ "vaddps %%ymm1,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm3,%%ymm0,%%ymm0 \n"
+ "vaddps (%[p0],%[off4],1),%%ymm0,%%ymm0\n"
+ "lea 0x20(%[p0]),%[p0] \n"
+ "vmovups %%ymm0,(%[dp]) \n"
+ "lea 0x20(%[dp]),%[dp] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovss (%[p0]),%%xmm0 \n"
+ "vmulss (%[p0],%[off1],1),%%xmm4,%%xmm1\n"
+ "vmulss (%[p0],%[off2],1),%%xmm5,%%xmm2\n"
+ "vmulss (%[p0],%[off3],1),%%xmm4,%%xmm3\n"
+ "vaddss %%xmm1,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm3,%%xmm0,%%xmm0 \n"
+ "vaddss (%[p0],%[off4],1),%%xmm0,%%xmm0\n"
+ "lea 0x04(%[p0]),%[p0] \n"
+ "vmovss %%xmm0,(%[dp]) \n"
+ "lea 0x04(%[dp]),%[dp] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width)
+ : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3),
+ [off4] "r"(off4), [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+#else
+ int n = width & ~7;
+ ptrdiff_t idx = 0;
+ const float* tmp;
+ asm volatile(
+ "vbroadcastss %[k4],%%ymm4 \n"
+ "vbroadcastss %[k6],%%ymm5 \n"
+ "cmpl $0,%[n] \n"
+ "jle 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "mov %[s0],%[tmp] \n"
+ "vmovups (%[tmp],%[idx],4),%%ymm0 \n"
+ "mov %[s1],%[tmp] \n"
+ "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm1\n"
+ "mov %[s2],%[tmp] \n"
+ "vmulps (%[tmp],%[idx],4),%%ymm5,%%ymm2\n"
+ "mov %[s3],%[tmp] \n"
+ "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm3\n"
+ "vaddps %%ymm1,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vaddps %%ymm3,%%ymm0,%%ymm0 \n"
+ "mov %[s4],%[tmp] \n"
+ "vaddps (%[tmp],%[idx],4),%%ymm0,%%ymm0\n"
+ "add $0x8,%[idx] \n"
+ "vmovups %%ymm0,(%[dp]) \n"
+ "lea 0x20(%[dp]),%[dp] \n"
+ "cmpl %[n],%[idx] \n"
+ "jl 1b \n"
+
+ "2: \n"
+ "cmpl %[width],%[idx] \n"
+ "jge 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "mov %[s0],%[tmp] \n"
+ "vmovss (%[tmp],%[idx],4),%%xmm0 \n"
+ "mov %[s1],%[tmp] \n"
+ "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm1\n"
+ "mov %[s2],%[tmp] \n"
+ "vmulss (%[tmp],%[idx],4),%%xmm5,%%xmm2\n"
+ "mov %[s3],%[tmp] \n"
+ "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm3\n"
+ "vaddss %%xmm1,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vaddss %%xmm3,%%xmm0,%%xmm0 \n"
+ "mov %[s4],%[tmp] \n"
+ "vaddss (%[tmp],%[idx],4),%%xmm0,%%xmm0\n"
+ "add $0x1,%[idx] \n"
+ "vmovss %%xmm0,(%[dp]) \n"
+ "lea 0x04(%[dp]),%[dp] \n"
+ "cmpl %[width],%[idx] \n"
+ "jl 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst)
+ : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3),
+ [s4] "m"(src4), [n] "m"(n), [width] "m"(width), [k4] "m"(kGaussF32_4),
+ [k6] "m"(kGaussF32_6)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+#endif
+}
+#endif // HAS_GAUSSCOL_F32_AVX2
+
+#ifdef HAS_SCALEMAXSAMPLES_AVX2
+float ScaleMaxSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width) {
+ float fmax;
+ asm volatile(
+ "vpxor %%xmm3,%%xmm3,%%xmm3 \n"
+ "vbroadcastss %[scale],%%ymm2 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vmaxps %%ymm3,%%ymm0,%%ymm3 \n"
+ "vmulps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovups %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "vextractf128 $0x1,%%ymm3,%%xmm1 \n"
+ "vmaxps %%xmm3,%%xmm1,%%xmm3 \n"
+ "vpshufd $0x4e,%%xmm3,%%xmm1 \n"
+ "vmaxps %%xmm3,%%xmm1,%%xmm3 \n"
+ "vpshufd $0x55,%%xmm3,%%xmm1 \n"
+ "vmaxss %%xmm3,%%xmm1,%%xmm3 \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovss (%[src]),%%xmm0 \n"
+ "lea 0x4(%[src]),%[src] \n"
+ "vmaxss %%xmm3,%%xmm0,%%xmm3 \n"
+ "vmulss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovss %%xmm0,(%[dst]) \n"
+ "lea 0x4(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vmovss %%xmm3,%[fmax] \n"
+ "vzeroupper \n"
+ : [src] "+r"(src),
+ [dst] "+r"(dst),
+ [width] "+r"(width),
+ [fmax] "=m"(fmax)
+ : [scale] "m"(scale)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ return fmax;
+}
+#endif // HAS_SCALEMAXSAMPLES_AVX2
+
+#ifdef HAS_SCALESUMSAMPLES_AVX2
+float ScaleSumSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width) {
+ float fsum;
+ asm volatile(
+ "vpxor %%xmm3,%%xmm3,%%xmm3 \n"
+ "vbroadcastss %[scale],%%ymm2 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vmulps %%ymm0,%%ymm0,%%ymm1 \n"
+ "vaddps %%ymm1,%%ymm3,%%ymm3 \n"
+ "vmulps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovups %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "vextractf128 $0x1,%%ymm3,%%xmm1 \n"
+ "vaddps %%xmm1,%%xmm3,%%xmm3 \n"
+ "vhaddps %%xmm3,%%xmm3,%%xmm3 \n"
+ "vhaddps %%xmm3,%%xmm3,%%xmm3 \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovss (%[src]),%%xmm0 \n"
+ "lea 0x4(%[src]),%[src] \n"
+ "vmulss %%xmm0,%%xmm0,%%xmm1 \n"
+ "vaddss %%xmm1,%%xmm3,%%xmm3 \n"
+ "vmulss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovss %%xmm0,(%[dst]) \n"
+ "lea 0x4(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vmovss %%xmm3,%[fsum] \n"
+ "vzeroupper \n"
+ : [src] "+r"(src),
+ [dst] "+r"(dst),
+ [width] "+r"(width),
+ [fsum] "=m"(fsum)
+ : [scale] "m"(scale)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ return fsum;
+}
+#endif // HAS_SCALESUMSAMPLES_AVX2
+
+#ifdef HAS_SCALESAMPLES_AVX2
+void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) {
+ asm volatile(
+ "vbroadcastss %[scale],%%ymm2 \n"
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovups (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vmulps %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovups %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovss (%[src]),%%xmm0 \n"
+ "lea 0x4(%[src]),%[src] \n"
+ "vmulss %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovss %%xmm0,(%[dst]) \n"
+ "lea 0x4(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width)
+ : [scale] "m"(scale)
+ : "memory", "cc", "xmm0", "xmm2");
+}
+#endif // HAS_SCALESAMPLES_AVX2
+
+#ifdef HAS_ARGBCOLORTABLEROW_AVX2
+// Transform ARGB pixels with color table.
+void ARGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[dst]),%%ymm0 \n"
+ "vpand %%ymm6,%%ymm0,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $8,%%ymm0,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n"
+ "vpslld $8,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $16,%%ymm0,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n"
+ "vpslld $16,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $24,%%ymm0,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n"
+ "vpslld $24,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vmovdqu %%ymm2,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+ "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n"
+ "vpsrldq $12,%%xmm5,%%xmm5 \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovd (%[dst]),%%xmm0 \n"
+ "vpand %%xmm6,%%xmm0,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n"
+ "vpand %%xmm6,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $8,%%xmm0,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm1,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n"
+ "vpslld $8,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm3,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $16,%%xmm0,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm1,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n"
+ "vpslld $16,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm3,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $24,%%xmm0,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n"
+ "vpslld $24,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm3,%%xmm2,%%xmm2 \n"
+
+ "vmovd %%xmm2,(%[dst]) \n"
+ "lea 0x04(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [table] "r"(table_argb)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#endif // HAS_ARGBCOLORTABLEROW_AVX2
+
+#ifdef HAS_RGBCOLORTABLEROW_AVX2
+// Transform RGB pixels with color table.
+void RGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff
+ "sub $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[dst]),%%ymm0 \n"
+ "vpand %%ymm6,%%ymm0,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $8,%%ymm0,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n"
+ "vpslld $8,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $16,%%ymm0,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n"
+ "vpslld $16,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vpslld $24,%%ymm6,%%ymm4 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm0,%%ymm2,%%ymm2 \n"
+
+ "vmovdqu %%ymm2,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x8,%[width] \n"
+ "jle 99f \n"
+ "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n"
+ "vpsrldq $12,%%xmm5,%%xmm5 \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovd (%[dst]),%%xmm0 \n"
+ "vpand %%xmm6,%%xmm0,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n"
+ "vpand %%xmm6,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $8,%%xmm0,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm1,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n"
+ "vpslld $8,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm3,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $16,%%xmm0,%%xmm1 \n"
+ "vpand %%xmm6,%%xmm1,%%xmm1 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n"
+ "vpslld $16,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm3,%%xmm2,%%xmm2 \n"
+
+ "vpslld $24,%%xmm6,%%xmm4 \n"
+ "vpand %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm0,%%xmm2,%%xmm2 \n"
+
+ "vmovd %%xmm2,(%[dst]) \n"
+ "lea 0x04(%[dst]),%[dst] \n"
+ "sub $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [table] "r"(table_argb)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+}
+#endif // HAS_RGBCOLORTABLEROW_AVX2
+
+#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2
+// Transform RGB pixels with luma table.
+void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff) {
+ uint32_t c_br = lumacoeff & 0x00ff00ffu;
+ uint32_t c_ga = (lumacoeff >> 8) & 0x000000ffu;
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff
+#if defined(__x86_64__)
+ "vpbroadcastd %[c_br],%%ymm8 \n"
+ "vpbroadcastd %[c_ga],%%ymm9 \n"
+ "vpsrld $1,%%ymm6,%%ymm10 \n"
+ "vpslld $8,%%ymm10,%%ymm10 \n" // 0x00007f00
+ "vpsrld $6,%%ymm6,%%ymm11 \n"
+ "vpslld $3,%%ymm11,%%ymm11 \n" // 0x00000018
+#endif
+ "subl $0x8,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpsllw $8,%%ymm0,%%ymm1 \n"
+ "vpsrlw $8,%%ymm1,%%ymm1 \n" // [B, R]
+ "vpsrlw $8,%%ymm0,%%ymm2 \n" // [G, A]
+#if defined(__x86_64__)
+ "vpmaddwd %%ymm8,%%ymm1,%%ymm1 \n"
+ "vpmaddwd %%ymm9,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm10,%%ymm1,%%ymm1 \n"
+#else
+ "vpbroadcastd %[c_br],%%ymm3 \n"
+ "vpmaddwd %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpbroadcastd %[c_ga],%%ymm3 \n"
+ "vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpsrld $1,%%ymm6,%%ymm3 \n"
+ "vpslld $8,%%ymm3,%%ymm3 \n"
+ "vpand %%ymm3,%%ymm1,%%ymm1 \n"
+#endif
+
+ "vpand %%ymm6,%%ymm0,%%ymm3 \n"
+ "vpor %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpsrld $2,%%ymm3,%%ymm3 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm2\n"
+#if defined(__x86_64__)
+ "vpslld $3,%%ymm0,%%ymm4 \n"
+ "vpand %%ymm11,%%ymm4,%%ymm4 \n"
+#else
+ "vpslld $30,%%ymm0,%%ymm4 \n"
+ "vpsrld $27,%%ymm4,%%ymm4 \n"
+#endif
+ "vpsrlvd %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $8,%%ymm0,%%ymm3 \n"
+ "vpand %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpsrld $2,%%ymm3,%%ymm3 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm5\n"
+#if defined(__x86_64__)
+ "vpsrld $5,%%ymm0,%%ymm4 \n"
+ "vpand %%ymm11,%%ymm4,%%ymm4 \n"
+#else
+ "vpslld $22,%%ymm0,%%ymm4 \n"
+ "vpsrld $30,%%ymm4,%%ymm4 \n"
+ "vpslld $3,%%ymm4,%%ymm4 \n"
+#endif
+ "vpsrlvd %%ymm4,%%ymm5,%%ymm5 \n"
+ "vpand %%ymm6,%%ymm5,%%ymm5 \n"
+ "vpslld $8,%%ymm5,%%ymm5 \n"
+ "vpor %%ymm5,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $16,%%ymm0,%%ymm3 \n"
+ "vpand %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpsrld $2,%%ymm3,%%ymm3 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm5\n"
+#if defined(__x86_64__)
+ "vpsrld $13,%%ymm0,%%ymm4 \n"
+ "vpand %%ymm11,%%ymm4,%%ymm4 \n"
+#else
+ "vpslld $14,%%ymm0,%%ymm4 \n"
+ "vpsrld $30,%%ymm4,%%ymm4 \n"
+ "vpslld $3,%%ymm4,%%ymm4 \n"
+#endif
+ "vpsrlvd %%ymm4,%%ymm5,%%ymm5 \n"
+ "vpand %%ymm6,%%ymm5,%%ymm5 \n"
+ "vpslld $16,%%ymm5,%%ymm5 \n"
+ "vpor %%ymm5,%%ymm2,%%ymm2 \n"
+
+ "vpsrld $24,%%ymm0,%%ymm0 \n"
+ "vpslld $24,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm0,%%ymm2,%%ymm2 \n"
+
+ "vmovdqu %%ymm2,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "subl $0x8,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "addl $0x8,%[width] \n"
+ "jle 99f \n"
+ "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n"
+ "vpsrldq $12,%%xmm5,%%xmm5 \n"
+
+ LABELALIGN
+ "3: \n"
+ "vmovd (%[src]),%%xmm0 \n"
+ "lea 0x04(%[src]),%[src] \n"
+ "vpsllw $8,%%xmm0,%%xmm1 \n"
+ "vpsrlw $8,%%xmm1,%%xmm1 \n"
+ "vpsrlw $8,%%xmm0,%%xmm2 \n"
+#if defined(__x86_64__)
+ "vpmaddwd %%xmm8,%%xmm1,%%xmm1 \n"
+ "vpmaddwd %%xmm9,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm2,%%xmm1,%%xmm1 \n"
+ "vpand %%xmm10,%%xmm1,%%xmm1 \n"
+#else
+ "vpbroadcastd %[c_br],%%xmm3 \n"
+ "vpmaddwd %%xmm3,%%xmm1,%%xmm1 \n"
+ "vpbroadcastd %[c_ga],%%xmm3 \n"
+ "vpmaddwd %%xmm3,%%xmm2,%%xmm2 \n"
+ "vpaddd %%xmm2,%%xmm1,%%xmm1 \n"
+ "vpsrld $1,%%xmm6,%%xmm3 \n"
+ "vpslld $8,%%xmm3,%%xmm3 \n"
+ "vpand %%xmm3,%%xmm1,%%xmm1 \n"
+#endif
+
+ "vpand %%xmm6,%%xmm0,%%xmm3 \n"
+ "vpor %%xmm1,%%xmm3,%%xmm3 \n"
+ "vpsrld $2,%%xmm3,%%xmm3 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm2\n"
+#if defined(__x86_64__)
+ "vpslld $3,%%xmm0,%%xmm4 \n"
+ "vpand %%xmm11,%%xmm4,%%xmm4 \n"
+#else
+ "vpslld $30,%%xmm0,%%xmm4 \n"
+ "vpsrld $27,%%xmm4,%%xmm4 \n"
+#endif
+ "vpsrlvd %%xmm4,%%xmm2,%%xmm2 \n"
+ "vpand %%xmm6,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $8,%%xmm0,%%xmm3 \n"
+ "vpand %%xmm6,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm1,%%xmm3,%%xmm3 \n"
+ "vpsrld $2,%%xmm3,%%xmm3 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm4\n"
+#if defined(__x86_64__)
+ "vpsrld $5,%%xmm0,%%xmm3 \n"
+ "vpand %%xmm11,%%xmm3,%%xmm3 \n"
+#else
+ "vpslld $22,%%xmm0,%%xmm3 \n"
+ "vpsrld $30,%%xmm3,%%xmm3 \n"
+ "vpslld $3,%%xmm3,%%xmm3 \n"
+#endif
+ "vpsrlvd %%xmm3,%%xmm4,%%xmm4 \n"
+ "vpand %%xmm6,%%xmm4,%%xmm4 \n"
+ "vpslld $8,%%xmm4,%%xmm4 \n"
+ "vpor %%xmm4,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $16,%%xmm0,%%xmm3 \n"
+ "vpand %%xmm6,%%xmm3,%%xmm3 \n"
+ "vpor %%xmm1,%%xmm3,%%xmm3 \n"
+ "vpsrld $2,%%xmm3,%%xmm3 \n"
+ "vmovdqa %%xmm5,%%xmm7 \n"
+ "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm4\n"
+#if defined(__x86_64__)
+ "vpsrld $13,%%xmm0,%%xmm3 \n"
+ "vpand %%xmm11,%%xmm3,%%xmm3 \n"
+#else
+ "vpslld $14,%%xmm0,%%xmm3 \n"
+ "vpsrld $30,%%xmm3,%%xmm3 \n"
+ "vpslld $3,%%xmm3,%%xmm3 \n"
+#endif
+ "vpsrlvd %%xmm3,%%xmm4,%%xmm4 \n"
+ "vpand %%xmm6,%%xmm4,%%xmm4 \n"
+ "vpslld $16,%%xmm4,%%xmm4 \n"
+ "vpor %%xmm4,%%xmm2,%%xmm2 \n"
+
+ "vpsrld $24,%%xmm0,%%xmm0 \n"
+ "vpslld $24,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm0,%%xmm2,%%xmm2 \n"
+
+ "vmovd %%xmm2,(%[dst]) \n"
+ "lea 0x04(%[dst]),%[dst] \n"
+ "subl $0x1,%[width] \n"
+ "jg 3b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+rm"(width)
+ : [luma] "r"(luma), [c_br] "m"(c_br), [c_ga] "m"(c_ga)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10", "xmm11"
+#endif
+ );
+}
+#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2
+
+#if defined(HAS_NV21TOYUV24ROW_SSSE3) || defined(HAS_NV21TOYUV24ROW_AVX2)
+static const uvec8 kYUV24Shuffle[3] = {
+ {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12},
+ {9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15},
+ {2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}};
+#endif
+
+#ifdef HAS_NV21TOYUV24ROW_SSSE3
+// Convert biplanar NV21 to packed YUV24
+// NV21 has VU in memory for chroma.
+// YUV24 is VUY in memory
+void NV21ToYUV24Row_SSSE3(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "movdqa (%4),%%xmm4 \n" // 3 shuffler constants
+ "movdqa 16(%4),%%xmm5 \n"
+ "movdqa 32(%4),%%xmm6 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm2 \n" // load 16 Y values
+ "movdqu (%0,%1),%%xmm3 \n" // load 8 VU values
+ "lea 16(%0),%0 \n"
+ "movdqa %%xmm2,%%xmm0 \n"
+ "movdqa %%xmm2,%%xmm1 \n"
+ "shufps $0x44,%%xmm3,%%xmm0 \n" // Y 0..7, UV 0..3
+ "shufps $0x99,%%xmm3,%%xmm1 \n" // Y 4..11, UV 2..5
+ "shufps $0xee,%%xmm3,%%xmm2 \n" // Y 8..15, UV 4..7
+ "pshufb %%xmm4, %%xmm0 \n" // weave into YUV24
+ "pshufb %%xmm5, %%xmm1 \n"
+ "pshufb %%xmm6, %%xmm2 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "movdqu %%xmm1,16(%2) \n"
+ "movdqu %%xmm2,32(%2) \n"
+ "lea 48(%2),%2 \n"
+ "sub $16,%3 \n" // 16 pixels per loop
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_vu), // %1
+ "+r"(dst_yuv24), // %2
+ "+r"(width) // %3
+ : "r"(&kYUV24Shuffle[0]) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_NV21TOYUV24ROW_SSSE3
+
+#ifdef HAS_NV21TOYUV24ROW_AVX2
+// Convert biplanar NV21 to packed YUV24
+// NV21 has VU in memory for chroma.
+// YUV24 is VUY in memory
+void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "vbroadcasti128 (%4),%%ymm4 \n" // 3 shuffler constants
+ "vbroadcasti128 16(%4),%%ymm5 \n"
+ "vbroadcasti128 32(%4),%%ymm6 \n"
+
+ "1: \n"
+ "vmovdqu (%0),%%ymm2 \n" // load 32 Y values
+ "vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values
+ "lea 32(%0),%0 \n"
+ "vshufps $0x44,%%ymm3,%%ymm2,%%ymm0 \n" // Y 0..7, UV 0..3
+ "vshufps $0x99,%%ymm3,%%ymm2,%%ymm1 \n" // Y 4..11, UV 2..5
+ "vshufps $0xee,%%ymm3,%%ymm2,%%ymm2 \n" // Y 8..15, UV 4..7
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" // weave into YUV24
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vperm2i128 $0x30,%%ymm0,%%ymm2,%%ymm0 \n"
+ "vperm2i128 $0x31,%%ymm2,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm3,(%2) \n"
+ "vmovdqu %%ymm0,32(%2) \n"
+ "vmovdqu %%ymm1,64(%2) \n"
+ "lea 96(%2),%2 \n"
+ "sub $32,%3 \n" // 32 pixels per loop
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_vu), // %1
+ "+r"(dst_yuv24), // %2
+ "+r"(width) // %3
+ : "r"(&kYUV24Shuffle[0]) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif // HAS_NV21TOYUV24ROW_AVX2
+
+#ifdef HAS_NV21TOYUV24ROW_AVX512VBMI
+static const uint8_t kNV21ToYUV24Perm_AVX512VBMI[3][64] = {
+ {64, 65, 0, 64, 65, 1, 66, 67, 2, 66, 67, 3, 68, 69, 4, 68,
+ 69, 5, 70, 71, 6, 70, 71, 7, 72, 73, 8, 72, 73, 9, 74, 75,
+ 10, 74, 75, 11, 76, 77, 12, 76, 77, 13, 78, 79, 14, 78, 79, 15,
+ 80, 81, 16, 80, 81, 17, 82, 83, 18, 82, 83, 19, 84, 85, 20, 84},
+ {85, 21, 86, 87, 22, 86, 87, 23, 88, 89, 24, 88, 89,
+ 25, 90, 91, 26, 90, 91, 27, 92, 93, 28, 92, 93, 29,
+ 94, 95, 30, 94, 95, 31, 96, 97, 32, 96, 97, 33, 98,
+ 99, 34, 98, 99, 35, 100, 101, 36, 100, 101, 37, 102, 103,
+ 38, 102, 103, 39, 104, 105, 40, 104, 105, 41, 106, 107},
+ {42, 106, 107, 43, 108, 109, 44, 108, 109, 45, 110, 111, 46,
+ 110, 111, 47, 112, 113, 48, 112, 113, 49, 114, 115, 50, 114,
+ 115, 51, 116, 117, 52, 116, 117, 53, 118, 119, 54, 118, 119,
+ 55, 120, 121, 56, 120, 121, 57, 122, 123, 58, 122, 123, 59,
+ 124, 125, 60, 124, 125, 61, 126, 127, 62, 126, 127, 63}};
+
+LIBYUV_TARGET_AVX512VBMI
+void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ uintptr_t temp;
+ uintptr_t mask;
+ asm volatile(
+ "sub %[src_y],%[src_vu] \n"
+ "vmovdqu8 (%[perm]),%%zmm4 \n"
+ "vmovdqu8 64(%[perm]),%%zmm5 \n"
+ "vmovdqu8 128(%[perm]),%%zmm16 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src_y]),%%zmm2 \n" // load 64 Y values
+ "vmovdqu8 (%[src_y],%[src_vu],1),%%zmm3 \n" // load 32 VU pairs
+ "lea 0x40(%[src_y]),%[src_y] \n"
+ "vmovdqa64 %%zmm2,%%zmm0 \n"
+ "vmovdqa64 %%zmm2,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm4,%%zmm0 \n"
+ "vpermt2b %%zmm3,%%zmm5,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_yuv24]) \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst_yuv24]) \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst_yuv24]) \n"
+ "lea 0xc0(%[dst_yuv24]),%[dst_yuv24] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "lea 1(%q[width]),%k[mask] \n"
+ "and $-2,%k[mask] \n"
+ "bzhi %q[mask],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k2 \n"
+ "vmovdqu8 (%[src_y]),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[src_y],%[src_vu],1),%%zmm3%{%%k2%}%{z%} \n"
+ "lea (%q[width],%q[width],2),%k[width] \n"
+ "bzhi %q[width],%[temp],%[mask] \n"
+ "kmovq %[mask],%%k1 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k2 \n"
+ "xor %k[mask],%k[mask] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp],%[mask] \n"
+ "bzhi %q[width],%[mask],%[mask] \n"
+ "kmovq %[mask],%%k3 \n"
+ "vmovdqa64 %%zmm2,%%zmm0 \n"
+ "vmovdqa64 %%zmm2,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm4,%%zmm0 \n"
+ "vpermt2b %%zmm3,%%zmm5,%%zmm1 \n"
+ "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n"
+ "vmovdqu8 %%zmm0,(%[dst_yuv24])%{%%k1%} \n"
+ "vmovdqu8 %%zmm1,0x40(%[dst_yuv24])%{%%k2%} \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst_yuv24])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_y] "+r"(src_y), [src_vu] "+r"(src_vu), [dst_yuv24] "+r"(dst_yuv24),
+ [width] "+r"(width), [temp] "=&r"(temp), [mask] "=&r"(mask)
+ : [perm] "r"(&kNV21ToYUV24Perm_AVX512VBMI[0][0])
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "k1", "k2", "k3");
+}
+#endif // HAS_NV21TOYUV24ROW_AVX512VBMI
+
+#if defined(HAS_SWAPUVROW_SSSE3) || defined(HAS_SWAPUVROW_AVX2)
+
+// Shuffle table for reversing the bytes.
+static const uvec8 kShuffleUVToVU = {1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u,
+ 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
+#endif
+
+#ifdef HAS_SWAPUVROW_SSSE3
+// Convert UV plane of NV12 to VU of NV21.
+void SwapUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ asm volatile("movdqu %3,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_vu), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleUVToVU) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
+}
+#endif // HAS_SWAPUVROW_SSSE3
+
+#ifdef HAS_SWAPUVROW_AVX2
+void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ asm volatile("vbroadcasti128 %3,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_vu), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleUVToVU) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
+}
+#endif // HAS_SWAPUVROW_AVX2
+
+#ifdef HAS_HALFMERGEUVROW_SSSE3
+void HalfMergeUVRow_SSSE3(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
+ "pxor %%xmm5,%%xmm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // load 16 U values
+ "movdqu (%1),%%xmm1 \n" // load 16 V values
+ "movdqu 0(%0,%4,1),%%xmm2 \n" // 16 from next row
+ "movdqu 0(%1,%5,1),%%xmm3 \n"
+ "lea 0x10(%0),%0 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n" // half size
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm3 \n"
+ "lea 0x10(%1),%1 \n"
+ "paddw %%xmm2,%%xmm0 \n"
+ "paddw %%xmm3,%%xmm1 \n"
+ "psrlw $0x1,%%xmm0 \n"
+ "psrlw $0x1,%%xmm1 \n"
+ "pavgw %%xmm5,%%xmm0 \n"
+ "pavgw %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "packuswb %%xmm1,%%xmm1 \n"
+ "punpcklbw %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n" // store 8 UV pixels
+ "lea 0x10(%2),%2 \n"
+ "sub $0x10,%3 \n" // 16 src pixels per loop
+ "jg 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(src_stride_u)), // %4
+ "r"((ptrdiff_t)(src_stride_v)) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_HALFMERGEUVROW_SSSE3
+
+#ifdef HAS_HALFMERGEUVROW_AVX2
+void HalfMergeUVRow_AVX2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // load 32 U values
+ "vmovdqu (%1),%%ymm1 \n" // load 32 V values
+ "vmovdqu 0(%0,%4,1),%%ymm2 \n" // 32 from next row
+ "vmovdqu 0(%1,%5,1),%%ymm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" // half size
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "lea 0x20(%1),%1 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x1,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x1,%%ymm1,%%ymm1 \n"
+ "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpavgw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%2) \n" // store 16 UV pixels
+ "lea 0x20(%2),%2 \n"
+ "sub $0x20,%3 \n" // 32 src pixels per loop
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"((ptrdiff_t)(src_stride_u)), // %4
+ "r"((ptrdiff_t)(src_stride_v)) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_HALFMERGEUVROW_AVX2
+
+#ifdef HAS_CONVERT8TO8ROW_AVX2
+void Convert8To8Row_AVX2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "vmovd %3,%%xmm2 \n"
+ "vmovd %4,%%xmm3 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
+ "vpbroadcastb %%xmm3,%%ymm3 \n"
+ "vpxor %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsllw $8,%%ymm2,%%ymm2 \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpunpckhbw %%ymm4,%%ymm0,%%ymm1 \n" // mutates
+ "vpunpcklbw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // unmutates
+ "vpaddb %%ymm3,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0,%1) \n"
+ "add $0x20,%0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_CONVERT8TO8ROW_AVX2
+
+#ifdef HAS_CONVERT8TO8ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert8To8Row_AVX512BW(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw %k[scale],%%zmm2 \n"
+ "vpbroadcastb %k[bias],%%zmm3 \n"
+ "vpxord %%zmm4,%%zmm4,%%zmm4 \n"
+ "vpsllw $8,%%zmm2,%%zmm2 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n"
+ "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddb %%zmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovq %[temp],%%k1 \n"
+ "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n"
+ "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddb %%zmm3,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_y), [dst] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [scale] "r"(scale), [bias] "r"(bias)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1");
+}
+#endif // HAS_CONVERT8TO8ROW_AVX512BW
+
+// ===========================================================================
+// Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI)
+// ===========================================================================
+
+#ifdef HAS_AYUVTOYROW_AVX512VBMI
+static const uint8_t kPermAYUVToY32[64] = {
+ 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62,
+ 66, 70, 74, 78, 82, 86, 90, 94, 98, 102, 106, 110, 114, 118, 122, 126,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv,
+ uint8_t* dst_y,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vmovdqu8 %[perm],%%zmm16 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src]),%%zmm1 \n"
+ "lea 0x80(%[src]),%[src] \n"
+ "vmovdqa64 %%zmm16,%%zmm2 \n"
+ "vpermt2b %%zmm1,%%zmm2,%%zmm0 \n"
+ "vmovdqu8 %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k3 \n"
+ "kmovw %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k3,%%k2 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src]),%%zmm1%{%%k2%}%{z%}\n"
+ "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n"
+ "vmovdqu8 %%ymm0,(%[dst])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [perm] "m"(kPermAYUVToY32)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "k1", "k2", "k3");
+}
+#endif // HAS_AYUVTOYROW_AVX512VBMI
+
+#if defined(HAS_AYUVTOUVROW_AVX512VBMI) || defined(HAS_AYUVTOVUROW_AVX512VBMI)
+static const uint8_t kPermAYUVToUV[64] = {
+ 1, 5, 0, 4, 9, 13, 8, 12, 17, 21, 16, 20, 25,
+ 29, 24, 28, 33, 37, 32, 36, 41, 45, 40, 44, 49, 53,
+ 48, 52, 57, 61, 56, 60, 65, 69, 64, 68, 73, 77, 72,
+ 76, 81, 85, 80, 84, 89, 93, 88, 92, 97, 101, 96, 100,
+ 105, 109, 104, 108, 113, 117, 112, 116, 121, 125, 120, 124};
+
+static const uint8_t kPermAYUVToVU[64] = {
+ 0, 4, 1, 5, 8, 12, 9, 13, 16, 20, 17, 21, 24,
+ 28, 25, 29, 32, 36, 33, 37, 40, 44, 41, 45, 48, 52,
+ 49, 53, 56, 60, 57, 61, 64, 68, 65, 69, 72, 76, 73,
+ 77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101,
+ 104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125};
+
+LIBYUV_TARGET_AVX512VBMI
+static inline void AYUVToUVOrVURow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ const uint8_t* perm_table,
+ int width) {
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "vmovdqu8 (%[perm]),%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n"
+ "vpsrlw $15,%%zmm18,%%zmm18 \n"
+ "vpsllw $8,%%zmm18,%%zmm19 \n"
+ "vpord %%zmm19,%%zmm18,%%zmm18 \n" // 0x0101
+ "vpxord %%zmm19,%%zmm19,%%zmm19 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src]),%%zmm1 \n"
+ "vmovdqu8 (%[src],%[stride]),%%zmm2 \n"
+ "vmovdqu8 0x40(%[src],%[stride]),%%zmm3 \n"
+ "lea 0x80(%[src]),%[src] \n"
+ "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n"
+ "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n"
+ "vpmaddubsw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm19,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp1] \n"
+ "kmovd %k[temp1],%%k3 \n"
+ "kmovw %k[temp1],%%k1 \n"
+ "kshiftrd $16,%%k3,%%k2 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src]),%%zmm1%{%%k2%}%{z%}\n"
+ "vmovdqu32 (%[src],%[stride]),%%zmm2%{%%k1%}%{z%}\n"
+ "vmovdqu32 0x40(%[src],%[stride]),%%zmm3%{%%k2%}%{z%}\n"
+ "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n"
+ "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n"
+ // If width is odd, duplicate pixel (width - 1) into pixel width in dword
+ // (width >> 1).
+ "mov %k[width],%k[temp1] \n"
+ "and $1,%k[temp1] \n"
+ "mov %k[width],%k[temp2] \n"
+ "shr $1,%k[temp2] \n"
+ "shlx %k[temp2],%k[temp1],%k[temp1] \n"
+ "kmovw %k[temp1],%%k4 \n"
+ "vpslld $8,%%zmm0,%%zmm1 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0%{%%k4%} \n"
+ "vpslld $8,%%zmm2,%%zmm3 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2%{%%k4%} \n"
+ "vpmaddubsw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm19,%%zmm0,%%zmm0 \n"
+ "lea 1(%q[width]),%k[temp2] \n"
+ "and $-2,%k[temp2] \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[temp2],%[temp1],%[temp1] \n"
+ "kmovd %k[temp1],%%k3 \n"
+ "vpmovwb %%zmm0,(%[dst])%{%%k3%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ayuv), [dst] "+r"(dst_uv), [width] "+r"(width),
+ [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [stride] "r"((ptrdiff_t)src_stride_ayuv), [perm] "r"(perm_table)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm18",
+ "xmm19", "k1", "k2", "k3", "k4");
+}
+#endif // defined(HAS_AYUVTOUVROW_AVX512VBMI) ||
+ // defined(HAS_AYUVTOVUROW_AVX512VBMI)
+
+#ifdef HAS_AYUVTOUVROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ AYUVToUVOrVURow_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_uv, kPermAYUVToUV,
+ width);
+}
+#endif // HAS_AYUVTOUVROW_AVX512VBMI
+
+#ifdef HAS_AYUVTOVUROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ AYUVToUVOrVURow_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_vu, kPermAYUVToVU,
+ width);
+}
+#endif // HAS_AYUVTOVUROW_AVX512VBMI
+
+#ifdef HAS_AYUVTOYROW_AVX2
+void OMITFP AYUVToYRow_AVX2(const uint8_t* src_ayuv,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile(
+ "vpcmpeqd %%ymm2,%%ymm2,%%ymm2 \n"
+ "vpsrld $24,%%ymm2,%%ymm2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src]),%%ymm1 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpsrld $16,%%ymm0,%%ymm0 \n"
+ "vpsrld $16,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [width] "+r"(width)
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_AYUVTOYROW_AVX2
+
+#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2)
+static const uint8_t kShufAYUVToUV16[16] = {
+ 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+
+static const uint8_t kShufAYUVToVU16[16] = {
+ 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+
+static inline void AYUVToUVOrVURow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ const uint8_t* shuf_table,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 (%[shuf]),%%ymm6 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpsrlw $15,%%ymm7,%%ymm7 \n"
+ "vpsllw $8,%%ymm7,%%ymm5 \n"
+ "vpor %%ymm5,%%ymm7,%%ymm7 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src]),%%ymm1 \n"
+ "vmovdqu (%[src],%[stride]),%%ymm2 \n"
+ "vmovdqu 0x20(%[src],%[stride]),%%ymm3 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpshufb %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpunpcklqdq %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpsrlw $1,%%ymm0,%%ymm0 \n"
+ "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vextracti128 $1,%%ymm0,%%xmm1 \n"
+ "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ayuv), [dst] "+r"(dst_uv), [width] "+rm"(width)
+ : [stride] "r"((ptrdiff_t)src_stride_ayuv), [shuf] "r"(shuf_table)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7");
+}
+#endif // defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2)
+
+#ifdef HAS_AYUVTOUVROW_AVX2
+void OMITFP AYUVToUVRow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_uv, kShufAYUVToUV16,
+ width);
+}
+#endif // HAS_AYUVTOUVROW_AVX2
+
+#ifdef HAS_AYUVTOVUROW_AVX2
+void OMITFP AYUVToVURow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_vu, kShufAYUVToVU16,
+ width);
+}
+#endif // HAS_AYUVTOVUROW_AVX2
+
+// ===========================================================================
+// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW)
+// ===========================================================================
+
+#ifdef HAS_AR30TOARGBROW_AVX2
+void OMITFP AR30ToARGBRow_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint32_t kC0000000 = 0xc0000000u;
+ static const uint32_t k300000FF = 0x300000ffu;
+ static const uint32_t k0C00FF00 = 0x0c00ff00u;
+ static const uint32_t k03FF0000 = 0x03ff0000u;
+ asm volatile(
+ "vpbroadcastd %[m0],%%ymm4 \n"
+ "vpbroadcastd %[m1],%%ymm5 \n"
+ "vpbroadcastd %[m2],%%ymm6 \n"
+ "vpbroadcastd %[m3],%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpsrld $2,%%ymm0,%%ymm1 \n"
+ "vpsrld $4,%%ymm0,%%ymm2 \n"
+ "vpsrld $6,%%ymm0,%%ymm3 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpand %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00),
+ [m3] "m"(k03FF0000)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif // HAS_AR30TOARGBROW_AVX2
+
+#ifdef HAS_AR30TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint32_t kC0000000 = 0xc0000000u;
+ static const uint32_t k300000FF = 0x300000ffu;
+ static const uint32_t k0C00FF00 = 0x0c00ff00u;
+ static const uint32_t k03FF0000 = 0x03ff0000u;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd %[m0],%%zmm16 \n"
+ "vpbroadcastd %[m1],%%zmm17 \n"
+ "vpbroadcastd %[m2],%%zmm18 \n"
+ "vpbroadcastd %[m3],%%zmm19 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpsrld $2,%%zmm0,%%zmm1 \n"
+ "vpsrld $4,%%zmm0,%%zmm2 \n"
+ "vpsrld $6,%%zmm0,%%zmm3 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm19,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrld $2,%%zmm0,%%zmm1 \n"
+ "vpsrld $4,%%zmm0,%%zmm2 \n"
+ "vpsrld $6,%%zmm0,%%zmm3 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm19,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00),
+ [m3] "m"(k03FF0000)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17",
+ "xmm18", "xmm19", "k1");
+}
+#endif // HAS_AR30TOARGBROW_AVX512BW
+
+#if defined(HAS_AR30TOABGRROW_AVX2) || defined(HAS_AR30TOABGRROW_AVX512BW)
+static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+#endif
+
+#ifdef HAS_AR30TOABGRROW_AVX2
+void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_abgr,
+ int width) {
+ static const uint32_t kC0000000 = 0xc0000000u;
+ static const uint32_t k300000FF = 0x300000ffu;
+ static const uint32_t k0C00FF00 = 0x0c00ff00u;
+ static const uint32_t k03FF0000 = 0x03ff0000u;
+ asm volatile(
+ "vpbroadcastd %[m0],%%ymm4 \n"
+ "vpbroadcastd %[m1],%%ymm5 \n"
+ "vpbroadcastd %[m2],%%ymm6 \n"
+#if defined(__x86_64__)
+ "vpbroadcastd %[m3],%%ymm8 \n"
+#endif
+ "vbroadcasti128 %[shuf],%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpsrld $2,%%ymm0,%%ymm1 \n"
+ "vpsrld $4,%%ymm0,%%ymm2 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpsrld $6,%%ymm0,%%ymm3 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+#if defined(__x86_64__)
+ "vpand %%ymm8,%%ymm3,%%ymm3 \n"
+#else
+ "vpbroadcastd %[m3],%%ymm1 \n"
+ "vpand %%ymm1,%%ymm3,%%ymm3 \n"
+#endif
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm7,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [width] "+r"(width)
+ : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00),
+ [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8"
+#endif
+ );
+}
+#endif // HAS_AR30TOABGRROW_AVX2
+
+#ifdef HAS_AR30TOABGRROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_abgr,
+ int width) {
+ static const uint32_t kC0000000 = 0xc0000000u;
+ static const uint32_t k300000FF = 0x300000ffu;
+ static const uint32_t k0C00FF00 = 0x0c00ff00u;
+ static const uint32_t k03FF0000 = 0x03ff0000u;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd %[m0],%%zmm16 \n"
+ "vpbroadcastd %[m1],%%zmm17 \n"
+ "vpbroadcastd %[m2],%%zmm18 \n"
+ "vpbroadcastd %[m3],%%zmm19 \n"
+ "vbroadcasti32x4 %[shuf],%%zmm20 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpsrld $2,%%zmm0,%%zmm1 \n"
+ "vpsrld $4,%%zmm0,%%zmm2 \n"
+ "vpsrld $6,%%zmm0,%%zmm3 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm19,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm20,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrld $2,%%zmm0,%%zmm1 \n"
+ "vpsrld $4,%%zmm0,%%zmm2 \n"
+ "vpsrld $6,%%zmm0,%%zmm3 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpandd %%zmm19,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpshufb %%zmm20,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00),
+ [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17",
+ "xmm18", "xmm19", "xmm20", "k1");
+}
+#endif // HAS_AR30TOABGRROW_AVX512BW
+
+#ifdef HAS_AR30TOAB30ROW_AVX2
+void OMITFP AR30ToAB30Row_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_ab30,
+ int width) {
+ static const uint32_t kC00FFC00 = 0xc00ffc00u;
+ static const uint32_t k3FF00000 = 0x3ff00000u;
+ static const uint32_t k000003FF = 0x000003ffu;
+ asm volatile(
+ "vpbroadcastd %[m0],%%ymm3 \n"
+ "vpbroadcastd %[m1],%%ymm4 \n"
+ "vpbroadcastd %[m2],%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpslld $20,%%ymm0,%%ymm1 \n"
+ "vpsrld $20,%%ymm0,%%ymm2 \n"
+ "vpand %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [width] "+r"(width)
+ : [m0] "m"(kC00FFC00), [m1] "m"(k3FF00000), [m2] "m"(k000003FF)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_AR30TOAB30ROW_AVX2
+
+#ifdef HAS_AR30TOAB30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_ab30,
+ int width) {
+ static const uint32_t kC00FFC00 = 0xc00ffc00u;
+ static const uint32_t k3FF00000 = 0x3ff00000u;
+ static const uint32_t k000003FF = 0x000003ffu;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd %[m0],%%zmm16 \n"
+ "vpbroadcastd %[m1],%%zmm17 \n"
+ "vpbroadcastd %[m2],%%zmm18 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpslld $20,%%zmm0,%%zmm1 \n"
+ "vpsrld $20,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpslld $20,%%zmm0,%%zmm1 \n"
+ "vpsrld $20,%%zmm0,%%zmm2 \n"
+ "vpandd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [m0] "m"(kC00FFC00), [m1] "m"(k3FF00000), [m2] "m"(k000003FF)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18",
+ "k1");
+}
+#endif // HAS_AR30TOAB30ROW_AVX512BW
+
+#ifdef HAS_MERGEXR30ROW_10_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP MergeXR30Row_10_AVX512BW(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ static const uint32_t kMax10 = 1023u;
+ static const uint32_t kAlpha30 = 0xc0000000u;
+ uintptr_t temp;
+ (void)depth;
+ asm volatile(
+ "vpbroadcastd %[max10],%%zmm16 \n"
+ "vpbroadcastd %[alpha],%%zmm17 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxwd (%[r]),%%zmm0 \n"
+ "vpmovzxwd (%[g]),%%zmm1 \n"
+ "vpmovzxwd (%[b]),%%zmm2 \n"
+ "lea 0x20(%[r]),%[r] \n"
+ "lea 0x20(%[g]),%[g] \n"
+ "lea 0x20(%[b]),%[b] \n"
+ "vpminud %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpminud %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpminud %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpslld $20,%%zmm0,%%zmm0 \n"
+ "vpslld $10,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vpmovzxwd (%[r]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmovzxwd (%[g]),%%zmm1%{%%k1%}%{z%} \n"
+ "vpmovzxwd (%[b]),%%zmm2%{%%k1%}%{z%} \n"
+ "vpminud %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpminud %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpminud %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpslld $20,%%zmm0,%%zmm0 \n"
+ "vpslld $10,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [r] "+r"(src_r), [g] "+r"(src_g), [b] "+r"(src_b), [dst] "+r"(dst_ar30),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [max10] "m"(kMax10), [alpha] "m"(kAlpha30)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "k1");
+}
+#endif // HAS_MERGEXR30ROW_10_AVX512BW
+
+#if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+static inline void ARGBOrABGRToAR30Row_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ const uvec8* shuf_rb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 (%[shuf]),%%zmm2 \n"
+ "vpbroadcastd %[mul_rb],%%zmm3 \n"
+ "vpbroadcastd %[mask_rb],%%zmm4 \n"
+ "vpbroadcastd %[mask_ag],%%zmm5 \n"
+ "vpbroadcastd %[mul_ag],%%zmm6 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpshufb %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpslld $10,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpshufb %%zmm2,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm4,%%zmm1,%%zmm1 \n"
+ "vpslld $10,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ :
+ [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp)
+ : [shuf] "r"(shuf_rb), [mul_rb] "m"(kMulRB10), [mask_rb] "m"(kMaskRB10),
+ [mask_ag] "m"(kMaskAG10), [mul_ag] "m"(kMulAG10)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "k1");
+}
+#endif
+
+#ifdef HAS_ARGBTOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBToAR30Row_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ int width) {
+ ARGBOrABGRToAR30Row_AVX512BW(src, dst, &kShuffleRB30, width);
+}
+#endif // HAS_ARGBTOAR30ROW_AVX512BW
+
+#ifdef HAS_ABGRTOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ABGRToAR30Row_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ int width) {
+ ARGBOrABGRToAR30Row_AVX512BW(src, dst, &kShuffleBR30, width);
+}
+#endif // HAS_ABGRTOAR30ROW_AVX512BW
+
+// ===========================================================================
+// Module 3: Planar / Channel Pack & Mirror / Interpolate_16 (AVX512VBMI / BW)
+// ===========================================================================
+
+#ifdef HAS_MERGERGBROW_AVX512VBMI
+static const uint8_t kPermMergeRGB64_0[64] = {
+ 0x00, 0x20, 0x40, 0x01, 0x21, 0x41, 0x02, 0x22, 0x42, 0x03, 0x23,
+ 0x43, 0x04, 0x24, 0x44, 0x05, 0x25, 0x45, 0x06, 0x26, 0x46, 0x07,
+ 0x27, 0x47, 0x08, 0x28, 0x48, 0x09, 0x29, 0x49, 0x0a, 0x2a, 0x4a,
+ 0x0b, 0x2b, 0x4b, 0x0c, 0x2c, 0x4c, 0x0d, 0x2d, 0x4d, 0x0e, 0x2e,
+ 0x4e, 0x0f, 0x2f, 0x4f, 0x10, 0x30, 0x50, 0x11, 0x31, 0x51, 0x12,
+ 0x32, 0x52, 0x13, 0x33, 0x53, 0x14, 0x34, 0x54, 0x15};
+
+static const uint8_t kPermMergeRGB64_1[64] = {
+ 0x35, 0x55, 0x16, 0x36, 0x56, 0x17, 0x37, 0x57, 0x18, 0x38, 0x58,
+ 0x19, 0x39, 0x59, 0x1a, 0x3a, 0x5a, 0x1b, 0x3b, 0x5b, 0x1c, 0x3c,
+ 0x5c, 0x1d, 0x3d, 0x5d, 0x1e, 0x3e, 0x5e, 0x1f, 0x3f, 0x5f, 0x60,
+ 0x00, 0x20, 0x61, 0x01, 0x21, 0x62, 0x02, 0x22, 0x63, 0x03, 0x23,
+ 0x64, 0x04, 0x24, 0x65, 0x05, 0x25, 0x66, 0x06, 0x26, 0x67, 0x07,
+ 0x27, 0x68, 0x08, 0x28, 0x69, 0x09, 0x29, 0x6a, 0x0a};
+
+static const uint8_t kPermMergeRGB64_2[64] = {
+ 0x6a, 0x2b, 0x4b, 0x6b, 0x2c, 0x4c, 0x6c, 0x2d, 0x4d, 0x6d, 0x2e,
+ 0x4e, 0x6e, 0x2f, 0x4f, 0x6f, 0x30, 0x50, 0x70, 0x31, 0x51, 0x71,
+ 0x32, 0x52, 0x72, 0x33, 0x53, 0x73, 0x34, 0x54, 0x74, 0x35, 0x55,
+ 0x75, 0x36, 0x56, 0x76, 0x37, 0x57, 0x77, 0x38, 0x58, 0x78, 0x39,
+ 0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c,
+ 0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f};
+
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP MergeRGBRow_AVX512VBMI(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "vmovdqu8 %[m_rgb0],%%zmm16 \n"
+ "vmovdqu8 %[m_rgb1],%%zmm17 \n"
+ "vmovdqu8 %[m_rgb2],%%zmm18 \n"
+ "movabs $0xb6db6db600000000,%[temp1] \n"
+ "kmovq %[temp1],%%k2 \n"
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ // 64-pixel loop (192 bytes out = 3 full 64B zmm stores, zero cache
+ // splits).
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[r]),%%ymm0 \n"
+ "vinserti64x4 $1,(%[g]),%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%[b]),%%ymm2 \n"
+ "vinserti64x4 $1,0x20(%[r]),%%zmm2,%%zmm2 \n"
+ "vmovdqu8 0x20(%[g]),%%ymm4 \n"
+ "vinserti64x4 $1,0x20(%[b]),%%zmm4,%%zmm4 \n"
+ "lea 0x40(%[r]),%[r] \n"
+ "lea 0x40(%[g]),%[g] \n"
+ "lea 0x40(%[b]),%[b] \n"
+ "vmovdqa64 %%zmm0,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n"
+ "vpermb %%zmm4,%%zmm17,%%zmm0%{%%k2%} \n"
+ "vpermt2b %%zmm4,%%zmm18,%%zmm2 \n"
+ "vmovdqu8 %%zmm3,(%[dst]) \n"
+ "vmovdqu8 %%zmm0,0x40(%[dst]) \n"
+ "vmovdqu8 %%zmm2,0x80(%[dst]) \n"
+ "lea 0xc0(%[dst]),%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jl 3f \n"
+ "vmovdqu8 (%[r]),%%ymm0 \n"
+ "vinserti64x4 $1,(%[g]),%%zmm0,%%zmm0 \n"
+ "vmovdqu8 (%[b]),%%ymm2 \n"
+ "lea 0x20(%[r]),%[r] \n"
+ "lea 0x20(%[g]),%[g] \n"
+ "lea 0x20(%[b]),%[b] \n"
+ "vmovdqa64 %%zmm0,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm3,(%[dst]) \n"
+ "vmovdqu8 %%ymm0,0x40(%[dst]) \n"
+ "lea 0x60(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+
+ "3: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovd %k[temp2],%%k1 \n"
+ "vmovdqu8 (%[r]),%%ymm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[g]),%%ymm1%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[b]),%%ymm2%{%%k1%}%{z%} \n"
+ "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n"
+ "lea (%q[width],%q[width],2),%k[width]\n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovq %[temp2],%%k3 \n"
+ "xor %k[temp2],%k[temp2] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp1],%[temp2] \n"
+ "bzhi %q[width],%[temp2],%[temp2] \n"
+ "kmovd %k[temp2],%%k4 \n"
+ "vmovdqa64 %%zmm0,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n"
+ "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n"
+ "vmovdqu8 %%zmm3,(%[dst])%{%%k3%} \n"
+ "vmovdqu8 %%ymm0,0x40(%[dst])%{%%k4%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [r] "+r"(src_r), [g] "+r"(src_g), [b] "+r"(src_b), [dst] "+r"(dst_rgb),
+ [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [m_rgb0] "m"(kPermMergeRGB64_0), [m_rgb1] "m"(kPermMergeRGB64_1),
+ [m_rgb2] "m"(kPermMergeRGB64_2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
+ "xmm17", "xmm18", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_MERGERGBROW_AVX512VBMI
+
+#ifdef HAS_RAWTORGBAROW_AVX512VBMI
+static const uint8_t kPermRAWToRGBA0[64] = {
+ 0, 2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, 0, 11, 10, 9,
+ 0, 14, 13, 12, 0, 17, 16, 15, 0, 20, 19, 18, 0, 23, 22, 21,
+ 0, 26, 25, 24, 0, 29, 28, 27, 0, 32, 31, 30, 0, 35, 34, 33,
+ 0, 38, 37, 36, 0, 41, 40, 39, 0, 44, 43, 42, 0, 47, 46, 45};
+
+static const uint8_t kPermRAWToRGBA1[64] = {
+ 0, 50, 49, 48, 0, 53, 52, 51, 0, 56, 55, 54, 0, 59, 58, 57,
+ 0, 62, 61, 60, 0, 65, 64, 63, 0, 68, 67, 66, 0, 71, 70, 69,
+ 0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81,
+ 0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93};
+
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgba,
+ int width) {
+ static const uint32_t kAlphaLow = 0x000000ffu;
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "vmovdqu8 %[p0],%%zmm16 \n"
+ "vmovdqu8 %[p1],%%zmm17 \n"
+ "vpbroadcastd %[alpha],%%zmm18 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[src]),%%zmm0 \n"
+ "vmovdqu8 0x40(%[src]),%%ymm1 \n"
+ "lea 0x60(%[src]),%[src] \n"
+ "vpermb %%zmm0,%%zmm16,%%zmm2 \n"
+ "vpermt2b %%zmm1,%%zmm17,%%zmm0 \n"
+ "vpord %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm18,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm2,(%[dst]) \n"
+ "vmovdqu32 %%zmm0,0x40(%[dst]) \n"
+ "lea 0x80(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovw %k[temp2],%%k1 \n"
+ "kmovd %k[temp2],%%k2 \n"
+ "kshiftrd $16,%%k2,%%k2 \n"
+ "lea (%q[width],%q[width],2),%k[width]\n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovq %[temp2],%%k3 \n"
+ "xor %k[temp2],%k[temp2] \n"
+ "sub $0x40,%k[width] \n"
+ "cmovg %[temp1],%[temp2] \n"
+ "bzhi %q[width],%[temp2],%[temp2] \n"
+ "kmovd %k[temp2],%%k4 \n"
+ "vmovdqu8 (%[src]),%%zmm0%{%%k3%}%{z%} \n"
+ "vmovdqu8 0x40(%[src]),%%ymm1%{%%k4%}%{z%}\n"
+ "vpermb %%zmm0,%%zmm16,%%zmm2 \n"
+ "vpermt2b %%zmm1,%%zmm17,%%zmm0 \n"
+ "vpord %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm18,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n"
+ "vmovdqu32 %%zmm0,0x40(%[dst])%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_raw), [dst] "+r"(dst_rgba), [width] "+r"(width),
+ [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [p0] "m"(kPermRAWToRGBA0), [p1] "m"(kPermRAWToRGBA1),
+ [alpha] "m"(kAlphaLow)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18", "k1",
+ "k2", "k3", "k4");
+}
+#endif // HAS_RAWTORGBAROW_AVX512VBMI
+
+#ifdef HAS_SWAPUVROW_AVX512BW
+static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6,
+ 9, 8, 11, 10, 13, 12, 15, 14};
+
+LIBYUV_TARGET_AVX512BW
+void OMITFP SwapUVRow_AVX512BW(const uint8_t* src_uv,
+ uint8_t* dst_vu,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 %[swap],%%zmm16 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu16 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpshufb %%zmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpshufb %%zmm16,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_uv), [dst] "+r"(dst_vu), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [swap] "m"(kSwapUV16)
+ : "memory", "cc", "xmm0", "xmm16", "k1");
+}
+#endif // HAS_SWAPUVROW_AVX512BW
+
+#ifdef HAS_MIRRORROW_16_AVX512BW
+static const uint16_t kMirror16Idx[32] = {
+ 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
+ 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
+
+LIBYUV_TARGET_AVX512BW
+void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src,
+ uint16_t* dst,
+ int width) {
+ intptr_t width64 = width;
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "vmovdqu16 %[rev],%%zmm16 \n"
+ "lea (%[src],%q[width],2),%[src] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "sub $0x40,%[src] \n"
+ "vmovdqu16 (%[src]),%%zmm0 \n"
+ "vpermw %%zmm0,%%zmm16,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst]) \n"
+ "add $0x40,%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $32,%k[temp2] \n"
+ "sub %k[width],%k[temp2] \n"
+ "vpbroadcastw %k[temp2],%%zmm17 \n"
+ "vpsubw %%zmm17,%%zmm16,%%zmm17 \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp1] \n"
+ "kmovd %k[temp1],%%k1 \n"
+ "neg %q[width] \n"
+ "vmovdqu16 (%[src],%q[width],2),%%zmm0%{%%k1%}%{z%}\n"
+ "vpermw %%zmm0,%%zmm17,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width64),
+ [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [rev] "m"(kMirror16Idx)
+ : "memory", "cc", "xmm0", "xmm16", "xmm17", "k1");
+}
+#endif // HAS_MIRRORROW_16_AVX512BW
+
+#ifdef HAS_HALFMERGEUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP HalfMergeUVRow_AVX512BW(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+ "vpsrlw $15,%%zmm16,%%zmm16 \n"
+ "vpsllw $8,%%zmm16,%%zmm17 \n"
+ "vpord %%zmm17,%%zmm16,%%zmm16 \n" // 0x0101
+ "vpxord %%zmm17,%%zmm17,%%zmm17 \n" // 0
+ "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n"
+ "vpsrlw $8,%%zmm18,%%zmm18 \n" // 0x00ff
+ "sub $0x40,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%[u]),%%zmm0 \n"
+ "vmovdqu8 (%[u],%[su]),%%zmm1 \n"
+ "vmovdqu8 (%[v]),%%zmm2 \n"
+ "vmovdqu8 (%[v],%[sv]),%%zmm3 \n"
+ "lea 0x40(%[u]),%[u] \n"
+ "lea 0x40(%[v]),%[v] \n"
+ "vpmaddubsw %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm2,%%zmm2 \n"
+ "vpavgw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsllw $8,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "vmovdqu8 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x40,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x40,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovq %[temp2],%%k1 \n"
+ "vmovdqu8 (%[u]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[u],%[su]),%%zmm1%{%%k1%}%{z%}\n"
+ "vmovdqu8 (%[v]),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqu8 (%[v],%[sv]),%%zmm3%{%%k1%}%{z%}\n"
+ // If width is odd, duplicate byte (width - 1) into high byte of word
+ // (width >> 1).
+ "mov %k[width],%k[temp1] \n"
+ "and $1,%k[temp1] \n"
+ "mov %k[width],%k[temp2] \n"
+ "shr $1,%k[temp2] \n"
+ "shlx %k[temp2],%k[temp1],%k[temp1] \n"
+ "kmovd %k[temp1],%%k2 \n"
+ "vpandd %%zmm18,%%zmm0,%%zmm4 \n"
+ "vpmullw %%zmm16,%%zmm4,%%zmm0%{%%k2%} \n"
+ "vpandd %%zmm18,%%zmm1,%%zmm4 \n"
+ "vpmullw %%zmm16,%%zmm4,%%zmm1%{%%k2%} \n"
+ "vpandd %%zmm18,%%zmm2,%%zmm4 \n"
+ "vpmullw %%zmm16,%%zmm4,%%zmm2%{%%k2%} \n"
+ "vpandd %%zmm18,%%zmm3,%%zmm4 \n"
+ "vpmullw %%zmm16,%%zmm4,%%zmm3%{%%k2%} \n"
+ "vpmaddubsw %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm16,%%zmm1,%%zmm1 \n"
+ "vpmaddubsw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm2,%%zmm2 \n"
+ "vpavgw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsllw $8,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm2,%%zmm0,%%zmm0 \n"
+ "lea 1(%q[width]),%k[temp2] \n"
+ "shr $1,%k[temp2] \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[temp2],%[temp1],%[temp1] \n"
+ "kmovd %k[temp1],%%k2 \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [u] "+r"(src_u), [v] "+r"(src_v), [dst] "+r"(dst_uv),
+ [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [su] "r"((ptrdiff_t)src_stride_u), [sv] "r"((ptrdiff_t)src_stride_v)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
+ "xmm17", "xmm18", "k1", "k2");
+}
+#endif // HAS_HALFMERGEUVROW_AVX512BW
+
+#ifdef HAS_INTERPOLATEROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ int y_fractions = (y1_fraction << 16) + y0_fraction;
+ uintptr_t temp1;
+ uintptr_t temp2;
+ asm volatile(
+ "cmpl $0x0,%[y1] \n"
+ "je 10f \n"
+ "cmpl $0x80,%[y1] \n"
+ "je 20f \n"
+
+ "vpbroadcastd %[y_frac],%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpsllw $15,%%zmm17,%%zmm17 \n" // 0x8000
+ "mov $0x800080,%k[temp1] \n"
+ "vpbroadcastd %k[temp1],%%zmm18 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ // 32-pixel general blend loop (vpmaddwd + vpackusdw).
+ LABELALIGN
+ "1: \n"
+ "vmovdqu16 (%[src]),%%zmm0 \n"
+ "vmovdqu16 (%[src1]),%%zmm1 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "lea 0x40(%[src1]),%[src1] \n"
+ "vpunpcklwd %%zmm1,%%zmm0,%%zmm2 \n"
+ "vpunpckhwd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpsubw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpsrad $8,%%zmm2,%%zmm2 \n"
+ "vpsrad $8,%%zmm0,%%zmm0 \n"
+ "vpackusdw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovd %k[temp2],%%k1 \n"
+ "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[src1]),%%zmm1%{%%k1%}%{z%} \n"
+ "vpunpcklwd %%zmm1,%%zmm0,%%zmm2 \n"
+ "vpunpckhwd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpsubw %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsubw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpsrad $8,%%zmm2,%%zmm2 \n"
+ "vpsrad $8,%%zmm0,%%zmm0 \n"
+ "vpackusdw %%zmm0,%%zmm2,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // y1 == 0 fast copy path (32 uint16_t per zmm).
+ "10: \n"
+ "sub $0x20,%[width] \n"
+ "jl 12f \n" LABELALIGN
+ "11: \n"
+ "vmovdqu16 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vmovdqu16 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 11b \n"
+ "12: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovd %k[temp2],%%k1 \n"
+ "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // y1 == 128 fast 50/50 average path (vpavgw on 32 uint16_t per zmm).
+ "20: \n"
+ "sub $0x20,%[width] \n"
+ "jl 22f \n" LABELALIGN
+ "21: \n"
+ "vmovdqu16 (%[src]),%%zmm0 \n"
+ "vpavgw (%[src1]),%%zmm0,%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "lea 0x40(%[src1]),%[src1] \n"
+ "vmovdqu16 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 21b \n"
+ "22: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp1] \n"
+ "bzhi %q[width],%[temp1],%[temp2] \n"
+ "kmovd %k[temp2],%%k1 \n"
+ "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 (%[src1]),%%zmm1%{%%k1%}%{z%} \n"
+ "vpavgw %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_ptr), [src] "+r"(src_ptr), [src1] "+r"(src_ptr1),
+ [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2)
+ : [y1] "r"(y1_fraction), [y_frac] "r"(y_fractions)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18",
+ "k1");
+}
+#endif // HAS_INTERPOLATEROW_16_AVX512BW
+
+// ===========================================================================
+// Module 4: NV12 / NV21 & I422Alpha to ARGB / RGB24 (AVX512BW / AVX512VBMI)
+// ===========================================================================
+
+#define READYUVNV12_AVX512BW \
+ "vmovdqu16 (%[uv_buf]),%%ymm3 \n" \
+ "lea 0x20(%[uv_buf]),%[uv_buf] \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+#define READYUVNV12_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu16 (%[uv_buf]),%%ymm3%{%%k2%}%{z%} \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+#define READYUVNV21_AVX512BW \
+ "vmovdqu16 (%[uv_buf]),%%ymm3 \n" \
+ "lea 0x20(%[uv_buf]),%[uv_buf] \n" \
+ "vpsllw $8,%%ymm3,%%ymm1 \n" \
+ "vpsrlw $8,%%ymm3,%%ymm3 \n" \
+ "vpor %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+#define READYUVNV21_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu16 (%[uv_buf]),%%ymm3%{%%k2%}%{z%} \n" \
+ "vpsllw $8,%%ymm3,%%ymm1 \n" \
+ "vpsrlw $8,%%ymm3,%%ymm3 \n" \
+ "vpor %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \
+ "vpsllw $8,%%zmm4,%%zmm3 \n" \
+ "vpord %%zmm3,%%zmm4,%%zmm4 \n" \
+ "vmovdqa64 %%zmm1,%%zmm3 \n"
+
+#ifdef HAS_NV12TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUVNV12_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUVNV12_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_NV12TOARGBROW_AVX512BW
+
+#ifdef HAS_NV21TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUVNV21_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUVNV21_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(vu_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_NV21TOARGBROW_AVX512BW
+
+#ifdef HAS_NV12TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vmovdqu32 %[kMaskBG],%%zmm20 \n"
+ "vmovdqu32 %[kMaskDST0],%%zmm21 \n"
+ "vmovdqu32 %[kMaskDST1],%%zmm22 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUVNV12_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_AVX512VBMI
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUVNV12_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_MASK_AVX512VBMI
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_rgb24] "+r"(dst_rgb24),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kMaskBG] "m"(kMaskBG),
+ [kMaskDST0] "m"(kMaskDST0),
+ [kMaskDST1] "m"(kMaskDST1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm20", "xmm21", "xmm22", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_NV12TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_NV21TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vmovdqu32 %[kMaskBG],%%zmm20 \n"
+ "vmovdqu32 %[kMaskDST0],%%zmm21 \n"
+ "vmovdqu32 %[kMaskDST1],%%zmm22 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUVNV21_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_AVX512VBMI
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUVNV21_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_MASK_AVX512VBMI
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(vu_buf),
+ [dst_rgb24] "+r"(dst_rgb24),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kMaskBG] "m"(kMaskBG),
+ [kMaskDST0] "m"(kMaskDST0),
+ [kMaskDST1] "m"(kMaskDST1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm20", "xmm21", "xmm22", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_NV21TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_I422ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ READYUV422_AVX512BW
+ "vpmovzxbw (%[a_buf]),%%zmm5 \n"
+ "lea 0x20(%[a_buf]),%[a_buf] \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_I422ALPHATOARGBROW_AVX512BW
+
+// ===========================================================================
+// Module 5: I400ToARGB & ARGB Shaders (Attenuate, Multiply, Gray, Blend)
+// ===========================================================================
+
+#ifdef HAS_I400TOARGBROW_AVX512BW
+static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7};
+static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+
+LIBYUV_TARGET_AVX512BW
+void OMITFP I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastw 96(%[yuv]),%%zmm16 \n" // kYToRgb
+ "vpbroadcastw 128(%[yuv]),%%zmm17 \n" // kYBiasToRgb
+ "vmovdqu64 %[lin],%%zmm18 \n"
+ "vbroadcasti32x4 %[bcast],%%zmm19 \n"
+ "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n"
+ "vpslld $24,%%zmm20,%%zmm20 \n" // 0xff000000
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%[y]),%%zmm0 \n"
+ "lea 0x20(%[y]),%[y] \n"
+ "vpsllw $8,%%zmm0,%%zmm1 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpaddsw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpsraw $6,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpermq %%zmm0,%%zmm18,%%zmm0 \n"
+ "vpmovzxbd %%xmm0,%%zmm1 \n"
+ "vextracti128 $1,%%ymm0,%%xmm0 \n"
+ "vpmovzxbd %%xmm0,%%zmm0 \n"
+ "vpshufb %%zmm19,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm19,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm20,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm1,(%[dst]) \n"
+ "vmovdqu32 %%zmm0,0x40(%[dst]) \n"
+ "lea 0x80(%[dst]),%[dst] \n"
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "kshiftrd $16,%%k1,%%k2 \n"
+ "vpmovzxbw (%[y]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsllw $8,%%zmm0,%%zmm1 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm16,%%zmm0,%%zmm0 \n"
+ "vpaddsw %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpsraw $6,%%zmm0,%%zmm0 \n"
+ "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n"
+ "vpermq %%zmm0,%%zmm18,%%zmm0 \n"
+ "vpmovzxbd %%xmm0,%%zmm1 \n"
+ "vextracti128 $1,%%ymm0,%%xmm0 \n"
+ "vpmovzxbd %%xmm0,%%zmm0 \n"
+ "vpshufb %%zmm19,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm19,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm20,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm1,(%[dst])%{%%k1%} \n"
+ "vmovdqu32 %%zmm0,0x40(%[dst])%{%%k2%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y] "+r"(y_buf), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuv] "r"(yuvconstants), [lin] "m"(kLinearizePackQ),
+ [bcast] "m"(kBcastGrayByte)
+ : "memory", "cc", "xmm0", "xmm1", "xmm16", "xmm17", "xmm18", "xmm19",
+ "xmm20", "k1", "k2");
+}
+#endif // HAS_I400TOARGBROW_AVX512BW
+
+#ifdef HAS_ARGBATTENUATEROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 %[shuf],%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpslld $24,%%zmm17,%%zmm17 \n" // 0xff000000
+ "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n"
+ "vpsrlw $8,%%zmm18,%%zmm18 \n" // 0x00ff
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm6 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpunpcklbw %%zmm17,%%zmm6,%%zmm0 \n"
+ "vpunpckhbw %%zmm17,%%zmm6,%%zmm1 \n"
+ "vpshufb %%zmm16,%%zmm0,%%zmm2 \n"
+ "vpshufb %%zmm16,%%zmm1,%%zmm3 \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmullw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpaddw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpsrlw $8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $8,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm6,%%zmm1 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm6%{%%k1%}%{z%} \n"
+ "vpunpcklbw %%zmm17,%%zmm6,%%zmm0 \n"
+ "vpunpckhbw %%zmm17,%%zmm6,%%zmm1 \n"
+ "vpshufb %%zmm16,%%zmm0,%%zmm2 \n"
+ "vpshufb %%zmm16,%%zmm1,%%zmm3 \n"
+ "vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmullw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpaddw %%zmm18,%%zmm0,%%zmm0 \n"
+ "vpaddw %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpsrlw $8,%%zmm0,%%zmm0 \n"
+ "vpsrlw $8,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm17,%%zmm6,%%zmm1 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [shuf] "m"(kAttenuateShuffle_AVX2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm6", "xmm16",
+ "xmm17", "xmm18", "k1");
+}
+#endif // HAS_ARGBATTENUATEROW_AVX512BW
+
+#ifdef HAS_ARGBMULTIPLYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpxord %%zmm16,%%zmm16,%%zmm16 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src0]),%%zmm1 \n"
+ "vmovdqu32 (%[src1]),%%zmm3 \n"
+ "lea 0x40(%[src0]),%[src0] \n"
+ "lea 0x40(%[src1]),%[src1] \n"
+ "vpunpcklbw %%zmm1,%%zmm16,%%zmm0 \n"
+ "vpunpckhbw %%zmm1,%%zmm16,%%zmm1 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpunpcklbw %%zmm16,%%zmm3,%%zmm2 \n"
+ "vpunpckhbw %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpmulhrsw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhrsw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src0]),%%zmm1%{%%k1%}%{z%} \n"
+ "vmovdqu32 (%[src1]),%%zmm3%{%%k1%}%{z%} \n"
+ "vpunpcklbw %%zmm1,%%zmm16,%%zmm0 \n"
+ "vpunpckhbw %%zmm1,%%zmm16,%%zmm1 \n"
+ "vpsrlw $1,%%zmm0,%%zmm0 \n"
+ "vpsrlw $1,%%zmm1,%%zmm1 \n"
+ "vpunpcklbw %%zmm16,%%zmm3,%%zmm2 \n"
+ "vpunpckhbw %%zmm16,%%zmm3,%%zmm3 \n"
+ "vpmulhrsw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhrsw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "k1");
+}
+#endif // HAS_ARGBMULTIPLYROW_AVX512BW
+
+#ifdef HAS_ARGBGRAYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0}
+ static const uint32_t kOne16 = 0x00010001u;
+ static const uint32_t kBias = 32896u;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd %[coeff],%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpsllw $7,%%zmm17,%%zmm17 \n"
+ "vpbroadcastb %%xmm17,%%zmm17 \n" // 0x80808080
+ "vpbroadcastd %[one16],%%zmm18 \n"
+ "vpbroadcastd %[bias],%%zmm19 \n"
+ "vbroadcasti32x4 %[shuf],%%zmm20 \n"
+ "vpternlogd $0xff,%%zmm21,%%zmm21,%%zmm21 \n"
+ "vpslld $24,%%zmm21,%%zmm21 \n" // 0xff000000
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpsubb %%zmm17,%%zmm0,%%zmm1 \n"
+ "vpmaddubsw %%zmm1,%%zmm16,%%zmm1 \n"
+ "vpmaddwd %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm19,%%zmm1,%%zmm1 \n"
+ "vpsrld $8,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsubb %%zmm17,%%zmm0,%%zmm1 \n"
+ "vpmaddubsw %%zmm1,%%zmm16,%%zmm1 \n"
+ "vpmaddwd %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpaddd %%zmm19,%%zmm1,%%zmm1 \n"
+ "vpsrld $8,%%zmm1,%%zmm1 \n"
+ "vpshufb %%zmm20,%%zmm1,%%zmm1 \n"
+ "vpandd %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [coeff] "m"(kCoeff), [one16] "m"(kOne16), [bias] "m"(kBias),
+ [shuf] "m"(kShufGrayDword)
+ : "memory", "cc", "xmm0", "xmm1", "xmm16", "xmm17", "xmm18", "xmm19",
+ "xmm20", "xmm21", "k1");
+}
+#endif // HAS_ARGBGRAYROW_AVX512BW
+
+#ifdef HAS_ARGBGRAYROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void OMITFP ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0}
+ static const uint32_t kBias = 32896u;
+ uintptr_t temp;
+ asm volatile(
+ "vpbroadcastd %[coeff],%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpslld $24,%%zmm17,%%zmm21 \n" // 0xff000000
+ "vpsllw $7,%%zmm17,%%zmm17 \n"
+ "vpbroadcastb %%xmm17,%%zmm17 \n" // 0x80808080
+ "vpbroadcastd %[bias],%%zmm19 \n"
+ "vbroadcasti32x4 %[shuf],%%zmm20 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpxorq %%zmm17,%%zmm0,%%zmm1 \n"
+ "vmovdqa64 %%zmm19,%%zmm2 \n"
+ "vpdpbusd %%zmm1,%%zmm16,%%zmm2 \n"
+ "vpsrld $8,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpternlogd $0xf8,%%zmm21,%%zmm0,%%zmm2 \n"
+ "vmovdqu32 %%zmm2,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpxorq %%zmm17,%%zmm0,%%zmm1 \n"
+ "vmovdqa64 %%zmm19,%%zmm2 \n"
+ "vpdpbusd %%zmm1,%%zmm16,%%zmm2 \n"
+ "vpsrld $8,%%zmm2,%%zmm2 \n"
+ "vpshufb %%zmm20,%%zmm2,%%zmm2 \n"
+ "vpternlogd $0xf8,%%zmm21,%%zmm0,%%zmm2 \n"
+ "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [coeff] "m"(kCoeff), [bias] "m"(kBias), [shuf] "m"(kShufGrayDword)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm19",
+ "xmm20", "xmm21", "k1");
+}
+#endif // HAS_ARGBGRAYROW_AVX512VNNI
+
+#ifdef HAS_ARGBBLENDROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vbroadcasti32x4 %[shuf],%%zmm16 \n"
+ "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
+ "vpslld $24,%%zmm17,%%zmm17 \n" // 0xff000000
+ "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n"
+ "vpsllw $8,%%zmm18,%%zmm18 \n" // 0xff00
+ "vpternlogd $0xff,%%zmm19,%%zmm19,%%zmm19 \n"
+ "vpsrlw $8,%%zmm19,%%zmm19 \n" // 0x00ff
+ "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n"
+ "vpsrlw $15,%%zmm20,%%zmm20 \n" // 0x0001
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movq (%0),%%xmm0 \n"
- "movq 0x2(%0),%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm0 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "psubw %%xmm1,%%xmm0 \n"
- "movq 0x00(%0,%1,1),%%xmm1 \n"
- "movq 0x02(%0,%1,1),%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "psubw %%xmm2,%%xmm1 \n"
- "movq 0x00(%0,%2,1),%%xmm2 \n"
- "movq 0x02(%0,%2,1),%%xmm3 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm3 \n"
- "psubw %%xmm3,%%xmm2 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm1,%%xmm0 \n"
- "paddw %%xmm1,%%xmm0 \n"
- "pxor %%xmm1,%%xmm1 \n"
- "psubw %%xmm0,%%xmm1 \n"
- "pmaxsw %%xmm1,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,0x00(%0,%3,1) \n"
- "lea 0x8(%0),%0 \n"
- "sub $0x8,%4 \n"
- "jg 1b \n"
- : "+r"(src_y0), // %0
- "+r"(src_y1), // %1
- "+r"(src_y2), // %2
- "+r"(dst_sobelx), // %3
- "+r"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "vmovdqu32 (%[src0]),%%zmm3 \n"
+ "vmovdqu32 (%[src1]),%%zmm2 \n"
+ "lea 0x40(%[src0]),%[src0] \n"
+ "lea 0x40(%[src1]),%[src1] \n"
+ "vmovdqa64 %%zmm3,%%zmm0 \n"
+ "vpxord %%zmm17,%%zmm3,%%zmm3 \n"
+ "vpshufb %%zmm16,%%zmm3,%%zmm3 \n"
+ "vmovdqa64 %%zmm2,%%zmm1 \n"
+ "vpandd %%zmm19,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpmullw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlw $8,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpmullw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpsrlw $8,%%zmm2,%%zmm2 \n"
+ "vpaddusb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpaddusb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src0]),%%zmm3%{%%k1%}%{z%} \n"
+ "vmovdqu32 (%[src1]),%%zmm2%{%%k1%}%{z%} \n"
+ "vmovdqa64 %%zmm3,%%zmm0 \n"
+ "vpxord %%zmm17,%%zmm3,%%zmm3 \n"
+ "vpshufb %%zmm16,%%zmm3,%%zmm3 \n"
+ "vmovdqa64 %%zmm2,%%zmm1 \n"
+ "vpandd %%zmm19,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm20,%%zmm3,%%zmm3 \n"
+ "vpmullw %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlw $8,%%zmm1,%%zmm1 \n"
+ "vpord %%zmm17,%%zmm0,%%zmm0 \n"
+ "vpmullw %%zmm3,%%zmm1,%%zmm1 \n"
+ "vpsrlw $8,%%zmm2,%%zmm2 \n"
+ "vpaddusb %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpandd %%zmm18,%%zmm1,%%zmm1 \n"
+ "vpaddusb %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ : [shuf] "m"(kShuffleAlpha)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17",
+ "xmm18", "xmm19", "xmm20", "k1");
}
-#endif // HAS_SOBELXROW_SSE2
+#endif // HAS_ARGBBLENDROW_AVX512BW
-#ifdef HAS_SOBELYROW_SSE2
-// SobelY as a matrix is
-// -1 -2 -1
-// 0 0 0
-// 1 2 1
-void SobelYRow_SSE2(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
+#ifdef HAS_ARGBORROW_SSE2
+void ARGBOrRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
int width) {
asm volatile(
- "sub %0,%1 \n"
- "sub %0,%2 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ "movd %[value],%%xmm1 \n"
+ "pshufd $0x00,%%xmm1,%%xmm1 \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movq (%0),%%xmm0 \n"
- "movq 0x00(%0,%1,1),%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm0 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "psubw %%xmm1,%%xmm0 \n"
- "movq 0x1(%0),%%xmm1 \n"
- "movq 0x01(%0,%1,1),%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "psubw %%xmm2,%%xmm1 \n"
- "movq 0x2(%0),%%xmm2 \n"
- "movq 0x02(%0,%1,1),%%xmm3 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm3 \n"
- "psubw %%xmm3,%%xmm2 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm1,%%xmm0 \n"
- "paddw %%xmm1,%%xmm0 \n"
- "pxor %%xmm1,%%xmm1 \n"
- "psubw %%xmm0,%%xmm1 \n"
- "pmaxsw %%xmm1,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,0x00(%0,%2,1) \n"
- "lea 0x8(%0),%0 \n"
- "sub $0x8,%3 \n"
- "jg 1b \n"
- : "+r"(src_y0), // %0
- "+r"(src_y1), // %1
- "+r"(dst_sobely), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "movdqu (%[src]),%%xmm0 \n"
+ "lea 0x10(%[src]),%[src] \n"
+ "por %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "sub $0x4,%[width] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [value] "r"(value)
+ : "memory", "cc", "xmm0", "xmm1");
}
-#endif // HAS_SOBELYROW_SSE2
+#endif // HAS_ARGBORROW_SSE2
-#ifdef HAS_SOBELROW_SSE2
-// Adds Sobel X and Sobel Y and stores Sobel into ARGB.
-// A = 255
-// R = Sobel
-// G = Sobel
-// B = Sobel
-void SobelRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
+#ifdef HAS_ARGBORROW_AVX2
+void ARGBOrRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
asm volatile(
- "sub %0,%1 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "pslld $0x18,%%xmm5 \n"
+ "vmovd %[value],%%xmm1 \n"
+ "vpbroadcastd %%xmm1,%%ymm1 \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%1,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "punpcklbw %%xmm0,%%xmm2 \n"
- "punpckhbw %%xmm0,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "punpcklwd %%xmm2,%%xmm1 \n"
- "punpckhwd %%xmm2,%%xmm2 \n"
- "por %%xmm5,%%xmm1 \n"
- "por %%xmm5,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm3 \n"
- "punpcklwd %%xmm0,%%xmm3 \n"
- "punpckhwd %%xmm0,%%xmm0 \n"
- "por %%xmm5,%%xmm3 \n"
- "por %%xmm5,%%xmm0 \n"
- "movdqu %%xmm1,(%2) \n"
- "movdqu %%xmm2,0x10(%2) \n"
- "movdqu %%xmm3,0x20(%2) \n"
- "movdqu %%xmm0,0x30(%2) \n"
- "lea 0x40(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_sobelx), // %0
- "+r"(src_sobely), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "vmovdqu (%[src]),%%ymm0 \n"
+ "lea 0x20(%[src]),%[src] \n"
+ "vpor %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width)
+ : [value] "r"(value)
+ : "memory", "cc", "xmm0", "xmm1");
}
-#endif // HAS_SOBELROW_SSE2
+#endif // HAS_ARGBORROW_AVX2
-#ifdef HAS_SOBELTOPLANEROW_SSE2
-// Adds Sobel X and Sobel Y and stores Sobel into a plane.
-void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_y,
- int width) {
+#ifdef HAS_ARGBORROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBOrRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "sub %0,%1 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "pslld $0x18,%%xmm5 \n"
+ "vpbroadcastd %[value],%%zmm1 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%1,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_sobelx), // %0
- "+r"(src_sobely), // %1
- "+r"(dst_y), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1");
+ "vmovdqu32 (%[src]),%%zmm0 \n"
+ "lea 0x40(%[src]),%[src] \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpord %%zmm1,%%zmm0,%%zmm0 \n"
+ "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [value] "r"(value)
+ : "memory", "cc", "xmm0", "xmm1", "k1");
}
-#endif // HAS_SOBELTOPLANEROW_SSE2
+#endif // HAS_ARGBORROW_AVX512BW
-#ifdef HAS_SOBELXYROW_SSE2
-// Mixes Sobel X, Sobel Y and Sobel into ARGB.
-// A = 255
-// R = Sobel X
-// G = Sobel
-// B = Sobel Y
-void SobelXYRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
+#ifdef HAS_ARGBALPHABLENDROW_SSSE3
+void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "sub %0,%1 \n"
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "pabsb %%xmm5,%%xmm4 \n" // 0x01010101...
+ "psllw $7,%%xmm4 \n"
+ "movdqa %%xmm4,%%xmm6 \n" // xmm6 = 0x8080...
+ "psrlw $7,%%xmm4 \n" // xmm4 = 0x0101...
+ "psubw %%xmm4,%%xmm6 \n" // 0x7f7f...
+ "psubw %%xmm5,%%xmm6 \n" // 0x7f80...
+ "psubw %%xmm5,%%xmm6 \n" // xmm6 = 0x7f81...
- // 8 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%1,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "paddusb %%xmm1,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm3 \n"
- "punpcklbw %%xmm5,%%xmm3 \n"
- "punpckhbw %%xmm5,%%xmm0 \n"
- "movdqa %%xmm1,%%xmm4 \n"
- "punpcklbw %%xmm2,%%xmm4 \n"
- "punpckhbw %%xmm2,%%xmm1 \n"
- "movdqa %%xmm4,%%xmm6 \n"
- "punpcklwd %%xmm3,%%xmm6 \n"
- "punpckhwd %%xmm3,%%xmm4 \n"
- "movdqa %%xmm1,%%xmm7 \n"
- "punpcklwd %%xmm0,%%xmm7 \n"
- "punpckhwd %%xmm0,%%xmm1 \n"
- "movdqu %%xmm6,(%2) \n"
- "movdqu %%xmm4,0x10(%2) \n"
- "movdqu %%xmm7,0x20(%2) \n"
- "movdqu %%xmm1,0x30(%2) \n"
- "lea 0x40(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_sobelx), // %0
- "+r"(src_sobely), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
+ "movdqu (%[src0]),%%xmm0 \n"
+ "movdqu (%[src1]),%%xmm1 \n"
+ "lea 0x10(%[src0]),%[src0] \n"
+ "lea 0x10(%[src1]),%[src1] \n"
+
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "psrld $24,%%xmm2 \n" // sa
+ "psrld $24,%%xmm3 \n" // da
+
+ "pcmpeqb %%xmm5,%%xmm5 \n" // -1
+ "movdqa %%xmm2,%%xmm7 \n"
+ "pmullw %%xmm3,%%xmm7 \n" // da * sa
+ "psubw %%xmm5,%%xmm7 \n" // da * sa + 1
+ "pmulhuw %%xmm4,%%xmm7 \n" // (da * sa) / 255
+
+ "paddd %%xmm3,%%xmm5 \n" // da - 1
+ "psrld $24,%%xmm5 \n" // t = (da==0) ? 255 : 0
+
+ "paddw %%xmm2,%%xmm3 \n" // da + sa
+ "psubw %%xmm7,%%xmm3 \n" // da2
+
+ "por %%xmm5,%%xmm2 \n" // sa_eff
+ "movdqa %%xmm3,%%xmm7 \n"
+ "por %%xmm5,%%xmm7 \n" // den_eff
+
+ "movdqa %%xmm2,%%xmm5 \n"
+ "pslld $8,%%xmm2 \n"
+ "psubd %%xmm5,%%xmm2 \n" // sa_eff * 255
+
+ "cvtdq2ps %%xmm2,%%xmm2 \n"
+ "cvtdq2ps %%xmm7,%%xmm7 \n"
+ "divps %%xmm7,%%xmm2 \n"
+ "cvttps2dq %%xmm2,%%xmm2 \n" // ratio
+
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psllw $8,%%xmm5 \n" // 0xff00
+
+ "pmullw %%xmm4,%%xmm2 \n"
+ "pshufhw $0xa0,%%xmm2,%%xmm2 \n"
+ "pshuflw $0xa0,%%xmm2,%%xmm2 \n"
+ "pxor %%xmm5,%%xmm2 \n"
+ "movdqa %%xmm2,%%xmm7 \n"
+ "punpcklwd %%xmm2,%%xmm2 \n"
+ "punpckhwd %%xmm7,%%xmm7 \n"
+
+ "movdqa %%xmm4,%%xmm5 \n"
+ "psllw $7,%%xmm5 \n" // 0x8080
+ "psubb %%xmm5,%%xmm0 \n"
+ "psubb %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm0,%%xmm5 \n"
+ "punpcklbw %%xmm1,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm5 \n"
+
+ "pmaddubsw %%xmm0,%%xmm2 \n"
+ "pmaddubsw %%xmm5,%%xmm7 \n"
+ "paddw %%xmm6,%%xmm2 \n"
+ "paddw %%xmm6,%%xmm7 \n"
+ "pmulhuw %%xmm4,%%xmm2 \n"
+ "pmulhuw %%xmm4,%%xmm7 \n"
+ "packuswb %%xmm7,%%xmm2 \n"
+
+ "pslld $8,%%xmm2 \n"
+ "psrld $8,%%xmm2 \n"
+ "pslld $24,%%xmm3 \n"
+ "por %%xmm3,%%xmm2 \n"
+
+ "movdqu %%xmm2,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "sub $0x4,%[width] \n"
+ "jg 1b \n"
+ : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [width] "+rm"(width)
:
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-#endif // HAS_SOBELXYROW_SSE2
+#endif // HAS_ARGBALPHABLENDROW_SSSE3
-#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2
-// Creates a table of cumulative sums where each value is a sum of all values
-// above and to the left of the value, inclusive of the value.
-void ComputeCumulativeSumRow_SSE2(const uint8_t* row,
- int32_t* cumsum,
- const int32_t* previous_cumsum,
- int width) {
+#ifdef HAS_ARGBALPHABLENDROW_AVX2
+void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "pxor %%xmm0,%%xmm0 \n"
- "pxor %%xmm1,%%xmm1 \n"
- "sub $0x4,%3 \n"
- "jl 49f \n"
- "test $0xf,%1 \n"
- "jne 49f \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpabsb %%ymm5,%%ymm4 \n" // 0x0101...
+ "vpsllw $7,%%ymm4,%%ymm6 \n" // 0x8080...
+ "vpsubw %%ymm4,%%ymm6,%%ymm6 \n" // 0x7f7f...
+ "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f80...
+ "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f81...
- // 4 pixel loop.
LABELALIGN
- "40: \n"
- "movdqu (%0),%%xmm2 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm2,%%xmm4 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "punpcklwd %%xmm1,%%xmm2 \n"
- "punpckhwd %%xmm1,%%xmm3 \n"
- "punpckhbw %%xmm1,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "punpcklwd %%xmm1,%%xmm4 \n"
- "punpckhwd %%xmm1,%%xmm5 \n"
- "paddd %%xmm2,%%xmm0 \n"
- "movdqu (%2),%%xmm2 \n"
- "paddd %%xmm0,%%xmm2 \n"
- "paddd %%xmm3,%%xmm0 \n"
- "movdqu 0x10(%2),%%xmm3 \n"
- "paddd %%xmm0,%%xmm3 \n"
- "paddd %%xmm4,%%xmm0 \n"
- "movdqu 0x20(%2),%%xmm4 \n"
- "paddd %%xmm0,%%xmm4 \n"
- "paddd %%xmm5,%%xmm0 \n"
- "movdqu 0x30(%2),%%xmm5 \n"
- "lea 0x40(%2),%2 \n"
- "paddd %%xmm0,%%xmm5 \n"
- "movdqu %%xmm2,(%1) \n"
- "movdqu %%xmm3,0x10(%1) \n"
- "movdqu %%xmm4,0x20(%1) \n"
- "movdqu %%xmm5,0x30(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x4,%3 \n"
- "jge 40b \n"
-
- "49: \n"
- "add $0x3,%3 \n"
- "jl 19f \n"
+ "1: \n"
+ "vmovdqu (%[src0]),%%ymm0 \n"
+ "vmovdqu (%[src1]),%%ymm1 \n"
+ "lea 0x20(%[src0]),%[src0] \n"
+ "lea 0x20(%[src1]),%[src1] \n"
- // 1 pixel loop.
- LABELALIGN
- "10: \n"
- "movd (%0),%%xmm2 \n"
- "lea 0x4(%0),%0 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "punpcklwd %%xmm1,%%xmm2 \n"
- "paddd %%xmm2,%%xmm0 \n"
- "movdqu (%2),%%xmm2 \n"
- "lea 0x10(%2),%2 \n"
- "paddd %%xmm0,%%xmm2 \n"
- "movdqu %%xmm2,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x1,%3 \n"
- "jge 10b \n"
-
- "19: \n"
- : "+r"(row), // %0
- "+r"(cumsum), // %1
- "+r"(previous_cumsum), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
-}
-#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2
+ "vpsrld $24,%%ymm0,%%ymm2 \n" // sa
+ "vpsrld $24,%%ymm1,%%ymm3 \n" // da
-#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2
-void CumulativeSumToAverageRow_SSE2(const int32_t* topleft,
- const int32_t* botleft,
- int width,
- int area,
- uint8_t* dst,
- int count) {
- asm volatile(
- "movd %5,%%xmm5 \n"
- "cvtdq2ps %%xmm5,%%xmm5 \n"
- "rcpss %%xmm5,%%xmm4 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
- "sub $0x4,%3 \n"
- "jl 49f \n"
- "cmpl $0x80,%5 \n"
- "ja 40f \n"
-
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "pcmpeqb %%xmm6,%%xmm6 \n"
- "psrld $0x10,%%xmm6 \n"
- "cvtdq2ps %%xmm6,%%xmm6 \n"
- "addps %%xmm6,%%xmm5 \n"
- "mulps %%xmm4,%%xmm5 \n"
- "cvtps2dq %%xmm5,%%xmm5 \n"
- "packssdw %%xmm5,%%xmm5 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // -1
+ "vpmullw %%ymm3,%%ymm2,%%ymm7 \n" // da * sa
+ "vpsubw %%ymm5,%%ymm7,%%ymm7 \n" // da * sa + 1
+ "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n" // (da * sa) / 255
- // 4 pixel small loop.
- LABELALIGN
- "4: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "psubd 0x00(%0,%4,4),%%xmm0 \n"
- "psubd 0x10(%0,%4,4),%%xmm1 \n"
- "psubd 0x20(%0,%4,4),%%xmm2 \n"
- "psubd 0x30(%0,%4,4),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "psubd (%1),%%xmm0 \n"
- "psubd 0x10(%1),%%xmm1 \n"
- "psubd 0x20(%1),%%xmm2 \n"
- "psubd 0x30(%1),%%xmm3 \n"
- "paddd 0x00(%1,%4,4),%%xmm0 \n"
- "paddd 0x10(%1,%4,4),%%xmm1 \n"
- "paddd 0x20(%1,%4,4),%%xmm2 \n"
- "paddd 0x30(%1,%4,4),%%xmm3 \n"
- "lea 0x40(%1),%1 \n"
- "packssdw %%xmm1,%%xmm0 \n"
- "packssdw %%xmm3,%%xmm2 \n"
- "pmulhuw %%xmm5,%%xmm0 \n"
- "pmulhuw %%xmm5,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jge 4b \n"
- "jmp 49f \n"
-
- // 4 pixel loop
- LABELALIGN
- "40: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "psubd 0x00(%0,%4,4),%%xmm0 \n"
- "psubd 0x10(%0,%4,4),%%xmm1 \n"
- "psubd 0x20(%0,%4,4),%%xmm2 \n"
- "psubd 0x30(%0,%4,4),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "psubd (%1),%%xmm0 \n"
- "psubd 0x10(%1),%%xmm1 \n"
- "psubd 0x20(%1),%%xmm2 \n"
- "psubd 0x30(%1),%%xmm3 \n"
- "paddd 0x00(%1,%4,4),%%xmm0 \n"
- "paddd 0x10(%1,%4,4),%%xmm1 \n"
- "paddd 0x20(%1,%4,4),%%xmm2 \n"
- "paddd 0x30(%1,%4,4),%%xmm3 \n"
- "lea 0x40(%1),%1 \n"
- "cvtdq2ps %%xmm0,%%xmm0 \n"
- "cvtdq2ps %%xmm1,%%xmm1 \n"
- "mulps %%xmm4,%%xmm0 \n"
- "mulps %%xmm4,%%xmm1 \n"
- "cvtdq2ps %%xmm2,%%xmm2 \n"
- "cvtdq2ps %%xmm3,%%xmm3 \n"
- "mulps %%xmm4,%%xmm2 \n"
- "mulps %%xmm4,%%xmm3 \n"
- "cvtps2dq %%xmm0,%%xmm0 \n"
- "cvtps2dq %%xmm1,%%xmm1 \n"
- "cvtps2dq %%xmm2,%%xmm2 \n"
- "cvtps2dq %%xmm3,%%xmm3 \n"
- "packssdw %%xmm1,%%xmm0 \n"
- "packssdw %%xmm3,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jge 40b \n"
-
- "49: \n"
- "add $0x3,%3 \n"
- "jl 19f \n"
-
- // 1 pixel loop
- LABELALIGN
- "10: \n"
- "movdqu (%0),%%xmm0 \n"
- "psubd 0x00(%0,%4,4),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "psubd (%1),%%xmm0 \n"
- "paddd 0x00(%1,%4,4),%%xmm0 \n"
- "lea 0x10(%1),%1 \n"
- "cvtdq2ps %%xmm0,%%xmm0 \n"
- "mulps %%xmm4,%%xmm0 \n"
- "cvtps2dq %%xmm0,%%xmm0 \n"
- "packssdw %%xmm0,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movd %%xmm0,(%2) \n"
- "lea 0x4(%2),%2 \n"
- "sub $0x1,%3 \n"
- "jge 10b \n"
- "19: \n"
- : "+r"(topleft), // %0
- "+r"(botleft), // %1
- "+r"(dst), // %2
- "+rm"(count) // %3
- : "r"((intptr_t)(width)), // %4
- "rm"(area) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
-}
-#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2
+ "vpaddd %%ymm3,%%ymm5,%%ymm5 \n" // da - 1
+ "vpsrld $24,%%ymm5,%%ymm5 \n" // t = (da==0) ? 255 : 0
-#ifdef HAS_ARGBAFFINEROW_SSE2
-// Copy ARGB pixels from source image with slope to a row of destination.
-LIBYUV_API
-void ARGBAffineRow_SSE2(const uint8_t* src_argb,
- int src_argb_stride,
- uint8_t* dst_argb,
- const float* src_dudv,
- int width) {
- intptr_t src_argb_stride_temp = src_argb_stride;
- intptr_t temp;
- asm volatile(
- "movq (%3),%%xmm2 \n"
- "movq 0x08(%3),%%xmm7 \n"
- "shl $0x10,%1 \n"
- "add $0x4,%1 \n"
- "movd %1,%%xmm5 \n"
- "sub $0x4,%4 \n"
- "jl 49f \n"
-
- "pshufd $0x44,%%xmm7,%%xmm7 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "movdqa %%xmm2,%%xmm0 \n"
- "addps %%xmm7,%%xmm0 \n"
- "movlhps %%xmm0,%%xmm2 \n"
- "movdqa %%xmm7,%%xmm4 \n"
- "addps %%xmm4,%%xmm4 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "addps %%xmm4,%%xmm3 \n"
- "addps %%xmm4,%%xmm4 \n"
-
- // 4 pixel loop
- LABELALIGN
- "40: \n"
- "cvttps2dq %%xmm2,%%xmm0 \n" // x,y float->int first 2
- "cvttps2dq %%xmm3,%%xmm1 \n" // x,y float->int next 2
- "packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts
- "pmaddwd %%xmm5,%%xmm0 \n" // off = x*4 + y*stride
- "movd %%xmm0,%k1 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
- "movd %%xmm0,%k5 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
- "movd 0x00(%0,%1,1),%%xmm1 \n"
- "movd 0x00(%0,%5,1),%%xmm6 \n"
- "punpckldq %%xmm6,%%xmm1 \n"
- "addps %%xmm4,%%xmm2 \n"
- "movq %%xmm1,(%2) \n"
- "movd %%xmm0,%k1 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
- "movd %%xmm0,%k5 \n"
- "movd 0x00(%0,%1,1),%%xmm0 \n"
- "movd 0x00(%0,%5,1),%%xmm6 \n"
- "punpckldq %%xmm6,%%xmm0 \n"
- "addps %%xmm4,%%xmm3 \n"
- "movq %%xmm0,0x08(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%4 \n"
- "jge 40b \n"
-
- "49: \n"
- "add $0x3,%4 \n"
- "jl 19f \n"
-
- // 1 pixel loop
- LABELALIGN
- "10: \n"
- "cvttps2dq %%xmm2,%%xmm0 \n"
- "packssdw %%xmm0,%%xmm0 \n"
- "pmaddwd %%xmm5,%%xmm0 \n"
- "addps %%xmm7,%%xmm2 \n"
- "movd %%xmm0,%k1 \n"
- "movd 0x00(%0,%1,1),%%xmm0 \n"
- "movd %%xmm0,(%2) \n"
- "lea 0x04(%2),%2 \n"
- "sub $0x1,%4 \n"
- "jge 10b \n"
- "19: \n"
- : "+r"(src_argb), // %0
- "+r"(src_argb_stride_temp), // %1
- "+r"(dst_argb), // %2
- "+r"(src_dudv), // %3
- "+rm"(width), // %4
- "=&r"(temp) // %5
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" // da + sa
+ "vpsubw %%ymm7,%%ymm3,%%ymm3 \n" // da2
+
+ "vpor %%ymm5,%%ymm2,%%ymm2 \n" // sa_eff
+ "vpor %%ymm5,%%ymm3,%%ymm7 \n" // den_eff
+
+ "vpslld $8,%%ymm2,%%ymm5 \n"
+ "vpsubd %%ymm2,%%ymm5,%%ymm2 \n" // sa_eff * 255
+
+ "vcvtdq2ps %%ymm2,%%ymm2 \n"
+ "vcvtdq2ps %%ymm7,%%ymm7 \n"
+ "vdivps %%ymm7,%%ymm2,%%ymm2 \n"
+ "vcvttps2dq %%ymm2,%%ymm2 \n" // ratio
+
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsllw $8,%%ymm5,%%ymm5 \n" // 0xff00
+
+ "vpmullw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpshufhw $0xa0,%%ymm2,%%ymm2 \n"
+ "vpshuflw $0xa0,%%ymm2,%%ymm2 \n"
+ "vpxor %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpunpckhwd %%ymm2,%%ymm2,%%ymm7 \n"
+ "vpunpcklwd %%ymm2,%%ymm2,%%ymm2 \n"
+
+ "vpsllw $7,%%ymm4,%%ymm5 \n" // 0x8080
+ "vpsubb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpsubb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+
+ "vpmaddubsw %%ymm0,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm5,%%ymm7,%%ymm7 \n"
+ "vpaddw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpaddw %%ymm6,%%ymm7,%%ymm7 \n"
+ "vpmulhuw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n"
+ "vpackuswb %%ymm7,%%ymm2,%%ymm2 \n"
+
+ "vpslld $8,%%ymm2,%%ymm2 \n"
+ "vpsrld $8,%%ymm2,%%ymm2 \n"
+ "vpslld $24,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm3,%%ymm2,%%ymm2 \n"
+
+ "vmovdqu %%ymm2,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [width] "+rm"(width)
:
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-#endif // HAS_ARGBAFFINEROW_SSE2
+#endif // HAS_ARGBALPHABLENDROW_AVX2
+
+#ifdef HAS_ARGBALPHABLENDROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n" // -1
+ "vpabsb %%zmm20,%%zmm16 \n" // zmm16 = 0x0101...
+ "vpsllw $7,%%zmm16,%%zmm21 \n" // zmm21 = 0x8080...
+ "vpsubw %%zmm16,%%zmm21,%%zmm18 \n" // 0x7f7f...
+ "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // 0x7f80...
+ "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // zmm18 = 0x7f81...
+ "vpsllw $8,%%zmm20,%%zmm17 \n" // zmm17 = 0xff00...
-#ifdef HAS_INTERPOLATEROW_SSSE3
-// Bilinear filter 16x2 -> 16x1
-void InterpolateRow_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- asm volatile(
- "sub %1,%0 \n"
- "cmp $0x0,%3 \n"
- "je 100f \n"
- "cmp $0x80,%3 \n"
- "je 50f \n"
-
- "movd %3,%%xmm0 \n"
- "neg %3 \n"
- "add $0x100,%3 \n"
- "movd %3,%%xmm5 \n"
- "punpcklbw %%xmm0,%%xmm5 \n"
- "punpcklwd %%xmm5,%%xmm5 \n"
- "pshufd $0x0,%%xmm5,%%xmm5 \n"
- "mov $0x80808080,%%eax \n"
- "movd %%eax,%%xmm4 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
+ "sub $0x10,%[width] \n"
+ "jl 2f \n"
- // General purpose row blend.
LABELALIGN
"1: \n"
- "movdqu (%1),%%xmm0 \n"
- "movdqu 0x00(%1,%4,1),%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm2,%%xmm0 \n"
- "punpckhbw %%xmm2,%%xmm1 \n"
- "psubb %%xmm4,%%xmm0 \n"
- "psubb %%xmm4,%%xmm1 \n"
- "movdqa %%xmm5,%%xmm2 \n"
- "movdqa %%xmm5,%%xmm3 \n"
- "pmaddubsw %%xmm0,%%xmm2 \n"
- "pmaddubsw %%xmm1,%%xmm3 \n"
- "paddw %%xmm4,%%xmm2 \n"
- "paddw %%xmm4,%%xmm3 \n"
- "psrlw $0x8,%%xmm2 \n"
- "psrlw $0x8,%%xmm3 \n"
- "packuswb %%xmm3,%%xmm2 \n"
- "movdqu %%xmm2,0x00(%1,%0,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "jmp 99f \n"
+ "vmovdqu32 (%[src0]),%%zmm0 \n"
+ "vmovdqu32 (%[src1]),%%zmm1 \n"
+ "lea 0x40(%[src0]),%[src0] \n"
+ "lea 0x40(%[src1]),%[src1] \n"
+
+ "vpsrld $24,%%zmm0,%%zmm2 \n" // sa
+ "vpsrld $24,%%zmm1,%%zmm3 \n" // da
+
+ "vpmullw %%zmm3,%%zmm2,%%zmm7 \n"
+ "vpsubw %%zmm20,%%zmm7,%%zmm7 \n" // da * sa + 1
+ "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n"
+
+ "vpaddd %%zmm3,%%zmm20,%%zmm5 \n" // da - 1
+ "vpsrld $24,%%zmm5,%%zmm5 \n" // t
+
+ "vpaddw %%zmm2,%%zmm3,%%zmm3 \n"
+ "vpsubw %%zmm7,%%zmm3,%%zmm3 \n" // da2
+
+ "vpord %%zmm5,%%zmm2,%%zmm2 \n" // sa_eff
+ "vpord %%zmm5,%%zmm3,%%zmm7 \n" // den_eff
+ "vpslld $8,%%zmm2,%%zmm6 \n"
+ "vpsubd %%zmm2,%%zmm6,%%zmm2 \n" // sa_eff * 255
+
+ "vcvtdq2ps %%zmm2,%%zmm2 \n"
+ "vcvtdq2ps %%zmm7,%%zmm7 \n"
+ "vdivps %%zmm7,%%zmm2,%%zmm2 \n"
+ "vcvttps2dq %%zmm2,%%zmm2 \n"
+
+ "vpmullw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpshufhw $0xa0,%%zmm2,%%zmm2 \n"
+ "vpshuflw $0xa0,%%zmm2,%%zmm2 \n"
+ "vpxord %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n"
+ "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n"
+
+ "vpsubb %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n"
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n"
+
+ "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n"
+ "vpaddw %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm7,%%zmm7 \n"
+ "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n"
+ "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n"
+
+ "vpslld $8,%%zmm2,%%zmm2 \n"
+ "vpsrld $8,%%zmm2,%%zmm2 \n"
+ "vpslld $24,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+
+ "vmovdqu32 %%zmm2,(%[dst]) \n"
+ "lea 0x40(%[dst]),%[dst] \n"
+ "sub $0x10,%[width] \n"
+ "jge 1b \n"
- // Blend 50 / 50.
- LABELALIGN
- "50: \n"
- "movdqu (%1),%%xmm0 \n"
- "movdqu 0x00(%1,%4,1),%%xmm1 \n"
- "pavgb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,0x00(%1,%0,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 50b \n"
- "jmp 99f \n"
+ "2: \n"
+ "add $0x10,%[width] \n"
+ "jle 99f \n"
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k1 \n"
+ "vmovdqu32 (%[src0]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 (%[src1]),%%zmm1%{%%k1%}%{z%} \n"
+
+ "vpsrld $24,%%zmm0,%%zmm2 \n"
+ "vpsrld $24,%%zmm1,%%zmm3 \n"
+
+ "vpmullw %%zmm3,%%zmm2,%%zmm7 \n"
+ "vpsubw %%zmm20,%%zmm7,%%zmm7 \n"
+ "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n"
+
+ "vpaddd %%zmm3,%%zmm20,%%zmm5 \n"
+ "vpsrld $24,%%zmm5,%%zmm5 \n"
+
+ "vpaddw %%zmm2,%%zmm3,%%zmm3 \n"
+ "vpsubw %%zmm7,%%zmm3,%%zmm3 \n"
+
+ "vpord %%zmm5,%%zmm2,%%zmm2 \n"
+ "vpord %%zmm5,%%zmm3,%%zmm7 \n"
+ "vpslld $8,%%zmm2,%%zmm6 \n"
+ "vpsubd %%zmm2,%%zmm6,%%zmm2 \n"
+
+ "vcvtdq2ps %%zmm2,%%zmm2 \n"
+ "vcvtdq2ps %%zmm7,%%zmm7 \n"
+ "vdivps %%zmm7,%%zmm2,%%zmm2 \n"
+ "vcvttps2dq %%zmm2,%%zmm2 \n"
+
+ "vpmullw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpshufhw $0xa0,%%zmm2,%%zmm2 \n"
+ "vpshuflw $0xa0,%%zmm2,%%zmm2 \n"
+ "vpxord %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n"
+ "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n"
+
+ "vpsubb %%zmm21,%%zmm0,%%zmm0 \n"
+ "vpsubb %%zmm21,%%zmm1,%%zmm1 \n"
+ "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n"
+ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n"
+
+ "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n"
+ "vpaddw %%zmm18,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm7,%%zmm7 \n"
+ "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n"
+ "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n"
+ "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n"
+
+ "vpslld $8,%%zmm2,%%zmm2 \n"
+ "vpsrld $8,%%zmm2,%%zmm2 \n"
+ "vpslld $24,%%zmm3,%%zmm3 \n"
+ "vpord %%zmm3,%%zmm2,%%zmm2 \n"
+
+ "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [width] "+r"(width), [temp] "=&r"(temp)
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7",
+ "xmm16", "xmm17", "xmm18", "xmm20", "xmm21", "k1");
+}
+#endif // HAS_ARGBALPHABLENDROW_AVX512BW
+
+// ===========================================================================
+// Module 6: Extended YUV (I444, YUY2, UYVY, I210, I212, I410, P210, P410)
+// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI)
+// ===========================================================================
+
+#define READYUV444_AVX512BW \
+ "vpmovzxbw (%[u_buf]),%%zmm3 \n" \
+ "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \
+ "lea 0x20(%[u_buf]),%[u_buf] \n" \
+ "vpsllw $8,%%zmm1,%%zmm1 \n" \
+ "vpord %%zmm1,%%zmm3,%%zmm3 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4 \n" \
+ "lea 0x20(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $8,%%zmm4,%%zmm1 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV444_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "mov $-1,%[temp] \n" \
+ "vpmovzxbw (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \
+ "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \
+ "vpsllw $8,%%zmm1,%%zmm1 \n" \
+ "vpord %%zmm1,%%zmm3,%%zmm3 \n" \
+ "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "vpsllw $8,%%zmm4,%%zmm1 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define STORERGBA_AVX512BW \
+ "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \
+ "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \
+ "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \
+ "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \
+ "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \
+ "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \
+ "vmovups %%zmm0,(%[dst_rgba]) \n" \
+ "vmovups %%zmm1,0x40(%[dst_rgba]) \n" \
+ "lea 0x80(%[dst_rgba]),%[dst_rgba] \n"
+
+#define STORERGBA_MASK_AVX512BW \
+ "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \
+ "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \
+ "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \
+ "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \
+ "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \
+ "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \
+ "kshiftrd $16,%%k1,%%k2 \n" \
+ "vmovdqu32 %%zmm0,(%[dst_rgba])%{%%k1%} \n" \
+ "vmovdqu32 %%zmm1,0x40(%[dst_rgba])%{%%k2%} \n"
+
+#define READYUV210_AVX512BW \
+ "vmovdqu16 (%[u_buf]),%%ymm3 \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \
+ "lea 0x20(%[u_buf]),%[u_buf] \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm1,%%ymm1 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4 \n" \
+ "lea 0x40(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $6,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $4,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV210_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \
+ "vpsrlw $2,%%ymm3,%%ymm3 \n" \
+ "vpsrlw $2,%%ymm1,%%ymm1 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \
+ "vpsllw $6,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $4,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV212_AVX512BW \
+ "vmovdqu16 (%[u_buf]),%%ymm3 \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \
+ "lea 0x20(%[u_buf]),%[u_buf] \n" \
+ "vpsraw $4,%%ymm3,%%ymm3 \n" \
+ "vpsraw $4,%%ymm1,%%ymm1 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4 \n" \
+ "lea 0x40(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $4,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $8,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV212_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \
+ "vpsraw $4,%%ymm3,%%ymm3 \n" \
+ "vpsraw $4,%%ymm1,%%ymm1 \n" \
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \
+ "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \
+ "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \
+ "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \
+ "vpsllw $4,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $8,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV410_AVX512BW \
+ "vmovdqu16 (%[u_buf]),%%zmm3 \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \
+ "lea 0x40(%[u_buf]),%[u_buf] \n" \
+ "vpsrlw $2,%%zmm3,%%zmm3 \n" \
+ "vpsrlw $2,%%zmm1,%%zmm1 \n" \
+ "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \
+ "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \
+ "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4 \n" \
+ "lea 0x40(%[y_buf]),%[y_buf] \n" \
+ "vpsllw $6,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $4,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READYUV410_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vmovdqu16 (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \
+ "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \
+ "vpsrlw $2,%%zmm3,%%zmm3 \n" \
+ "vpsrlw $2,%%zmm1,%%zmm1 \n" \
+ "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \
+ "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \
+ "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "vpsllw $6,%%zmm4,%%zmm1 \n" \
+ "vpsrlw $4,%%zmm4,%%zmm4 \n" \
+ "vpord %%zmm1,%%zmm4,%%zmm4 \n"
+
+#define READP210_AVX512BW \
+ "vmovdqu16 (%[y_buf]),%%zmm4 \n" \
+ "lea 0x40(%[y_buf]),%[y_buf] \n" \
+ "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \
+ "lea 0x40(%[uv_buf]),%[uv_buf] \n" \
+ "vpshufb %%zmm14,%%zmm3,%%zmm3 \n"
+
+#define READP210_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "mov $-1,%[temp] \n" \
+ "lea 1(%q[width]),%k[y_buf] \n" \
+ "shr $1,%k[y_buf] \n" \
+ "bzhi %q[y_buf],%[temp],%[y_buf] \n" \
+ "kmovw %k[y_buf],%%k2 \n" \
+ "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k2%}%{z%} \n" \
+ "vpshufb %%zmm14,%%zmm3,%%zmm3 \n"
+
+#define READP410_AVX512BW \
+ "vmovdqu16 (%[y_buf]),%%zmm4 \n" \
+ "lea 0x40(%[y_buf]),%[y_buf] \n" \
+ "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \
+ "vmovdqu32 0x40(%[uv_buf]),%%zmm1 \n" \
+ "lea 0x80(%[uv_buf]),%[uv_buf] \n" \
+ "vpsrlw $8,%%zmm3,%%zmm3 \n" \
+ "vpsrlw $8,%%zmm1,%%zmm1 \n" \
+ "vpmovwb %%zmm3,%%ymm3 \n" \
+ "vpmovwb %%zmm1,%%ymm1 \n" \
+ "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n"
+
+#define READP410_MASK_AVX512BW \
+ "mov $-1,%[temp] \n" \
+ "bzhi %q[width],%[temp],%[temp] \n" \
+ "kmovd %k[temp],%%k1 \n" \
+ "kshiftrd $16,%%k1,%%k2 \n" \
+ "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
+ "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k1%}%{z%} \n" \
+ "vmovdqu32 0x40(%[uv_buf]),%%zmm1%{%%k2%}%{z%} \n" \
+ "vpsrlw $8,%%zmm3,%%zmm3 \n" \
+ "vpsrlw $8,%%zmm1,%%zmm1 \n" \
+ "vpmovwb %%zmm3,%%ymm3 \n" \
+ "vpmovwb %%zmm1,%%ymm1 \n" \
+ "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n"
+
+#ifdef HAS_I444TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // Blend 100 / 0 - Copy row unchanged.
LABELALIGN
- "100: \n"
- "movdqu (%1),%%xmm0 \n"
- "movdqu %%xmm0,0x00(%1,%0,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 100b \n"
+ "1: \n"
+ READYUV444_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV444_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
"99: \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+rm"(dst_width), // %2
- "+r"(source_y_fraction) // %3
- : "r"((intptr_t)(src_stride)) // %4
- : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_INTERPOLATEROW_SSSE3
-
-#ifdef HAS_INTERPOLATEROW_AVX2
-// Bilinear filter 32x2 -> 32x1
-void InterpolateRow_AVX2(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
+#endif // HAS_I444TOARGBROW_AVX512BW
+
+#ifdef HAS_I444ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "cmp $0x0,%3 \n"
- "je 100f \n"
- "sub %1,%0 \n"
- "cmp $0x80,%3 \n"
- "je 50f \n"
-
- "vmovd %3,%%xmm0 \n"
- "neg %3 \n"
- "add $0x100,%3 \n"
- "vmovd %3,%%xmm5 \n"
- "vpunpcklbw %%xmm0,%%xmm5,%%xmm5 \n"
- "vpunpcklwd %%xmm5,%%xmm5,%%xmm5 \n"
- "vbroadcastss %%xmm5,%%ymm5 \n"
- "mov $0x80808080,%%eax \n"
- "vmovd %%eax,%%xmm4 \n"
- "vbroadcastss %%xmm4,%%ymm4 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // General purpose row blend.
LABELALIGN
"1: \n"
- "vmovdqu (%1),%%ymm0 \n"
- "vmovdqu 0x00(%1,%4,1),%%ymm2 \n"
- "vpunpckhbw %%ymm2,%%ymm0,%%ymm1 \n"
- "vpunpcklbw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsubb %%ymm4,%%ymm1,%%ymm1 \n"
- "vpsubb %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm1,%%ymm5,%%ymm1 \n"
- "vpmaddubsw %%ymm0,%%ymm5,%%ymm0 \n"
- "vpaddw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpaddw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "jmp 99f \n"
+ READYUV444_AVX512BW
+ "vpmovzxbw (%[a_buf]),%%zmm5 \n"
+ "lea 0x20(%[a_buf]),%[a_buf] \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
- // Blend 50 / 50.
- LABELALIGN
- "50: \n"
- "vmovdqu (%1),%%ymm0 \n"
- "vpavgb 0x00(%1,%4,1),%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,0x00(%1,%0,1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 50b \n"
- "jmp 99f \n"
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
- // Blend 100 / 0 - Copy row unchanged.
- LABELALIGN
- "100: \n"
- "rep movsb \n"
- "jmp 999f \n"
+ READYUV444_MASK_AVX512BW
+ "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
"99: \n"
"vzeroupper \n"
- "999: \n"
- : "+D"(dst_ptr), // %0
- "+S"(src_ptr), // %1
- "+cm"(dst_width), // %2
- "+r"(source_y_fraction) // %3
- : "r"((intptr_t)(src_stride)) // %4
- : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_INTERPOLATEROW_AVX2
-
-#ifdef HAS_ARGBSHUFFLEROW_SSSE3
-// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
-void ARGBShuffleRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
+#endif // HAS_I444ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_I444TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
-
- "movdqu (%3),%%xmm5 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vmovdqu32 %[kMaskBG],%%zmm20 \n"
+ "vmovdqu32 %[kMaskDST0],%%zmm21 \n"
+ "vmovdqu32 %[kMaskDST1],%%zmm22 \n"
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(shuffler) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
-}
-#endif // HAS_ARGBSHUFFLEROW_SSSE3
+ READYUV444_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_AVX512VBMI
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_ARGBSHUFFLEROW_AVX2
-// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
-void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
- asm volatile(
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV444_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGB24_MASK_AVX512VBMI
- "vbroadcastf128 (%3),%%ymm5 \n"
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_rgb24] "+r"(dst_rgb24),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kMaskBG] "m"(kMaskBG),
+ [kMaskDST0] "m"(kMaskDST0),
+ [kMaskDST1] "m"(kMaskDST1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm20", "xmm21", "xmm22", "k1", "k3", "k4");
+}
+#endif // HAS_I444TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_I422TORGBAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGBA_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STORERGBA_MASK_AVX512BW
+
+ "99: \n"
"vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(shuffler) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_rgba] "+r"(dst_rgba),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_ARGBSHUFFLEROW_AVX2
-
-#ifdef HAS_I422TOYUY2ROW_SSE2
-void I422ToYUY2Row_SSE2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_yuy2,
- int width) {
+#endif // HAS_I422TORGBAROW_AVX512BW
+
+#ifdef HAS_YUY2TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
-
- "sub %1,%2 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vbroadcasti32x4 %[kShufY],%%zmm14 \n"
+ "vbroadcasti32x4 %[kShufUV],%%zmm15 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "movq (%1),%%xmm2 \n"
- "movq 0x00(%1,%2,1),%%xmm1 \n"
- "add $0x8,%1 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "movdqu (%0),%%xmm0 \n"
- "add $0x10,%0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm2,%%xmm0 \n"
- "punpckhbw %%xmm2,%%xmm1 \n"
- "movdqu %%xmm0,(%3) \n"
- "movdqu %%xmm1,0x10(%3) \n"
- "lea 0x20(%3),%3 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_yuy2), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
-}
-#endif // HAS_I422TOYUY2ROW_SSE2
+ "vmovdqu32 (%[src_yuy2]),%%zmm1 \n"
+ "lea 0x40(%[src_yuy2]),%[src_yuy2] \n"
+ "vpshufb %%zmm14,%%zmm1,%%zmm4 \n"
+ "vpshufb %%zmm15,%%zmm1,%%zmm3 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_I422TOUYVYROW_SSE2
-void I422ToUYVYRow_SSE2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uyvy,
- int width) {
- asm volatile(
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "mov $-1,%[temp] \n"
+ "add $1,%k[width] \n"
+ "shr $1,%k[width] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k2 \n"
+ "vmovdqu32 (%[src_yuy2]),%%zmm1%{%%k2%}%{z%}\n"
+ "vpshufb %%zmm14,%%zmm1,%%zmm4 \n"
+ "vpshufb %%zmm15,%%zmm1,%%zmm3 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
- "sub %1,%2 \n"
+ "99: \n"
+ "vzeroupper \n"
+ : [src_yuy2] "+r"(src_yuy2),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kShufY] "m"(kShuffleYUY2Y),
+ [kShufUV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2");
+}
+#endif // HAS_YUY2TOARGBROW_AVX512BW
+
+#ifdef HAS_UYVYTOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vbroadcasti32x4 %[kShufY],%%zmm14 \n"
+ "vbroadcasti32x4 %[kShufUV],%%zmm15 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "movq (%1),%%xmm2 \n"
- "movq 0x00(%1,%2,1),%%xmm1 \n"
- "add $0x8,%1 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "add $0x10,%0 \n"
- "punpcklbw %%xmm0,%%xmm1 \n"
- "punpckhbw %%xmm0,%%xmm2 \n"
- "movdqu %%xmm1,(%3) \n"
- "movdqu %%xmm2,0x10(%3) \n"
- "lea 0x20(%3),%3 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_uyvy), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
-}
-#endif // HAS_I422TOUYVYROW_SSE2
+ "vmovdqu32 (%[src_uyvy]),%%zmm1 \n"
+ "lea 0x40(%[src_uyvy]),%[src_uyvy] \n"
+ "vpshufb %%zmm14,%%zmm1,%%zmm4 \n"
+ "vpshufb %%zmm15,%%zmm1,%%zmm3 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_I422TOYUY2ROW_AVX2
-void I422ToYUY2Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_yuy2,
- int width) {
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ "mov $-1,%[temp] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovd %k[temp],%%k1 \n"
+ "mov $-1,%[temp] \n"
+ "add $1,%k[width] \n"
+ "shr $1,%k[width] \n"
+ "bzhi %q[width],%[temp],%[temp] \n"
+ "kmovw %k[temp],%%k2 \n"
+ "vmovdqu32 (%[src_uyvy]),%%zmm1%{%%k2%}%{z%}\n"
+ "vpshufb %%zmm14,%%zmm1,%%zmm4 \n"
+ "vpshufb %%zmm15,%%zmm1,%%zmm3 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_uyvy] "+r"(src_uyvy),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kShufY] "m"(kShuffleUYVYY),
+ [kShufUV] "m"(kShuffleUYVYUV)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2");
+}
+#endif // HAS_UYVYTOARGBROW_AVX512BW
+
+#ifdef HAS_I210TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
-
- "sub %1,%2 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "vpmovzxbw (%1),%%ymm1 \n"
- "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
- "add $0x10,%1 \n"
- "vpsllw $0x8,%%ymm2,%%ymm2 \n"
- "vpor %%ymm1,%%ymm2,%%ymm2 \n"
- "vmovdqu (%0),%%ymm0 \n"
- "add $0x20,%0 \n"
- "vpunpcklbw %%ymm2,%%ymm0,%%ymm1 \n"
- "vpunpckhbw %%ymm2,%%ymm0,%%ymm2 \n"
- "vextractf128 $0x0,%%ymm1,(%3) \n"
- "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
- "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
- "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
- "lea 0x40(%3),%3 \n"
- "sub $0x20,%4 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_yuy2), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
-}
-#endif // HAS_I422TOYUY2ROW_AVX2
+ READYUV210_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_I422TOUYVYROW_AVX2
-void I422ToUYVYRow_AVX2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uyvy,
- int width) {
- asm volatile(
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
- "sub %1,%2 \n"
+ READYUV210_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
- LABELALIGN
- "1: \n"
- "vpmovzxbw (%1),%%ymm1 \n"
- "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
- "add $0x10,%1 \n"
- "vpsllw $0x8,%%ymm2,%%ymm2 \n"
- "vpor %%ymm1,%%ymm2,%%ymm2 \n"
- "vmovdqu (%0),%%ymm0 \n"
- "add $0x20,%0 \n"
- "vpunpcklbw %%ymm0,%%ymm2,%%ymm1 \n"
- "vpunpckhbw %%ymm0,%%ymm2,%%ymm2 \n"
- "vextractf128 $0x0,%%ymm1,(%3) \n"
- "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
- "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
- "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
- "lea 0x40(%3),%3 \n"
- "sub $0x20,%4 \n"
- "jg 1b \n"
+ "99: \n"
"vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_uyvy), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_I422TOUYVYROW_AVX2
-
-#ifdef HAS_ARGBPOLYNOMIALROW_SSE2
-void ARGBPolynomialRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const float* poly,
- int width) {
+#endif // HAS_I210TOARGBROW_AVX512BW
+
+#ifdef HAS_I210TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n"
+ "vpsllw $10,%%zmm5,%%zmm19 \n"
+ "vpsllw $14,%%zmm5,%%zmm5 \n"
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- "pxor %%xmm3,%%xmm3 \n"
-
- // 2 pixel loop.
LABELALIGN
"1: \n"
- "movq (%0),%%xmm0 \n"
- "lea 0x8(%0),%0 \n"
- "punpcklbw %%xmm3,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm4 \n"
- "punpcklwd %%xmm3,%%xmm0 \n"
- "punpckhwd %%xmm3,%%xmm4 \n"
- "cvtdq2ps %%xmm0,%%xmm0 \n"
- "cvtdq2ps %%xmm4,%%xmm4 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "mulps 0x10(%3),%%xmm0 \n"
- "mulps 0x10(%3),%%xmm4 \n"
- "addps (%3),%%xmm0 \n"
- "addps (%3),%%xmm4 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "movdqa %%xmm5,%%xmm6 \n"
- "mulps %%xmm1,%%xmm2 \n"
- "mulps %%xmm5,%%xmm6 \n"
- "mulps %%xmm2,%%xmm1 \n"
- "mulps %%xmm6,%%xmm5 \n"
- "mulps 0x20(%3),%%xmm2 \n"
- "mulps 0x20(%3),%%xmm6 \n"
- "mulps 0x30(%3),%%xmm1 \n"
- "mulps 0x30(%3),%%xmm5 \n"
- "addps %%xmm2,%%xmm0 \n"
- "addps %%xmm6,%%xmm4 \n"
- "addps %%xmm1,%%xmm0 \n"
- "addps %%xmm5,%%xmm4 \n"
- "cvttps2dq %%xmm0,%%xmm0 \n"
- "cvttps2dq %%xmm4,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x2,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(poly) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
-}
-#endif // HAS_ARGBPOLYNOMIALROW_SSE2
+ READYUV210_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
-void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const float* poly,
- int width) {
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV210_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_ar30] "+r"(dst_ar30),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [permar30_0] "r"(kPermAR30_0),
+ [permar30_1] "r"(kPermAR30_1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2");
+}
+#endif // HAS_I210TOAR30ROW_AVX512BW
+
+#ifdef HAS_I210ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastf128 (%3),%%ymm4 \n"
- "vbroadcastf128 0x10(%3),%%ymm5 \n"
- "vbroadcastf128 0x20(%3),%%ymm6 \n"
- "vbroadcastf128 0x30(%3),%%ymm7 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // 2 pixel loop.
LABELALIGN
"1: \n"
- "vpmovzxbd (%0),%%ymm0 \n" // 2 ARGB pixels
- "lea 0x8(%0),%0 \n"
- "vcvtdq2ps %%ymm0,%%ymm0 \n" // X 8 floats
- "vmulps %%ymm0,%%ymm0,%%ymm2 \n" // X * X
- "vmulps %%ymm7,%%ymm0,%%ymm3 \n" // C3 * X
- "vfmadd132ps %%ymm5,%%ymm4,%%ymm0 \n" // result = C0 + C1 * X
- "vfmadd231ps %%ymm6,%%ymm2,%%ymm0 \n" // result += C2 * X * X
- "vfmadd231ps %%ymm3,%%ymm2,%%ymm0 \n" // result += C3 * X * X *
- // X
- "vcvttps2dq %%ymm0,%%ymm0 \n"
- "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
- "vmovq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x2,%2 \n"
- "jg 1b \n"
+ READYUV210_AVX512BW
+ "vmovdqu16 (%[a_buf]),%%zmm5 \n"
+ "lea 0x40(%[a_buf]),%[a_buf] \n"
+ "vpsrlw $2,%%zmm5,%%zmm5 \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV210_MASK_AVX512BW
+ "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n"
+ "vpsrlw $2,%%zmm5,%%zmm5 \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
"vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(poly) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_ARGBPOLYNOMIALROW_AVX2
-
-#ifdef HAS_HALFFLOATROW_SSE2
-static float kScaleBias = 1.9259299444e-34f;
-void HalfFloatRow_SSE2(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- scale *= kScaleBias;
+#endif // HAS_I210ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_I212TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "movd %3,%%xmm4 \n"
- "pshufd $0x0,%%xmm4,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
- "sub %0,%1 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // 16 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm2 \n" // 8 shorts
- "add $0x10,%0 \n"
- "movdqa %%xmm2,%%xmm3 \n"
- "punpcklwd %%xmm5,%%xmm2 \n" // 8 ints in xmm2/1
- "cvtdq2ps %%xmm2,%%xmm2 \n" // 8 floats
- "punpckhwd %%xmm5,%%xmm3 \n"
- "cvtdq2ps %%xmm3,%%xmm3 \n"
- "mulps %%xmm4,%%xmm2 \n"
- "mulps %%xmm4,%%xmm3 \n"
- "psrld $0xd,%%xmm2 \n"
- "psrld $0xd,%%xmm3 \n"
- "packssdw %%xmm3,%%xmm2 \n"
- "movdqu %%xmm2,-0x10(%0,%1,1) \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(scale) // %3
- : "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5");
-}
-#endif // HAS_HALFFLOATROW_SSE2
+ READYUV212_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_HALFFLOATROW_AVX2
-void HalfFloatRow_AVX2(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- scale *= kScaleBias;
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV212_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_I212TOARGBROW_AVX512BW
+
+#ifdef HAS_I212TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastss %3, %%ymm4 \n"
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
- "sub %0,%1 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n"
+ "vpsllw $10,%%zmm5,%%zmm19 \n"
+ "vpsllw $14,%%zmm5,%%zmm5 \n"
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // 16 pixel loop.
LABELALIGN
"1: \n"
- "vmovdqu (%0),%%ymm2 \n" // 16 shorts
- "add $0x20,%0 \n"
- "vpunpckhwd %%ymm5,%%ymm2,%%ymm3 \n" // mutates
- "vpunpcklwd %%ymm5,%%ymm2,%%ymm2 \n"
- "vcvtdq2ps %%ymm3,%%ymm3 \n"
- "vcvtdq2ps %%ymm2,%%ymm2 \n"
- "vmulps %%ymm3,%%ymm4,%%ymm3 \n"
- "vmulps %%ymm2,%%ymm4,%%ymm2 \n"
- "vpsrld $0xd,%%ymm3,%%ymm3 \n"
- "vpsrld $0xd,%%ymm2,%%ymm2 \n"
- "vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // unmutates
- "vmovdqu %%ymm2,-0x20(%0,%1,1) \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ READYUV212_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+ READYUV212_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
+
+ "99: \n"
"vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
-#if defined(__x86_64__)
- : "x"(scale) // %3
-#else
- : "m"(scale) // %3
-#endif
- : "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_ar30] "+r"(dst_ar30),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [permar30_0] "r"(kPermAR30_0),
+ [permar30_1] "r"(kPermAR30_1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2");
}
-#endif // HAS_HALFFLOATROW_AVX2
-
-#ifdef HAS_HALFFLOATROW_F16C
-void HalfFloatRow_F16C(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
+#endif // HAS_I212TOAR30ROW_AVX512BW
+
+#ifdef HAS_I410TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "vbroadcastss %3, %%ymm4 \n"
- "sub %0,%1 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // 16 pixel loop.
LABELALIGN
"1: \n"
- "vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints
- "vpmovzxwd 0x10(%0),%%ymm3 \n"
- "vcvtdq2ps %%ymm2,%%ymm2 \n"
- "vcvtdq2ps %%ymm3,%%ymm3 \n"
- "vmulps %%ymm2,%%ymm4,%%ymm2 \n"
- "vmulps %%ymm3,%%ymm4,%%ymm3 \n"
- "vcvtps2ph $3, %%ymm2, %%xmm2 \n"
- "vcvtps2ph $3, %%ymm3, %%xmm3 \n"
- "vmovdqu %%xmm2,0x00(%0,%1,1) \n"
- "vmovdqu %%xmm3,0x10(%0,%1,1) \n"
- "add $0x20,%0 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ READYUV410_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV410_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
"vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
-#if defined(__x86_64__)
- : "x"(scale) // %3
-#else
- : "m"(scale) // %3
-#endif
- : "memory", "cc", "xmm2", "xmm3", "xmm4");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
}
-#endif // HAS_HALFFLOATROW_F16C
-
-#ifdef HAS_HALFFLOATROW_F16C
-void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) {
+#endif // HAS_I410TOARGBROW_AVX512BW
+
+#ifdef HAS_I410TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "sub %0,%1 \n"
- // 16 pixel loop.
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n"
+ "vpsllw $10,%%zmm5,%%zmm19 \n"
+ "vpsllw $14,%%zmm5,%%zmm5 \n"
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
LABELALIGN
"1: \n"
- "vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints
- "vpmovzxwd 0x10(%0),%%ymm3 \n"
- "vcvtdq2ps %%ymm2,%%ymm2 \n"
- "vcvtdq2ps %%ymm3,%%ymm3 \n"
- "vcvtps2ph $3, %%ymm2, %%xmm2 \n"
- "vcvtps2ph $3, %%ymm3, %%xmm3 \n"
- "vmovdqu %%xmm2,0x00(%0,%1,1) \n"
- "vmovdqu %%xmm3,0x10(%0,%1,1) \n"
- "add $0x20,%0 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ READYUV410_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV410_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
+
+ "99: \n"
"vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- :
- : "memory", "cc", "xmm2", "xmm3");
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [dst_ar30] "+r"(dst_ar30),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [permar30_0] "r"(kPermAR30_0),
+ [permar30_1] "r"(kPermAR30_1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2");
}
-#endif // HAS_HALFFLOATROW_F16C
-
-#ifdef HAS_ARGBCOLORTABLEROW_X86
-// Tranform ARGB pixels with color table.
-void ARGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width) {
- uintptr_t pixel_temp;
+#endif // HAS_I410TOAR30ROW_AVX512BW
+
+#ifdef HAS_I410ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- // 1 pixel loop.
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "sub %[u_buf],%[v_buf] \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
LABELALIGN
"1: \n"
- "movzb (%0),%1 \n"
- "lea 0x4(%0),%0 \n"
- "movzb 0x00(%3,%1,4),%1 \n"
- "mov %b1,-0x4(%0) \n"
- "movzb -0x3(%0),%1 \n"
- "movzb 0x01(%3,%1,4),%1 \n"
- "mov %b1,-0x3(%0) \n"
- "movzb -0x2(%0),%1 \n"
- "movzb 0x02(%3,%1,4),%1 \n"
- "mov %b1,-0x2(%0) \n"
- "movzb -0x1(%0),%1 \n"
- "movzb 0x03(%3,%1,4),%1 \n"
- "mov %b1,-0x1(%0) \n"
- "dec %2 \n"
- "jg 1b \n"
- : "+r"(dst_argb), // %0
- "=&d"(pixel_temp), // %1
- "+r"(width) // %2
- : "r"(table_argb) // %3
- : "memory", "cc");
-}
-#endif // HAS_ARGBCOLORTABLEROW_X86
+ READYUV410_AVX512BW
+ "vmovdqu16 (%[a_buf]),%%zmm5 \n"
+ "lea 0x40(%[a_buf]),%[a_buf] \n"
+ "vpsrlw $2,%%zmm5,%%zmm5 \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_RGBCOLORTABLEROW_X86
-// Tranform RGB pixels with color table.
-void RGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width) {
- uintptr_t pixel_temp;
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READYUV410_MASK_AVX512BW
+ "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n"
+ "vpsrlw $2,%%zmm5,%%zmm5 \n"
+ "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n"
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [u_buf] "+r"(u_buf),
+ [v_buf] "+r"(v_buf),
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_I410ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_P210TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- // 1 pixel loop.
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vbroadcasti32x4 %[kShufUV],%%zmm14 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
+
LABELALIGN
"1: \n"
- "movzb (%0),%1 \n"
- "lea 0x4(%0),%0 \n"
- "movzb 0x00(%3,%1,4),%1 \n"
- "mov %b1,-0x4(%0) \n"
- "movzb -0x3(%0),%1 \n"
- "movzb 0x01(%3,%1,4),%1 \n"
- "mov %b1,-0x3(%0) \n"
- "movzb -0x2(%0),%1 \n"
- "movzb 0x02(%3,%1,4),%1 \n"
- "mov %b1,-0x2(%0) \n"
- "dec %2 \n"
- "jg 1b \n"
- : "+r"(dst_argb), // %0
- "=&d"(pixel_temp), // %1
- "+r"(width) // %2
- : "r"(table_argb) // %3
- : "memory", "cc");
-}
-#endif // HAS_RGBCOLORTABLEROW_X86
+ READP210_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3
-// Tranform RGB pixels with luma table.
-void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- const uint8_t* luma,
- uint32_t lumacoeff) {
- uintptr_t pixel_temp;
- uintptr_t table_temp;
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
+
+ READP210_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [kShufUV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm14", "k1", "k2");
+}
+#endif // HAS_P210TOARGBROW_AVX512BW
+
+#ifdef HAS_P210TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
asm volatile(
- "movd %6,%%xmm3 \n"
- "pshufd $0x0,%%xmm3,%%xmm3 \n"
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psllw $0x8,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vbroadcasti32x4 %[kShufUV],%%zmm14 \n"
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n"
+ "vpsllw $10,%%zmm5,%%zmm19 \n"
+ "vpsllw $14,%%zmm5,%%zmm5 \n"
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
- // 4 pixel loop.
LABELALIGN
"1: \n"
- "movdqu (%2),%%xmm0 \n"
- "pmaddubsw %%xmm3,%%xmm0 \n"
- "phaddw %%xmm0,%%xmm0 \n"
- "pand %%xmm4,%%xmm0 \n"
- "punpcklwd %%xmm5,%%xmm0 \n"
- "movd %%xmm0,%k1 \n" // 32 bit offset
- "add %5,%1 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
-
- "movzb (%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,(%3) \n"
- "movzb 0x1(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x1(%3) \n"
- "movzb 0x2(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x2(%3) \n"
- "movzb 0x3(%2),%0 \n"
- "mov %b0,0x3(%3) \n"
-
- "movd %%xmm0,%k1 \n" // 32 bit offset
- "add %5,%1 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
-
- "movzb 0x4(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x4(%3) \n"
- "movzb 0x5(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x5(%3) \n"
- "movzb 0x6(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x6(%3) \n"
- "movzb 0x7(%2),%0 \n"
- "mov %b0,0x7(%3) \n"
-
- "movd %%xmm0,%k1 \n" // 32 bit offset
- "add %5,%1 \n"
- "pshufd $0x39,%%xmm0,%%xmm0 \n"
-
- "movzb 0x8(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x8(%3) \n"
- "movzb 0x9(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0x9(%3) \n"
- "movzb 0xa(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0xa(%3) \n"
- "movzb 0xb(%2),%0 \n"
- "mov %b0,0xb(%3) \n"
-
- "movd %%xmm0,%k1 \n" // 32 bit offset
- "add %5,%1 \n"
-
- "movzb 0xc(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0xc(%3) \n"
- "movzb 0xd(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0xd(%3) \n"
- "movzb 0xe(%2),%0 \n"
- "movzb 0x00(%1,%0,1),%0 \n"
- "mov %b0,0xe(%3) \n"
- "movzb 0xf(%2),%0 \n"
- "mov %b0,0xf(%3) \n"
- "lea 0x10(%2),%2 \n"
- "lea 0x10(%3),%3 \n"
- "sub $0x4,%4 \n"
- "jg 1b \n"
- : "=&d"(pixel_temp), // %0
- "=&a"(table_temp), // %1
- "+r"(src_argb), // %2
- "+r"(dst_argb), // %3
- "+rm"(width) // %4
- : "r"(luma), // %5
- "rm"(lumacoeff) // %6
- : "memory", "cc", "xmm0", "xmm3", "xmm4", "xmm5");
-}
-#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3
+ READP210_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_NV21TOYUV24ROW_AVX2
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
-// begin NV21ToYUV24Row_C avx2 constants
-static const ulvec8 kBLEND0 = {0x80, 0x00, 0x80, 0x80, 0x00, 0x80, 0x80, 0x00,
- 0x80, 0x80, 0x00, 0x80, 0x80, 0x00, 0x80, 0x80,
- 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80,
- 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00};
-
-static const ulvec8 kBLEND1 = {0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00,
- 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00,
- 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00,
- 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80};
-
-static const ulvec8 kBLEND2 = {0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00,
- 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80,
- 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00, 0x00,
- 0x80, 0x00, 0x00, 0x80, 0x00, 0x00, 0x80, 0x00};
-
-static const ulvec8 kSHUF0 = {0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80, 0x02, 0x0d,
- 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f, 0x80, 0x05,
- 0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80, 0x02, 0x0d,
- 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f, 0x80, 0x05};
-
-static const ulvec8 kSHUF1 = {0x80, 0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80, 0x02,
- 0x0d, 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f, 0x80,
- 0x80, 0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80, 0x02,
- 0x0d, 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f, 0x80};
-
-static const ulvec8 kSHUF2 = {0x0a, 0x80, 0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80,
- 0x02, 0x0d, 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f,
- 0x0a, 0x80, 0x00, 0x0b, 0x80, 0x01, 0x0c, 0x80,
- 0x02, 0x0d, 0x80, 0x03, 0x0e, 0x80, 0x04, 0x0f};
-
-static const ulvec8 kSHUF3 = {0x80, 0x80, 0x06, 0x80, 0x80, 0x07, 0x80, 0x80,
- 0x08, 0x80, 0x80, 0x09, 0x80, 0x80, 0x0a, 0x80,
- 0x80, 0x80, 0x06, 0x80, 0x80, 0x07, 0x80, 0x80,
- 0x08, 0x80, 0x80, 0x09, 0x80, 0x80, 0x0a, 0x80};
-
-static const ulvec8 kSHUF4 = {0x05, 0x80, 0x80, 0x06, 0x80, 0x80, 0x07, 0x80,
- 0x80, 0x08, 0x80, 0x80, 0x09, 0x80, 0x80, 0x0a,
- 0x05, 0x80, 0x80, 0x06, 0x80, 0x80, 0x07, 0x80,
- 0x80, 0x08, 0x80, 0x80, 0x09, 0x80, 0x80, 0x0a};
-
-static const ulvec8 kSHUF5 = {0x80, 0x05, 0x80, 0x80, 0x06, 0x80, 0x80, 0x07,
- 0x80, 0x80, 0x08, 0x80, 0x80, 0x09, 0x80, 0x80,
- 0x80, 0x05, 0x80, 0x80, 0x06, 0x80, 0x80, 0x07,
- 0x80, 0x80, 0x08, 0x80, 0x80, 0x09, 0x80, 0x80};
-
-// NV21ToYUV24Row_AVX2
-void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_yuv24,
- int width) {
- uint8_t* src_y_ptr;
- uint64_t src_offset = 0;
- uint64_t width64;
-
- width64 = width;
- src_y_ptr = (uint8_t*)src_y;
-
- asm volatile(
- "vmovdqu %5, %%ymm0 \n" // init blend value
- "vmovdqu %6, %%ymm1 \n" // init blend value
- "vmovdqu %7, %%ymm2 \n" // init blend value
- // "sub $0x20, %3 \n" //sub 32 from width for final loop
-
- LABELALIGN
- "1: \n" // label 1
- "vmovdqu (%0,%4), %%ymm3 \n" // src_y
- "vmovdqu 1(%1,%4), %%ymm4 \n" // src_uv+1
- "vmovdqu (%1), %%ymm5 \n" // src_uv
- "vpshufb %8, %%ymm3, %%ymm13 \n" // y, kSHUF0 for shuf
- "vpshufb %9, %%ymm4, %%ymm14 \n" // uv+1, kSHUF1 for
- // shuf
- "vpshufb %10, %%ymm5, %%ymm15 \n" // uv, kSHUF2 for
- // shuf
- "vpshufb %11, %%ymm3, %%ymm3 \n" // y kSHUF3 for shuf
- "vpshufb %12, %%ymm4, %%ymm4 \n" // uv+1 kSHUF4 for
- // shuf
- "vpblendvb %%ymm0, %%ymm14, %%ymm13, %%ymm12 \n" // blend 0
- "vpblendvb %%ymm0, %%ymm13, %%ymm14, %%ymm14 \n" // blend 0
- "vpblendvb %%ymm2, %%ymm15, %%ymm12, %%ymm12 \n" // blend 2
- "vpblendvb %%ymm1, %%ymm15, %%ymm14, %%ymm13 \n" // blend 1
- "vpshufb %13, %%ymm5, %%ymm15 \n" // shuffle const
- "vpor %%ymm4, %%ymm3, %%ymm5 \n" // get results
- "vmovdqu %%ymm12, 0x20(%2) \n" // store dst_yuv+20h
- "vpor %%ymm15, %%ymm5, %%ymm3 \n" // get results
- "add $0x20, %4 \n" // add to src buffer
- // ptr
- "vinserti128 $0x1, %%xmm3, %%ymm13, %%ymm4 \n" // insert
- "vperm2i128 $0x31, %%ymm13, %%ymm3, %%ymm5 \n" // insert
- "vmovdqu %%ymm4, (%2) \n" // store dst_yuv
- "vmovdqu %%ymm5, 0x40(%2) \n" // store dst_yuv+40h
- "add $0x60,%2 \n" // add to dst buffer
- // ptr
- // "cmp %3, %4 \n" //(width64 -
- // 32 bytes) and src_offset
- "sub $0x20,%3 \n" // 32 pixels per loop
- "jg 1b \n"
- "vzeroupper \n" // sse-avx2
- // transistions
-
- : "+r"(src_y), //%0
- "+r"(src_vu), //%1
- "+r"(dst_yuv24), //%2
- "+r"(width64), //%3
- "+r"(src_offset) //%4
- : "m"(kBLEND0), //%5
- "m"(kBLEND1), //%6
- "m"(kBLEND2), //%7
- "m"(kSHUF0), //%8
- "m"(kSHUF1), //%9
- "m"(kSHUF2), //%10
- "m"(kSHUF3), //%11
- "m"(kSHUF4), //%12
- "m"(kSHUF5) //%13
- : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm12",
- "xmm13", "xmm14", "xmm15");
+ READP210_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
+
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_ar30] "+r"(dst_ar30),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [permar30_0] "r"(kPermAR30_0),
+ [permar30_1] "r"(kPermAR30_1),
+ [kShufUV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7", "xmm14", "xmm19", "xmm20", "xmm21",
+ "k1", "k2");
}
-#endif // HAS_NV21TOYUV24ROW_AVX2
+#endif // HAS_P210TOAR30ROW_AVX512BW
+
+#ifdef HAS_P410TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
-#ifdef HAS_SWAPUVROW_SSSE3
+ LABELALIGN
+ "1: \n"
+ READP410_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-// Shuffle table for reversing the bytes.
-static const uvec8 kShuffleUVToVU = {1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u,
- 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
-// Convert UV plane of NV12 to VU of NV21.
-void SwapUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
- asm volatile(
+ READP410_MASK_AVX512BW
+ YUVTORGB_AVX512BW(yuvconstants)
+ STOREARGB_MASK_AVX512BW
- "movdqu %3,%%xmm5 \n"
+ "99: \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_argb] "+r"(dst_argb),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "k1", "k2");
+}
+#endif // HAS_P410TOARGBROW_AVX512BW
+
+#ifdef HAS_P410TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uintptr_t temp;
+ asm volatile(
+ YUVTORGB_SETUP_AVX512BW(yuvconstants)
+ "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
+ "vpsrlw $6,%%zmm5,%%zmm7 \n"
+ "vpsllw $4,%%zmm7,%%zmm7 \n"
+ "vpsllw $10,%%zmm5,%%zmm19 \n"
+ "vpsllw $14,%%zmm5,%%zmm5 \n"
+ "vpxord %%zmm6,%%zmm6,%%zmm6 \n"
+ "vmovdqu16 (%[permar30_0]),%%zmm20 \n"
+ "vmovdqu16 (%[permar30_1]),%%zmm21 \n"
+ "sub $0x20,%[width] \n"
+ "jl 2f \n"
LABELALIGN
"1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_uv), // %0
- "+r"(dst_vu), // %1
- "+r"(width) // %2
- : "m"(kShuffleUVToVU) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
-}
-#endif // HAS_SWAPUVROW_SSSE3
+ READP410_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_AVX512BW
+ "sub $0x20,%[width] \n"
+ "jge 1b \n"
-#ifdef HAS_SWAPUVROW_AVX2
-void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
- asm volatile(
+ "2: \n"
+ "add $0x20,%[width] \n"
+ "jle 99f \n"
- "vbroadcastf128 %3,%%ymm5 \n"
+ READP410_MASK_AVX512BW
+ YUVTORGB16_AVX512BW(yuvconstants)
+ STOREAR30_MASK_AVX512BW
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
+ "99: \n"
"vzeroupper \n"
- : "+r"(src_uv), // %0
- "+r"(dst_vu), // %1
- "+r"(width) // %2
- : "m"(kShuffleUVToVU) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ : [y_buf] "+r"(y_buf),
+ [uv_buf] "+r"(uv_buf),
+ [dst_ar30] "+r"(dst_ar30),
+ [width] "+r"(width),
+ [temp] "=&r"(temp)
+ : [yuvconstants] "r"(yuvconstants),
+ [quadsplitperm] "r"(kSplitQuadWords),
+ [unperm] "r"(kUnpermuteAVX512),
+ [permar30_0] "r"(kPermAR30_0),
+ [permar30_1] "r"(kPermAR30_1)
+ : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2");
}
-#endif // HAS_SWAPUVROW_AVX2
+#endif // HAS_P410TOAR30ROW_AVX512BW
-#endif // defined(__x86_64__) || defined(__i386__)
+#endif // !defined(LIBYUV_DISABLE_X86) &&
+ // (defined(__x86_64__) || defined(__i386__)) &&
+ // !defined(LIBYUV_ENABLE_ROWWIN)
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/source/row_hvx.cc b/libs/libyuv/source/row_hvx.cc
new file mode 100644
index 0000000000..9ed5bfac19
--- /dev/null
+++ b/libs/libyuv/source/row_hvx.cc
@@ -0,0 +1,1863 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/row.h"
+
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#include <hexagon_protos.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <string.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+static __inline HVX_Vector vloada(const void* ptr) {
+ HVX_Vector v;
+ memcpy(&v, (const HVX_Vector*)ptr, sizeof(v));
+ return v;
+}
+
+static __inline HVX_UVector vloadu(const void* ptr) {
+ HVX_UVector v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+}
+
+static __inline void vstoreu(void* ptr, HVX_UVector v) {
+ memcpy(ptr, &v, sizeof(v));
+}
+
+static __inline void vstoreu_partial(void* dst, HVX_Vector v, int n) {
+ if (n >= 128) {
+ vstoreu(dst, v);
+ } else if (n > 0) {
+ const uintptr_t addr = (uintptr_t)dst;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector value = Q6_V_vlalign_VVR(v, v, addr);
+ HVX_VectorPred ql_not = Q6_Q_vsetq_R(addr);
+ HVX_VectorPred qr = Q6_Q_vsetq2_R(offset + (uintptr_t)n);
+ if (offset + (uintptr_t)n > 128) {
+ Q6_vmem_QRIV(qr, (uint8_t*)dst + 128, value);
+ qr = Q6_Q_vcmp_eq_VbVb(value, value);
+ }
+ ql_not = Q6_Q_or_QQn(ql_not, qr);
+ Q6_vmem_QnRIV(ql_not, dst, value);
+ }
+}
+
+static __inline HVX_Vector vloadu_partial_zero(const void* src, int n) {
+ if (n >= 128) {
+ return vloadu(src);
+ }
+ if (n <= 0) {
+ return Q6_V_vzero();
+ }
+ const uintptr_t addr = (uintptr_t)src;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector v0 = vloada(src);
+ HVX_Vector v1 =
+ (offset + (uintptr_t)n > 128) ? vloada((const uint8_t*)src + 128) : v0;
+ HVX_Vector v = Q6_V_valign_VVR(v1, v0, addr);
+ return Q6_V_vmux_QVV(Q6_Q_vsetq_R(n), v, Q6_V_vzero());
+}
+
+void CopyRow_HVX(const uint8_t* src, uint8_t* dst, int count) {
+ while (count >= 128) {
+ vstoreu(dst, vloadu(src));
+ src += 128;
+ dst += 128;
+ count -= 128;
+ }
+ if (count > 0) {
+ vstoreu_partial(dst, vloadu_partial_zero(src, count), count);
+ }
+}
+
+void CopyRow_16_HVX(const uint16_t* src, uint16_t* dst, int count) {
+ CopyRow_HVX((const uint8_t*)src, (uint8_t*)dst, count * 2);
+}
+
+void SetRow_HVX(uint8_t* dst, uint8_t v8, int width) {
+ HVX_Vector v = Q6_Vb_vsplat_R(v8);
+ while (width >= 128) {
+ vstoreu(dst, v);
+ dst += 128;
+ width -= 128;
+ }
+ if (width > 0) {
+ vstoreu_partial(dst, v, width);
+ }
+}
+
+void ARGBSetRow_HVX(uint8_t* dst_argb, uint32_t v32, int width) {
+ HVX_Vector v = Q6_V_vsplat_R(v32);
+ int bytes = width * 4;
+ while (bytes >= 128) {
+ vstoreu(dst_argb, v);
+ dst_argb += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ vstoreu_partial(dst_argb, v, bytes);
+ }
+}
+
+void ARGBOrRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ HVX_Vector v = Q6_V_vsplat_R(value);
+ int bytes = width * 4;
+ while (bytes >= 128) {
+ HVX_Vector s = vloadu(src_argb);
+ vstoreu(dst_argb, Q6_V_vor_VV(s, v));
+ src_argb += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ HVX_Vector s = vloadu_partial_zero(src_argb, bytes);
+ vstoreu_partial(dst_argb, Q6_V_vor_VV(s, v), bytes);
+ }
+}
+
+void MirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7F);
+ src += width;
+ while (width >= 128) {
+ src -= 128;
+ HVX_Vector v = vloadu(src);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ vstoreu(dst, v);
+ dst += 128;
+ width -= 128;
+ }
+ if (width > 0) {
+ src -= width;
+ HVX_Vector v = vloadu_partial_zero(src, width);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ v = Q6_V_valign_VVR(v, v, 128 - width);
+ vstoreu_partial(dst, v, width);
+ }
+}
+
+void MirrorRow_16_HVX(const uint16_t* src, uint16_t* dst, int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7E);
+ const uint8_t* s = (const uint8_t*)(src + width);
+ uint8_t* d = (uint8_t*)dst;
+ int bytes = width * 2;
+ while (bytes >= 128) {
+ s -= 128;
+ HVX_Vector v = vloadu(s);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ vstoreu(d, v);
+ d += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ s -= bytes;
+ HVX_Vector v = vloadu_partial_zero(s, bytes);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ v = Q6_V_valign_VVR(v, v, 128 - bytes);
+ vstoreu_partial(d, v, bytes);
+ }
+}
+
+void MirrorUVRow_HVX(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ MirrorRow_16_HVX((const uint16_t*)src_uv, (uint16_t*)dst_uv, width);
+}
+
+void MirrorSplitUVRow_HVX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7F);
+ src_uv += width * 2;
+ while (width >= 128) {
+ src_uv -= 256;
+ HVX_Vector s0 = vloadu(src_uv);
+ HVX_Vector s1 = vloadu(src_uv + 128);
+ HVX_VectorPair uv = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_Vector u = Q6_V_vdelta_VV(Q6_V_lo_W(uv), ctrl);
+ HVX_Vector v = Q6_V_vdelta_VV(Q6_V_hi_W(uv), ctrl);
+ vstoreu(dst_u, u);
+ vstoreu(dst_v, v);
+ dst_u += 128;
+ dst_v += 128;
+ width -= 128;
+ }
+ if (width > 0) {
+ int src_bytes = width * 2;
+ src_uv -= src_bytes;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uv + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair uv = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_Vector u = Q6_V_vdelta_VV(Q6_V_lo_W(uv), ctrl);
+ HVX_Vector v = Q6_V_vdelta_VV(Q6_V_hi_W(uv), ctrl);
+ u = Q6_V_valign_VVR(u, u, 128 - width);
+ v = Q6_V_valign_VVR(v, v, 128 - width);
+ vstoreu_partial(dst_u, u, width);
+ vstoreu_partial(dst_v, v, width);
+ }
+}
+
+void ARGBMirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7C);
+ int bytes = width * 4;
+ src += bytes;
+ while (bytes >= 128) {
+ src -= 128;
+ HVX_Vector v = vloadu(src);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ vstoreu(dst, v);
+ dst += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ src -= bytes;
+ HVX_Vector v = vloadu_partial_zero(src, bytes);
+ v = Q6_V_vdelta_VV(v, ctrl);
+ v = Q6_V_valign_VVR(v, v, 128 - bytes);
+ vstoreu_partial(dst, v, bytes);
+ }
+}
+
+void SplitUVRow_HVX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uv + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair uv = Q6_W_vdeal_VVR(s1, s0, -1);
+ vstoreu_partial(dst_u, Q6_V_lo_W(uv), w);
+ vstoreu_partial(dst_v, Q6_V_hi_W(uv), w);
+ src_uv += 256;
+ dst_u += 128;
+ dst_v += 128;
+ width -= 128;
+ }
+}
+
+void MergeUVRow_HVX(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector u = vloadu_partial_zero(src_u, w);
+ HVX_Vector v = vloadu_partial_zero(src_v, w);
+ HVX_VectorPair uv = Q6_W_vshuff_VVR(v, u, -1);
+ int dst_bytes = w * 2;
+ vstoreu_partial(dst_uv, Q6_V_lo_W(uv), dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_uv + 128, Q6_V_hi_W(uv), dst_bytes - 128);
+ }
+ src_u += 128;
+ src_v += 128;
+ dst_uv += 256;
+ width -= 128;
+ }
+}
+
+void SplitUVRow_16_HVX(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ while (width > 0) {
+ int w = width >= 64 ? 64 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero((const uint8_t*)src_uv + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair uv = Q6_W_vdeal_VVR(s1, s0, -2);
+ HVX_Vector u = Q6_Vuh_vlsr_VuhR(Q6_V_lo_W(uv), shift);
+ HVX_Vector v = Q6_Vuh_vlsr_VuhR(Q6_V_hi_W(uv), shift);
+ vstoreu_partial(dst_u, u, w * 2);
+ vstoreu_partial(dst_v, v, w * 2);
+ src_uv += 128;
+ dst_u += 64;
+ dst_v += 64;
+ width -= 64;
+ }
+}
+
+void MergeUVRow_16_HVX(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ while (width > 0) {
+ int w = width >= 64 ? 64 : width;
+ HVX_Vector u = vloadu_partial_zero(src_u, w * 2);
+ HVX_Vector v = vloadu_partial_zero(src_v, w * 2);
+ u = Q6_Vh_vasl_VhR(u, shift);
+ v = Q6_Vh_vasl_VhR(v, shift);
+ HVX_VectorPair uv = Q6_W_vshuff_VVR(v, u, -2);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_uv, Q6_V_lo_W(uv), dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial((uint8_t*)dst_uv + 128, Q6_V_hi_W(uv), dst_bytes - 128);
+ }
+ src_u += 64;
+ src_v += 64;
+ dst_uv += 128;
+ width -= 64;
+ }
+}
+
+void SplitARGBRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_argb + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_argb + 384, src_bytes - 384)
+ : Q6_V_vzero();
+ HVX_VectorPair p01 = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_VectorPair p23 = Q6_W_vdeal_VVR(s3, s2, -1);
+ HVX_VectorPair br = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -1);
+ HVX_VectorPair ga = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -1);
+ vstoreu_partial(dst_b, Q6_V_lo_W(br), w);
+ vstoreu_partial(dst_g, Q6_V_lo_W(ga), w);
+ vstoreu_partial(dst_r, Q6_V_hi_W(br), w);
+ vstoreu_partial(dst_a, Q6_V_hi_W(ga), w);
+ src_argb += 512;
+ dst_b += 128;
+ dst_g += 128;
+ dst_r += 128;
+ dst_a += 128;
+ width -= 128;
+ }
+}
+
+void MergeARGBRow_HVX(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector b = vloadu_partial_zero(src_b, w);
+ HVX_Vector g = vloadu_partial_zero(src_g, w);
+ HVX_Vector r = vloadu_partial_zero(src_r, w);
+ HVX_Vector a = vloadu_partial_zero(src_a, w);
+ HVX_VectorPair br = Q6_W_vshuff_VVR(r, b, -1);
+ HVX_VectorPair ga = Q6_W_vshuff_VVR(a, g, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ga), Q6_V_lo_W(br), -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ga), Q6_V_hi_W(br), -1);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_argb, Q6_V_lo_W(p01),
+ dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_argb + 128, Q6_V_hi_W(p01),
+ dst_bytes > 256 ? 128 : dst_bytes - 128);
+ }
+ if (dst_bytes > 256) {
+ vstoreu_partial(dst_argb + 256, Q6_V_lo_W(p23),
+ dst_bytes > 384 ? 128 : dst_bytes - 256);
+ }
+ if (dst_bytes > 384) {
+ vstoreu_partial(dst_argb + 384, Q6_V_hi_W(p23), dst_bytes - 384);
+ }
+ src_b += 128;
+ src_g += 128;
+ src_r += 128;
+ src_a += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void SplitXRGBRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_argb + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_argb + 384, src_bytes - 384)
+ : Q6_V_vzero();
+ HVX_VectorPair p01 = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_VectorPair p23 = Q6_W_vdeal_VVR(s3, s2, -1);
+ HVX_VectorPair br = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -1);
+ HVX_VectorPair ga = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -1);
+ vstoreu_partial(dst_b, Q6_V_lo_W(br), w);
+ vstoreu_partial(dst_g, Q6_V_lo_W(ga), w);
+ vstoreu_partial(dst_r, Q6_V_hi_W(br), w);
+ src_argb += 512;
+ dst_b += 128;
+ dst_g += 128;
+ dst_r += 128;
+ width -= 128;
+ }
+}
+
+void MergeXRGBRow_HVX(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector b = vloadu_partial_zero(src_b, w);
+ HVX_Vector g = vloadu_partial_zero(src_g, w);
+ HVX_Vector r = vloadu_partial_zero(src_r, w);
+ HVX_VectorPair br = Q6_W_vshuff_VVR(r, b, -1);
+ HVX_VectorPair ga = Q6_W_vshuff_VVR(a, g, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ga), Q6_V_lo_W(br), -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ga), Q6_V_hi_W(br), -1);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_argb, Q6_V_lo_W(p01),
+ dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_argb + 128, Q6_V_hi_W(p01),
+ dst_bytes > 256 ? 128 : dst_bytes - 128);
+ }
+ if (dst_bytes > 256) {
+ vstoreu_partial(dst_argb + 256, Q6_V_lo_W(p23),
+ dst_bytes > 384 ? 128 : dst_bytes - 256);
+ }
+ if (dst_bytes > 384) {
+ vstoreu_partial(dst_argb + 384, Q6_V_hi_W(p23), dst_bytes - 384);
+ }
+ src_b += 128;
+ src_g += 128;
+ src_r += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void SwapUVRow_HVX(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x01);
+ int bytes = width * 2;
+ while (bytes >= 128) {
+ HVX_Vector v = vloadu(src_uv);
+ vstoreu(dst_vu, Q6_V_vdelta_VV(v, ctrl));
+ src_uv += 128;
+ dst_vu += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ HVX_Vector v = vloadu_partial_zero(src_uv, bytes);
+ vstoreu_partial(dst_vu, Q6_V_vdelta_VV(v, ctrl), bytes);
+ }
+}
+
+void InterpolateRow_HVX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ if (y1_fraction == 0) {
+ CopyRow_HVX(src_ptr, dst_ptr, width);
+ return;
+ }
+ if (y1_fraction == 128) {
+ while (width >= 128) {
+ HVX_Vector s0 = vloadu(src_ptr);
+ HVX_Vector s1 = vloadu(src_ptr1);
+ vstoreu(dst_ptr, Q6_Vub_vavg_VubVub_rnd(s0, s1));
+ src_ptr += 128;
+ src_ptr1 += 128;
+ dst_ptr += 128;
+ width -= 128;
+ }
+ if (width > 0) {
+ HVX_Vector s0 = vloadu_partial_zero(src_ptr, width);
+ HVX_Vector s1 = vloadu_partial_zero(src_ptr1, width);
+ vstoreu_partial(dst_ptr, Q6_Vub_vavg_VubVub_rnd(s0, s1), width);
+ }
+ return;
+ }
+ uint32_t y0_32 = (uint32_t)y0_fraction * 0x01010101u;
+ uint32_t y1_32 = (uint32_t)y1_fraction * 0x01010101u;
+ HVX_Vector v128 = Q6_Vh_vsplat_R(128);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector s0 = vloadu_partial_zero(src_ptr, w);
+ HVX_Vector s1 = vloadu_partial_zero(src_ptr1, w);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubRub(s0, y0_32);
+ p = Q6_Wuh_vmpyacc_WuhVubRub(p, s1, y1_32);
+ HVX_Vector lo = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_lo_W(p), v128);
+ HVX_Vector hi = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_hi_W(p), v128);
+ HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo);
+ vstoreu_partial(dst_ptr, out, w);
+ src_ptr += 128;
+ src_ptr1 += 128;
+ dst_ptr += 128;
+ width -= 128;
+ }
+}
+
+void InterpolateRow_16_HVX(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ if (y1_fraction == 0) {
+ CopyRow_16_HVX(src_ptr, dst_ptr, width);
+ return;
+ }
+ if (y1_fraction == 128) {
+ int bytes = width * 2;
+ const uint8_t* s0_p = (const uint8_t*)src_ptr;
+ const uint8_t* s1_p = (const uint8_t*)src_ptr1;
+ uint8_t* d_p = (uint8_t*)dst_ptr;
+ while (bytes >= 128) {
+ HVX_Vector s0 = vloadu(s0_p);
+ HVX_Vector s1 = vloadu(s1_p);
+ vstoreu(d_p, Q6_Vuh_vavg_VuhVuh_rnd(s0, s1));
+ s0_p += 128;
+ s1_p += 128;
+ d_p += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ HVX_Vector s0 = vloadu_partial_zero(s0_p, bytes);
+ HVX_Vector s1 = vloadu_partial_zero(s1_p, bytes);
+ vstoreu_partial(d_p, Q6_Vuh_vavg_VuhVuh_rnd(s0, s1), bytes);
+ }
+ return;
+ }
+ uint32_t y0_32 = ((uint32_t)y0_fraction << 16) | (uint32_t)y0_fraction;
+ uint32_t y1_32 = ((uint32_t)y1_fraction << 16) | (uint32_t)y1_fraction;
+ HVX_Vector v128 = Q6_V_vsplat_R(128);
+ while (width > 0) {
+ int w = width >= 64 ? 64 : width;
+ int bytes = w * 2;
+ HVX_Vector s0 = vloadu_partial_zero(src_ptr, bytes);
+ HVX_Vector s1 = vloadu_partial_zero(src_ptr1, bytes);
+ HVX_VectorPair p = Q6_Wuw_vmpy_VuhRuh(s0, y0_32);
+ p = Q6_Wuw_vmpyacc_WuwVuhRuh(p, s1, y1_32);
+ HVX_Vector lo = Q6_Vuw_vlsr_VuwR(Q6_Vw_vadd_VwVw(Q6_V_lo_W(p), v128), 8);
+ HVX_Vector hi = Q6_Vuw_vlsr_VuwR(Q6_Vw_vadd_VwVw(Q6_V_hi_W(p), v128), 8);
+ HVX_Vector out = Q6_Vh_vshuffe_VhVh(hi, lo);
+ vstoreu_partial(dst_ptr, out, bytes);
+ src_ptr += 64;
+ src_ptr1 += 64;
+ dst_ptr += 64;
+ width -= 64;
+ }
+}
+
+void BlendPlaneRow_HVX(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ HVX_Vector v255 = Q6_Vh_vsplat_R(255);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector f = vloadu_partial_zero(src0, w);
+ HVX_Vector b = vloadu_partial_zero(src1, w);
+ HVX_Vector a = vloadu_partial_zero(alpha, w);
+ HVX_Vector not_a = Q6_V_vnot_V(a);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubVub(f, a);
+ p = Q6_Wuh_vmpyacc_WuhVubVub(p, b, not_a);
+ HVX_Vector lo = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_lo_W(p), v255);
+ HVX_Vector hi = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_hi_W(p), v255);
+ HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo);
+ vstoreu_partial(dst, out, w);
+ src0 += 128;
+ src1 += 128;
+ alpha += 128;
+ dst += 128;
+ width -= 128;
+ }
+}
+
+void ARGBBlendRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ HVX_Vector v255 = Q6_Vh_vsplat_R(255);
+ HVX_Vector v_alpha_mask = Q6_V_vsplat_R(0xFF000000u);
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector fg = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector bg = vloadu_partial_zero(src_argb1, b_cnt);
+ HVX_Vector t = Q6_Vb_vshuffo_VbVb(fg, fg);
+ HVX_Vector a_vec = Q6_Vh_vshuffo_VhVh(t, t);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubVub(bg, a_vec);
+ HVX_Vector lo = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_lo_W(p), v255);
+ HVX_Vector hi = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_hi_W(p), v255);
+ HVX_Vector atten_bg = Q6_Vb_vshuffo_VbVb(hi, lo);
+ HVX_Vector scaled_bg = Q6_Vub_vsub_VubVub_sat(bg, atten_bg);
+ HVX_Vector out = Q6_Vub_vadd_VubVub_sat(scaled_bg, fg);
+ out = Q6_V_vor_VV(out, v_alpha_mask);
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src_argb += 128;
+ src_argb1 += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ HVX_Vector vzero = Q6_V_vzero();
+ HVX_Vector v0001 = Q6_Vh_vsplat_R(1);
+ HVX_Vector v00ff = Q6_Vh_vsplat_R(255);
+ HVX_Vector vffff = Q6_Vh_vsplat_R(-1);
+ HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu);
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector fg = vloadu_partial_zero(src_argb0, b_cnt);
+ HVX_Vector bg = vloadu_partial_zero(src_argb1, b_cnt);
+
+ // Broadcast sa and da to all 4 byte lanes of each 32-bit pixel.
+ HVX_Vector t_fg = Q6_Vb_vshuffo_VbVb(fg, fg);
+ HVX_Vector sa_vec = Q6_Vh_vshuffo_VhVh(t_fg, t_fg);
+ HVX_Vector t_bg = Q6_Vb_vshuffo_VbVb(bg, bg);
+ HVX_Vector da_vec = Q6_Vh_vshuffo_VhVh(t_bg, t_bg);
+
+ // Both uint16_t halfwords of each pixel hold sa and da (0..255).
+ HVX_Vector sa16 = Q6_V_vand_VV(sa_vec, v00ff);
+ HVX_Vector da16 = Q6_V_vand_VV(da_vec, v00ff);
+
+ // x = da * sa in both uint16_t halfwords of each pixel.
+ HVX_Vector x = Q6_V_lo_W(Q6_Wuh_vmpy_VubVub(da_vec, sa_vec));
+ // prod_div255 = (x + 1 + (x >> 8)) >> 8.
+ HVX_Vector prod_div255 = Q6_Vuh_vlsr_VuhR(
+ Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(x, v0001), Q6_Vuh_vlsr_VuhR(x, 8)), 8);
+ // out_a = da + sa - (da * sa) / 255.
+ HVX_Vector out_a16 =
+ Q6_Vh_vadd_VhVh(da16, Q6_Vh_vsub_VhVh(sa16, prod_div255));
+
+ // 8-step restoring division: ratio = (da == 0) ? 255 : (sa * 255) / out_a.
+ HVX_Vector rem = Q6_Vh_vsub_VhVh(Q6_Vh_vasl_VhR(sa16, 8), sa16);
+ HVX_VectorPred q_da_zero = Q6_Q_vcmp_eq_VhVh(da16, vzero);
+ rem = Q6_V_vmux_QVV(q_da_zero, vffff, rem);
+ HVX_Vector ratio = vzero;
+
+ for (int bit = 7; bit >= 0; --bit) {
+ HVX_Vector d_sh = Q6_Vh_vasl_VhR(out_a16, bit);
+ HVX_VectorPred q_gt = Q6_Q_vcmp_gt_VuhVuh(d_sh, rem);
+ rem = Q6_V_vmux_QVV(q_gt, rem, Q6_Vh_vsub_VhVh(rem, d_sh));
+ ratio = Q6_V_vmux_QVV(q_gt, ratio,
+ Q6_V_vor_VV(ratio, Q6_Vb_vsplat_R(1 << bit)));
+ }
+
+ // Exact RGB lerp: (bg * (255 - ratio) + fg * ratio) / 255.
+ HVX_Vector inv_ratio = Q6_V_vnot_V(ratio);
+ HVX_VectorPair p_rgb = Q6_Wuh_vmpy_VubVub(fg, ratio);
+ p_rgb = Q6_Wuh_vmpyacc_WuhVubVub(p_rgb, bg, inv_ratio);
+ HVX_Vector lo = Q6_V_lo_W(p_rgb);
+ HVX_Vector hi = Q6_V_hi_W(p_rgb);
+ lo = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(lo, v0001), Q6_Vuh_vlsr_VuhR(lo, 8));
+ hi = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(hi, v0001), Q6_Vuh_vlsr_VuhR(hi, 8));
+ HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo);
+ out =
+ Q6_V_vor_VV(Q6_V_vand_VV(out, v_rgb_mask), Q6_Vw_vasl_VwR(out_a16, 24));
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src_argb0 += 128;
+ src_argb1 += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBAttenuateRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ HVX_Vector v255 = Q6_Vh_vsplat_R(255);
+ HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu);
+ HVX_Vector v_a_mask = Q6_V_vsplat_R(0xFF000000u);
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector src = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector t = Q6_Vb_vshuffo_VbVb(src, src);
+ HVX_Vector a_vec = Q6_Vh_vshuffo_VhVh(t, t);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubVub(src, a_vec);
+ HVX_Vector lo = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_lo_W(p), v255);
+ HVX_Vector hi = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_hi_W(p), v255);
+ HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo);
+ out =
+ Q6_V_vor_VV(Q6_V_vand_VV(out, v_rgb_mask), Q6_V_vand_VV(src, v_a_mask));
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src_argb += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBMultiplyRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ HVX_Vector v128 = Q6_Vh_vsplat_R(128);
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector s0 = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector s1 = vloadu_partial_zero(src_argb1, b_cnt);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubVub(s0, s1);
+ HVX_Vector lo = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_lo_W(p), v128);
+ HVX_Vector hi = Q6_Vuh_vadd_VuhVuh_sat(Q6_V_hi_W(p), v128);
+ HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo);
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src_argb += 128;
+ src_argb1 += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBAddRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector s0 = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector s1 = vloadu_partial_zero(src_argb1, b_cnt);
+ vstoreu_partial(dst_argb, Q6_Vub_vadd_VubVub_sat(s0, s1), b_cnt);
+ src_argb += 128;
+ src_argb1 += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBSubtractRow_HVX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector s0 = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector s1 = vloadu_partial_zero(src_argb1, b_cnt);
+ vstoreu_partial(dst_argb, Q6_Vub_vsub_VubVub_sat(s0, s1), b_cnt);
+ src_argb += 128;
+ src_argb1 += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBExtractAlphaRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_argb + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_argb + 384, src_bytes - 384)
+ : Q6_V_vzero();
+ HVX_Vector ga01 = Q6_V_hi_W(Q6_W_vdeal_VVR(s1, s0, -1));
+ HVX_Vector ga23 = Q6_V_hi_W(Q6_W_vdeal_VVR(s3, s2, -1));
+ HVX_Vector a = Q6_V_hi_W(Q6_W_vdeal_VVR(ga23, ga01, -1));
+ vstoreu_partial(dst_a, a, w);
+ src_argb += 512;
+ dst_a += 128;
+ width -= 128;
+ }
+}
+
+void ARGBCopyAlphaRow_HVX(const uint8_t* src, uint8_t* dst, int width) {
+ HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu);
+ HVX_Vector v_a_mask = Q6_V_vsplat_R(0xFF000000u);
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector s = vloadu_partial_zero(src, b_cnt);
+ HVX_Vector d = vloadu_partial_zero(dst, b_cnt);
+ HVX_Vector out =
+ Q6_V_vor_VV(Q6_V_vand_VV(d, v_rgb_mask), Q6_V_vand_VV(s, v_a_mask));
+ vstoreu_partial(dst, out, b_cnt);
+ src += 128;
+ dst += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBCopyYToAlphaRow_HVX(const uint8_t* src, uint8_t* dst, int width) {
+ HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu);
+ HVX_Vector z = Q6_V_vzero();
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector y = vloadu_partial_zero(src, w);
+ HVX_VectorPair ga = Q6_W_vshuff_VVR(y, z, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ga), z, -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ga), z, -1);
+ int dst_bytes = w * 4;
+ int b0 = dst_bytes > 128 ? 128 : dst_bytes;
+ HVX_Vector d0 = vloadu_partial_zero(dst, b0);
+ vstoreu_partial(
+ dst, Q6_V_vor_VV(Q6_V_vand_VV(d0, v_rgb_mask), Q6_V_lo_W(p01)), b0);
+ if (dst_bytes > 128) {
+ int b1 = dst_bytes > 256 ? 128 : dst_bytes - 128;
+ HVX_Vector d1 = vloadu_partial_zero(dst + 128, b1);
+ vstoreu_partial(dst + 128,
+ Q6_V_vor_VV(Q6_V_vand_VV(d1, v_rgb_mask), Q6_V_hi_W(p01)),
+ b1);
+ }
+ if (dst_bytes > 256) {
+ int b2 = dst_bytes > 384 ? 128 : dst_bytes - 256;
+ HVX_Vector d2 = vloadu_partial_zero(dst + 256, b2);
+ vstoreu_partial(dst + 256,
+ Q6_V_vor_VV(Q6_V_vand_VV(d2, v_rgb_mask), Q6_V_lo_W(p23)),
+ b2);
+ }
+ if (dst_bytes > 384) {
+ int b3 = dst_bytes - 384;
+ HVX_Vector d3 = vloadu_partial_zero(dst + 384, b3);
+ vstoreu_partial(dst + 384,
+ Q6_V_vor_VV(Q6_V_vand_VV(d3, v_rgb_mask), Q6_V_hi_W(p23)),
+ b3);
+ }
+ src += 128;
+ dst += 512;
+ width -= 128;
+ }
+}
+
+void Convert16To8Row_HVX(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale;
+ HVX_Vector v255 = Q6_Vh_vsplat_R(255);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_y, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero((const uint8_t*)src_y + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair p0 = Q6_Wuw_vmpy_VuhRuh(s0, scale32);
+ HVX_VectorPair p1 = Q6_Wuw_vmpy_VuhRuh(s1, scale32);
+ HVX_Vector h0 = Q6_Vuh_vmin_VuhVuh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p0), Q6_V_lo_W(p0)), v255);
+ HVX_Vector h1 = Q6_Vuh_vmin_VuhVuh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p1), Q6_V_lo_W(p1)), v255);
+ HVX_Vector out = Q6_Vb_vpacke_VhVh(h1, h0);
+ vstoreu_partial(dst_y, out, w);
+ src_y += 128;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void Convert8To16Row_HVX(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ int shift = 16 - bits;
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector s = vloadu_partial_zero(src_y, w);
+ HVX_VectorPair d16 = Q6_W_vshuff_VVR(s, s, -1);
+ HVX_Vector d0 = Q6_Vuh_vlsr_VuhR(Q6_V_lo_W(d16), shift);
+ HVX_Vector d1 = Q6_Vuh_vlsr_VuhR(Q6_V_hi_W(d16), shift);
+ int dst_bytes = w * 2;
+ vstoreu_partial(dst_y, d0, dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial((uint8_t*)dst_y + 128, d1, dst_bytes - 128);
+ }
+ src_y += 128;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void Convert8To8Row_HVX(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ uint32_t scale32 = (uint32_t)(uint8_t)scale * 0x01010101u;
+ HVX_Vector vbias = Q6_Vb_vsplat_R(bias);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector s = vloadu_partial_zero(src_y, w);
+ HVX_VectorPair p = Q6_Wuh_vmpy_VubRub(s, scale32);
+ HVX_Vector out =
+ Q6_Vb_vadd_VbVb(Q6_Vb_vshuffo_VbVb(Q6_V_hi_W(p), Q6_V_lo_W(p)), vbias);
+ vstoreu_partial(dst_y, out, w);
+ src_y += 128;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void MultiplyRow_16_HVX(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale;
+ while (width > 0) {
+ int w = width >= 64 ? 64 : width;
+ int bytes = w * 2;
+ HVX_Vector s = vloadu_partial_zero(src_y, bytes);
+ HVX_VectorPair p = Q6_Wuw_vmpy_VuhRuh(s, scale32);
+ HVX_Vector out = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(p), Q6_V_lo_W(p));
+ vstoreu_partial(dst_y, out, bytes);
+ src_y += 64;
+ dst_y += 64;
+ width -= 64;
+ }
+}
+
+void DivideRow_16_HVX(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale;
+ while (width > 0) {
+ int w = width >= 64 ? 64 : width;
+ int bytes = w * 2;
+ HVX_Vector s = vloadu_partial_zero(src_y, bytes);
+ HVX_VectorPair p = Q6_Wuw_vmpy_VuhRuh(s, scale32);
+ HVX_Vector out = Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p), Q6_V_lo_W(p));
+ vstoreu_partial(dst_y, out, bytes);
+ src_y += 64;
+ dst_y += 64;
+ width -= 64;
+ }
+}
+
+void HalfRow_16To8_HVX(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale;
+ HVX_Vector v255 = Q6_Vh_vsplat_R(255);
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero((const uint8_t*)src_uv + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t0 =
+ vloadu_partial_zero(src_uv1, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero((const uint8_t*)src_uv1 + 128,
+ src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector a0 = Q6_Vuh_vavg_VuhVuh_rnd(s0, t0);
+ HVX_Vector a1 = Q6_Vuh_vavg_VuhVuh_rnd(s1, t1);
+ HVX_VectorPair p0 = Q6_Wuw_vmpy_VuhRuh(a0, scale32);
+ HVX_VectorPair p1 = Q6_Wuw_vmpy_VuhRuh(a1, scale32);
+ HVX_Vector h0 = Q6_Vuh_vmin_VuhVuh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p0), Q6_V_lo_W(p0)), v255);
+ HVX_Vector h1 = Q6_Vuh_vmin_VuhVuh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p1), Q6_V_lo_W(p1)), v255);
+ HVX_Vector out = Q6_Vb_vpacke_VhVh(h1, h0);
+ vstoreu_partial(dst_uv, out, w);
+ src_uv += 128;
+ src_uv1 += 128;
+ dst_uv += 128;
+ width -= 128;
+ }
+}
+
+void YUY2ToYRow_HVX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_yuy2, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_yuy2 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -1);
+ vstoreu_partial(dst_y, Q6_V_lo_W(d), w);
+ src_yuy2 += 256;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void YUY2ToUV422Row_HVX(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int uv_width = (width + 1) >> 1;
+ while (uv_width > 0) {
+ int w = uv_width >= 64 ? 64 : uv_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_yuy2, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_yuy2 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector uv = Q6_V_hi_W(Q6_W_vdeal_VVR(s1, s0, -1));
+ HVX_VectorPair u_v = Q6_W_vdeal_VVR(uv, uv, -1);
+ vstoreu_partial(dst_u, Q6_V_lo_W(u_v), w);
+ vstoreu_partial(dst_v, Q6_V_hi_W(u_v), w);
+ src_yuy2 += 256;
+ dst_u += 64;
+ dst_v += 64;
+ uv_width -= 64;
+ }
+}
+
+void YUY2ToUVRow_HVX(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2;
+ int uv_width = (width + 1) >> 1;
+ while (uv_width > 0) {
+ int w = uv_width >= 64 ? 64 : uv_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_yuy2, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_yuy2 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t0 =
+ vloadu_partial_zero(src_yuy2_1, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(src_yuy2_1 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector a0 = Q6_Vub_vavg_VubVub_rnd(s0, t0);
+ HVX_Vector a1 = Q6_Vub_vavg_VubVub_rnd(s1, t1);
+ HVX_Vector uv = Q6_V_hi_W(Q6_W_vdeal_VVR(a1, a0, -1));
+ HVX_VectorPair u_v = Q6_W_vdeal_VVR(uv, uv, -1);
+ vstoreu_partial(dst_u, Q6_V_lo_W(u_v), w);
+ vstoreu_partial(dst_v, Q6_V_hi_W(u_v), w);
+ src_yuy2 += 256;
+ src_yuy2_1 += 256;
+ dst_u += 64;
+ dst_v += 64;
+ uv_width -= 64;
+ }
+}
+
+void UYVYToYRow_HVX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uyvy, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uyvy + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -1);
+ vstoreu_partial(dst_y, Q6_V_hi_W(d), w);
+ src_uyvy += 256;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void UYVYToUV422Row_HVX(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int uv_width = (width + 1) >> 1;
+ while (uv_width > 0) {
+ int w = uv_width >= 64 ? 64 : uv_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uyvy, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uyvy + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector uv = Q6_V_lo_W(Q6_W_vdeal_VVR(s1, s0, -1));
+ HVX_VectorPair u_v = Q6_W_vdeal_VVR(uv, uv, -1);
+ vstoreu_partial(dst_u, Q6_V_lo_W(u_v), w);
+ vstoreu_partial(dst_v, Q6_V_hi_W(u_v), w);
+ src_uyvy += 256;
+ dst_u += 64;
+ dst_v += 64;
+ uv_width -= 64;
+ }
+}
+
+void UYVYToUVRow_HVX(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy;
+ int uv_width = (width + 1) >> 1;
+ while (uv_width > 0) {
+ int w = uv_width >= 64 ? 64 : uv_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_uyvy, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uyvy + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t0 =
+ vloadu_partial_zero(src_uyvy_1, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(src_uyvy_1 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector a0 = Q6_Vub_vavg_VubVub_rnd(s0, t0);
+ HVX_Vector a1 = Q6_Vub_vavg_VubVub_rnd(s1, t1);
+ HVX_Vector uv = Q6_V_lo_W(Q6_W_vdeal_VVR(a1, a0, -1));
+ HVX_VectorPair u_v = Q6_W_vdeal_VVR(uv, uv, -1);
+ vstoreu_partial(dst_u, Q6_V_lo_W(u_v), w);
+ vstoreu_partial(dst_v, Q6_V_hi_W(u_v), w);
+ src_uyvy += 256;
+ src_uyvy_1 += 256;
+ dst_u += 64;
+ dst_v += 64;
+ uv_width -= 64;
+ }
+}
+
+void J400ToARGBRow_HVX(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_VectorPair yy = Q6_W_vshuff_VVR(y, y, -1);
+ HVX_VectorPair ya = Q6_W_vshuff_VVR(a, y, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ya), Q6_V_lo_W(yy), -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ya), Q6_V_hi_W(yy), -1);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_argb, Q6_V_lo_W(p01),
+ dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_argb + 128, Q6_V_hi_W(p01),
+ dst_bytes > 256 ? 128 : dst_bytes - 128);
+ }
+ if (dst_bytes > 256) {
+ vstoreu_partial(dst_argb + 256, Q6_V_lo_W(p23),
+ dst_bytes > 384 ? 128 : dst_bytes - 256);
+ }
+ if (dst_bytes > 384) {
+ vstoreu_partial(dst_argb + 384, Q6_V_hi_W(p23), dst_bytes - 384);
+ }
+ src_y += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void I400ToARGBRow_HVX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uint16_t yg = (uint16_t)yuvconstants->kYToRgb[0];
+ int16_t yb = yuvconstants->kYBiasToRgb[0];
+ uint32_t yg32 = ((uint32_t)yg << 16) | yg;
+ HVX_Vector v_yb = Q6_Vh_vsplat_R(yb);
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector y_even_0101 = Q6_Vb_vshuffe_VbVb(y, y);
+ HVX_Vector y_odd_0101 = Q6_Vb_vshuffo_VbVb(y, y);
+ HVX_VectorPair p_even = Q6_Wuw_vmpy_VuhRuh(y_even_0101, yg32);
+ HVX_VectorPair p_odd = Q6_Wuw_vmpy_VuhRuh(y_odd_0101, yg32);
+ HVX_Vector y_lo = Q6_Vh_vasr_VhR(
+ Q6_Vh_vadd_VhVh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p_even), Q6_V_lo_W(p_even)), v_yb),
+ 6);
+ HVX_Vector y_hi = Q6_Vh_vasr_VhR(
+ Q6_Vh_vadd_VhVh(Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p_odd), Q6_V_lo_W(p_odd)),
+ v_yb),
+ 6);
+ HVX_Vector y_out = Q6_Vub_vsat_VhVh(y_hi, y_lo);
+ HVX_VectorPair yy = Q6_W_vshuff_VVR(y_out, y_out, -1);
+ HVX_VectorPair ya = Q6_W_vshuff_VVR(a, y_out, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ya), Q6_V_lo_W(yy), -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ya), Q6_V_hi_W(yy), -1);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_argb, Q6_V_lo_W(p01),
+ dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_argb + 128, Q6_V_hi_W(p01),
+ dst_bytes > 256 ? 128 : dst_bytes - 128);
+ }
+ if (dst_bytes > 256) {
+ vstoreu_partial(dst_argb + 256, Q6_V_lo_W(p23),
+ dst_bytes > 384 ? 128 : dst_bytes - 256);
+ }
+ if (dst_bytes > 384) {
+ vstoreu_partial(dst_argb + 384, Q6_V_hi_W(p23), dst_bytes - 384);
+ }
+ src_y += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void ARGBToABGRRow_HVX(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
+ HVX_Vector ctrl = Q6_V_vsplat_R(0x00020002);
+ int bytes = width * 4;
+ while (bytes >= 128) {
+ vstoreu(dst_abgr, Q6_V_vdelta_VV(vloadu(src_argb), ctrl));
+ src_argb += 128;
+ dst_abgr += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ vstoreu_partial(dst_abgr,
+ Q6_V_vdelta_VV(vloadu_partial_zero(src_argb, bytes), ctrl),
+ bytes);
+ }
+}
+
+void ARGBToBGRARow_HVX(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
+ HVX_Vector ctrl = Q6_Vb_vsplat_R(0x03);
+ int bytes = width * 4;
+ while (bytes >= 128) {
+ vstoreu(dst_bgra, Q6_V_vdelta_VV(vloadu(src_argb), ctrl));
+ src_argb += 128;
+ dst_bgra += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ vstoreu_partial(dst_bgra,
+ Q6_V_vdelta_VV(vloadu_partial_zero(src_argb, bytes), ctrl),
+ bytes);
+ }
+}
+
+void ARGBToRGBARow_HVX(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector v = vloadu_partial_zero(src_argb, b_cnt);
+ HVX_Vector out = Q6_V_vor_VV(Q6_Vw_vasl_VwR(v, 8), Q6_Vuw_vlsr_VuwR(v, 24));
+ vstoreu_partial(dst_rgba, out, b_cnt);
+ src_argb += 128;
+ dst_rgba += 128;
+ bytes -= 128;
+ }
+}
+
+void RGBAToARGBRow_HVX(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
+ int bytes = width * 4;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector v = vloadu_partial_zero(src_rgba, b_cnt);
+ HVX_Vector out = Q6_V_vor_VV(Q6_Vuw_vlsr_VuwR(v, 8), Q6_Vw_vasl_VwR(v, 24));
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src_rgba += 128;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void ARGBToAR64Row_HVX(const uint8_t* src_argb, uint16_t* dst_ar64, int width) {
+ Convert8To16Row_HVX(src_argb, dst_ar64, 16, width * 4);
+}
+
+void ARGBToAB64Row_HVX(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
+ HVX_Vector ctrl = Q6_V_vsplat_R(0x00020002);
+ int bytes = width * 4;
+ uint8_t* dst = (uint8_t*)dst_ab64;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ HVX_Vector v = Q6_V_vdelta_VV(vloadu_partial_zero(src_argb, b_cnt), ctrl);
+ HVX_VectorPair d16 = Q6_W_vshuff_VVR(v, v, -1);
+ int dst_bytes = b_cnt * 2;
+ vstoreu_partial(dst, Q6_V_lo_W(d16), dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst + 128, Q6_V_hi_W(d16), dst_bytes - 128);
+ }
+ src_argb += 128;
+ dst += 256;
+ bytes -= 128;
+ }
+}
+
+void AR64ToARGBRow_HVX(const uint16_t* src_ar64, uint8_t* dst_argb, int width) {
+ int bytes = width * 4;
+ const uint8_t* src = (const uint8_t*)src_ar64;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ int src_bytes = b_cnt * 2;
+ HVX_Vector s0 = vloadu_partial_zero(src, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ vstoreu_partial(dst_argb, Q6_Vb_vpacko_VhVh(s1, s0), b_cnt);
+ src += 256;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void AB64ToARGBRow_HVX(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
+ HVX_Vector ctrl = Q6_V_vsplat_R(0x00020002);
+ int bytes = width * 4;
+ const uint8_t* src = (const uint8_t*)src_ab64;
+ while (bytes > 0) {
+ int b_cnt = bytes >= 128 ? 128 : bytes;
+ int src_bytes = b_cnt * 2;
+ HVX_Vector s0 = vloadu_partial_zero(src, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector out = Q6_V_vdelta_VV(Q6_Vb_vpacko_VhVh(s1, s0), ctrl);
+ vstoreu_partial(dst_argb, out, b_cnt);
+ src += 256;
+ dst_argb += 128;
+ bytes -= 128;
+ }
+}
+
+void AR64ToAB64Row_HVX(const uint16_t* src_ar64,
+ uint16_t* dst_ab64,
+ int width) {
+ HVX_Vector ctrl = Q6_V_vsplat_R(0x00000404);
+ int bytes = width * 8;
+ const uint8_t* src = (const uint8_t*)src_ar64;
+ uint8_t* dst = (uint8_t*)dst_ab64;
+ while (bytes >= 128) {
+ vstoreu(dst, Q6_V_vdelta_VV(vloadu(src), ctrl));
+ src += 128;
+ dst += 128;
+ bytes -= 128;
+ }
+ if (bytes > 0) {
+ vstoreu_partial(dst, Q6_V_vdelta_VV(vloadu_partial_zero(src, bytes), ctrl),
+ bytes);
+ }
+}
+
+static __inline void YuvToRgb_HVX(HVX_Vector y,
+ HVX_Vector u,
+ HVX_Vector v,
+ HVX_Vector* b_out,
+ HVX_Vector* g_out,
+ HVX_Vector* r_out,
+ const struct YuvConstants* yuvconstants) {
+ uint8_t ub = yuvconstants->kUVToB[0];
+ uint8_t ug = yuvconstants->kUVToG[0];
+ uint8_t vg = yuvconstants->kUVToG[1];
+ uint8_t vr = yuvconstants->kUVToR[1];
+ uint16_t yg = (uint16_t)yuvconstants->kYToRgb[0];
+ int16_t yb = yuvconstants->kYBiasToRgb[0];
+
+ uint32_t yg32 = ((uint32_t)yg << 16) | yg;
+ HVX_Vector v_yb = Q6_Vh_vsplat_R(yb);
+ HVX_Vector v_ub = Q6_Vb_vsplat_R(ub);
+ HVX_Vector v_ug = Q6_Vb_vsplat_R(ug);
+ HVX_Vector v_vg = Q6_Vb_vsplat_R(vg);
+ HVX_Vector v_vr = Q6_Vb_vsplat_R(vr);
+ HVX_Vector v80 = Q6_Vb_vsplat_R(0x80);
+
+ HVX_Vector ui = Q6_Vb_vsub_VbVb(u, v80);
+ HVX_Vector vi = Q6_Vb_vsub_VbVb(v, v80);
+
+ HVX_Vector y_even_0101 = Q6_Vb_vshuffe_VbVb(y, y);
+ HVX_Vector y_odd_0101 = Q6_Vb_vshuffo_VbVb(y, y);
+ HVX_VectorPair p_even = Q6_Wuw_vmpy_VuhRuh(y_even_0101, yg32);
+ HVX_VectorPair p_odd = Q6_Wuw_vmpy_VuhRuh(y_odd_0101, yg32);
+ HVX_Vector y1_lo = Q6_Vh_vadd_VhVh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p_even), Q6_V_lo_W(p_even)), v_yb);
+ HVX_Vector y1_hi = Q6_Vh_vadd_VhVh(
+ Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(p_odd), Q6_V_lo_W(p_odd)), v_yb);
+
+ HVX_VectorPair b_uv = Q6_Wh_vmpy_VubVb(v_ub, ui);
+ HVX_Vector b_lo =
+ Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh_sat(y1_lo, Q6_V_lo_W(b_uv)), 6);
+ HVX_Vector b_hi =
+ Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh_sat(y1_hi, Q6_V_hi_W(b_uv)), 6);
+ *b_out = Q6_Vub_vsat_VhVh(b_hi, b_lo);
+
+ HVX_VectorPair g_uv =
+ Q6_Wh_vmpyacc_WhVubVb(Q6_Wh_vmpy_VubVb(v_ug, ui), v_vg, vi);
+ HVX_Vector g_lo =
+ Q6_Vh_vasr_VhR(Q6_Vh_vsub_VhVh_sat(y1_lo, Q6_V_lo_W(g_uv)), 6);
+ HVX_Vector g_hi =
+ Q6_Vh_vasr_VhR(Q6_Vh_vsub_VhVh_sat(y1_hi, Q6_V_hi_W(g_uv)), 6);
+ *g_out = Q6_Vub_vsat_VhVh(g_hi, g_lo);
+
+ HVX_VectorPair r_uv = Q6_Wh_vmpy_VubVb(v_vr, vi);
+ HVX_Vector r_lo =
+ Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh_sat(y1_lo, Q6_V_lo_W(r_uv)), 6);
+ HVX_Vector r_hi =
+ Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh_sat(y1_hi, Q6_V_hi_W(r_uv)), 6);
+ *r_out = Q6_Vub_vsat_VhVh(r_hi, r_lo);
+}
+
+static __inline void StoreARGB_HVX(uint8_t* dst_argb,
+ HVX_Vector b,
+ HVX_Vector g,
+ HVX_Vector r,
+ HVX_Vector a,
+ int w) {
+ HVX_VectorPair br = Q6_W_vshuff_VVR(r, b, -1);
+ HVX_VectorPair ga = Q6_W_vshuff_VVR(a, g, -1);
+ HVX_VectorPair p01 = Q6_W_vshuff_VVR(Q6_V_lo_W(ga), Q6_V_lo_W(br), -1);
+ HVX_VectorPair p23 = Q6_W_vshuff_VVR(Q6_V_hi_W(ga), Q6_V_hi_W(br), -1);
+ int dst_bytes = w * 4;
+ vstoreu_partial(dst_argb, Q6_V_lo_W(p01), dst_bytes > 128 ? 128 : dst_bytes);
+ if (dst_bytes > 128) {
+ vstoreu_partial(dst_argb + 128, Q6_V_hi_W(p01),
+ dst_bytes > 256 ? 128 : dst_bytes - 128);
+ }
+ if (dst_bytes > 256) {
+ vstoreu_partial(dst_argb + 256, Q6_V_lo_W(p23),
+ dst_bytes > 384 ? 128 : dst_bytes - 256);
+ }
+ if (dst_bytes > 384) {
+ vstoreu_partial(dst_argb + 384, Q6_V_hi_W(p23), dst_bytes - 384);
+ }
+}
+
+void I444ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector u = vloadu_partial_zero(src_u, w);
+ HVX_Vector v = vloadu_partial_zero(src_v, w);
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_u += 128;
+ src_v += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void I444AlphaToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector u = vloadu_partial_zero(src_u, w);
+ HVX_Vector v = vloadu_partial_zero(src_v, w);
+ HVX_Vector a = vloadu_partial_zero(src_a, w);
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_u += 128;
+ src_v += 128;
+ src_a += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void I422ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int uv_w = (w + 1) >> 1;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector u_sub = vloadu_partial_zero(src_u, uv_w);
+ HVX_Vector v_sub = vloadu_partial_zero(src_v, uv_w);
+ HVX_Vector u = Q6_V_lo_W(Q6_W_vshuff_VVR(u_sub, u_sub, -1));
+ HVX_Vector v = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sub, v_sub, -1));
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_u += 64;
+ src_v += 64;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void I422AlphaToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int uv_w = (w + 1) >> 1;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector u_sub = vloadu_partial_zero(src_u, uv_w);
+ HVX_Vector v_sub = vloadu_partial_zero(src_v, uv_w);
+ HVX_Vector a = vloadu_partial_zero(src_a, w);
+ HVX_Vector u = Q6_V_lo_W(Q6_W_vshuff_VVR(u_sub, u_sub, -1));
+ HVX_Vector v = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sub, v_sub, -1));
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_u += 64;
+ src_v += 64;
+ src_a += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void I422ToRGBARow_HVX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int uv_w = (w + 1) >> 1;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector u_sub = vloadu_partial_zero(src_u, uv_w);
+ HVX_Vector v_sub = vloadu_partial_zero(src_v, uv_w);
+ HVX_Vector u = Q6_V_lo_W(Q6_W_vshuff_VVR(u_sub, u_sub, -1));
+ HVX_Vector v = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sub, v_sub, -1));
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_rgba, a, b, g, r, w);
+ src_y += 128;
+ src_u += 64;
+ src_v += 64;
+ dst_rgba += 512;
+ width -= 128;
+ }
+}
+
+void NV12ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int uv_bytes = ((w + 1) >> 1) * 2;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector uv = vloadu_partial_zero(src_uv, uv_bytes);
+ HVX_VectorPair u_v = Q6_W_vdeal_VVR(uv, uv, -1);
+ HVX_Vector u_sub = Q6_V_lo_W(u_v);
+ HVX_Vector v_sub = Q6_V_hi_W(u_v);
+ HVX_Vector u = Q6_V_lo_W(Q6_W_vshuff_VVR(u_sub, u_sub, -1));
+ HVX_Vector v = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sub, v_sub, -1));
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_uv += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void NV21ToARGBRow_HVX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ HVX_Vector a = Q6_Vb_vsplat_R(0xff);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int uv_bytes = ((w + 1) >> 1) * 2;
+ HVX_Vector y = vloadu_partial_zero(src_y, w);
+ HVX_Vector vu = vloadu_partial_zero(src_vu, uv_bytes);
+ HVX_VectorPair v_u = Q6_W_vdeal_VVR(vu, vu, -1);
+ HVX_Vector v_sub = Q6_V_lo_W(v_u);
+ HVX_Vector u_sub = Q6_V_hi_W(v_u);
+ HVX_Vector u = Q6_V_lo_W(Q6_W_vshuff_VVR(u_sub, u_sub, -1));
+ HVX_Vector v = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sub, v_sub, -1));
+ HVX_Vector b, g, r;
+ YuvToRgb_HVX(y, u, v, &b, &g, &r, yuvconstants);
+ StoreARGB_HVX(dst_argb, b, g, r, a, w);
+ src_y += 128;
+ src_vu += 128;
+ dst_argb += 512;
+ width -= 128;
+ }
+}
+
+void ARGBToYMatrixRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ uint32_t coeff_y;
+ memcpy(&coeff_y, c->kRGBToY, sizeof(coeff_y));
+ HVX_Vector v_add_y = Q6_V_vsplat_R((uint32_t)c->kAddY[0]);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_argb + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_argb + 384, src_bytes - 384)
+ : Q6_V_vzero();
+ HVX_Vector y0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vadd_VwVw(Q6_Vuw_vrmpy_VubRub(s0, coeff_y), v_add_y), 8);
+ HVX_Vector y1 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vadd_VwVw(Q6_Vuw_vrmpy_VubRub(s1, coeff_y), v_add_y), 8);
+ HVX_Vector y2 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vadd_VwVw(Q6_Vuw_vrmpy_VubRub(s2, coeff_y), v_add_y), 8);
+ HVX_Vector y3 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vadd_VwVw(Q6_Vuw_vrmpy_VubRub(s3, coeff_y), v_add_y), 8);
+ HVX_Vector y01 = Q6_Vh_vpacke_VwVw(y1, y0);
+ HVX_Vector y23 = Q6_Vh_vpacke_VwVw(y3, y2);
+ HVX_Vector out = Q6_Vb_vpacke_VhVh(y23, y01);
+ vstoreu_partial(dst_y, out, w);
+ src_argb += 512;
+ dst_y += 128;
+ width -= 128;
+ }
+}
+
+void ARGBToUV444MatrixRow_HVX(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ int32_t coeff_u;
+ int32_t coeff_v;
+ memcpy(&coeff_u, c->kRGBToU, sizeof(coeff_u));
+ memcpy(&coeff_v, c->kRGBToV, sizeof(coeff_v));
+ HVX_Vector v_add_uv = Q6_V_vsplat_R((uint32_t)c->kAddUV[0]);
+ while (width > 0) {
+ int w = width >= 128 ? 128 : width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_argb + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_argb + 384, src_bytes - 384)
+ : Q6_V_vzero();
+
+ HVX_Vector u0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s0, coeff_u)), 8);
+ HVX_Vector u1 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s1, coeff_u)), 8);
+ HVX_Vector u2 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s2, coeff_u)), 8);
+ HVX_Vector u3 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s3, coeff_u)), 8);
+ HVX_Vector u_out =
+ Q6_Vb_vpacke_VhVh(Q6_Vh_vpacke_VwVw(u3, u2), Q6_Vh_vpacke_VwVw(u1, u0));
+ vstoreu_partial(dst_u, u_out, w);
+
+ HVX_Vector v0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s0, coeff_v)), 8);
+ HVX_Vector v1 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s1, coeff_v)), 8);
+ HVX_Vector v2 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s2, coeff_v)), 8);
+ HVX_Vector v3 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(s3, coeff_v)), 8);
+ HVX_Vector v_out =
+ Q6_Vb_vpacke_VhVh(Q6_Vh_vpacke_VwVw(v3, v2), Q6_Vh_vpacke_VwVw(v1, v0));
+ vstoreu_partial(dst_v, v_out, w);
+
+ src_argb += 512;
+ dst_u += 128;
+ dst_v += 128;
+ width -= 128;
+ }
+}
+
+void ARGBToUVMatrixRow_HVX(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ int32_t coeff_u;
+ int32_t coeff_v;
+ memcpy(&coeff_u, c->kRGBToU, sizeof(coeff_u));
+ memcpy(&coeff_v, c->kRGBToV, sizeof(coeff_v));
+ HVX_Vector v_add_uv = Q6_V_vsplat_R((uint32_t)c->kAddUV[0]);
+ HVX_Vector vtwo = Q6_Vh_vsplat_R(2);
+ const uint8_t* src_argb1 = src_argb + src_stride_argb;
+
+ int even_width = width & ~1;
+ while (even_width > 0) {
+ int w = even_width >= 64 ? 64 : even_width;
+ int uv_w = w >> 1;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t0 =
+ vloadu_partial_zero(src_argb1, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(src_argb1 + 128, src_bytes - 128)
+ : Q6_V_vzero();
+
+ HVX_VectorPair ds = Q6_W_vdeal_VVR(s1, s0, -4);
+ HVX_VectorPair dt = Q6_W_vdeal_VVR(t1, t0, -4);
+ HVX_VectorPair s_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(ds));
+ HVX_VectorPair s_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(ds));
+ HVX_VectorPair t_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(dt));
+ HVX_VectorPair t_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(dt));
+
+ HVX_Vector sum_lo = Q6_Vh_vadd_VhVh(Q6_V_lo_W(s_even), Q6_V_lo_W(s_odd));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_even));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_odd));
+ sum_lo = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_lo, vtwo), 2);
+
+ HVX_Vector sum_hi = Q6_Vh_vadd_VhVh(Q6_V_hi_W(s_even), Q6_V_hi_W(s_odd));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_even));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_odd));
+ sum_hi = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_hi, vtwo), 2);
+
+ HVX_Vector avg = Q6_Vub_vpack_VhVh_sat(sum_hi, sum_lo);
+ HVX_Vector u0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(avg, coeff_u)), 8);
+ HVX_Vector v0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(avg, coeff_v)), 8);
+ HVX_Vector u_out =
+ Q6_Vb_vpacke_VhVh(Q6_V_vzero(), Q6_Vh_vpacke_VwVw(Q6_V_vzero(), u0));
+ HVX_Vector v_out =
+ Q6_Vb_vpacke_VhVh(Q6_V_vzero(), Q6_Vh_vpacke_VwVw(Q6_V_vzero(), v0));
+ vstoreu_partial(dst_u, u_out, uv_w);
+ vstoreu_partial(dst_v, v_out, uv_w);
+
+ src_argb += w * 4;
+ src_argb1 += w * 4;
+ dst_u += uv_w;
+ dst_v += uv_w;
+ even_width -= w;
+ }
+ if (width & 1) {
+ HVX_Vector s0 = vloadu_partial_zero(src_argb, 4);
+ HVX_Vector t0 = vloadu_partial_zero(src_argb1, 4);
+ HVX_Vector avg = Q6_Vub_vavg_VubVub_rnd(s0, t0);
+ HVX_Vector u0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(avg, coeff_u)), 8);
+ HVX_Vector v0 = Q6_Vuw_vlsr_VuwR(
+ Q6_Vw_vsub_VwVw(v_add_uv, Q6_Vw_vrmpy_VubRb(avg, coeff_v)), 8);
+ vstoreu_partial(dst_u, u0, 1);
+ vstoreu_partial(dst_v, v0, 1);
+ }
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
diff --git a/libs/libyuv/source/row_lasx.cc b/libs/libyuv/source/row_lasx.cc
new file mode 100644
index 0000000000..12d267ad23
--- /dev/null
+++ b/libs/libyuv/source/row_lasx.cc
@@ -0,0 +1,3558 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+#include "libyuv/video_common.h"
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
+
+#include "libyuv/basic_types.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int count) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count)
+ :
+ : "memory", "$xr0");
+}
+
+void SetRow_LASX(uint8_t* dst, uint8_t v8, int count) {
+ asm volatile(
+ "xvreplgr2vr.b $xr0, %[val]\n\t"
+ "1:\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst), [cnt] "+r"(count)
+ : [val] "r"(v8)
+ : "memory", "$xr0");
+}
+
+void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width) {
+ asm volatile(
+ "xvreplgr2vr.w $xr0, %[val]\n\t"
+ "1:\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(v32)
+ : "memory", "$xr0");
+}
+
+void MirrorRow_LASX(const uint8_t* src_y, uint8_t* dst_y, int width) {
+ static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8,
+ 7, 6, 5, 4, 3, 2, 1, 0};
+ src_y += width - 32;
+ asm volatile(
+ "vld $vr1, %[rev], 0\n\t"
+ "xvpermi.q $xr1, $xr1, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 16\n\t"
+ "vld $vr31, %[src], 0\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [rev] "r"(kRev)
+ : "memory", "$xr0", "$xr1", "$xr31");
+}
+
+void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9,
+ 6, 7, 4, 5, 2, 3, 0, 1};
+ src_uv += width * 2 - 32;
+ asm volatile(
+ "vld $vr1, %[rev], 0\n\t"
+ "xvpermi.q $xr1, $xr1, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 16\n\t"
+ "vld $vr31, %[src], 0\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width)
+ : [rev] "r"(kRevUV)
+ : "memory", "$xr0", "$xr1", "$xr31");
+}
+
+void ARGBMirrorRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ src_argb += width * 4 - 32;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 16\n\t"
+ "vld $vr31, %[src], 0\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvshuf4i.w $xr0, $xr0, 0x1b\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr31");
+}
+
+void MirrorSplitUVRow_LASX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9,
+ 6, 7, 4, 5, 2, 3, 0, 1};
+ src_uv += width * 2 - 64;
+ asm volatile(
+ "vld $vr4, %[rev], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 48\n\t"
+ "vld $vr31, %[src], 16\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 32\n\t"
+ "vld $vr31, %[src], 0\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t"
+ "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvst $xr2, %[du], 0\n\t"
+ "xvst $xr3, %[dv], 0\n\t"
+ "addi.d %[src], %[src], -64\n\t"
+ "addi.d %[du], %[du], 32\n\t"
+ "addi.d %[dv], %[dv], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width)
+ : [rev] "r"(kRevUV)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void SplitUVRow_LASX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvst $xr2, %[du], 0\n\t"
+ "xvst $xr3, %[dv], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[du], %[du], 32\n\t"
+ "addi.d %[dv], %[dv], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31");
+}
+
+void MergeUVRow_LASX(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[su], 0\n\t"
+ "xvld $xr1, %[sv], 0\n\t"
+ "xvilvl.b $xr2, $xr1, $xr0\n\t"
+ "xvilvh.b $xr3, $xr1, $xr0\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr3, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[su], %[su], 32\n\t"
+ "addi.d %[sv], %[sv], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31");
+}
+
+void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickod.b $xr0, $xr1, $xr0\n\t"
+ "xvpickod.b $xr0, $xr0, $xr0\n\t"
+ "xvpermi.d $xr31, $xr0, 0xd8\n\t"
+ "vst $vr31, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr31");
+}
+
+void I422ToYUY2Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[sy], 0\n\t"
+ "vld $vr1, %[su], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "vld $vr2, %[sv], 0\n\t"
+ "xvpermi.d $xr2, $xr2, 0xd8\n\t"
+ "xvilvl.b $xr3, $xr2, $xr1\n\t"
+ "xvilvl.b $xr4, $xr3, $xr0\n\t"
+ "xvilvh.b $xr5, $xr3, $xr0\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr4, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr5, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 32\n\t"
+ "addi.d %[su], %[su], 16\n\t"
+ "addi.d %[sv], %[sv], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_yuy2), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31");
+}
+
+void I422ToUYVYRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[sy], 0\n\t"
+ "vld $vr1, %[su], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "vld $vr2, %[sv], 0\n\t"
+ "xvpermi.d $xr2, $xr2, 0xd8\n\t"
+ "xvilvl.b $xr3, $xr2, $xr1\n\t"
+ "xvilvl.b $xr4, $xr0, $xr3\n\t"
+ "xvilvh.b $xr5, $xr0, $xr3\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr4, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr5, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 32\n\t"
+ "addi.d %[su], %[su], 16\n\t"
+ "addi.d %[sv], %[sv], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_uyvy), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31");
+}
+
+void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr31");
+}
+
+void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickod.b $xr2, $xr1, $xr0\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr31");
+}
+
+void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickod.b $xr2, $xr1, $xr0\n\t"
+ "xvpickev.b $xr3, $xr2, $xr2\n\t"
+ "xvpickod.b $xr4, $xr2, $xr2\n\t"
+ "xvpermi.d $xr31, $xr3, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "xvpermi.d $xr31, $xr4, 0xd8\n\t"
+ "vst $vr31, %[dv], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void UYVYToUV422Row_LASX(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickev.b $xr3, $xr2, $xr2\n\t"
+ "xvpickod.b $xr4, $xr2, $xr2\n\t"
+ "xvpermi.d $xr31, $xr3, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "xvpermi.d $xr31, $xr4, 0xd8\n\t"
+ "vst $vr31, %[dv], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void YUY2ToUVRow_LASX(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[s0], 16\n\t"
+ "vld $vr31, %[s0], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "vld $vr2, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 32\n\t"
+ "xvpermi.q $xr2, $xr31, 0x02\n\t"
+ "vld $vr3, %[s1], 16\n\t"
+ "vld $vr31, %[s1], 48\n\t"
+ "xvpermi.q $xr3, $xr31, 0x02\n\t"
+ "xvpickod.b $xr0, $xr1, $xr0\n\t"
+ "xvpickod.b $xr2, $xr3, $xr2\n\t"
+ "xvavgr.bu $xr0, $xr0, $xr2\n\t"
+ "xvpickev.b $xr1, $xr0, $xr0\n\t"
+ "xvpickod.b $xr2, $xr0, $xr0\n\t"
+ "xvpermi.d $xr31, $xr1, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "xvpermi.d $xr31, $xr2, 0xd8\n\t"
+ "vst $vr31, %[dv], 0\n\t"
+ "addi.d %[s0], %[s0], 64\n\t"
+ "addi.d %[s1], %[s1], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31");
+}
+
+void UYVYToUVRow_LASX(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[s0], 16\n\t"
+ "vld $vr31, %[s0], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "vld $vr2, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 32\n\t"
+ "xvpermi.q $xr2, $xr31, 0x02\n\t"
+ "vld $vr3, %[s1], 16\n\t"
+ "vld $vr31, %[s1], 48\n\t"
+ "xvpermi.q $xr3, $xr31, 0x02\n\t"
+ "xvpickev.b $xr0, $xr1, $xr0\n\t"
+ "xvpickev.b $xr2, $xr3, $xr2\n\t"
+ "xvavgr.bu $xr0, $xr0, $xr2\n\t"
+ "xvpickev.b $xr1, $xr0, $xr0\n\t"
+ "xvpickod.b $xr2, $xr0, $xr0\n\t"
+ "xvpermi.d $xr31, $xr1, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "xvpermi.d $xr31, $xr2, 0xd8\n\t"
+ "vst $vr31, %[dv], 0\n\t"
+ "addi.d %[s0], %[s0], 64\n\t"
+ "addi.d %[s1], %[s1], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31");
+}
+
+void I422ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22",
+ "$xr23", "$xr24", "$xr31");
+}
+
+void I422ToRGBARow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr2, $xr5\n\t"
+ "xvilvl.b $xr7, $xr4, $xr3\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22",
+ "$xr23", "$xr24", "$xr31");
+}
+
+void I422AlphaToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "vld $vr5, %[sa], 0\n\t"
+ "xvpermi.d $xr5, $xr5, 0xd8\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[sa], %[sa], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22",
+ "$xr23", "$xr24", "$xr31");
+}
+
+void I422ToRGB24Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9,
+ 10, 12, 13, 14, 16, 17, 18, 20};
+ static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30,
+ 0, 0, 0, 0, 0, 0, 0, 0};
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "vld $vr12, %[s0], 0\n\t"
+ "xvpermi.q $xr12, $xr12, 0x00\n\t"
+ "vld $vr13, %[s1], 0\n\t"
+ "xvpermi.q $xr13, $xr13, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr4, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "xvshuf.b $xr10, $xr9, $xr8, $xr12\n\t"
+ "xvshuf.b $xr11, $xr9, $xr8, $xr13\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr10, 0x01\n\t"
+ "vst $vr31, %[dst], 24\n\t"
+ "vstelm.d $vr11, %[dst], 16, 0\n\t"
+ "xvpermi.q $xr31, $xr11, 0x01\n\t"
+ "vstelm.d $vr31, %[dst], 40, 0\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 48\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7",
+ "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr20", "$xr21",
+ "$xr22", "$xr23", "$xr24", "$xr31");
+}
+
+void I422ToRGB565Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr14, 31\n\t"
+ "xvsrli.h $xr14, $xr14, 8\n\t"
+ "xvldi $xr15, 63\n\t"
+ "xvsrli.h $xr15, $xr15, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvsrli.b $xr3, $xr3, 2\n\t"
+ "xvsrli.b $xr4, $xr4, 3\n\t"
+ "xvilvl.b $xr2, $xr2, $xr2\n\t"
+ "xvilvl.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr4, $xr4, $xr4\n\t"
+ "xvand.v $xr2, $xr2, $xr14\n\t"
+ "xvand.v $xr3, $xr3, $xr15\n\t"
+ "xvand.v $xr4, $xr4, $xr14\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr4, $xr4, 11\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvor.v $xr2, $xr2, $xr4\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11",
+ "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31");
+}
+
+void I422ToARGB4444Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr14, 15\n\t"
+ "xvsrli.h $xr14, $xr14, 8\n\t"
+ "xvldi $xr15, 240\n\t"
+ "xvsrli.h $xr15, $xr15, 8\n\t"
+ "xvslli.h $xr15, $xr15, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvsrli.b $xr2, $xr2, 4\n\t"
+ "xvsrli.b $xr3, $xr3, 4\n\t"
+ "xvsrli.b $xr4, $xr4, 4\n\t"
+ "xvilvl.b $xr2, $xr2, $xr2\n\t"
+ "xvilvl.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr4, $xr4, $xr4\n\t"
+ "xvand.v $xr2, $xr2, $xr14\n\t"
+ "xvand.v $xr3, $xr3, $xr14\n\t"
+ "xvand.v $xr4, $xr4, $xr14\n\t"
+ "xvslli.h $xr3, $xr3, 4\n\t"
+ "xvslli.h $xr4, $xr4, 8\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvor.v $xr2, $xr2, $xr4\n\t"
+ "xvor.v $xr2, $xr2, $xr15\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11",
+ "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31");
+}
+
+void I422ToARGB1555Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr14, 31\n\t"
+ "xvsrli.h $xr14, $xr14, 8\n\t"
+ "xvldi $xr15, 128\n\t"
+ "xvsrli.h $xr15, $xr15, 8\n\t"
+ "xvslli.h $xr15, $xr15, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvldrepl.d $xr10, %[su], 0\n\t"
+ "xvshuf4i.w $xr31, $xr10, 0x01\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "xvldrepl.d $xr11, %[sv], 0\n\t"
+ "xvshuf4i.w $xr31, $xr11, 0x01\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvilvl.b $xr10, $xr10, $xr10\n\t"
+ "xvilvl.b $xr11, $xr11, $xr11\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvsrli.b $xr3, $xr3, 3\n\t"
+ "xvsrli.b $xr4, $xr4, 3\n\t"
+ "xvilvl.b $xr2, $xr2, $xr2\n\t"
+ "xvilvl.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr4, $xr4, $xr4\n\t"
+ "xvand.v $xr2, $xr2, $xr14\n\t"
+ "xvand.v $xr3, $xr3, $xr14\n\t"
+ "xvand.v $xr4, $xr4, $xr14\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr4, $xr4, 10\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvor.v $xr2, $xr2, $xr4\n\t"
+ "xvor.v $xr2, $xr2, $xr15\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11",
+ "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31");
+}
+
+void NV12ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr1, %[suv], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "xvilvl.h $xr1, $xr1, $xr1\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[suv], %[suv], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24",
+ "$xr31");
+}
+
+void NV21ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr1, %[suv], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "xvshuf4i.b $xr1, $xr1, 0xb1\n\t"
+ "xvilvl.h $xr1, $xr1, $xr1\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[suv], %[suv], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24",
+ "$xr31");
+}
+
+void NV12ToRGB565Row_LASX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr14, 31\n\t"
+ "xvsrli.h $xr14, $xr14, 8\n\t"
+ "xvldi $xr15, 63\n\t"
+ "xvsrli.h $xr15, $xr15, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr1, %[suv], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "xvilvl.h $xr1, $xr1, $xr1\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvsrli.b $xr3, $xr3, 2\n\t"
+ "xvsrli.b $xr4, $xr4, 3\n\t"
+ "xvilvl.b $xr2, $xr2, $xr2\n\t"
+ "xvilvl.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr4, $xr4, $xr4\n\t"
+ "xvand.v $xr2, $xr2, $xr14\n\t"
+ "xvand.v $xr3, $xr3, $xr15\n\t"
+ "xvand.v $xr4, $xr4, $xr14\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr4, $xr4, 11\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvor.v $xr2, $xr2, $xr4\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[suv], %[suv], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr14", "$xr15",
+ "$xr20", "$xr21", "$xr22", "$xr23", "$xr24");
+}
+
+void I444ToARGBRow_LASX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr10, %[su], 0\n\t"
+ "xvpermi.d $xr10, $xr10, 0xd8\n\t"
+ "vld $vr11, %[sv], 0\n\t"
+ "xvpermi.d $xr11, $xr11, 0xd8\n\t"
+ "xvilvl.b $xr1, $xr11, $xr10\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 16\n\t"
+ "addi.d %[sv], %[sv], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22",
+ "$xr23", "$xr24", "$xr31");
+}
+
+void I400ToARGBRow_LASX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vld $vr23, %[c], 96\n\t"
+ "xvpermi.q $xr23, $xr23, 0x00\n\t"
+ "vld $vr24, %[c], 128\n\t"
+ "xvpermi.q $xr24, $xr24, 0x00\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvssrani.bu.h $xr2, $xr0, 6\n\t"
+ "xvilvl.b $xr6, $xr2, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr2\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9",
+ "$xr23", "$xr24", "$xr31");
+}
+
+void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr2, %[sy], 0\n\t"
+ "xvpermi.d $xr2, $xr2, 0xd8\n\t"
+ "xvilvl.b $xr6, $xr2, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr2\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr31");
+}
+
+void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "xvld $xr10, %[src], 0\n\t"
+ "xvpickev.b $xr0, $xr10, $xr10\n\t"
+ "xvpickod.b $xr1, $xr10, $xr10\n\t"
+ "xvilvl.h $xr1, $xr1, $xr1\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23",
+ "$xr24", "$xr31");
+}
+
+void UYVYToARGBRow_LASX(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "xvldrepl.w $xr20, %[c], 0\n\t"
+ "xvldrepl.w $xr21, %[c], 32\n\t"
+ "xvldrepl.w $xr22, %[c], 64\n\t"
+ "xvldrepl.h $xr23, %[c], 96\n\t"
+ "xvldrepl.h $xr24, %[c], 128\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "xvld $xr10, %[src], 0\n\t"
+ "xvpickod.b $xr0, $xr10, $xr10\n\t"
+ "xvpickev.b $xr1, $xr10, $xr10\n\t"
+ "xvilvl.h $xr1, $xr1, $xr1\n\t"
+ "xvilvl.b $xr0, $xr0, $xr0\n\t"
+ "xvxori.b $xr1, $xr1, 0x80\n\t"
+ "xvmuh.hu $xr0, $xr0, $xr23\n\t"
+ "xvadd.h $xr0, $xr0, $xr24\n\t"
+ "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t"
+ "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t"
+ "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t"
+ "xvsadd.h $xr2, $xr0, $xr2\n\t"
+ "xvsadd.h $xr4, $xr0, $xr4\n\t"
+ "xvssub.h $xr3, $xr0, $xr3\n\t"
+ "xvssrani.bu.h $xr2, $xr2, 6\n\t"
+ "xvssrani.bu.h $xr3, $xr3, 6\n\t"
+ "xvssrani.bu.h $xr4, $xr4, 6\n\t"
+ "xvilvl.b $xr6, $xr3, $xr2\n\t"
+ "xvilvl.b $xr7, $xr5, $xr4\n\t"
+ "xvilvl.h $xr8, $xr7, $xr6\n\t"
+ "xvilvh.h $xr9, $xr7, $xr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr8, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr9, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23",
+ "$xr24", "$xr31");
+}
+
+static LIBYUV_NOINLINE void ARGBTo24Helper_LASX(const uint8_t* src_argb,
+ uint8_t* dst_24,
+ const uint8_t* shuf,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 64\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 80\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "vld $vr2, %[src], 32\n\t"
+ "vld $vr31, %[src], 96\n\t"
+ "xvpermi.q $xr2, $xr31, 0x02\n\t"
+ "vld $vr3, %[src], 48\n\t"
+ "vld $vr31, %[src], 112\n\t"
+ "xvpermi.q $xr3, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t"
+ "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t"
+ "xvshuf.b $xr2, $xr2, $xr2, $xr4\n\t"
+ "xvshuf.b $xr3, $xr3, $xr3, $xr4\n\t"
+ "vstelm.d $vr0, %[dst], 0, 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vstelm.d $vr31, %[dst], 48, 0\n\t"
+ "vstelm.w $vr0, %[dst], 8, 2\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 56, 2\n\t"
+ "vstelm.w $vr1, %[dst], 12, 0\n\t"
+ "vstelm.w $vr1, %[dst], 16, 1\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 60, 0\n\t"
+ "vstelm.w $vr31, %[dst], 64, 1\n\t"
+ "vstelm.w $vr1, %[dst], 20, 2\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 68, 2\n\t"
+ "vstelm.d $vr2, %[dst], 24, 0\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vstelm.d $vr31, %[dst], 72, 0\n\t"
+ "vstelm.w $vr2, %[dst], 32, 2\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 80, 2\n\t"
+ "vstelm.w $vr3, %[dst], 36, 0\n\t"
+ "vstelm.w $vr3, %[dst], 40, 1\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 84, 0\n\t"
+ "vstelm.w $vr31, %[dst], 88, 1\n\t"
+ "vstelm.w $vr3, %[dst], 44, 2\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 92, 2\n\t"
+ "addi.d %[src], %[src], 128\n\t"
+ "addi.d %[dst], %[dst], 96\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void ARGBToRGB24Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width) {
+ static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9,
+ 10, 12, 13, 14, 0, 0, 0, 0};
+ ARGBTo24Helper_LASX(src_argb, dst_rgb24, kShuf, width);
+}
+
+void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
+ static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9,
+ 8, 14, 13, 12, 0, 0, 0, 0};
+ ARGBTo24Helper_LASX(src_argb, dst_raw, kShuf, width);
+}
+
+static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24,
+ uint8_t* dst_argb,
+ const uint8_t* shuf,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "xvpermi.q $xr5, $xr5, 0x00\n\t"
+ "xvldi $xr6, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 12\n\t"
+ "vld $vr31, %[src], 60\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "vld $vr2, %[src], 24\n\t"
+ "vld $vr31, %[src], 72\n\t"
+ "xvpermi.q $xr2, $xr31, 0x02\n\t"
+ "vld $vr3, %[src], 32\n\t"
+ "vld $vr31, %[src], 80\n\t"
+ "xvpermi.q $xr3, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr6, $xr0, $xr4\n\t"
+ "xvshuf.b $xr1, $xr6, $xr1, $xr4\n\t"
+ "xvshuf.b $xr2, $xr6, $xr2, $xr4\n\t"
+ "xvshuf.b $xr3, $xr6, $xr3, $xr5\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vst $vr31, %[dst], 64\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vst $vr31, %[dst], 80\n\t"
+ "vst $vr2, %[dst], 32\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vst $vr31, %[dst], 96\n\t"
+ "vst $vr3, %[dst], 48\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vst $vr31, %[dst], 112\n\t"
+ "addi.d %[src], %[src], 96\n\t"
+ "addi.d %[dst], %[dst], 128\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr31");
+}
+
+void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8,
+ 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8,
+ 9, 16, 10, 11, 12, 16, 13, 14, 15, 16};
+ RGB24ToARGBHelper_LASX(src_rgb24, dst_argb, kShuf, width);
+}
+
+void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6,
+ 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8,
+ 7, 16, 12, 11, 10, 16, 15, 14, 13, 16};
+ RGB24ToARGBHelper_LASX(src_raw, dst_argb, kShuf, width);
+}
+
+void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10,
+ 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7,
+ 12, 11, 10, 15, 14, 13, 0, 0, 0, 0};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "xvpermi.q $xr5, $xr5, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 12\n\t"
+ "vld $vr31, %[src], 60\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "vld $vr2, %[src], 24\n\t"
+ "vld $vr31, %[src], 72\n\t"
+ "xvpermi.q $xr2, $xr31, 0x02\n\t"
+ "vld $vr3, %[src], 32\n\t"
+ "vld $vr31, %[src], 80\n\t"
+ "xvpermi.q $xr3, $xr31, 0x02\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t"
+ "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t"
+ "xvshuf.b $xr2, $xr2, $xr2, $xr4\n\t"
+ "xvshuf.b $xr3, $xr3, $xr3, $xr5\n\t"
+ "vstelm.d $vr0, %[dst], 0, 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vstelm.d $vr31, %[dst], 48, 0\n\t"
+ "vstelm.w $vr0, %[dst], 8, 2\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 56, 2\n\t"
+ "vstelm.w $vr1, %[dst], 12, 0\n\t"
+ "vstelm.w $vr1, %[dst], 16, 1\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 60, 0\n\t"
+ "vstelm.w $vr31, %[dst], 64, 1\n\t"
+ "vstelm.w $vr1, %[dst], 20, 2\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 68, 2\n\t"
+ "vstelm.d $vr2, %[dst], 24, 0\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vstelm.d $vr31, %[dst], 72, 0\n\t"
+ "vstelm.w $vr2, %[dst], 32, 2\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 80, 2\n\t"
+ "vstelm.w $vr3, %[dst], 36, 0\n\t"
+ "vstelm.w $vr3, %[dst], 40, 1\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 84, 0\n\t"
+ "vstelm.w $vr31, %[dst], 88, 1\n\t"
+ "vstelm.w $vr3, %[dst], 44, 2\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vstelm.w $vr31, %[dst], 92, 2\n\t"
+ "addi.d %[src], %[src], 96\n\t"
+ "addi.d %[dst], %[dst], 96\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width)
+ : [shuf] "r"(kShuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31");
+}
+
+void ARGBToRGB565Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvsrli.b $xr3, $xr3, 2\n\t"
+ "xvand.v $xr4, $xr2, $xr6\n\t"
+ "xvsrli.h $xr2, $xr2, 8\n\t"
+ "xvand.v $xr3, $xr3, $xr7\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr2, $xr2, 11\n\t"
+ "xvor.v $xr4, $xr4, $xr3\n\t"
+ "xvor.v $xr4, $xr4, $xr2\n\t"
+ "xvst $xr4, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7",
+ "$xr31");
+}
+
+void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ asm volatile(
+ "xvreplgr2vr.w $xr5, %[dith]\n\t"
+ "xvilvl.b $xr5, $xr5, $xr5\n\t"
+ "xvilvl.h $xr5, $xr5, $xr5\n\t"
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvsadd.bu $xr0, $xr0, $xr5\n\t"
+ "xvpickev.b $xr2, $xr0, $xr0\n\t"
+ "xvpickod.b $xr3, $xr0, $xr0\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvsrli.b $xr3, $xr3, 2\n\t"
+ "xvand.v $xr4, $xr2, $xr6\n\t"
+ "xvsrli.h $xr2, $xr2, 8\n\t"
+ "xvand.v $xr3, $xr3, $xr7\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr2, $xr2, 11\n\t"
+ "xvor.v $xr4, $xr4, $xr3\n\t"
+ "xvor.v $xr4, $xr4, $xr2\n\t"
+ "xvpermi.d $xr31, $xr4, 0xd8\n\t"
+ "vst $vr31, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ : [dith] "r"(dither4)
+ : "memory", "$xr0", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7",
+ "$xr31");
+}
+
+void ARGBToARGB1555Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvsrli.b $xr2, $xr2, 3\n\t"
+ "xvand.v $xr4, $xr2, $xr6\n\t"
+ "xvsrli.h $xr2, $xr2, 8\n\t"
+ "xvsrli.h $xr5, $xr3, 15\n\t"
+ "xvsrli.b $xr3, $xr3, 3\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvslli.h $xr3, $xr3, 5\n\t"
+ "xvslli.h $xr2, $xr2, 10\n\t"
+ "xvslli.h $xr5, $xr5, 15\n\t"
+ "xvor.v $xr4, $xr4, $xr3\n\t"
+ "xvor.v $xr4, $xr4, $xr2\n\t"
+ "xvor.v $xr4, $xr4, $xr5\n\t"
+ "xvst $xr4, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr31");
+}
+
+void ARGBToARGB4444Row_LASX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvsrli.b $xr2, $xr2, 4\n\t"
+ "xvsrli.b $xr3, $xr3, 4\n\t"
+ "xvslli.b $xr3, $xr3, 4\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31");
+}
+
+void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr1, $xr1, $xr1\n\t"
+ "xvpickev.b $xr2, $xr2, $xr2\n\t"
+ "xvpickev.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr8, $xr2, $xr1\n\t"
+ "xvilvl.b $xr9, $xr5, $xr3\n\t"
+ "xvilvl.h $xr10, $xr9, $xr8\n\t"
+ "xvilvh.h $xr11, $xr9, $xr8\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr10, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr11, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr11, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr31");
+}
+
+void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr1, $xr1, $xr1\n\t"
+ "xvpickev.b $xr2, $xr2, $xr2\n\t"
+ "xvpickev.b $xr3, $xr3, $xr3\n\t"
+ "xvpickev.b $xr5, $xr5, $xr5\n\t"
+ "xvilvl.b $xr8, $xr2, $xr1\n\t"
+ "xvilvl.b $xr9, $xr5, $xr3\n\t"
+ "xvilvl.h $xr10, $xr9, $xr8\n\t"
+ "xvilvh.h $xr11, $xr9, $xr8\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr10, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr11, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr11, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr8", "$xr9", "$xr10", "$xr11", "$xr31");
+}
+
+void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 0x0f\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.b $xr2, $xr0, 4\n\t"
+ "xvslli.b $xr3, $xr1, 4\n\t"
+ "xvor.v $xr1, $xr1, $xr3\n\t"
+ "xvslli.b $xr3, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvilvl.b $xr4, $xr2, $xr1\n\t"
+ "xvilvh.b $xr5, $xr2, $xr1\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr4, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr5, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr31");
+}
+
+void ARGBMultiplyRow_LASX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t"
+ "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t"
+ "xvsrari.h $xr2, $xr2, 8\n\t"
+ "xvsrari.h $xr3, $xr3, 8\n\t"
+ "xvpackev.b $xr2, $xr3, $xr2\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3");
+}
+
+void ARGBAddRow_LASX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvsadd.bu $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1");
+}
+
+void ARGBSubtractRow_LASX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvssub.bu $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1");
+}
+
+void ARGBAttenuateRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16,
+ 11, 11, 11, 16, 15, 15, 15, 16};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "xvldi $xr6, 255\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvshuf.b $xr1, $xr5, $xr0, $xr4\n\t"
+ "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t"
+ "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t"
+ "xvadd.h $xr2, $xr2, $xr6\n\t"
+ "xvadd.h $xr3, $xr3, $xr6\n\t"
+ "xvpackod.b $xr2, $xr3, $xr2\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(kAlpha)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6");
+}
+
+void ARGBBlendRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7,
+ 11, 11, 11, 11, 15, 15, 15, 15};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "xvldi $xr5, 0xff\n\t"
+ "xvslli.w $xr5, $xr5, 24\n\t"
+ "xvldi $xr6, 1\n\t"
+ "xvslli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr8, 0\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvshuf.b $xr2, $xr0, $xr0, $xr4\n\t"
+ "xvilvl.b $xr3, $xr8, $xr2\n\t"
+ "xvilvh.b $xr2, $xr8, $xr2\n\t"
+ "xvsub.h $xr3, $xr6, $xr3\n\t"
+ "xvsub.h $xr2, $xr6, $xr2\n\t"
+ "xvilvl.b $xr7, $xr8, $xr1\n\t"
+ "xvilvh.b $xr1, $xr8, $xr1\n\t"
+ "xvmul.h $xr7, $xr7, $xr3\n\t"
+ "xvmul.h $xr1, $xr1, $xr2\n\t"
+ "xvsrlni.b.h $xr1, $xr7, 8\n\t"
+ "xvsadd.bu $xr0, $xr0, $xr1\n\t"
+ "xvor.v $xr0, $xr0, $xr5\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [shuf] "r"(kAlpha)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8");
+}
+
+void ARGBOrRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ asm volatile(
+ "xvreplgr2vr.w $xr1, %[val]\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvor.v $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(value)
+ : "memory", "$xr0", "$xr1");
+}
+
+void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4,
+ 8, 8, 8, 8, 12, 12, 12, 12};
+ asm volatile(
+ "vld $vr8, %[shuf], 0\n\t"
+ "xvpermi.q $xr8, $xr8, 0x00\n\t"
+ "xvldi $xr9, 1\n\t"
+ "xvsrli.h $xr10, $xr9, 8\n\t"
+ "xvldi $xr11, 0xff\n\t"
+ "xvsrli.w $xr11, $xr11, 8\n\t"
+ "xvldi $xr12, 0\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvsrli.w $xr2, $xr0, 24\n\t"
+ "xvsrli.w $xr3, $xr1, 24\n\t"
+ "xvmul.h $xr4, $xr3, $xr2\n\t"
+ "xvadd.h $xr4, $xr4, $xr10\n\t"
+ "xvmuh.hu $xr4, $xr4, $xr9\n\t"
+ "xvsub.w $xr4, $xr2, $xr4\n\t"
+ "xvadd.w $xr4, $xr3, $xr4\n\t"
+ "xvseq.w $xr5, $xr3, $xr12\n\t"
+ "xvsrli.w $xr5, $xr5, 24\n\t"
+ "xvor.v $xr6, $xr2, $xr5\n\t"
+ "xvor.v $xr7, $xr4, $xr5\n\t"
+ "xvslli.w $xr5, $xr6, 8\n\t"
+ "xvsub.w $xr5, $xr5, $xr6\n\t"
+ "xvffint.s.wu $xr5, $xr5\n\t"
+ "xvffint.s.wu $xr7, $xr7\n\t"
+ "xvfdiv.s $xr5, $xr5, $xr7\n\t"
+ "xvftintrz.wu.s $xr5, $xr5\n\t"
+ "xvshuf.b $xr5, $xr5, $xr5, $xr8\n\t"
+ "xvnor.v $xr6, $xr5, $xr5\n\t"
+ "xvmulwev.h.bu $xr2, $xr1, $xr6\n\t"
+ "xvmulwod.h.bu $xr3, $xr1, $xr6\n\t"
+ "xvmulwev.h.bu $xr7, $xr0, $xr5\n\t"
+ "xvmulwod.h.bu $xr0, $xr0, $xr5\n\t"
+ "xvadd.h $xr2, $xr2, $xr7\n\t"
+ "xvadd.h $xr3, $xr3, $xr0\n\t"
+ "xvadd.h $xr2, $xr2, $xr10\n\t"
+ "xvadd.h $xr3, $xr3, $xr10\n\t"
+ "xvmuh.hu $xr2, $xr2, $xr9\n\t"
+ "xvmuh.hu $xr3, $xr3, $xr9\n\t"
+ "xvslli.h $xr3, $xr3, 8\n\t"
+ "xvor.v $xr2, $xr2, $xr3\n\t"
+ "xvand.v $xr2, $xr2, $xr11\n\t"
+ "xvslli.w $xr4, $xr4, 24\n\t"
+ "xvor.v $xr2, $xr2, $xr4\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [shuf] "r"(kShufRatio)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12");
+}
+
+void ARGBShadeRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ uint32_t value) {
+ asm volatile(
+ "xvreplgr2vr.w $xr4, %[val]\n\t"
+ "xvilvl.b $xr5, $xr4, $xr4\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvilvl.b $xr1, $xr0, $xr0\n\t"
+ "xvilvh.b $xr2, $xr0, $xr0\n\t"
+ "xvmuh.hu $xr1, $xr1, $xr5\n\t"
+ "xvmuh.hu $xr2, $xr2, $xr5\n\t"
+ "xvsrlni.b.h $xr2, $xr1, 8\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(value)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5");
+}
+
+void ARGBShuffleRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(shuffler)
+ : "memory", "$xr0", "$xr4");
+}
+
+void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "xvldi $xr4, 29\n\t"
+ "xvldi $xr5, 150\n\t"
+ "xvldi $xr6, 77\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvmulwev.h.bu $xr7, $xr2, $xr4\n\t"
+ "xvmaddwod.h.bu $xr7, $xr2, $xr6\n\t"
+ "xvmaddwev.h.bu $xr7, $xr3, $xr5\n\t"
+ "xvsrlri.h $xr7, $xr7, 8\n\t"
+ "xvsrli.h $xr3, $xr3, 8\n\t"
+ "xvslli.h $xr3, $xr3, 8\n\t"
+ "xvor.v $xr3, $xr7, $xr3\n\t"
+ "xvslli.h $xr8, $xr7, 8\n\t"
+ "xvor.v $xr7, $xr7, $xr8\n\t"
+ "xvilvl.h $xr0, $xr3, $xr7\n\t"
+ "xvilvh.h $xr1, $xr3, $xr7\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr31");
+}
+
+void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width) {
+ asm volatile(
+ "xvldi $xr4, 17\n\t"
+ "xvldi $xr5, 68\n\t"
+ "xvldi $xr6, 35\n\t"
+ "xvldi $xr7, 22\n\t"
+ "xvldi $xr8, 88\n\t"
+ "xvldi $xr9, 45\n\t"
+ "xvldi $xr10, 24\n\t"
+ "xvldi $xr11, 98\n\t"
+ "xvldi $xr12, 50\n\t"
+ "1:\n\t"
+ "vld $vr0, %[dst], 0\n\t"
+ "vld $vr31, %[dst], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "vld $vr1, %[dst], 16\n\t"
+ "vld $vr31, %[dst], 48\n\t"
+ "xvpermi.q $xr1, $xr31, 0x02\n\t"
+ "xvpickev.b $xr2, $xr1, $xr0\n\t"
+ "xvpickod.b $xr3, $xr1, $xr0\n\t"
+ "xvmulwev.h.bu $xr13, $xr2, $xr4\n\t"
+ "xvmaddwod.h.bu $xr13, $xr2, $xr6\n\t"
+ "xvmaddwev.h.bu $xr13, $xr3, $xr5\n\t"
+ "xvmulwev.h.bu $xr14, $xr2, $xr7\n\t"
+ "xvmaddwod.h.bu $xr14, $xr2, $xr9\n\t"
+ "xvmaddwev.h.bu $xr14, $xr3, $xr8\n\t"
+ "xvmulwev.h.bu $xr15, $xr2, $xr10\n\t"
+ "xvmaddwod.h.bu $xr15, $xr2, $xr12\n\t"
+ "xvmaddwev.h.bu $xr15, $xr3, $xr11\n\t"
+ "xvsrli.h $xr13, $xr13, 7\n\t"
+ "xvsrli.h $xr14, $xr14, 7\n\t"
+ "xvsrli.h $xr15, $xr15, 7\n\t"
+ "xvsat.hu $xr13, $xr13, 7\n\t"
+ "xvsat.hu $xr14, $xr14, 7\n\t"
+ "xvsat.hu $xr15, $xr15, 7\n\t"
+ "xvpickev.b $xr13, $xr13, $xr13\n\t"
+ "xvpickev.b $xr14, $xr14, $xr14\n\t"
+ "xvpickev.b $xr15, $xr15, $xr15\n\t"
+ "xvpickod.b $xr3, $xr3, $xr3\n\t"
+ "xvilvl.b $xr2, $xr14, $xr13\n\t"
+ "xvilvl.b $xr3, $xr3, $xr15\n\t"
+ "xvilvl.h $xr0, $xr3, $xr2\n\t"
+ "xvilvh.h $xr1, $xr3, $xr2\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15", "$xr31");
+}
+
+void ARGBColorMatrixRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr4, %[mat], 0\n\t"
+ "vldrepl.w $vr31, %[mat], 0\n\t"
+ "xvpermi.q $xr4, $xr31, 0x02\n\t"
+ "vldrepl.w $vr5, %[mat], 4\n\t"
+ "vldrepl.w $vr31, %[mat], 4\n\t"
+ "xvpermi.q $xr5, $xr31, 0x02\n\t"
+ "vldrepl.w $vr6, %[mat], 8\n\t"
+ "vldrepl.w $vr31, %[mat], 8\n\t"
+ "xvpermi.q $xr6, $xr31, 0x02\n\t"
+ "vldrepl.w $vr7, %[mat], 12\n\t"
+ "vldrepl.w $vr31, %[mat], 12\n\t"
+ "xvpermi.q $xr7, $xr31, 0x02\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvmulwev.h.bu.b $xr8, $xr0, $xr4\n\t"
+ "xvmaddwod.h.bu.b $xr8, $xr0, $xr4\n\t"
+ "xvmulwev.h.bu.b $xr9, $xr0, $xr5\n\t"
+ "xvmaddwod.h.bu.b $xr9, $xr0, $xr5\n\t"
+ "xvmulwev.h.bu.b $xr10, $xr0, $xr6\n\t"
+ "xvmaddwod.h.bu.b $xr10, $xr0, $xr6\n\t"
+ "xvmulwev.h.bu.b $xr11, $xr0, $xr7\n\t"
+ "xvmaddwod.h.bu.b $xr11, $xr0, $xr7\n\t"
+ "xvhaddw.w.h $xr8, $xr8, $xr8\n\t"
+ "xvhaddw.w.h $xr9, $xr9, $xr9\n\t"
+ "xvhaddw.w.h $xr10, $xr10, $xr10\n\t"
+ "xvhaddw.w.h $xr11, $xr11, $xr11\n\t"
+ "xvilvl.w $xr12, $xr9, $xr8\n\t"
+ "xvilvh.w $xr13, $xr9, $xr8\n\t"
+ "xvilvl.w $xr14, $xr11, $xr10\n\t"
+ "xvilvh.w $xr15, $xr11, $xr10\n\t"
+ "xvilvl.d $xr8, $xr14, $xr12\n\t"
+ "xvilvh.d $xr9, $xr14, $xr12\n\t"
+ "xvilvl.d $xr10, $xr15, $xr13\n\t"
+ "xvilvh.d $xr11, $xr15, $xr13\n\t"
+ "xvssrani.h.w $xr9, $xr8, 6\n\t"
+ "xvssrani.h.w $xr11, $xr10, 6\n\t"
+ "xvssrani.bu.h $xr11, $xr9, 0\n\t"
+ "xvst $xr11, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [mat] "r"(matrix_argb)
+ : "memory", "$xr0", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9",
+ "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr31");
+}
+
+void ARGBQuantizeRow_LASX(uint8_t* dst_argb,
+ int scale,
+ int interval_size,
+ int interval_offset,
+ int width) {
+ asm volatile(
+ "xvreplgr2vr.h $xr4, %[scl]\n\t"
+ "xvreplgr2vr.b $xr5, %[isz]\n\t"
+ "xvreplgr2vr.b $xr6, %[ioff]\n\t"
+ "xvldi $xr7, 0xff\n\t"
+ "xvslli.w $xr7, $xr7, 24\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[dst], 0\n\t"
+ "xvilvl.b $xr1, $xr0, $xr0\n\t"
+ "xvilvh.b $xr2, $xr0, $xr0\n\t"
+ "xvsrli.h $xr1, $xr1, 8\n\t"
+ "xvsrli.h $xr2, $xr2, 8\n\t"
+ "xvmuh.hu $xr1, $xr1, $xr4\n\t"
+ "xvmuh.hu $xr2, $xr2, $xr4\n\t"
+ "xvsrlni.b.h $xr2, $xr1, 0\n\t"
+ "xvmul.b $xr2, $xr2, $xr5\n\t"
+ "xvadd.b $xr2, $xr2, $xr6\n\t"
+ "xvand.v $xr0, $xr0, $xr7\n\t"
+ "xvandn.v $xr2, $xr7, $xr2\n\t"
+ "xvor.v $xr2, $xr2, $xr0\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5", "$xr6", "$xr7");
+}
+
+void SobelRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "xvldi $xr4, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sx], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr1, %[sy], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "xvsadd.bu $xr0, $xr0, $xr1\n\t"
+ "xvilvl.b $xr2, $xr0, $xr0\n\t"
+ "xvilvl.b $xr3, $xr4, $xr0\n\t"
+ "xvilvl.h $xr0, $xr3, $xr2\n\t"
+ "xvilvh.h $xr1, $xr3, $xr2\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr0, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr1, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sx], %[sx], 16\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void SobelToPlaneRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[sx], 0\n\t"
+ "xvld $xr1, %[sy], 0\n\t"
+ "xvsadd.bu $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[sx], %[sx], 32\n\t"
+ "addi.d %[sy], %[sy], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1");
+}
+
+void SobelXYRow_LASX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "xvldi $xr4, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sx], 0\n\t"
+ "xvpermi.d $xr0, $xr0, 0xd8\n\t"
+ "vld $vr1, %[sy], 0\n\t"
+ "xvpermi.d $xr1, $xr1, 0xd8\n\t"
+ "xvsadd.bu $xr2, $xr0, $xr1\n\t"
+ "xvilvl.b $xr1, $xr2, $xr1\n\t"
+ "xvilvl.b $xr0, $xr4, $xr0\n\t"
+ "xvilvl.h $xr2, $xr0, $xr1\n\t"
+ "xvilvh.h $xr3, $xr0, $xr1\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "xvpermi.q $xr31, $xr2, 0x01\n\t"
+ "vst $vr31, %[dst], 32\n\t"
+ "vst $vr3, %[dst], 16\n\t"
+ "xvpermi.q $xr31, $xr3, 0x01\n\t"
+ "vst $vr31, %[dst], 48\n\t"
+ "addi.d %[sx], %[sx], 16\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31");
+}
+
+void InterpolateRow_LASX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int32_t source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ if (y1_fraction == 0) {
+ CopyRow_LASX(src_ptr, dst_ptr, width);
+ return;
+ }
+ if (y1_fraction == 128) {
+ asm volatile(
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvavgr.bu $xr0, $xr0, $xr1\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1");
+ return;
+ }
+ uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8));
+ asm volatile(
+ "xvreplgr2vr.h $xr3, %[yf]\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[s0], 0\n\t"
+ "xvld $xr1, %[s1], 0\n\t"
+ "xvilvl.b $xr2, $xr1, $xr0\n\t"
+ "xvilvh.b $xr4, $xr1, $xr0\n\t"
+ "xvmulwev.h.bu $xr0, $xr2, $xr3\n\t"
+ "xvmaddwod.h.bu $xr0, $xr2, $xr3\n\t"
+ "xvmulwev.h.bu $xr1, $xr4, $xr3\n\t"
+ "xvmaddwod.h.bu $xr1, $xr4, $xr3\n\t"
+ "xvsrarni.b.h $xr1, $xr0, 8\n\t"
+ "xvst $xr1, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr),
+ [cnt] "+r"(width)
+ : [yf] "r"(y_frac)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4");
+}
+
+void HalfFloatRow_LASX(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
+ asm volatile(
+ "xvldrepl.w $xr3, %[scale], 0\n\t"
+ "xvldrepl.w $xr5, %[bias], 0\n\t"
+ "xvldi $xr4, 0\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvilvl.h $xr1, $xr4, $xr0\n\t"
+ "xvilvh.h $xr2, $xr4, $xr0\n\t"
+ "xvffint.s.wu $xr1, $xr1\n\t"
+ "xvffint.s.wu $xr2, $xr2\n\t"
+ "xvfmul.s $xr1, $xr1, $xr3\n\t"
+ "xvfmul.s $xr2, $xr2, $xr3\n\t"
+ "xvfmul.s $xr1, $xr1, $xr5\n\t"
+ "xvfmul.s $xr2, $xr2, $xr5\n\t"
+ "xvsrlni.h.w $xr2, $xr1, 13\n\t"
+ "xvst $xr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
+ : [scale] "r"(&scale), [bias] "r"(&bias)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5");
+}
+
+void ARGBToYMatrixRow_LASX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "xvldrepl.b $xr0, %[c], 0\n\t"
+ "xvldrepl.b $xr1, %[c], 1\n\t"
+ "xvldrepl.b $xr2, %[c], 2\n\t"
+ "xvldrepl.b $xr3, %[c], 3\n\t"
+ "xvldrepl.h $xr4, %[c], 96\n\t"
+ "1:\n\t"
+ "vld $vr5, %[src], 0\n\t"
+ "vld $vr31, %[src], 32\n\t"
+ "xvpermi.q $xr5, $xr31, 0x02\n\t"
+ "vld $vr6, %[src], 16\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr6, $xr31, 0x02\n\t"
+ "xvor.v $xr9, $xr4, $xr4\n\t"
+ "xvpickev.b $xr7, $xr6, $xr5\n\t"
+ "xvpickod.b $xr8, $xr6, $xr5\n\t"
+ "xvmaddwev.h.bu $xr9, $xr7, $xr0\n\t"
+ "xvmaddwev.h.bu $xr9, $xr8, $xr1\n\t"
+ "xvmaddwod.h.bu $xr9, $xr7, $xr2\n\t"
+ "xvmaddwod.h.bu $xr9, $xr8, $xr3\n\t"
+ "xvpickod.b $xr9, $xr9, $xr9\n\t"
+ "xvpermi.d $xr31, $xr9, 0xd8\n\t"
+ "vst $vr31, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [c] "r"(c)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr31");
+}
+
+void RGBToYMatrixRow_LASX(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8,
+ 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8,
+ 9, 0, 10, 11, 12, 0, 13, 14, 15, 0};
+ asm volatile(
+ "xvldrepl.b $xr0, %[c], 0\n\t"
+ "xvldrepl.b $xr1, %[c], 1\n\t"
+ "xvldrepl.b $xr2, %[c], 2\n\t"
+ "xvldrepl.h $xr3, %[c], 96\n\t"
+ "vld $vr4, %[shuf], 0\n\t"
+ "xvpermi.q $xr4, $xr4, 0x00\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "xvpermi.q $xr5, $xr5, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr6, %[src], 0\n\t"
+ "vld $vr31, %[src], 48\n\t"
+ "xvpermi.q $xr6, $xr31, 0x02\n\t"
+ "vld $vr7, %[src], 12\n\t"
+ "vld $vr31, %[src], 60\n\t"
+ "xvpermi.q $xr7, $xr31, 0x02\n\t"
+ "vld $vr8, %[src], 24\n\t"
+ "vld $vr31, %[src], 72\n\t"
+ "xvpermi.q $xr8, $xr31, 0x02\n\t"
+ "vld $vr9, %[src], 32\n\t"
+ "vld $vr31, %[src], 80\n\t"
+ "xvpermi.q $xr9, $xr31, 0x02\n\t"
+ "xvshuf.b $xr6, $xr6, $xr6, $xr4\n\t"
+ "xvshuf.b $xr7, $xr7, $xr7, $xr4\n\t"
+ "xvshuf.b $xr8, $xr8, $xr8, $xr4\n\t"
+ "xvshuf.b $xr9, $xr9, $xr9, $xr5\n\t"
+ "xvor.v $xr12, $xr3, $xr3\n\t"
+ "xvor.v $xr13, $xr3, $xr3\n\t"
+ "xvpickev.b $xr10, $xr7, $xr6\n\t"
+ "xvpickod.b $xr11, $xr7, $xr6\n\t"
+ "xvmaddwev.h.bu $xr12, $xr10, $xr0\n\t"
+ "xvmaddwev.h.bu $xr12, $xr11, $xr1\n\t"
+ "xvmaddwod.h.bu $xr12, $xr10, $xr2\n\t"
+ "xvpickev.b $xr10, $xr9, $xr8\n\t"
+ "xvpickod.b $xr11, $xr9, $xr8\n\t"
+ "xvmaddwev.h.bu $xr13, $xr10, $xr0\n\t"
+ "xvmaddwev.h.bu $xr13, $xr11, $xr1\n\t"
+ "xvmaddwod.h.bu $xr13, $xr10, $xr2\n\t"
+ "xvpickod.b $xr12, $xr13, $xr12\n\t"
+ "xvst $xr12, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 96\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [c] "r"(c), [shuf] "r"(kShuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr31");
+}
+
+void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "xvldi $xr8, 25\n\t"
+ "xvsrli.h $xr8, $xr8, 8\n\t"
+ "xvldi $xr9, 129\n\t"
+ "xvsrli.h $xr9, $xr9, 8\n\t"
+ "xvldi $xr10, 66\n\t"
+ "xvsrli.h $xr10, $xr10, 8\n\t"
+ "xvldi $xr11, 33\n\t"
+ "xvsrli.h $xr11, $xr11, 8\n\t"
+ "xvslli.h $xr11, $xr11, 7\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvmul.h $xr1, $xr1, $xr8\n\t"
+ "xvmadd.h $xr1, $xr2, $xr9\n\t"
+ "xvmadd.h $xr1, $xr3, $xr10\n\t"
+ "xvadd.h $xr1, $xr1, $xr11\n\t"
+ "xvpickod.b $xr1, $xr1, $xr1\n\t"
+ "xvpermi.d $xr31, $xr1, 0xd8\n\t"
+ "vst $vr31, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7",
+ "$xr8", "$xr9", "$xr10", "$xr11", "$xr31");
+}
+
+void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile(
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr8, 25\n\t"
+ "xvsrli.h $xr8, $xr8, 8\n\t"
+ "xvldi $xr9, 129\n\t"
+ "xvsrli.h $xr9, $xr9, 8\n\t"
+ "xvldi $xr10, 66\n\t"
+ "xvsrli.h $xr10, $xr10, 8\n\t"
+ "xvldi $xr11, 33\n\t"
+ "xvsrli.h $xr11, $xr11, 8\n\t"
+ "xvslli.h $xr11, $xr11, 7\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[src], 0\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvmul.h $xr1, $xr1, $xr8\n\t"
+ "xvmadd.h $xr1, $xr2, $xr9\n\t"
+ "xvmadd.h $xr1, $xr3, $xr10\n\t"
+ "xvadd.h $xr1, $xr1, $xr11\n\t"
+ "xvpickod.b $xr1, $xr1, $xr1\n\t"
+ "xvpermi.d $xr31, $xr1, 0xd8\n\t"
+ "vst $vr31, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr8", "$xr9", "$xr10", "$xr11", "$xr31");
+}
+
+void RGB565ToUVRow_LASX(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565;
+ asm volatile(
+ "xvldi $xr20, 112\n\t"
+ "xvsrli.h $xr20, $xr20, 8\n\t"
+ "xvldi $xr21, 74\n\t"
+ "xvsrli.h $xr21, $xr21, 8\n\t"
+ "xvldi $xr22, 38\n\t"
+ "xvsrli.h $xr22, $xr22, 8\n\t"
+ "xvldi $xr23, 94\n\t"
+ "xvsrli.h $xr23, $xr23, 8\n\t"
+ "xvldi $xr24, 18\n\t"
+ "xvsrli.h $xr24, $xr24, 8\n\t"
+ "xvldi $xr25, 128\n\t"
+ "xvsrli.h $xr25, $xr25, 8\n\t"
+ "xvslli.h $xr25, $xr25, 8\n\t"
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvor.v $xr10, $xr1, $xr1\n\t"
+ "xvor.v $xr11, $xr2, $xr2\n\t"
+ "xvor.v $xr12, $xr3, $xr3\n\t"
+ "vld $vr0, %[s0], 16\n\t"
+ "vld $vr31, %[s0], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr10, $xr1, $xr10\n\t"
+ "xvpickev.b $xr11, $xr2, $xr11\n\t"
+ "xvpickev.b $xr12, $xr3, $xr12\n\t"
+ "vld $vr0, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvor.v $xr13, $xr1, $xr1\n\t"
+ "xvor.v $xr14, $xr2, $xr2\n\t"
+ "xvor.v $xr15, $xr3, $xr3\n\t"
+ "vld $vr0, %[s1], 16\n\t"
+ "vld $vr31, %[s1], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr7\n\t"
+ "xvsrli.h $xr3, $xr0, 11\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 2\n\t"
+ "xvsrli.h $xr2, $xr2, 4\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr5, $xr3, $xr15\n\t"
+ "xvpickev.b $xr4, $xr2, $xr14\n\t"
+ "xvpickev.b $xr3, $xr1, $xr13\n\t"
+ "xvor.v $xr0, $xr10, $xr10\n\t"
+ "xvor.v $xr1, $xr11, $xr11\n\t"
+ "xvor.v $xr2, $xr12, $xr12\n\t"
+ "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t"
+ "xvadd.h $xr0, $xr0, $xr3\n\t"
+ "xvsrari.h $xr0, $xr0, 2\n\t"
+ "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t"
+ "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t"
+ "xvadd.h $xr1, $xr1, $xr4\n\t"
+ "xvsrari.h $xr1, $xr1, 2\n\t"
+ "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t"
+ "xvadd.h $xr2, $xr2, $xr5\n\t"
+ "xvsrari.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr6, $xr25, $xr25\n\t"
+ "xvor.v $xr7, $xr25, $xr25\n\t"
+ "xvmadd.h $xr6, $xr0, $xr20\n\t"
+ "xvmsub.h $xr6, $xr1, $xr21\n\t"
+ "xvmsub.h $xr6, $xr2, $xr22\n\t"
+ "xvmadd.h $xr7, $xr2, $xr20\n\t"
+ "xvmsub.h $xr7, $xr1, $xr23\n\t"
+ "xvmsub.h $xr7, $xr0, $xr24\n\t"
+ "xvpickod.b $xr6, $xr7, $xr6\n\t"
+ "xvpermi.d $xr31, $xr6, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "xvpermi.q $xr31, $xr6, 0x01\n\t"
+ "vstelm.d $vr31, %[dv], 8, 1\n\t"
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "xvldi $xr7, 63\n\t"
+ "xvsrli.h $xr7, $xr7, 8\n\t"
+ "addi.d %[s0], %[s0], 64\n\t"
+ "addi.d %[s1], %[s1], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20",
+ "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31");
+}
+
+void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555;
+ asm volatile(
+ "xvldi $xr20, 112\n\t"
+ "xvsrli.h $xr20, $xr20, 8\n\t"
+ "xvldi $xr21, 74\n\t"
+ "xvsrli.h $xr21, $xr21, 8\n\t"
+ "xvldi $xr22, 38\n\t"
+ "xvsrli.h $xr22, $xr22, 8\n\t"
+ "xvldi $xr23, 94\n\t"
+ "xvsrli.h $xr23, $xr23, 8\n\t"
+ "xvldi $xr24, 18\n\t"
+ "xvsrli.h $xr24, $xr24, 8\n\t"
+ "xvldi $xr25, 128\n\t"
+ "xvsrli.h $xr25, $xr25, 8\n\t"
+ "xvslli.h $xr25, $xr25, 8\n\t"
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvor.v $xr10, $xr1, $xr1\n\t"
+ "xvor.v $xr11, $xr2, $xr2\n\t"
+ "xvor.v $xr12, $xr3, $xr3\n\t"
+ "vld $vr0, %[s0], 16\n\t"
+ "vld $vr31, %[s0], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr10, $xr1, $xr10\n\t"
+ "xvpickev.b $xr11, $xr2, $xr11\n\t"
+ "xvpickev.b $xr12, $xr3, $xr12\n\t"
+ "vld $vr0, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 32\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvor.v $xr13, $xr1, $xr1\n\t"
+ "xvor.v $xr14, $xr2, $xr2\n\t"
+ "xvor.v $xr15, $xr3, $xr3\n\t"
+ "vld $vr0, %[s1], 16\n\t"
+ "vld $vr31, %[s1], 48\n\t"
+ "xvpermi.q $xr0, $xr31, 0x02\n\t"
+ "xvand.v $xr1, $xr0, $xr6\n\t"
+ "xvsrli.h $xr2, $xr0, 5\n\t"
+ "xvand.v $xr2, $xr2, $xr6\n\t"
+ "xvsrli.h $xr3, $xr0, 10\n\t"
+ "xvand.v $xr3, $xr3, $xr6\n\t"
+ "xvsrai.h $xr5, $xr0, 15\n\t"
+ "xvslli.h $xr4, $xr1, 3\n\t"
+ "xvsrli.h $xr1, $xr1, 2\n\t"
+ "xvor.v $xr1, $xr4, $xr1\n\t"
+ "xvslli.h $xr4, $xr2, 3\n\t"
+ "xvsrli.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr2, $xr4, $xr2\n\t"
+ "xvslli.h $xr4, $xr3, 3\n\t"
+ "xvsrli.h $xr3, $xr3, 2\n\t"
+ "xvor.v $xr3, $xr4, $xr3\n\t"
+ "xvpickev.b $xr5, $xr3, $xr15\n\t"
+ "xvpickev.b $xr4, $xr2, $xr14\n\t"
+ "xvpickev.b $xr3, $xr1, $xr13\n\t"
+ "xvor.v $xr0, $xr10, $xr10\n\t"
+ "xvor.v $xr1, $xr11, $xr11\n\t"
+ "xvor.v $xr2, $xr12, $xr12\n\t"
+ "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t"
+ "xvadd.h $xr0, $xr0, $xr3\n\t"
+ "xvsrari.h $xr0, $xr0, 2\n\t"
+ "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t"
+ "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t"
+ "xvadd.h $xr1, $xr1, $xr4\n\t"
+ "xvsrari.h $xr1, $xr1, 2\n\t"
+ "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t"
+ "xvadd.h $xr2, $xr2, $xr5\n\t"
+ "xvsrari.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr6, $xr25, $xr25\n\t"
+ "xvor.v $xr7, $xr25, $xr25\n\t"
+ "xvmadd.h $xr6, $xr0, $xr20\n\t"
+ "xvmsub.h $xr6, $xr1, $xr21\n\t"
+ "xvmsub.h $xr6, $xr2, $xr22\n\t"
+ "xvmadd.h $xr7, $xr2, $xr20\n\t"
+ "xvmsub.h $xr7, $xr1, $xr23\n\t"
+ "xvmsub.h $xr7, $xr0, $xr24\n\t"
+ "xvpickod.b $xr6, $xr7, $xr6\n\t"
+ "xvpermi.d $xr31, $xr6, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "xvpermi.q $xr31, $xr6, 0x01\n\t"
+ "vstelm.d $vr31, %[dv], 8, 1\n\t"
+ "xvldi $xr6, 31\n\t"
+ "xvsrli.h $xr6, $xr6, 8\n\t"
+ "addi.d %[s0], %[s0], 64\n\t"
+ "addi.d %[s1], %[s1], 64\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20",
+ "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31");
+}
+
+static LIBYUV_NOINLINE void XRGBToUVHelper_LASX(const uint8_t* src,
+ int stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ const uint8_t* shuf,
+ int width) {
+ const uint8_t* next = src + stride;
+ asm volatile(
+ "xvldi $xr20, 112\n\t"
+ "xvsrli.h $xr20, $xr20, 8\n\t"
+ "xvldi $xr21, 74\n\t"
+ "xvsrli.h $xr21, $xr21, 8\n\t"
+ "xvldi $xr22, 38\n\t"
+ "xvsrli.h $xr22, $xr22, 8\n\t"
+ "xvldi $xr23, 94\n\t"
+ "xvsrli.h $xr23, $xr23, 8\n\t"
+ "xvldi $xr24, 18\n\t"
+ "xvsrli.h $xr24, $xr24, 8\n\t"
+ "xvldi $xr25, 128\n\t"
+ "xvsrli.h $xr25, $xr25, 8\n\t"
+ "xvslli.h $xr25, $xr25, 8\n\t"
+ "vld $vr16, %[shuf], 0\n\t"
+ "xvpermi.q $xr16, $xr16, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr8, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 64\n\t"
+ "xvpermi.q $xr8, $xr31, 0x02\n\t"
+ "vld $vr9, %[s0], 16\n\t"
+ "vld $vr31, %[s0], 80\n\t"
+ "xvpermi.q $xr9, $xr31, 0x02\n\t"
+ "vld $vr10, %[s0], 32\n\t"
+ "vld $vr31, %[s0], 96\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "vld $vr11, %[s0], 48\n\t"
+ "vld $vr31, %[s0], 112\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t"
+ "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t"
+ "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t"
+ "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t"
+ "xvpickev.b $xr12, $xr9, $xr8\n\t"
+ "xvpickod.b $xr13, $xr9, $xr8\n\t"
+ "xvpickev.b $xr14, $xr11, $xr10\n\t"
+ "xvpickod.b $xr15, $xr11, $xr10\n\t"
+ "xvpickev.b $xr0, $xr14, $xr12\n\t"
+ "xvpickod.b $xr2, $xr14, $xr12\n\t"
+ "xvpickev.b $xr1, $xr15, $xr13\n\t"
+ "vld $vr8, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 64\n\t"
+ "xvpermi.q $xr8, $xr31, 0x02\n\t"
+ "vld $vr9, %[s1], 16\n\t"
+ "vld $vr31, %[s1], 80\n\t"
+ "xvpermi.q $xr9, $xr31, 0x02\n\t"
+ "vld $vr10, %[s1], 32\n\t"
+ "vld $vr31, %[s1], 96\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "vld $vr11, %[s1], 48\n\t"
+ "vld $vr31, %[s1], 112\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t"
+ "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t"
+ "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t"
+ "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t"
+ "xvpickev.b $xr12, $xr9, $xr8\n\t"
+ "xvpickod.b $xr13, $xr9, $xr8\n\t"
+ "xvpickev.b $xr14, $xr11, $xr10\n\t"
+ "xvpickod.b $xr15, $xr11, $xr10\n\t"
+ "xvpickev.b $xr3, $xr14, $xr12\n\t"
+ "xvpickod.b $xr5, $xr14, $xr12\n\t"
+ "xvpickev.b $xr4, $xr15, $xr13\n\t"
+ "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t"
+ "xvadd.h $xr0, $xr0, $xr3\n\t"
+ "xvsrari.h $xr0, $xr0, 2\n\t"
+ "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t"
+ "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t"
+ "xvadd.h $xr1, $xr1, $xr4\n\t"
+ "xvsrari.h $xr1, $xr1, 2\n\t"
+ "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t"
+ "xvadd.h $xr2, $xr2, $xr5\n\t"
+ "xvsrari.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr6, $xr25, $xr25\n\t"
+ "xvor.v $xr7, $xr25, $xr25\n\t"
+ "xvmadd.h $xr6, $xr0, $xr20\n\t"
+ "xvmsub.h $xr6, $xr1, $xr21\n\t"
+ "xvmsub.h $xr6, $xr2, $xr22\n\t"
+ "xvmadd.h $xr7, $xr2, $xr20\n\t"
+ "xvmsub.h $xr7, $xr1, $xr23\n\t"
+ "xvmsub.h $xr7, $xr0, $xr24\n\t"
+ "xvpickod.b $xr6, $xr7, $xr6\n\t"
+ "xvpermi.d $xr31, $xr6, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "xvpermi.q $xr31, $xr6, 0x01\n\t"
+ "vstelm.d $vr31, %[dv], 8, 1\n\t"
+ "addi.d %[s0], %[s0], 128\n\t"
+ "addi.d %[s1], %[s1], 128\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15", "$xr16", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25",
+ "$xr31");
+}
+
+void BGRAToUVRow_LASX(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4,
+ 11, 10, 9, 8, 15, 14, 13, 12};
+ XRGBToUVHelper_LASX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width);
+}
+
+void ABGRToUVRow_LASX(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+ XRGBToUVHelper_LASX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width);
+}
+
+void RGBAToUVRow_LASX(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4,
+ 9, 10, 11, 8, 13, 14, 15, 12};
+ XRGBToUVHelper_LASX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width);
+}
+
+static LIBYUV_NOINLINE void RGB24ToUVHelper_LASX(const uint8_t* src,
+ int stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ const uint8_t* shuf,
+ int width) {
+ const uint8_t* next = src + stride;
+ asm volatile(
+ "xvldi $xr20, 112\n\t"
+ "xvsrli.h $xr20, $xr20, 8\n\t"
+ "xvldi $xr21, 74\n\t"
+ "xvsrli.h $xr21, $xr21, 8\n\t"
+ "xvldi $xr22, 38\n\t"
+ "xvsrli.h $xr22, $xr22, 8\n\t"
+ "xvldi $xr23, 94\n\t"
+ "xvsrli.h $xr23, $xr23, 8\n\t"
+ "xvldi $xr24, 18\n\t"
+ "xvsrli.h $xr24, $xr24, 8\n\t"
+ "xvldi $xr25, 128\n\t"
+ "xvsrli.h $xr25, $xr25, 8\n\t"
+ "xvslli.h $xr25, $xr25, 8\n\t"
+ "vld $vr16, %[shuf], 0\n\t"
+ "xvpermi.q $xr16, $xr16, 0x00\n\t"
+ "vld $vr17, %[shuf], 16\n\t"
+ "xvpermi.q $xr17, $xr17, 0x00\n\t"
+ "1:\n\t"
+ "vld $vr8, %[s0], 0\n\t"
+ "vld $vr31, %[s0], 48\n\t"
+ "xvpermi.q $xr8, $xr31, 0x02\n\t"
+ "vld $vr9, %[s0], 12\n\t"
+ "vld $vr31, %[s0], 60\n\t"
+ "xvpermi.q $xr9, $xr31, 0x02\n\t"
+ "vld $vr10, %[s0], 24\n\t"
+ "vld $vr31, %[s0], 72\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "vld $vr11, %[s0], 32\n\t"
+ "vld $vr31, %[s0], 80\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t"
+ "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t"
+ "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t"
+ "xvshuf.b $xr11, $xr11, $xr11, $xr17\n\t"
+ "xvpickev.b $xr12, $xr9, $xr8\n\t"
+ "xvpickod.b $xr13, $xr9, $xr8\n\t"
+ "xvpickev.b $xr14, $xr11, $xr10\n\t"
+ "xvpickod.b $xr15, $xr11, $xr10\n\t"
+ "xvpickev.b $xr0, $xr14, $xr12\n\t"
+ "xvpickod.b $xr2, $xr14, $xr12\n\t"
+ "xvpickev.b $xr1, $xr15, $xr13\n\t"
+ "vld $vr8, %[s1], 0\n\t"
+ "vld $vr31, %[s1], 48\n\t"
+ "xvpermi.q $xr8, $xr31, 0x02\n\t"
+ "vld $vr9, %[s1], 12\n\t"
+ "vld $vr31, %[s1], 60\n\t"
+ "xvpermi.q $xr9, $xr31, 0x02\n\t"
+ "vld $vr10, %[s1], 24\n\t"
+ "vld $vr31, %[s1], 72\n\t"
+ "xvpermi.q $xr10, $xr31, 0x02\n\t"
+ "vld $vr11, %[s1], 32\n\t"
+ "vld $vr31, %[s1], 80\n\t"
+ "xvpermi.q $xr11, $xr31, 0x02\n\t"
+ "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t"
+ "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t"
+ "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t"
+ "xvshuf.b $xr11, $xr11, $xr11, $xr17\n\t"
+ "xvpickev.b $xr12, $xr9, $xr8\n\t"
+ "xvpickod.b $xr13, $xr9, $xr8\n\t"
+ "xvpickev.b $xr14, $xr11, $xr10\n\t"
+ "xvpickod.b $xr15, $xr11, $xr10\n\t"
+ "xvpickev.b $xr3, $xr14, $xr12\n\t"
+ "xvpickod.b $xr5, $xr14, $xr12\n\t"
+ "xvpickev.b $xr4, $xr15, $xr13\n\t"
+ "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t"
+ "xvadd.h $xr0, $xr0, $xr3\n\t"
+ "xvsrari.h $xr0, $xr0, 2\n\t"
+ "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t"
+ "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t"
+ "xvadd.h $xr1, $xr1, $xr4\n\t"
+ "xvsrari.h $xr1, $xr1, 2\n\t"
+ "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t"
+ "xvadd.h $xr2, $xr2, $xr5\n\t"
+ "xvsrari.h $xr2, $xr2, 2\n\t"
+ "xvor.v $xr6, $xr25, $xr25\n\t"
+ "xvor.v $xr7, $xr25, $xr25\n\t"
+ "xvmadd.h $xr6, $xr0, $xr20\n\t"
+ "xvmsub.h $xr6, $xr1, $xr21\n\t"
+ "xvmsub.h $xr6, $xr2, $xr22\n\t"
+ "xvmadd.h $xr7, $xr2, $xr20\n\t"
+ "xvmsub.h $xr7, $xr1, $xr23\n\t"
+ "xvmsub.h $xr7, $xr0, $xr24\n\t"
+ "xvpickod.b $xr6, $xr7, $xr6\n\t"
+ "xvpermi.d $xr31, $xr6, 0xd8\n\t"
+ "vst $vr31, %[du], 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "xvpermi.q $xr31, $xr6, 0x01\n\t"
+ "vstelm.d $vr31, %[dv], 8, 1\n\t"
+ "addi.d %[s0], %[s0], 96\n\t"
+ "addi.d %[s1], %[s1], 96\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+ "$xr15", "$xr16", "$xr17", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24",
+ "$xr25", "$xr31");
+}
+
+void RGB24ToUVRow_LASX(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8,
+ 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8,
+ 9, 0, 10, 11, 12, 0, 13, 14, 15, 0};
+ RGB24ToUVHelper_LASX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width);
+}
+
+void RAWToUVRow_LASX(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6,
+ 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8,
+ 7, 0, 12, 11, 10, 0, 15, 14, 13, 0};
+ RGB24ToUVHelper_LASX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width);
+}
+
+#ifdef HAS_SSIM4X4ROW_LASX
+void Ssim4x4Row_LASX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const uint8_t* pa;
+ const uint8_t* pb;
+ int r;
+ asm volatile(
+ "1:\n\t"
+ "move %[pa], %[sa]\n\t"
+ "move %[pb], %[sb]\n\t"
+ "li.d %[r], 4\n\t"
+ "xvxor.v $xr8, $xr8, $xr8\n\t"
+ "xvxor.v $xr9, $xr9, $xr9\n\t"
+ "xvxor.v $xr10, $xr10, $xr10\n\t"
+ "xvxor.v $xr11, $xr11, $xr11\n\t"
+ "2:\n\t"
+ "xvld $xr0, %[pa], 0\n\t"
+ "xvld $xr1, %[pb], 0\n\t"
+ "add.d %[pa], %[pa], %[sta]\n\t"
+ "add.d %[pb], %[pb], %[stb]\n\t"
+ "xvhaddw.hu.bu $xr2, $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr3, $xr1, $xr1\n\t"
+ "xvadd.h $xr8, $xr8, $xr2\n\t"
+ "xvadd.h $xr9, $xr9, $xr3\n\t"
+ "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t"
+ "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t"
+ "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t"
+ "xvadd.w $xr11, $xr11, $xr2\n\t"
+ "xvadd.w $xr11, $xr11, $xr3\n\t"
+ "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t"
+ "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t"
+ "xvmulwev.h.bu $xr4, $xr1, $xr1\n\t"
+ "xvmulwod.h.bu $xr5, $xr1, $xr1\n\t"
+ "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t"
+ "xvhaddw.wu.hu $xr4, $xr4, $xr4\n\t"
+ "xvhaddw.wu.hu $xr5, $xr5, $xr5\n\t"
+ "xvadd.w $xr2, $xr2, $xr3\n\t"
+ "xvadd.w $xr4, $xr4, $xr5\n\t"
+ "xvadd.w $xr10, $xr10, $xr2\n\t"
+ "xvadd.w $xr10, $xr10, $xr4\n\t"
+ "addi.d %[r], %[r], -1\n\t"
+ "bnez %[r], 2b\n\t"
+ "xvhaddw.wu.hu $xr8, $xr8, $xr8\n\t"
+ "xvhaddw.wu.hu $xr9, $xr9, $xr9\n\t"
+ "xvilvl.w $xr0, $xr9, $xr8\n\t"
+ "xvilvh.w $xr1, $xr9, $xr8\n\t"
+ "xvilvl.w $xr2, $xr11, $xr10\n\t"
+ "xvilvh.w $xr3, $xr11, $xr10\n\t"
+ "xvilvl.d $xr4, $xr2, $xr0\n\t"
+ "xvilvh.d $xr5, $xr2, $xr0\n\t"
+ "xvilvl.d $xr6, $xr3, $xr1\n\t"
+ "xvilvh.d $xr7, $xr3, $xr1\n\t"
+ "xvor.v $xr0, $xr5, $xr5\n\t"
+ "xvpermi.q $xr0, $xr4, 0x20\n\t"
+ "xvpermi.q $xr5, $xr4, 0x31\n\t"
+ "xvor.v $xr1, $xr7, $xr7\n\t"
+ "xvpermi.q $xr1, $xr6, 0x20\n\t"
+ "xvpermi.q $xr7, $xr6, 0x31\n\t"
+ "xvst $xr0, %[dst], 0\n\t"
+ "xvst $xr1, %[dst], 32\n\t"
+ "xvst $xr5, %[dst], 64\n\t"
+ "xvst $xr7, %[dst], 96\n\t"
+ "addi.d %[sa], %[sa], 32\n\t"
+ "addi.d %[sb], %[sb], 32\n\t"
+ "addi.d %[dst], %[dst], 128\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width),
+ [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
+ : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7", "$xr8", "$xr9", "$xr10", "$xr11");
+}
+#endif // HAS_SSIM4X4ROW_LASX
+
+#ifdef HAS_SSIMWINDOWROW_LASX
+float SsimWindowRow_LASX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ uint32_t c1 = 26634;
+ uint32_t c2 = 239708;
+ uint32_t c0 = 0;
+ asm volatile(
+ "xvreplgr2vr.w $xr6, %[c0]\n\t"
+ "xvreplgr2vr.w $xr4, %[c1]\n\t"
+ "xvreplgr2vr.w $xr5, %[c2]\n\t"
+ "addi.d %[cnt], %[cnt], -2\n\t"
+ "bltz %[cnt], 2f\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[r0], 0\n\t"
+ "xvld $xr1, %[r0n], 0\n\t"
+ "xvld $xr2, %[r1], 0\n\t"
+ "xvld $xr3, %[r1n], 0\n\t"
+ "xvadd.w $xr0, $xr0, $xr1\n\t"
+ "xvadd.w $xr2, $xr2, $xr3\n\t"
+ "xvadd.w $xr0, $xr0, $xr2\n\t"
+ "xvshuf4i.w $xr1, $xr0, 0x00\n\t"
+ "xvshuf4i.w $xr2, $xr0, 0x55\n\t"
+ "xvshuf4i.w $xr3, $xr0, 0xaa\n\t"
+ "xvshuf4i.w $xr0, $xr0, 0xff\n\t"
+ "xvmul.w $xr7, $xr1, $xr2\n\t"
+ "xvslli.w $xr7, $xr7, 1\n\t"
+ "xvmul.w $xr1, $xr1, $xr1\n\t"
+ "xvmadd.w $xr1, $xr2, $xr2\n\t"
+ "xvslli.w $xr0, $xr0, 7\n\t"
+ "xvsub.w $xr0, $xr0, $xr7\n\t"
+ "xvadd.w $xr7, $xr7, $xr4\n\t"
+ "xvadd.w $xr0, $xr0, $xr5\n\t"
+ "xvslli.w $xr3, $xr3, 6\n\t"
+ "xvsub.w $xr3, $xr3, $xr1\n\t"
+ "xvadd.w $xr1, $xr1, $xr4\n\t"
+ "xvadd.w $xr3, $xr3, $xr5\n\t"
+ "xvffint.s.w $xr7, $xr7\n\t"
+ "xvffint.s.w $xr0, $xr0\n\t"
+ "xvffint.s.w $xr1, $xr1\n\t"
+ "xvffint.s.w $xr3, $xr3\n\t"
+ "xvfmul.s $xr7, $xr7, $xr0\n\t"
+ "xvfmul.s $xr1, $xr1, $xr3\n\t"
+ "xvfdiv.s $xr7, $xr7, $xr1\n\t"
+ "xvfadd.s $xr6, $xr6, $xr7\n\t"
+ "addi.d %[r0], %[r0], 32\n\t"
+ "addi.d %[r0n], %[r0n], 32\n\t"
+ "addi.d %[r1], %[r1], 32\n\t"
+ "addi.d %[r1n], %[r1n], 32\n\t"
+ "addi.d %[cnt], %[cnt], -2\n\t"
+ "bgez %[cnt], 1b\n\t"
+ "xvpermi.q $xr0, $xr6, 0x01\n\t"
+ "vfadd.s $vr6, $vr6, $vr0\n\t"
+ "2:\n\t"
+ "addi.d %[cnt], %[cnt], 2\n\t"
+ "beqz %[cnt], 3f\n\t"
+ "vld $vr0, %[r0], 0\n\t"
+ "vld $vr1, %[r0n], 0\n\t"
+ "vld $vr2, %[r1], 0\n\t"
+ "vld $vr3, %[r1n], 0\n\t"
+ "vadd.w $vr0, $vr0, $vr1\n\t"
+ "vadd.w $vr2, $vr2, $vr3\n\t"
+ "vadd.w $vr0, $vr0, $vr2\n\t"
+ "vshuf4i.w $vr1, $vr0, 0x00\n\t"
+ "vshuf4i.w $vr2, $vr0, 0x55\n\t"
+ "vshuf4i.w $vr3, $vr0, 0xaa\n\t"
+ "vshuf4i.w $vr0, $vr0, 0xff\n\t"
+ "vmul.w $vr7, $vr1, $vr2\n\t"
+ "vslli.w $vr7, $vr7, 1\n\t"
+ "vmul.w $vr1, $vr1, $vr1\n\t"
+ "vmadd.w $vr1, $vr2, $vr2\n\t"
+ "vslli.w $vr0, $vr0, 7\n\t"
+ "vsub.w $vr0, $vr0, $vr7\n\t"
+ "vadd.w $vr7, $vr7, $vr4\n\t"
+ "vadd.w $vr0, $vr0, $vr5\n\t"
+ "vslli.w $vr3, $vr3, 6\n\t"
+ "vsub.w $vr3, $vr3, $vr1\n\t"
+ "vadd.w $vr1, $vr1, $vr4\n\t"
+ "vadd.w $vr3, $vr3, $vr5\n\t"
+ "vffint.s.w $vr7, $vr7\n\t"
+ "vffint.s.w $vr0, $vr0\n\t"
+ "vffint.s.w $vr1, $vr1\n\t"
+ "vffint.s.w $vr3, $vr3\n\t"
+ "vfmul.s $vr7, $vr7, $vr0\n\t"
+ "vfmul.s $vr1, $vr1, $vr3\n\t"
+ "vfdiv.s $vr7, $vr7, $vr1\n\t"
+ "vfadd.s $vr6, $vr6, $vr7\n\t"
+ "3:\n\t"
+ "vstelm.w $vr6, %[tot], 0, 0\n\t"
+ : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1),
+ [r1n] "+r"(row1_next), [cnt] "+r"(num_windows)
+ : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total)
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+ "$xr7");
+ return total;
+}
+#endif // HAS_SSIMWINDOWROW_LASX
+
+#ifdef HAS_ARGBDETECTROW_LASX
+uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width) {
+ uint32_t diff = 0;
+ uint32_t tmp;
+ uint32_t rgb_mask = 0x00ffff00u;
+ asm volatile(
+ "xvreplgr2vr.w $xr2, %[rgb_mask]\n\t"
+ "xvldi $xr3, -1\n\t"
+ "addi.d %[w], %[w], -8\n\t"
+ "bltz %[w], 2f\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[argb], 0\n\t"
+ "addi.d %[argb], %[argb], 32\n\t"
+ "xvor.v $xr0, $xr0, $xr2\n\t"
+ "xvseq.b $xr0, $xr0, $xr3\n\t"
+ "xvmskltz.b $xr0, $xr0\n\t"
+ "xvpickve2gr.wu %[diff], $xr0, 0\n\t"
+ "xvpickve2gr.wu %[tmp], $xr0, 4\n\t"
+ "bstrins.w %[diff], %[tmp], 31, 16\n\t"
+ "nor %[diff], %[diff], $zero\n\t"
+ "bnez %[diff], 4f\n\t"
+ "addi.d %[w], %[w], -8\n\t"
+ "bgez %[w], 1b\n\t"
+ "2:\n\t"
+ "addi.d %[w], %[w], 8\n\t"
+ "beqz %[w], 4f\n\t"
+ "3:\n\t"
+ "vldrepl.w $vr0, %[argb], 0\n\t"
+ "addi.d %[argb], %[argb], 4\n\t"
+ "vor.v $vr0, $vr0, $vr2\n\t"
+ "vseq.b $vr0, $vr0, $vr3\n\t"
+ "vmskltz.b $vr0, $vr0\n\t"
+ "vpickve2gr.hu %[diff], $vr0, 0\n\t"
+ "nor %[diff], %[diff], $zero\n\t"
+ "bstrpick.w %[diff], %[diff], 15, 0\n\t"
+ "bnez %[diff], 4f\n\t"
+ "addi.d %[w], %[w], -1\n\t"
+ "bnez %[w], 3b\n\t"
+ "4:\n\t"
+ : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff), [tmp] "=&r"(tmp)
+ : [rgb_mask] "r"(rgb_mask)
+ : "memory", "$xr0", "$xr2", "$xr3");
+ if (diff) {
+ return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_LASX
+
+#ifdef HAS_HAMMINGDISTANCE_LASX
+uint32_t HammingDistance_LASX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ asm volatile(
+ "xvxor.v $xr4, $xr4, $xr4\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[sa], 0\n\t"
+ "xvld $xr1, %[sb], 0\n\t"
+ "xvxor.v $xr0, $xr0, $xr1\n\t"
+ "xvpcnt.b $xr0, $xr0\n\t"
+ "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.wu.hu $xr0, $xr0, $xr0\n\t"
+ "xvadd.w $xr4, $xr4, $xr0\n\t"
+ "addi.d %[sa], %[sa], 32\n\t"
+ "addi.d %[sb], %[sb], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t"
+ "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t"
+ "xvpermi.q $xr0, $xr4, 0x01\n\t"
+ "vadd.w $vr4, $vr4, $vr0\n\t"
+ "vpickve2gr.wu %[diff], $vr4, 0\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff)
+ :
+ : "memory", "$xr0", "$xr1", "$xr4");
+ return diff;
+}
+#endif // HAS_HAMMINGDISTANCE_LASX
+
+#ifdef HAS_SUMSQUAREERROR_LASX
+uint32_t SumSquareError_LASX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t sse;
+ asm volatile(
+ "xvxor.v $xr4, $xr4, $xr4\n\t"
+ "1:\n\t"
+ "xvld $xr0, %[sa], 0\n\t"
+ "xvld $xr1, %[sb], 0\n\t"
+ "xvabsd.bu $xr0, $xr0, $xr1\n\t"
+ "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t"
+ "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t"
+ "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t"
+ "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t"
+ "xvadd.w $xr4, $xr4, $xr2\n\t"
+ "xvadd.w $xr4, $xr4, $xr3\n\t"
+ "addi.d %[sa], %[sa], 32\n\t"
+ "addi.d %[sb], %[sb], 32\n\t"
+ "addi.d %[cnt], %[cnt], -32\n\t"
+ "bnez %[cnt], 1b\n\t"
+ "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t"
+ "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t"
+ "xvpermi.q $xr0, $xr4, 0x01\n\t"
+ "vadd.w $vr4, $vr4, $vr0\n\t"
+ "vpickve2gr.wu %[sse], $vr4, 0\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse)
+ :
+ : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_LASX
+
+#ifdef HAS_HASHDJB2_LASX
+static const uint32_t kDjb2Muls_LASX[16] = {
+ 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
+ 0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
+ 0x00008c61, 0x00000441, 0x00000021, 0x00000001,
+};
+
+uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ uint32_t tmp;
+ uint32_t c16 = 0x92d9e201u;
+ asm volatile(
+ "xvld $xr4, %[muls], 0\n\t"
+ "xvld $xr5, %[muls], 32\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vbsrl.v $vr1, $vr0, 8\n\t"
+ "vext2xv.wu.bu $xr0, $xr0\n\t"
+ "vext2xv.wu.bu $xr1, $xr1\n\t"
+ "xvmul.w $xr0, $xr0, $xr4\n\t"
+ "xvmadd.w $xr0, $xr1, $xr5\n\t"
+ "xvhaddw.du.wu $xr0, $xr0, $xr0\n\t"
+ "xvhaddw.qu.du $xr0, $xr0, $xr0\n\t"
+ "xvpermi.q $xr1, $xr0, 0x01\n\t"
+ "vadd.w $vr0, $vr0, $vr1\n\t"
+ "vpickve2gr.wu %[tmp], $vr0, 0\n\t"
+ "mul.w %[hash], %[hash], %[c16]\n\t"
+ "add.w %[hash], %[hash], %[tmp]\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
+ : [muls] "r"(kDjb2Muls_LASX), [c16] "r"(c16)
+ : "memory", "$xr0", "$xr1", "$xr4", "$xr5");
+ return hash;
+}
+#endif // HAS_HASHDJB2_LASX
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
diff --git a/libs/libyuv/source/row_lsx.cc b/libs/libyuv/source/row_lsx.cc
new file mode 100644
index 0000000000..cb0ad41378
--- /dev/null
+++ b/libs/libyuv/source/row_lsx.cc
@@ -0,0 +1,3112 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+#include "libyuv/video_common.h"
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+
+#include "libyuv/basic_types.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int count) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count)
+ :
+ : "memory", "$vr0");
+}
+
+void SetRow_LSX(uint8_t* dst, uint8_t v8, int count) {
+ asm volatile(
+ "vreplgr2vr.b $vr0, %[val]\n\t"
+ "1:\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst), [cnt] "+r"(count)
+ : [val] "r"(v8)
+ : "memory", "$vr0");
+}
+
+void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width) {
+ asm volatile(
+ "vreplgr2vr.w $vr0, %[val]\n\t"
+ "1:\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(v32)
+ : "memory", "$vr0");
+}
+
+void MirrorRow_LSX(const uint8_t* src_y, uint8_t* dst_y, int width) {
+ static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8,
+ 7, 6, 5, 4, 3, 2, 1, 0};
+ src_y += width - 16;
+ asm volatile(
+ "vld $vr1, %[rev], 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [rev] "r"(kRev)
+ : "memory", "$vr0", "$vr1");
+}
+
+void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9,
+ 6, 7, 4, 5, 2, 3, 0, 1};
+ src_uv += width * 2 - 16;
+ asm volatile(
+ "vld $vr1, %[rev], 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width)
+ : [rev] "r"(kRevUV)
+ : "memory", "$vr0", "$vr1");
+}
+
+void ARGBMirrorRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ src_argb += width * 4 - 16;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vshuf4i.w $vr0, $vr0, 0x1b\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], -16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0");
+}
+
+void MirrorSplitUVRow_LSX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9,
+ 6, 7, 4, 5, 2, 3, 0, 1};
+ src_uv += width * 2 - 32;
+ asm volatile(
+ "vld $vr4, %[rev], 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 16\n\t"
+ "vld $vr1, %[src], 0\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t"
+ "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vst $vr2, %[du], 0\n\t"
+ "vst $vr3, %[dv], 0\n\t"
+ "addi.d %[src], %[src], -32\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width)
+ : [rev] "r"(kRevUV)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void SplitUVRow_LSX(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vst $vr2, %[du], 0\n\t"
+ "vst $vr3, %[dv], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[du], %[du], 16\n\t"
+ "addi.d %[dv], %[dv], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void MergeUVRow_LSX(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[su], 0\n\t"
+ "vld $vr1, %[sv], 0\n\t"
+ "vilvl.b $vr2, $vr1, $vr0\n\t"
+ "vilvh.b $vr3, $vr1, $vr0\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "vst $vr3, %[dst], 16\n\t"
+ "addi.d %[su], %[su], 16\n\t"
+ "addi.d %[sv], %[sv], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ :
+ [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickod.b $vr0, $vr1, $vr0\n\t"
+ "vpickod.b $vr0, $vr0, $vr0\n\t"
+ "vstelm.d $vr0, %[dst], 0, 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 8\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void I422ToYUY2Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr1, %[su], 0\n\t"
+ "vldrepl.d $vr2, %[sv], 0\n\t"
+ "vilvl.b $vr3, $vr2, $vr1\n\t"
+ "vilvl.b $vr4, $vr3, $vr0\n\t"
+ "vilvh.b $vr5, $vr3, $vr0\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_yuy2), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
+}
+
+void I422ToUYVYRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr1, %[su], 0\n\t"
+ "vldrepl.d $vr2, %[sv], 0\n\t"
+ "vilvl.b $vr3, $vr2, $vr1\n\t"
+ "vilvl.b $vr4, $vr0, $vr3\n\t"
+ "vilvh.b $vr5, $vr0, $vr3\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_uyvy), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
+}
+
+void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2");
+}
+
+void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickod.b $vr2, $vr1, $vr0\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2");
+}
+
+void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickod.b $vr2, $vr1, $vr0\n\t"
+ "vpickev.b $vr3, $vr2, $vr2\n\t"
+ "vpickod.b $vr4, $vr2, $vr2\n\t"
+ "vstelm.d $vr3, %[du], 0, 0\n\t"
+ "vstelm.d $vr4, %[dv], 0, 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void UYVYToUV422Row_LSX(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickev.b $vr3, $vr2, $vr2\n\t"
+ "vpickod.b $vr4, $vr2, $vr2\n\t"
+ "vstelm.d $vr3, %[du], 0, 0\n\t"
+ "vstelm.d $vr4, %[dv], 0, 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void YUY2ToUVRow_LSX(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s0], 16\n\t"
+ "vld $vr2, %[s1], 0\n\t"
+ "vld $vr3, %[s1], 16\n\t"
+ "vpickod.b $vr0, $vr1, $vr0\n\t"
+ "vpickod.b $vr2, $vr3, $vr2\n\t"
+ "vavgr.bu $vr0, $vr0, $vr2\n\t"
+ "vpickev.b $vr1, $vr0, $vr0\n\t"
+ "vpickod.b $vr2, $vr0, $vr0\n\t"
+ "vstelm.d $vr1, %[du], 0, 0\n\t"
+ "vstelm.d $vr2, %[dv], 0, 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void UYVYToUVRow_LSX(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy;
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s0], 16\n\t"
+ "vld $vr2, %[s1], 0\n\t"
+ "vld $vr3, %[s1], 16\n\t"
+ "vpickev.b $vr0, $vr1, $vr0\n\t"
+ "vpickev.b $vr2, $vr3, $vr2\n\t"
+ "vavgr.bu $vr0, $vr0, $vr2\n\t"
+ "vpickev.b $vr1, $vr0, $vr0\n\t"
+ "vpickod.b $vr2, $vr0, $vr0\n\t"
+ "vstelm.d $vr1, %[du], 0, 0\n\t"
+ "vstelm.d $vr2, %[dv], 0, 0\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void I422ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22",
+ "$vr23", "$vr24");
+}
+
+void I422ToRGBARow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr2, $vr5\n\t"
+ "vilvl.b $vr7, $vr4, $vr3\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22",
+ "$vr23", "$vr24");
+}
+
+void I422AlphaToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vldrepl.d $vr5, %[sa], 0\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[sa], %[sa], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22",
+ "$vr23", "$vr24");
+}
+
+void I422ToRGB24Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9,
+ 10, 12, 13, 14, 16, 17, 18, 20};
+ static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30,
+ 0, 0, 0, 0, 0, 0, 0, 0};
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vld $vr12, %[s0], 0\n\t"
+ "vld $vr13, %[s1], 0\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr4, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vshuf.b $vr10, $vr9, $vr8, $vr12\n\t"
+ "vshuf.b $vr11, $vr9, $vr8, $vr13\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "vstelm.d $vr11, %[dst], 16, 0\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 24\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7",
+ "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20", "$vr21",
+ "$vr22", "$vr23", "$vr24");
+}
+
+void I422ToRGB565Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr14, 31\n\t"
+ "vsrli.h $vr14, $vr14, 8\n\t"
+ "vldi $vr15, 63\n\t"
+ "vsrli.h $vr15, $vr15, 8\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vsrli.b $vr3, $vr3, 2\n\t"
+ "vsrli.b $vr4, $vr4, 3\n\t"
+ "vilvl.b $vr2, $vr2, $vr2\n\t"
+ "vilvl.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr4, $vr4, $vr4\n\t"
+ "vand.v $vr2, $vr2, $vr14\n\t"
+ "vand.v $vr3, $vr3, $vr15\n\t"
+ "vand.v $vr4, $vr4, $vr14\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr4, $vr4, 11\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vor.v $vr2, $vr2, $vr4\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11",
+ "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void I422ToARGB4444Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr14, 15\n\t"
+ "vsrli.h $vr14, $vr14, 8\n\t"
+ "vldi $vr15, 240\n\t"
+ "vsrli.h $vr15, $vr15, 8\n\t"
+ "vslli.h $vr15, $vr15, 8\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vsrli.b $vr2, $vr2, 4\n\t"
+ "vsrli.b $vr3, $vr3, 4\n\t"
+ "vsrli.b $vr4, $vr4, 4\n\t"
+ "vilvl.b $vr2, $vr2, $vr2\n\t"
+ "vilvl.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr4, $vr4, $vr4\n\t"
+ "vand.v $vr2, $vr2, $vr14\n\t"
+ "vand.v $vr3, $vr3, $vr14\n\t"
+ "vand.v $vr4, $vr4, $vr14\n\t"
+ "vslli.h $vr3, $vr3, 4\n\t"
+ "vslli.h $vr4, $vr4, 8\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vor.v $vr2, $vr2, $vr4\n\t"
+ "vor.v $vr2, $vr2, $vr15\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11",
+ "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void I422ToARGB1555Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr14, 31\n\t"
+ "vsrli.h $vr14, $vr14, 8\n\t"
+ "vldi $vr15, 128\n\t"
+ "vsrli.h $vr15, $vr15, 8\n\t"
+ "vslli.h $vr15, $vr15, 8\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.w $vr10, %[su], 0\n\t"
+ "vldrepl.w $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr10, $vr10, $vr10\n\t"
+ "vilvl.b $vr11, $vr11, $vr11\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vsrli.b $vr3, $vr3, 3\n\t"
+ "vsrli.b $vr4, $vr4, 3\n\t"
+ "vilvl.b $vr2, $vr2, $vr2\n\t"
+ "vilvl.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr4, $vr4, $vr4\n\t"
+ "vand.v $vr2, $vr2, $vr14\n\t"
+ "vand.v $vr3, $vr3, $vr14\n\t"
+ "vand.v $vr4, $vr4, $vr14\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr4, $vr4, 10\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vor.v $vr2, $vr2, $vr4\n\t"
+ "vor.v $vr2, $vr2, $vr15\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 4\n\t"
+ "addi.d %[sv], %[sv], 4\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11",
+ "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void NV12ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr1, %[suv], 0\n\t"
+ "vilvl.h $vr1, $vr1, $vr1\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[suv], %[suv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void NV21ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr1, %[suv], 0\n\t"
+ "vshuf4i.b $vr1, $vr1, 0xb1\n\t"
+ "vilvl.h $vr1, $vr1, $vr1\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[suv], %[suv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void NV12ToRGB565Row_LSX(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr14, 31\n\t"
+ "vsrli.h $vr14, $vr14, 8\n\t"
+ "vldi $vr15, 63\n\t"
+ "vsrli.h $vr15, $vr15, 8\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr1, %[suv], 0\n\t"
+ "vilvl.h $vr1, $vr1, $vr1\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vsrli.b $vr3, $vr3, 2\n\t"
+ "vsrli.b $vr4, $vr4, 3\n\t"
+ "vilvl.b $vr2, $vr2, $vr2\n\t"
+ "vilvl.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr4, $vr4, $vr4\n\t"
+ "vand.v $vr2, $vr2, $vr14\n\t"
+ "vand.v $vr3, $vr3, $vr15\n\t"
+ "vand.v $vr4, $vr4, $vr14\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr4, $vr4, 11\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vor.v $vr2, $vr2, $vr4\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[suv], %[suv], 8\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr14", "$vr15",
+ "$vr20", "$vr21", "$vr22", "$vr23", "$vr24");
+}
+
+void I444ToARGBRow_LSX(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vldrepl.d $vr10, %[su], 0\n\t"
+ "vldrepl.d $vr11, %[sv], 0\n\t"
+ "vilvl.b $vr1, $vr11, $vr10\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[su], %[su], 8\n\t"
+ "addi.d %[sv], %[sv], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v),
+ [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22",
+ "$vr23", "$vr24");
+}
+
+void I400ToARGBRow_LSX(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vld $vr23, %[c], 96\n\t"
+ "vld $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sy], 0\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vssrani.bu.h $vr2, $vr0, 6\n\t"
+ "vilvl.b $vr6, $vr2, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr2\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9",
+ "$vr23", "$vr24");
+}
+
+void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr2, %[sy], 0\n\t"
+ "vilvl.b $vr6, $vr2, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr2\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9");
+}
+
+void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr10, %[src], 0\n\t"
+ "vpickev.b $vr0, $vr10, $vr10\n\t"
+ "vpickod.b $vr1, $vr10, $vr10\n\t"
+ "vilvl.h $vr1, $vr1, $vr1\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23",
+ "$vr24");
+}
+
+void UYVYToARGBRow_LSX(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr20, %[c], 0\n\t"
+ "vldrepl.w $vr21, %[c], 32\n\t"
+ "vldrepl.w $vr22, %[c], 64\n\t"
+ "vldrepl.h $vr23, %[c], 96\n\t"
+ "vldrepl.h $vr24, %[c], 128\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr10, %[src], 0\n\t"
+ "vpickod.b $vr0, $vr10, $vr10\n\t"
+ "vpickev.b $vr1, $vr10, $vr10\n\t"
+ "vilvl.h $vr1, $vr1, $vr1\n\t"
+ "vilvl.b $vr0, $vr0, $vr0\n\t"
+ "vxori.b $vr1, $vr1, 0x80\n\t"
+ "vmuh.hu $vr0, $vr0, $vr23\n\t"
+ "vadd.h $vr0, $vr0, $vr24\n\t"
+ "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t"
+ "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t"
+ "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t"
+ "vsadd.h $vr2, $vr0, $vr2\n\t"
+ "vsadd.h $vr4, $vr0, $vr4\n\t"
+ "vssub.h $vr3, $vr0, $vr3\n\t"
+ "vssrani.bu.h $vr2, $vr2, 6\n\t"
+ "vssrani.bu.h $vr3, $vr3, 6\n\t"
+ "vssrani.bu.h $vr4, $vr4, 6\n\t"
+ "vilvl.b $vr6, $vr3, $vr2\n\t"
+ "vilvl.b $vr7, $vr5, $vr4\n\t"
+ "vilvl.h $vr8, $vr7, $vr6\n\t"
+ "vilvh.h $vr9, $vr7, $vr6\n\t"
+ "vst $vr8, %[dst], 0\n\t"
+ "vst $vr9, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [c] "r"(yuvconstants)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23",
+ "$vr24");
+}
+
+static LIBYUV_NOINLINE void ARGBTo24Helper_LSX(const uint8_t* src_argb,
+ uint8_t* dst_24,
+ const uint8_t* shuf,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vld $vr2, %[src], 32\n\t"
+ "vld $vr3, %[src], 48\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t"
+ "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t"
+ "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t"
+ "vshuf.b $vr3, $vr3, $vr3, $vr4\n\t"
+ "vstelm.d $vr0, %[dst], 0, 0\n\t"
+ "vstelm.w $vr0, %[dst], 8, 2\n\t"
+ "vstelm.w $vr1, %[dst], 12, 0\n\t"
+ "vstelm.w $vr1, %[dst], 16, 1\n\t"
+ "vstelm.w $vr1, %[dst], 20, 2\n\t"
+ "vstelm.d $vr2, %[dst], 24, 0\n\t"
+ "vstelm.w $vr2, %[dst], 32, 2\n\t"
+ "vstelm.w $vr3, %[dst], 36, 0\n\t"
+ "vstelm.w $vr3, %[dst], 40, 1\n\t"
+ "vstelm.w $vr3, %[dst], 44, 2\n\t"
+ "addi.d %[src], %[src], 64\n\t"
+ "addi.d %[dst], %[dst], 48\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void ARGBToRGB24Row_LSX(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width) {
+ static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9,
+ 10, 12, 13, 14, 0, 0, 0, 0};
+ ARGBTo24Helper_LSX(src_argb, dst_rgb24, kShuf, width);
+}
+
+void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
+ static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9,
+ 8, 14, 13, 12, 0, 0, 0, 0};
+ ARGBTo24Helper_LSX(src_argb, dst_raw, kShuf, width);
+}
+
+static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24,
+ uint8_t* dst_argb,
+ const uint8_t* shuf,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "vldi $vr6, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 12\n\t"
+ "vld $vr2, %[src], 24\n\t"
+ "vld $vr3, %[src], 32\n\t"
+ "vshuf.b $vr0, $vr6, $vr0, $vr4\n\t"
+ "vshuf.b $vr1, $vr6, $vr1, $vr4\n\t"
+ "vshuf.b $vr2, $vr6, $vr2, $vr4\n\t"
+ "vshuf.b $vr3, $vr6, $vr3, $vr5\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "vst $vr2, %[dst], 32\n\t"
+ "vst $vr3, %[dst], 48\n\t"
+ "addi.d %[src], %[src], 48\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6");
+}
+
+void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8,
+ 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8,
+ 9, 16, 10, 11, 12, 16, 13, 14, 15, 16};
+ RGB24ToARGBHelper_LSX(src_rgb24, dst_argb, kShuf, width);
+}
+
+void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6,
+ 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8,
+ 7, 16, 12, 11, 10, 16, 15, 14, 13, 16};
+ RGB24ToARGBHelper_LSX(src_raw, dst_argb, kShuf, width);
+}
+
+void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10,
+ 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7,
+ 12, 11, 10, 15, 14, 13, 0, 0, 0, 0};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 12\n\t"
+ "vld $vr2, %[src], 24\n\t"
+ "vld $vr3, %[src], 32\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t"
+ "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t"
+ "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t"
+ "vshuf.b $vr3, $vr3, $vr3, $vr5\n\t"
+ "vstelm.d $vr0, %[dst], 0, 0\n\t"
+ "vstelm.w $vr0, %[dst], 8, 2\n\t"
+ "vstelm.w $vr1, %[dst], 12, 0\n\t"
+ "vstelm.w $vr1, %[dst], 16, 1\n\t"
+ "vstelm.w $vr1, %[dst], 20, 2\n\t"
+ "vstelm.d $vr2, %[dst], 24, 0\n\t"
+ "vstelm.w $vr2, %[dst], 32, 2\n\t"
+ "vstelm.w $vr3, %[dst], 36, 0\n\t"
+ "vstelm.w $vr3, %[dst], 40, 1\n\t"
+ "vstelm.w $vr3, %[dst], 44, 2\n\t"
+ "addi.d %[src], %[src], 48\n\t"
+ "addi.d %[dst], %[dst], 48\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width)
+ : [shuf] "r"(kShuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
+}
+
+void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vsrli.b $vr3, $vr3, 2\n\t"
+ "vand.v $vr4, $vr2, $vr6\n\t"
+ "vsrli.h $vr2, $vr2, 8\n\t"
+ "vand.v $vr3, $vr3, $vr7\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr2, $vr2, 11\n\t"
+ "vor.v $vr4, $vr4, $vr3\n\t"
+ "vor.v $vr4, $vr4, $vr2\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7");
+}
+
+void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ asm volatile(
+ "vreplgr2vr.w $vr5, %[dith]\n\t"
+ "vilvl.b $vr5, $vr5, $vr5\n\t"
+ "vilvl.h $vr5, $vr5, $vr5\n\t"
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vsadd.bu $vr0, $vr0, $vr5\n\t"
+ "vpickev.b $vr2, $vr0, $vr0\n\t"
+ "vpickod.b $vr3, $vr0, $vr0\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vsrli.b $vr3, $vr3, 2\n\t"
+ "vand.v $vr4, $vr2, $vr6\n\t"
+ "vsrli.h $vr2, $vr2, 8\n\t"
+ "vand.v $vr3, $vr3, $vr7\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr2, $vr2, 11\n\t"
+ "vor.v $vr4, $vr4, $vr3\n\t"
+ "vor.v $vr4, $vr4, $vr2\n\t"
+ "vstelm.d $vr4, %[dst], 0, 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 8\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ : [dith] "r"(dither4)
+ : "memory", "$vr0", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7");
+}
+
+void ARGBToARGB1555Row_LSX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vsrli.b $vr2, $vr2, 3\n\t"
+ "vand.v $vr4, $vr2, $vr6\n\t"
+ "vsrli.h $vr2, $vr2, 8\n\t"
+ "vsrli.h $vr5, $vr3, 15\n\t"
+ "vsrli.b $vr3, $vr3, 3\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vslli.h $vr3, $vr3, 5\n\t"
+ "vslli.h $vr2, $vr2, 10\n\t"
+ "vslli.h $vr5, $vr5, 15\n\t"
+ "vor.v $vr4, $vr4, $vr3\n\t"
+ "vor.v $vr4, $vr4, $vr2\n\t"
+ "vor.v $vr4, $vr4, $vr5\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6");
+}
+
+void ARGBToARGB4444Row_LSX(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vsrli.b $vr2, $vr2, 4\n\t"
+ "vsrli.b $vr3, $vr3, 4\n\t"
+ "vslli.b $vr3, $vr3, 4\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr1, $vr1, $vr1\n\t"
+ "vpickev.b $vr2, $vr2, $vr2\n\t"
+ "vpickev.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr8, $vr2, $vr1\n\t"
+ "vilvl.b $vr9, $vr5, $vr3\n\t"
+ "vilvl.h $vr10, $vr9, $vr8\n\t"
+ "vilvh.h $vr11, $vr9, $vr8\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "vst $vr11, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11");
+}
+
+void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr1, $vr1, $vr1\n\t"
+ "vpickev.b $vr2, $vr2, $vr2\n\t"
+ "vpickev.b $vr3, $vr3, $vr3\n\t"
+ "vpickev.b $vr5, $vr5, $vr5\n\t"
+ "vilvl.b $vr8, $vr2, $vr1\n\t"
+ "vilvl.b $vr9, $vr5, $vr3\n\t"
+ "vilvl.h $vr10, $vr9, $vr8\n\t"
+ "vilvh.h $vr11, $vr9, $vr8\n\t"
+ "vst $vr10, %[dst], 0\n\t"
+ "vst $vr11, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr8", "$vr9", "$vr10", "$vr11");
+}
+
+void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vldi $vr6, 0x0f\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.b $vr2, $vr0, 4\n\t"
+ "vslli.b $vr3, $vr1, 4\n\t"
+ "vor.v $vr1, $vr1, $vr3\n\t"
+ "vslli.b $vr3, $vr2, 4\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vilvl.b $vr4, $vr2, $vr1\n\t"
+ "vilvh.b $vr5, $vr2, $vr1\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6");
+}
+
+void ARGBMultiplyRow_LSX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vmulwev.h.bu $vr2, $vr0, $vr1\n\t"
+ "vmulwod.h.bu $vr3, $vr0, $vr1\n\t"
+ "vsrari.h $vr2, $vr2, 8\n\t"
+ "vsrari.h $vr3, $vr3, 8\n\t"
+ "vpackev.b $vr2, $vr3, $vr2\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void ARGBAddRow_LSX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vsadd.bu $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void ARGBSubtractRow_LSX(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vssub.bu $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void ARGBAttenuateRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16,
+ 11, 11, 11, 16, 15, 15, 15, 16};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "vldi $vr6, 255\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vshuf.b $vr1, $vr5, $vr0, $vr4\n\t"
+ "vmulwev.h.bu $vr2, $vr0, $vr1\n\t"
+ "vmulwod.h.bu $vr3, $vr0, $vr1\n\t"
+ "vadd.h $vr2, $vr2, $vr6\n\t"
+ "vadd.h $vr3, $vr3, $vr6\n\t"
+ "vpackod.b $vr2, $vr3, $vr2\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(kAlpha)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6");
+}
+
+void ARGBBlendRow_LSX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7,
+ 11, 11, 11, 11, 15, 15, 15, 15};
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "vldi $vr5, 0xff\n\t"
+ "vslli.w $vr5, $vr5, 24\n\t"
+ "vldi $vr6, 1\n\t"
+ "vslli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr8, 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vshuf.b $vr2, $vr0, $vr0, $vr4\n\t"
+ "vilvl.b $vr3, $vr8, $vr2\n\t"
+ "vilvh.b $vr2, $vr8, $vr2\n\t"
+ "vsub.h $vr3, $vr6, $vr3\n\t"
+ "vsub.h $vr2, $vr6, $vr2\n\t"
+ "vilvl.b $vr7, $vr8, $vr1\n\t"
+ "vilvh.b $vr1, $vr8, $vr1\n\t"
+ "vmul.h $vr7, $vr7, $vr3\n\t"
+ "vmul.h $vr1, $vr1, $vr2\n\t"
+ "vsrlni.b.h $vr1, $vr7, 8\n\t"
+ "vsadd.bu $vr0, $vr0, $vr1\n\t"
+ "vor.v $vr0, $vr0, $vr5\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [shuf] "r"(kAlpha)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8");
+}
+
+void ARGBOrRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ asm volatile(
+ "vreplgr2vr.w $vr1, %[val]\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vor.v $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(value)
+ : "memory", "$vr0", "$vr1");
+}
+
+void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4,
+ 8, 8, 8, 8, 12, 12, 12, 12};
+ asm volatile(
+ "vld $vr8, %[shuf], 0\n\t"
+ "vldi $vr9, 1\n\t"
+ "vsrli.h $vr10, $vr9, 8\n\t"
+ "vldi $vr11, 0xff\n\t"
+ "vsrli.w $vr11, $vr11, 8\n\t"
+ "vldi $vr12, 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vsrli.w $vr2, $vr0, 24\n\t"
+ "vsrli.w $vr3, $vr1, 24\n\t"
+ "vmul.h $vr4, $vr3, $vr2\n\t"
+ "vadd.h $vr4, $vr4, $vr10\n\t"
+ "vmuh.hu $vr4, $vr4, $vr9\n\t"
+ "vsub.w $vr4, $vr2, $vr4\n\t"
+ "vadd.w $vr4, $vr3, $vr4\n\t"
+ "vseq.w $vr5, $vr3, $vr12\n\t"
+ "vsrli.w $vr5, $vr5, 24\n\t"
+ "vor.v $vr6, $vr2, $vr5\n\t"
+ "vor.v $vr7, $vr4, $vr5\n\t"
+ "vslli.w $vr5, $vr6, 8\n\t"
+ "vsub.w $vr5, $vr5, $vr6\n\t"
+ "vffint.s.wu $vr5, $vr5\n\t"
+ "vffint.s.wu $vr7, $vr7\n\t"
+ "vfdiv.s $vr5, $vr5, $vr7\n\t"
+ "vftintrz.wu.s $vr5, $vr5\n\t"
+ "vshuf.b $vr5, $vr5, $vr5, $vr8\n\t"
+ "vnor.v $vr6, $vr5, $vr5\n\t"
+ "vmulwev.h.bu $vr2, $vr1, $vr6\n\t"
+ "vmulwod.h.bu $vr3, $vr1, $vr6\n\t"
+ "vmulwev.h.bu $vr7, $vr0, $vr5\n\t"
+ "vmulwod.h.bu $vr0, $vr0, $vr5\n\t"
+ "vadd.h $vr2, $vr2, $vr7\n\t"
+ "vadd.h $vr3, $vr3, $vr0\n\t"
+ "vadd.h $vr2, $vr2, $vr10\n\t"
+ "vadd.h $vr3, $vr3, $vr10\n\t"
+ "vmuh.hu $vr2, $vr2, $vr9\n\t"
+ "vmuh.hu $vr3, $vr3, $vr9\n\t"
+ "vslli.h $vr3, $vr3, 8\n\t"
+ "vor.v $vr2, $vr2, $vr3\n\t"
+ "vand.v $vr2, $vr2, $vr11\n\t"
+ "vslli.w $vr4, $vr4, 24\n\t"
+ "vor.v $vr2, $vr2, $vr4\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ : [shuf] "r"(kShufRatio)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12");
+}
+
+void ARGBShadeRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ uint32_t value) {
+ asm volatile(
+ "vreplgr2vr.w $vr4, %[val]\n\t"
+ "vilvl.b $vr5, $vr4, $vr4\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vilvl.b $vr1, $vr0, $vr0\n\t"
+ "vilvh.b $vr2, $vr0, $vr0\n\t"
+ "vmuh.hu $vr1, $vr1, $vr5\n\t"
+ "vmuh.hu $vr2, $vr2, $vr5\n\t"
+ "vsrlni.b.h $vr2, $vr1, 8\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [val] "r"(value)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5");
+}
+
+void ARGBShuffleRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ asm volatile(
+ "vld $vr4, %[shuf], 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [shuf] "r"(shuffler)
+ : "memory", "$vr0", "$vr4");
+}
+
+void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vldi $vr4, 29\n\t"
+ "vldi $vr5, 150\n\t"
+ "vldi $vr6, 77\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vld $vr1, %[src], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vmulwev.h.bu $vr7, $vr2, $vr4\n\t"
+ "vmaddwod.h.bu $vr7, $vr2, $vr6\n\t"
+ "vmaddwev.h.bu $vr7, $vr3, $vr5\n\t"
+ "vsrlri.h $vr7, $vr7, 8\n\t"
+ "vsrli.h $vr3, $vr3, 8\n\t"
+ "vslli.h $vr3, $vr3, 8\n\t"
+ "vor.v $vr3, $vr7, $vr3\n\t"
+ "vslli.h $vr8, $vr7, 8\n\t"
+ "vor.v $vr7, $vr7, $vr8\n\t"
+ "vilvl.h $vr0, $vr3, $vr7\n\t"
+ "vilvh.h $vr1, $vr3, $vr7\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8");
+}
+
+void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vldi $vr4, 17\n\t"
+ "vldi $vr5, 68\n\t"
+ "vldi $vr6, 35\n\t"
+ "vldi $vr7, 22\n\t"
+ "vldi $vr8, 88\n\t"
+ "vldi $vr9, 45\n\t"
+ "vldi $vr10, 24\n\t"
+ "vldi $vr11, 98\n\t"
+ "vldi $vr12, 50\n\t"
+ "1:\n\t"
+ "vld $vr0, %[dst], 0\n\t"
+ "vld $vr1, %[dst], 16\n\t"
+ "vpickev.b $vr2, $vr1, $vr0\n\t"
+ "vpickod.b $vr3, $vr1, $vr0\n\t"
+ "vmulwev.h.bu $vr13, $vr2, $vr4\n\t"
+ "vmaddwod.h.bu $vr13, $vr2, $vr6\n\t"
+ "vmaddwev.h.bu $vr13, $vr3, $vr5\n\t"
+ "vmulwev.h.bu $vr14, $vr2, $vr7\n\t"
+ "vmaddwod.h.bu $vr14, $vr2, $vr9\n\t"
+ "vmaddwev.h.bu $vr14, $vr3, $vr8\n\t"
+ "vmulwev.h.bu $vr15, $vr2, $vr10\n\t"
+ "vmaddwod.h.bu $vr15, $vr2, $vr12\n\t"
+ "vmaddwev.h.bu $vr15, $vr3, $vr11\n\t"
+ "vsrli.h $vr13, $vr13, 7\n\t"
+ "vsrli.h $vr14, $vr14, 7\n\t"
+ "vsrli.h $vr15, $vr15, 7\n\t"
+ "vsat.hu $vr13, $vr13, 7\n\t"
+ "vsat.hu $vr14, $vr14, 7\n\t"
+ "vsat.hu $vr15, $vr15, 7\n\t"
+ "vpickev.b $vr13, $vr13, $vr13\n\t"
+ "vpickev.b $vr14, $vr14, $vr14\n\t"
+ "vpickev.b $vr15, $vr15, $vr15\n\t"
+ "vpickod.b $vr3, $vr3, $vr3\n\t"
+ "vilvl.b $vr2, $vr14, $vr13\n\t"
+ "vilvl.b $vr3, $vr3, $vr15\n\t"
+ "vilvl.h $vr0, $vr3, $vr2\n\t"
+ "vilvh.h $vr1, $vr3, $vr2\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15");
+}
+
+void ARGBColorMatrixRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ asm volatile(
+ "vldrepl.w $vr4, %[mat], 0\n\t"
+ "vldrepl.w $vr5, %[mat], 4\n\t"
+ "vldrepl.w $vr6, %[mat], 8\n\t"
+ "vldrepl.w $vr7, %[mat], 12\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vmulwev.h.bu.b $vr8, $vr0, $vr4\n\t"
+ "vmaddwod.h.bu.b $vr8, $vr0, $vr4\n\t"
+ "vmulwev.h.bu.b $vr9, $vr0, $vr5\n\t"
+ "vmaddwod.h.bu.b $vr9, $vr0, $vr5\n\t"
+ "vmulwev.h.bu.b $vr10, $vr0, $vr6\n\t"
+ "vmaddwod.h.bu.b $vr10, $vr0, $vr6\n\t"
+ "vmulwev.h.bu.b $vr11, $vr0, $vr7\n\t"
+ "vmaddwod.h.bu.b $vr11, $vr0, $vr7\n\t"
+ "vhaddw.w.h $vr8, $vr8, $vr8\n\t"
+ "vhaddw.w.h $vr9, $vr9, $vr9\n\t"
+ "vhaddw.w.h $vr10, $vr10, $vr10\n\t"
+ "vhaddw.w.h $vr11, $vr11, $vr11\n\t"
+ "vilvl.w $vr12, $vr9, $vr8\n\t"
+ "vilvh.w $vr13, $vr9, $vr8\n\t"
+ "vilvl.w $vr14, $vr11, $vr10\n\t"
+ "vilvh.w $vr15, $vr11, $vr10\n\t"
+ "vilvl.d $vr8, $vr14, $vr12\n\t"
+ "vilvh.d $vr9, $vr14, $vr12\n\t"
+ "vilvl.d $vr10, $vr15, $vr13\n\t"
+ "vilvh.d $vr11, $vr15, $vr13\n\t"
+ "vssrani.h.w $vr9, $vr8, 6\n\t"
+ "vssrani.h.w $vr11, $vr10, 6\n\t"
+ "vssrani.bu.h $vr11, $vr9, 0\n\t"
+ "vst $vr11, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [mat] "r"(matrix_argb)
+ : "memory", "$vr0", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9",
+ "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15");
+}
+
+void ARGBQuantizeRow_LSX(uint8_t* dst_argb,
+ int scale,
+ int interval_size,
+ int interval_offset,
+ int width) {
+ asm volatile(
+ "vreplgr2vr.h $vr4, %[scl]\n\t"
+ "vreplgr2vr.b $vr5, %[isz]\n\t"
+ "vreplgr2vr.b $vr6, %[ioff]\n\t"
+ "vldi $vr7, 0xff\n\t"
+ "vslli.w $vr7, $vr7, 24\n\t"
+ "1:\n\t"
+ "vld $vr0, %[dst], 0\n\t"
+ "vilvl.b $vr1, $vr0, $vr0\n\t"
+ "vilvh.b $vr2, $vr0, $vr0\n\t"
+ "vsrli.h $vr1, $vr1, 8\n\t"
+ "vsrli.h $vr2, $vr2, 8\n\t"
+ "vmuh.hu $vr1, $vr1, $vr4\n\t"
+ "vmuh.hu $vr2, $vr2, $vr4\n\t"
+ "vsrlni.b.h $vr2, $vr1, 0\n\t"
+ "vmul.b $vr2, $vr2, $vr5\n\t"
+ "vadd.b $vr2, $vr2, $vr6\n\t"
+ "vand.v $vr0, $vr0, $vr7\n\t"
+ "vandn.v $vr2, $vr7, $vr2\n\t"
+ "vor.v $vr2, $vr2, $vr0\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -4\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [dst] "+r"(dst_argb), [cnt] "+r"(width)
+ : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5", "$vr6", "$vr7");
+}
+
+void SobelRow_LSX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vldi $vr4, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sx], 0\n\t"
+ "vldrepl.d $vr1, %[sy], 0\n\t"
+ "vsadd.bu $vr0, $vr0, $vr1\n\t"
+ "vilvl.b $vr2, $vr0, $vr0\n\t"
+ "vilvl.b $vr3, $vr4, $vr0\n\t"
+ "vilvl.h $vr0, $vr3, $vr2\n\t"
+ "vilvh.h $vr1, $vr3, $vr2\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "vst $vr1, %[dst], 16\n\t"
+ "addi.d %[sx], %[sx], 8\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void SobelToPlaneRow_LSX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[sx], 0\n\t"
+ "vld $vr1, %[sy], 0\n\t"
+ "vsadd.bu $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[sx], %[sx], 16\n\t"
+ "addi.d %[sy], %[sy], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void SobelXYRow_LSX(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vldi $vr4, 0xff\n\t"
+ "1:\n\t"
+ "vldrepl.d $vr0, %[sx], 0\n\t"
+ "vldrepl.d $vr1, %[sy], 0\n\t"
+ "vsadd.bu $vr2, $vr0, $vr1\n\t"
+ "vilvl.b $vr1, $vr2, $vr1\n\t"
+ "vilvl.b $vr0, $vr4, $vr0\n\t"
+ "vilvl.h $vr2, $vr0, $vr1\n\t"
+ "vilvh.h $vr3, $vr0, $vr1\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "vst $vr3, %[dst], 16\n\t"
+ "addi.d %[sx], %[sx], 8\n\t"
+ "addi.d %[sy], %[sy], 8\n\t"
+ "addi.d %[dst], %[dst], 32\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void InterpolateRow_LSX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int32_t source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ if (y1_fraction == 0) {
+ CopyRow_LSX(src_ptr, dst_ptr, width);
+ return;
+ }
+ if (y1_fraction == 128) {
+ asm volatile(
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vavgr.bu $vr0, $vr0, $vr1\n\t"
+ "vst $vr0, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr),
+ [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1");
+ return;
+ }
+ uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8));
+ asm volatile(
+ "vreplgr2vr.h $vr3, %[yf]\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vld $vr1, %[s1], 0\n\t"
+ "vilvl.b $vr2, $vr1, $vr0\n\t"
+ "vilvh.b $vr4, $vr1, $vr0\n\t"
+ "vmulwev.h.bu $vr0, $vr2, $vr3\n\t"
+ "vmaddwod.h.bu $vr0, $vr2, $vr3\n\t"
+ "vmulwev.h.bu $vr1, $vr4, $vr3\n\t"
+ "vmaddwod.h.bu $vr1, $vr4, $vr3\n\t"
+ "vsrarni.b.h $vr1, $vr0, 8\n\t"
+ "vst $vr1, %[dst], 0\n\t"
+ "addi.d %[s0], %[s0], 16\n\t"
+ "addi.d %[s1], %[s1], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr),
+ [cnt] "+r"(width)
+ : [yf] "r"(y_frac)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+}
+
+void HalfFloatRow_LSX(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
+ asm volatile(
+ "vldrepl.w $vr3, %[scale], 0\n\t"
+ "vldrepl.w $vr5, %[bias], 0\n\t"
+ "vldi $vr4, 0\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vilvl.h $vr1, $vr4, $vr0\n\t"
+ "vilvh.h $vr2, $vr4, $vr0\n\t"
+ "vffint.s.wu $vr1, $vr1\n\t"
+ "vffint.s.wu $vr2, $vr2\n\t"
+ "vfmul.s $vr1, $vr1, $vr3\n\t"
+ "vfmul.s $vr2, $vr2, $vr3\n\t"
+ "vfmul.s $vr1, $vr1, $vr5\n\t"
+ "vfmul.s $vr2, $vr2, $vr5\n\t"
+ "vsrlni.h.w $vr2, $vr1, 13\n\t"
+ "vst $vr2, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
+ : [scale] "r"(&scale), [bias] "r"(&bias)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
+}
+
+void ARGBToYMatrixRow_LSX(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "vldrepl.b $vr0, %[c], 0\n\t"
+ "vldrepl.b $vr1, %[c], 1\n\t"
+ "vldrepl.b $vr2, %[c], 2\n\t"
+ "vldrepl.b $vr3, %[c], 3\n\t"
+ "vldrepl.h $vr4, %[c], 96\n\t"
+ "1:\n\t"
+ "vld $vr5, %[src], 0\n\t"
+ "vld $vr6, %[src], 16\n\t"
+ "vor.v $vr9, $vr4, $vr4\n\t"
+ "vpickev.b $vr7, $vr6, $vr5\n\t"
+ "vpickod.b $vr8, $vr6, $vr5\n\t"
+ "vmaddwev.h.bu $vr9, $vr7, $vr0\n\t"
+ "vmaddwev.h.bu $vr9, $vr8, $vr1\n\t"
+ "vmaddwod.h.bu $vr9, $vr7, $vr2\n\t"
+ "vmaddwod.h.bu $vr9, $vr8, $vr3\n\t"
+ "vpickod.b $vr9, $vr9, $vr9\n\t"
+ "vstelm.d $vr9, %[dst], 0, 0\n\t"
+ "addi.d %[src], %[src], 32\n\t"
+ "addi.d %[dst], %[dst], 8\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [c] "r"(c)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9");
+}
+
+void RGBToYMatrixRow_LSX(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8,
+ 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8,
+ 9, 0, 10, 11, 12, 0, 13, 14, 15, 0};
+ asm volatile(
+ "vldrepl.b $vr0, %[c], 0\n\t"
+ "vldrepl.b $vr1, %[c], 1\n\t"
+ "vldrepl.b $vr2, %[c], 2\n\t"
+ "vldrepl.h $vr3, %[c], 96\n\t"
+ "vld $vr4, %[shuf], 0\n\t"
+ "vld $vr5, %[shuf], 16\n\t"
+ "1:\n\t"
+ "vld $vr6, %[src], 0\n\t"
+ "vld $vr7, %[src], 12\n\t"
+ "vld $vr8, %[src], 24\n\t"
+ "vld $vr9, %[src], 32\n\t"
+ "vshuf.b $vr6, $vr6, $vr6, $vr4\n\t"
+ "vshuf.b $vr7, $vr7, $vr7, $vr4\n\t"
+ "vshuf.b $vr8, $vr8, $vr8, $vr4\n\t"
+ "vshuf.b $vr9, $vr9, $vr9, $vr5\n\t"
+ "vor.v $vr12, $vr3, $vr3\n\t"
+ "vor.v $vr13, $vr3, $vr3\n\t"
+ "vpickev.b $vr10, $vr7, $vr6\n\t"
+ "vpickod.b $vr11, $vr7, $vr6\n\t"
+ "vmaddwev.h.bu $vr12, $vr10, $vr0\n\t"
+ "vmaddwev.h.bu $vr12, $vr11, $vr1\n\t"
+ "vmaddwod.h.bu $vr12, $vr10, $vr2\n\t"
+ "vpickev.b $vr10, $vr9, $vr8\n\t"
+ "vpickod.b $vr11, $vr9, $vr8\n\t"
+ "vmaddwev.h.bu $vr13, $vr10, $vr0\n\t"
+ "vmaddwev.h.bu $vr13, $vr11, $vr1\n\t"
+ "vmaddwod.h.bu $vr13, $vr10, $vr2\n\t"
+ "vpickod.b $vr12, $vr13, $vr12\n\t"
+ "vst $vr12, %[dst], 0\n\t"
+ "addi.d %[src], %[src], 48\n\t"
+ "addi.d %[dst], %[dst], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ : [c] "r"(c), [shuf] "r"(kShuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13");
+}
+
+void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "vldi $vr8, 25\n\t"
+ "vsrli.h $vr8, $vr8, 8\n\t"
+ "vldi $vr9, 129\n\t"
+ "vsrli.h $vr9, $vr9, 8\n\t"
+ "vldi $vr10, 66\n\t"
+ "vsrli.h $vr10, $vr10, 8\n\t"
+ "vldi $vr11, 33\n\t"
+ "vsrli.h $vr11, $vr11, 8\n\t"
+ "vslli.h $vr11, $vr11, 7\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vmul.h $vr1, $vr1, $vr8\n\t"
+ "vmadd.h $vr1, $vr2, $vr9\n\t"
+ "vmadd.h $vr1, $vr3, $vr10\n\t"
+ "vadd.h $vr1, $vr1, $vr11\n\t"
+ "vpickod.b $vr1, $vr1, $vr1\n\t"
+ "vstelm.d $vr1, %[dst], 0, 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 8\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7",
+ "$vr8", "$vr9", "$vr10", "$vr11");
+}
+
+void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width) {
+ asm volatile(
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr8, 25\n\t"
+ "vsrli.h $vr8, $vr8, 8\n\t"
+ "vldi $vr9, 129\n\t"
+ "vsrli.h $vr9, $vr9, 8\n\t"
+ "vldi $vr10, 66\n\t"
+ "vsrli.h $vr10, $vr10, 8\n\t"
+ "vldi $vr11, 33\n\t"
+ "vsrli.h $vr11, $vr11, 8\n\t"
+ "vslli.h $vr11, $vr11, 7\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vmul.h $vr1, $vr1, $vr8\n\t"
+ "vmadd.h $vr1, $vr2, $vr9\n\t"
+ "vmadd.h $vr1, $vr3, $vr10\n\t"
+ "vadd.h $vr1, $vr1, $vr11\n\t"
+ "vpickod.b $vr1, $vr1, $vr1\n\t"
+ "vstelm.d $vr1, %[dst], 0, 0\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[dst], %[dst], 8\n\t"
+ "addi.d %[cnt], %[cnt], -8\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr8", "$vr9", "$vr10", "$vr11");
+}
+
+void RGB565ToUVRow_LSX(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565;
+ asm volatile(
+ "vldi $vr20, 112\n\t"
+ "vsrli.h $vr20, $vr20, 8\n\t"
+ "vldi $vr21, 74\n\t"
+ "vsrli.h $vr21, $vr21, 8\n\t"
+ "vldi $vr22, 38\n\t"
+ "vsrli.h $vr22, $vr22, 8\n\t"
+ "vldi $vr23, 94\n\t"
+ "vsrli.h $vr23, $vr23, 8\n\t"
+ "vldi $vr24, 18\n\t"
+ "vsrli.h $vr24, $vr24, 8\n\t"
+ "vldi $vr25, 128\n\t"
+ "vsrli.h $vr25, $vr25, 8\n\t"
+ "vslli.h $vr25, $vr25, 8\n\t"
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vor.v $vr10, $vr1, $vr1\n\t"
+ "vor.v $vr11, $vr2, $vr2\n\t"
+ "vor.v $vr12, $vr3, $vr3\n\t"
+ "vld $vr0, %[s0], 16\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr10, $vr1, $vr10\n\t"
+ "vpickev.b $vr11, $vr2, $vr11\n\t"
+ "vpickev.b $vr12, $vr3, $vr12\n\t"
+ "vld $vr0, %[s1], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vor.v $vr13, $vr1, $vr1\n\t"
+ "vor.v $vr14, $vr2, $vr2\n\t"
+ "vor.v $vr15, $vr3, $vr3\n\t"
+ "vld $vr0, %[s1], 16\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr7\n\t"
+ "vsrli.h $vr3, $vr0, 11\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 2\n\t"
+ "vsrli.h $vr2, $vr2, 4\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr5, $vr3, $vr15\n\t"
+ "vpickev.b $vr4, $vr2, $vr14\n\t"
+ "vpickev.b $vr3, $vr1, $vr13\n\t"
+ "vor.v $vr0, $vr10, $vr10\n\t"
+ "vor.v $vr1, $vr11, $vr11\n\t"
+ "vor.v $vr2, $vr12, $vr12\n\t"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t"
+ "vadd.h $vr0, $vr0, $vr3\n\t"
+ "vsrari.h $vr0, $vr0, 2\n\t"
+ "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t"
+ "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t"
+ "vadd.h $vr1, $vr1, $vr4\n\t"
+ "vsrari.h $vr1, $vr1, 2\n\t"
+ "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t"
+ "vadd.h $vr2, $vr2, $vr5\n\t"
+ "vsrari.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr6, $vr25, $vr25\n\t"
+ "vor.v $vr7, $vr25, $vr25\n\t"
+ "vmadd.h $vr6, $vr0, $vr20\n\t"
+ "vmsub.h $vr6, $vr1, $vr21\n\t"
+ "vmsub.h $vr6, $vr2, $vr22\n\t"
+ "vmadd.h $vr7, $vr2, $vr20\n\t"
+ "vmsub.h $vr7, $vr1, $vr23\n\t"
+ "vmsub.h $vr7, $vr0, $vr24\n\t"
+ "vpickod.b $vr6, $vr7, $vr6\n\t"
+ "vstelm.d $vr6, %[du], 0, 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "vldi $vr7, 63\n\t"
+ "vsrli.h $vr7, $vr7, 8\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20",
+ "$vr21", "$vr22", "$vr23", "$vr24", "$vr25");
+}
+
+void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555;
+ asm volatile(
+ "vldi $vr20, 112\n\t"
+ "vsrli.h $vr20, $vr20, 8\n\t"
+ "vldi $vr21, 74\n\t"
+ "vsrli.h $vr21, $vr21, 8\n\t"
+ "vldi $vr22, 38\n\t"
+ "vsrli.h $vr22, $vr22, 8\n\t"
+ "vldi $vr23, 94\n\t"
+ "vsrli.h $vr23, $vr23, 8\n\t"
+ "vldi $vr24, 18\n\t"
+ "vsrli.h $vr24, $vr24, 8\n\t"
+ "vldi $vr25, 128\n\t"
+ "vsrli.h $vr25, $vr25, 8\n\t"
+ "vslli.h $vr25, $vr25, 8\n\t"
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "1:\n\t"
+ "vld $vr0, %[s0], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vor.v $vr10, $vr1, $vr1\n\t"
+ "vor.v $vr11, $vr2, $vr2\n\t"
+ "vor.v $vr12, $vr3, $vr3\n\t"
+ "vld $vr0, %[s0], 16\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr10, $vr1, $vr10\n\t"
+ "vpickev.b $vr11, $vr2, $vr11\n\t"
+ "vpickev.b $vr12, $vr3, $vr12\n\t"
+ "vld $vr0, %[s1], 0\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vor.v $vr13, $vr1, $vr1\n\t"
+ "vor.v $vr14, $vr2, $vr2\n\t"
+ "vor.v $vr15, $vr3, $vr3\n\t"
+ "vld $vr0, %[s1], 16\n\t"
+ "vand.v $vr1, $vr0, $vr6\n\t"
+ "vsrli.h $vr2, $vr0, 5\n\t"
+ "vand.v $vr2, $vr2, $vr6\n\t"
+ "vsrli.h $vr3, $vr0, 10\n\t"
+ "vand.v $vr3, $vr3, $vr6\n\t"
+ "vsrai.h $vr5, $vr0, 15\n\t"
+ "vslli.h $vr4, $vr1, 3\n\t"
+ "vsrli.h $vr1, $vr1, 2\n\t"
+ "vor.v $vr1, $vr4, $vr1\n\t"
+ "vslli.h $vr4, $vr2, 3\n\t"
+ "vsrli.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr2, $vr4, $vr2\n\t"
+ "vslli.h $vr4, $vr3, 3\n\t"
+ "vsrli.h $vr3, $vr3, 2\n\t"
+ "vor.v $vr3, $vr4, $vr3\n\t"
+ "vpickev.b $vr5, $vr3, $vr15\n\t"
+ "vpickev.b $vr4, $vr2, $vr14\n\t"
+ "vpickev.b $vr3, $vr1, $vr13\n\t"
+ "vor.v $vr0, $vr10, $vr10\n\t"
+ "vor.v $vr1, $vr11, $vr11\n\t"
+ "vor.v $vr2, $vr12, $vr12\n\t"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t"
+ "vadd.h $vr0, $vr0, $vr3\n\t"
+ "vsrari.h $vr0, $vr0, 2\n\t"
+ "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t"
+ "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t"
+ "vadd.h $vr1, $vr1, $vr4\n\t"
+ "vsrari.h $vr1, $vr1, 2\n\t"
+ "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t"
+ "vadd.h $vr2, $vr2, $vr5\n\t"
+ "vsrari.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr6, $vr25, $vr25\n\t"
+ "vor.v $vr7, $vr25, $vr25\n\t"
+ "vmadd.h $vr6, $vr0, $vr20\n\t"
+ "vmsub.h $vr6, $vr1, $vr21\n\t"
+ "vmsub.h $vr6, $vr2, $vr22\n\t"
+ "vmadd.h $vr7, $vr2, $vr20\n\t"
+ "vmsub.h $vr7, $vr1, $vr23\n\t"
+ "vmsub.h $vr7, $vr0, $vr24\n\t"
+ "vpickod.b $vr6, $vr7, $vr6\n\t"
+ "vstelm.d $vr6, %[du], 0, 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "vldi $vr6, 31\n\t"
+ "vsrli.h $vr6, $vr6, 8\n\t"
+ "addi.d %[s0], %[s0], 32\n\t"
+ "addi.d %[s1], %[s1], 32\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u),
+ [dv] "+r"(dst_v), [cnt] "+r"(width)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20",
+ "$vr21", "$vr22", "$vr23", "$vr24", "$vr25");
+}
+
+static LIBYUV_NOINLINE void XRGBToUVHelper_LSX(const uint8_t* src,
+ int stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ const uint8_t* shuf,
+ int width) {
+ const uint8_t* next = src + stride;
+ asm volatile(
+ "vldi $vr20, 112\n\t"
+ "vsrli.h $vr20, $vr20, 8\n\t"
+ "vldi $vr21, 74\n\t"
+ "vsrli.h $vr21, $vr21, 8\n\t"
+ "vldi $vr22, 38\n\t"
+ "vsrli.h $vr22, $vr22, 8\n\t"
+ "vldi $vr23, 94\n\t"
+ "vsrli.h $vr23, $vr23, 8\n\t"
+ "vldi $vr24, 18\n\t"
+ "vsrli.h $vr24, $vr24, 8\n\t"
+ "vldi $vr25, 128\n\t"
+ "vsrli.h $vr25, $vr25, 8\n\t"
+ "vslli.h $vr25, $vr25, 8\n\t"
+ "vld $vr16, %[shuf], 0\n\t"
+ "1:\n\t"
+ "vld $vr8, %[s0], 0\n\t"
+ "vld $vr9, %[s0], 16\n\t"
+ "vld $vr10, %[s0], 32\n\t"
+ "vld $vr11, %[s0], 48\n\t"
+ "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t"
+ "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t"
+ "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t"
+ "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t"
+ "vpickev.b $vr12, $vr9, $vr8\n\t"
+ "vpickod.b $vr13, $vr9, $vr8\n\t"
+ "vpickev.b $vr14, $vr11, $vr10\n\t"
+ "vpickod.b $vr15, $vr11, $vr10\n\t"
+ "vpickev.b $vr0, $vr14, $vr12\n\t"
+ "vpickod.b $vr2, $vr14, $vr12\n\t"
+ "vpickev.b $vr1, $vr15, $vr13\n\t"
+ "vld $vr8, %[s1], 0\n\t"
+ "vld $vr9, %[s1], 16\n\t"
+ "vld $vr10, %[s1], 32\n\t"
+ "vld $vr11, %[s1], 48\n\t"
+ "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t"
+ "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t"
+ "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t"
+ "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t"
+ "vpickev.b $vr12, $vr9, $vr8\n\t"
+ "vpickod.b $vr13, $vr9, $vr8\n\t"
+ "vpickev.b $vr14, $vr11, $vr10\n\t"
+ "vpickod.b $vr15, $vr11, $vr10\n\t"
+ "vpickev.b $vr3, $vr14, $vr12\n\t"
+ "vpickod.b $vr5, $vr14, $vr12\n\t"
+ "vpickev.b $vr4, $vr15, $vr13\n\t"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t"
+ "vadd.h $vr0, $vr0, $vr3\n\t"
+ "vsrari.h $vr0, $vr0, 2\n\t"
+ "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t"
+ "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t"
+ "vadd.h $vr1, $vr1, $vr4\n\t"
+ "vsrari.h $vr1, $vr1, 2\n\t"
+ "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t"
+ "vadd.h $vr2, $vr2, $vr5\n\t"
+ "vsrari.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr6, $vr25, $vr25\n\t"
+ "vor.v $vr7, $vr25, $vr25\n\t"
+ "vmadd.h $vr6, $vr0, $vr20\n\t"
+ "vmsub.h $vr6, $vr1, $vr21\n\t"
+ "vmsub.h $vr6, $vr2, $vr22\n\t"
+ "vmadd.h $vr7, $vr2, $vr20\n\t"
+ "vmsub.h $vr7, $vr1, $vr23\n\t"
+ "vmsub.h $vr7, $vr0, $vr24\n\t"
+ "vpickod.b $vr6, $vr7, $vr6\n\t"
+ "vstelm.d $vr6, %[du], 0, 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "addi.d %[s0], %[s0], 64\n\t"
+ "addi.d %[s1], %[s1], 64\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15", "$vr16", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", "$vr25");
+}
+
+void BGRAToUVRow_LSX(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4,
+ 11, 10, 9, 8, 15, 14, 13, 12};
+ XRGBToUVHelper_LSX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width);
+}
+
+void ABGRToUVRow_LSX(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+ XRGBToUVHelper_LSX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width);
+}
+
+void RGBAToUVRow_LSX(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4,
+ 9, 10, 11, 8, 13, 14, 15, 12};
+ XRGBToUVHelper_LSX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width);
+}
+
+static LIBYUV_NOINLINE void RGB24ToUVHelper_LSX(const uint8_t* src,
+ int stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ const uint8_t* shuf,
+ int width) {
+ const uint8_t* next = src + stride;
+ asm volatile(
+ "vldi $vr20, 112\n\t"
+ "vsrli.h $vr20, $vr20, 8\n\t"
+ "vldi $vr21, 74\n\t"
+ "vsrli.h $vr21, $vr21, 8\n\t"
+ "vldi $vr22, 38\n\t"
+ "vsrli.h $vr22, $vr22, 8\n\t"
+ "vldi $vr23, 94\n\t"
+ "vsrli.h $vr23, $vr23, 8\n\t"
+ "vldi $vr24, 18\n\t"
+ "vsrli.h $vr24, $vr24, 8\n\t"
+ "vldi $vr25, 128\n\t"
+ "vsrli.h $vr25, $vr25, 8\n\t"
+ "vslli.h $vr25, $vr25, 8\n\t"
+ "vld $vr16, %[shuf], 0\n\t"
+ "vld $vr17, %[shuf], 16\n\t"
+ "1:\n\t"
+ "vld $vr8, %[s0], 0\n\t"
+ "vld $vr9, %[s0], 12\n\t"
+ "vld $vr10, %[s0], 24\n\t"
+ "vld $vr11, %[s0], 32\n\t"
+ "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t"
+ "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t"
+ "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t"
+ "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t"
+ "vpickev.b $vr12, $vr9, $vr8\n\t"
+ "vpickod.b $vr13, $vr9, $vr8\n\t"
+ "vpickev.b $vr14, $vr11, $vr10\n\t"
+ "vpickod.b $vr15, $vr11, $vr10\n\t"
+ "vpickev.b $vr0, $vr14, $vr12\n\t"
+ "vpickod.b $vr2, $vr14, $vr12\n\t"
+ "vpickev.b $vr1, $vr15, $vr13\n\t"
+ "vld $vr8, %[s1], 0\n\t"
+ "vld $vr9, %[s1], 12\n\t"
+ "vld $vr10, %[s1], 24\n\t"
+ "vld $vr11, %[s1], 32\n\t"
+ "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t"
+ "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t"
+ "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t"
+ "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t"
+ "vpickev.b $vr12, $vr9, $vr8\n\t"
+ "vpickod.b $vr13, $vr9, $vr8\n\t"
+ "vpickev.b $vr14, $vr11, $vr10\n\t"
+ "vpickod.b $vr15, $vr11, $vr10\n\t"
+ "vpickev.b $vr3, $vr14, $vr12\n\t"
+ "vpickod.b $vr5, $vr14, $vr12\n\t"
+ "vpickev.b $vr4, $vr15, $vr13\n\t"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t"
+ "vadd.h $vr0, $vr0, $vr3\n\t"
+ "vsrari.h $vr0, $vr0, 2\n\t"
+ "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t"
+ "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t"
+ "vadd.h $vr1, $vr1, $vr4\n\t"
+ "vsrari.h $vr1, $vr1, 2\n\t"
+ "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t"
+ "vadd.h $vr2, $vr2, $vr5\n\t"
+ "vsrari.h $vr2, $vr2, 2\n\t"
+ "vor.v $vr6, $vr25, $vr25\n\t"
+ "vor.v $vr7, $vr25, $vr25\n\t"
+ "vmadd.h $vr6, $vr0, $vr20\n\t"
+ "vmsub.h $vr6, $vr1, $vr21\n\t"
+ "vmsub.h $vr6, $vr2, $vr22\n\t"
+ "vmadd.h $vr7, $vr2, $vr20\n\t"
+ "vmsub.h $vr7, $vr1, $vr23\n\t"
+ "vmsub.h $vr7, $vr0, $vr24\n\t"
+ "vpickod.b $vr6, $vr7, $vr6\n\t"
+ "vstelm.d $vr6, %[du], 0, 0\n\t"
+ "vstelm.d $vr6, %[dv], 0, 1\n\t"
+ "addi.d %[s0], %[s0], 48\n\t"
+ "addi.d %[s1], %[s1], 48\n\t"
+ "addi.d %[du], %[du], 8\n\t"
+ "addi.d %[dv], %[dv], 8\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v),
+ [cnt] "+r"(width)
+ : [shuf] "r"(shuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14",
+ "$vr15", "$vr16", "$vr17", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24",
+ "$vr25");
+}
+
+void RGB24ToUVRow_LSX(const uint8_t* src_rgb24,
+ int src_stride_rgb24,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8,
+ 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8,
+ 9, 0, 10, 11, 12, 0, 13, 14, 15, 0};
+ RGB24ToUVHelper_LSX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width);
+}
+
+void RAWToUVRow_LSX(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6,
+ 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8,
+ 7, 0, 12, 11, 10, 0, 15, 14, 13, 0};
+ RGB24ToUVHelper_LSX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width);
+}
+
+#ifdef HAS_SSIM4X4ROW_LSX
+void Ssim4x4Row_LSX(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ const uint8_t* pa;
+ const uint8_t* pb;
+ int r;
+ asm volatile(
+ "1:\n\t"
+ "move %[pa], %[sa]\n\t"
+ "move %[pb], %[sb]\n\t"
+ "li.d %[r], 4\n\t"
+ "vxor.v $vr8, $vr8, $vr8\n\t"
+ "vxor.v $vr9, $vr9, $vr9\n\t"
+ "vxor.v $vr10, $vr10, $vr10\n\t"
+ "vxor.v $vr11, $vr11, $vr11\n\t"
+ "2:\n\t"
+ "vld $vr0, %[pa], 0\n\t"
+ "vld $vr1, %[pb], 0\n\t"
+ "add.d %[pa], %[pa], %[sta]\n\t"
+ "add.d %[pb], %[pb], %[stb]\n\t"
+ "vhaddw.hu.bu $vr2, $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr3, $vr1, $vr1\n\t"
+ "vadd.h $vr8, $vr8, $vr2\n\t"
+ "vadd.h $vr9, $vr9, $vr3\n\t"
+ "vmulwev.h.bu $vr2, $vr0, $vr1\n\t"
+ "vmulwod.h.bu $vr3, $vr0, $vr1\n\t"
+ "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t"
+ "vadd.w $vr11, $vr11, $vr2\n\t"
+ "vadd.w $vr11, $vr11, $vr3\n\t"
+ "vmulwev.h.bu $vr2, $vr0, $vr0\n\t"
+ "vmulwod.h.bu $vr3, $vr0, $vr0\n\t"
+ "vmulwev.h.bu $vr4, $vr1, $vr1\n\t"
+ "vmulwod.h.bu $vr5, $vr1, $vr1\n\t"
+ "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t"
+ "vhaddw.wu.hu $vr4, $vr4, $vr4\n\t"
+ "vhaddw.wu.hu $vr5, $vr5, $vr5\n\t"
+ "vadd.w $vr2, $vr2, $vr3\n\t"
+ "vadd.w $vr4, $vr4, $vr5\n\t"
+ "vadd.w $vr10, $vr10, $vr2\n\t"
+ "vadd.w $vr10, $vr10, $vr4\n\t"
+ "addi.d %[r], %[r], -1\n\t"
+ "bnez %[r], 2b\n\t"
+ "vhaddw.wu.hu $vr8, $vr8, $vr8\n\t"
+ "vhaddw.wu.hu $vr9, $vr9, $vr9\n\t"
+ "vilvl.w $vr0, $vr9, $vr8\n\t"
+ "vilvh.w $vr1, $vr9, $vr8\n\t"
+ "vilvl.w $vr2, $vr11, $vr10\n\t"
+ "vilvh.w $vr3, $vr11, $vr10\n\t"
+ "vilvl.d $vr4, $vr2, $vr0\n\t"
+ "vilvh.d $vr5, $vr2, $vr0\n\t"
+ "vilvl.d $vr6, $vr3, $vr1\n\t"
+ "vilvh.d $vr7, $vr3, $vr1\n\t"
+ "vst $vr4, %[dst], 0\n\t"
+ "vst $vr5, %[dst], 16\n\t"
+ "vst $vr6, %[dst], 32\n\t"
+ "vst $vr7, %[dst], 48\n\t"
+ "addi.d %[sa], %[sa], 16\n\t"
+ "addi.d %[sb], %[sb], 16\n\t"
+ "addi.d %[dst], %[dst], 64\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width),
+ [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
+ : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11");
+}
+#endif // HAS_SSIM4X4ROW_LSX
+
+#ifdef HAS_SSIMWINDOWROW_LSX
+float SsimWindowRow_LSX(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ uint32_t c1 = 26634;
+ uint32_t c2 = 239708;
+ uint32_t c0 = 0;
+ asm volatile(
+ "vreplgr2vr.w $vr6, %[c0]\n\t"
+ "vreplgr2vr.w $vr4, %[c1]\n\t"
+ "vreplgr2vr.w $vr5, %[c2]\n\t"
+ "1:\n\t"
+ "vld $vr0, %[r0], 0\n\t"
+ "vld $vr1, %[r0n], 0\n\t"
+ "vld $vr2, %[r1], 0\n\t"
+ "vld $vr3, %[r1n], 0\n\t"
+ "vadd.w $vr0, $vr0, $vr1\n\t"
+ "vadd.w $vr2, $vr2, $vr3\n\t"
+ "vadd.w $vr0, $vr0, $vr2\n\t"
+ "vshuf4i.w $vr1, $vr0, 0x00\n\t"
+ "vshuf4i.w $vr2, $vr0, 0x55\n\t"
+ "vshuf4i.w $vr3, $vr0, 0xaa\n\t"
+ "vshuf4i.w $vr0, $vr0, 0xff\n\t"
+ "vmul.w $vr7, $vr1, $vr2\n\t"
+ "vslli.w $vr7, $vr7, 1\n\t"
+ "vmul.w $vr1, $vr1, $vr1\n\t"
+ "vmadd.w $vr1, $vr2, $vr2\n\t"
+ "vslli.w $vr0, $vr0, 7\n\t"
+ "vsub.w $vr0, $vr0, $vr7\n\t"
+ "vadd.w $vr7, $vr7, $vr4\n\t"
+ "vadd.w $vr0, $vr0, $vr5\n\t"
+ "vslli.w $vr3, $vr3, 6\n\t"
+ "vsub.w $vr3, $vr3, $vr1\n\t"
+ "vadd.w $vr1, $vr1, $vr4\n\t"
+ "vadd.w $vr3, $vr3, $vr5\n\t"
+ "vffint.s.w $vr7, $vr7\n\t"
+ "vffint.s.w $vr0, $vr0\n\t"
+ "vffint.s.w $vr1, $vr1\n\t"
+ "vffint.s.w $vr3, $vr3\n\t"
+ "vfmul.s $vr7, $vr7, $vr0\n\t"
+ "vfmul.s $vr1, $vr1, $vr3\n\t"
+ "vfdiv.s $vr7, $vr7, $vr1\n\t"
+ "vfadd.s $vr6, $vr6, $vr7\n\t"
+ "addi.d %[r0], %[r0], 16\n\t"
+ "addi.d %[r0n], %[r0n], 16\n\t"
+ "addi.d %[r1], %[r1], 16\n\t"
+ "addi.d %[r1n], %[r1n], 16\n\t"
+ "addi.d %[cnt], %[cnt], -1\n\t"
+ "bnez %[cnt], 1b\n\t"
+ "vstelm.w $vr6, %[tot], 0, 0\n\t"
+ : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1),
+ [r1n] "+r"(row1_next), [cnt] "+r"(num_windows)
+ : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+ return total;
+}
+#endif // HAS_SSIMWINDOWROW_LSX
+
+#ifdef HAS_ARGBDETECTROW_LSX
+uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width) {
+ uint32_t diff = 0;
+ uint32_t rgb_mask = 0x00ffff00u;
+ asm volatile(
+ "vreplgr2vr.w $vr2, %[rgb_mask]\n\t"
+ "vldi $vr3, -1\n\t"
+ "addi.d %[w], %[w], -4\n\t"
+ "bltz %[w], 2f\n\t"
+ "1:\n\t"
+ "vld $vr0, %[argb], 0\n\t"
+ "addi.d %[argb], %[argb], 16\n\t"
+ "vor.v $vr0, $vr0, $vr2\n\t"
+ "vseq.b $vr0, $vr0, $vr3\n\t"
+ "vmskltz.b $vr0, $vr0\n\t"
+ "vpickve2gr.hu %[diff], $vr0, 0\n\t"
+ "nor %[diff], %[diff], $zero\n\t"
+ "bstrpick.w %[diff], %[diff], 15, 0\n\t"
+ "bnez %[diff], 4f\n\t"
+ "addi.d %[w], %[w], -4\n\t"
+ "bgez %[w], 1b\n\t"
+ "2:\n\t"
+ "addi.d %[w], %[w], 4\n\t"
+ "beqz %[w], 4f\n\t"
+ "3:\n\t"
+ "vldrepl.w $vr0, %[argb], 0\n\t"
+ "addi.d %[argb], %[argb], 4\n\t"
+ "vor.v $vr0, $vr0, $vr2\n\t"
+ "vseq.b $vr0, $vr0, $vr3\n\t"
+ "vmskltz.b $vr0, $vr0\n\t"
+ "vpickve2gr.hu %[diff], $vr0, 0\n\t"
+ "nor %[diff], %[diff], $zero\n\t"
+ "bstrpick.w %[diff], %[diff], 15, 0\n\t"
+ "bnez %[diff], 4f\n\t"
+ "addi.d %[w], %[w], -1\n\t"
+ "bnez %[w], 3b\n\t"
+ "4:\n\t"
+ : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff)
+ : [rgb_mask] "r"(rgb_mask)
+ : "memory", "$vr0", "$vr2", "$vr3");
+ if (diff) {
+ return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_LSX
+
+#ifdef HAS_HAMMINGDISTANCE_LSX
+uint32_t HammingDistance_LSX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ asm volatile(
+ "vxor.v $vr4, $vr4, $vr4\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sa], 0\n\t"
+ "vld $vr1, %[sb], 0\n\t"
+ "vxor.v $vr0, $vr0, $vr1\n\t"
+ "vpcnt.b $vr0, $vr0\n\t"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.wu.hu $vr0, $vr0, $vr0\n\t"
+ "vadd.w $vr4, $vr4, $vr0\n\t"
+ "addi.d %[sa], %[sa], 16\n\t"
+ "addi.d %[sb], %[sb], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ "vhaddw.du.wu $vr4, $vr4, $vr4\n\t"
+ "vhaddw.qu.du $vr4, $vr4, $vr4\n\t"
+ "vpickve2gr.wu %[diff], $vr4, 0\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff)
+ :
+ : "memory", "$vr0", "$vr1", "$vr4");
+ return diff;
+}
+#endif // HAS_HAMMINGDISTANCE_LSX
+
+#ifdef HAS_SUMSQUAREERROR_LSX
+uint32_t SumSquareError_LSX(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t sse;
+ asm volatile(
+ "vxor.v $vr4, $vr4, $vr4\n\t"
+ "1:\n\t"
+ "vld $vr0, %[sa], 0\n\t"
+ "vld $vr1, %[sb], 0\n\t"
+ "vabsd.bu $vr0, $vr0, $vr1\n\t"
+ "vmulwev.h.bu $vr2, $vr0, $vr0\n\t"
+ "vmulwod.h.bu $vr3, $vr0, $vr0\n\t"
+ "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t"
+ "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t"
+ "vadd.w $vr4, $vr4, $vr2\n\t"
+ "vadd.w $vr4, $vr4, $vr3\n\t"
+ "addi.d %[sa], %[sa], 16\n\t"
+ "addi.d %[sb], %[sb], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ "vhaddw.du.wu $vr4, $vr4, $vr4\n\t"
+ "vhaddw.qu.du $vr4, $vr4, $vr4\n\t"
+ "vpickve2gr.wu %[sse], $vr4, 0\n\t"
+ : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse)
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_LSX
+
+#ifdef HAS_HASHDJB2_LSX
+static const uint32_t kDjb2Muls_LSX[16] = {
+ 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
+ 0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
+ 0x00008c61, 0x00000441, 0x00000021, 0x00000001,
+};
+
+uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ uint32_t tmp;
+ uint32_t c16 = 0x92d9e201u;
+ asm volatile(
+ "vld $vr4, %[muls], 0\n\t"
+ "vld $vr5, %[muls], 16\n\t"
+ "vld $vr6, %[muls], 32\n\t"
+ "vld $vr7, %[muls], 48\n\t"
+ "1:\n\t"
+ "vld $vr0, %[src], 0\n\t"
+ "vsllwil.hu.bu $vr1, $vr0, 0\n\t"
+ "vexth.hu.bu $vr2, $vr0\n\t"
+ "vsllwil.wu.hu $vr0, $vr1, 0\n\t"
+ "vexth.wu.hu $vr1, $vr1\n\t"
+ "vsllwil.wu.hu $vr3, $vr2, 0\n\t"
+ "vexth.wu.hu $vr2, $vr2\n\t"
+ "vmul.w $vr0, $vr0, $vr4\n\t"
+ "vmadd.w $vr0, $vr1, $vr5\n\t"
+ "vmadd.w $vr0, $vr3, $vr6\n\t"
+ "vmadd.w $vr0, $vr2, $vr7\n\t"
+ "vhaddw.du.wu $vr0, $vr0, $vr0\n\t"
+ "vhaddw.qu.du $vr0, $vr0, $vr0\n\t"
+ "vpickve2gr.wu %[tmp], $vr0, 0\n\t"
+ "mul.w %[hash], %[hash], %[c16]\n\t"
+ "add.w %[hash], %[hash], %[tmp]\n\t"
+ "addi.d %[src], %[src], 16\n\t"
+ "addi.d %[cnt], %[cnt], -16\n\t"
+ "bnez %[cnt], 1b\n\t"
+ : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
+ : [muls] "r"(kDjb2Muls_LSX), [c16] "r"(c16)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+ return hash;
+}
+#endif // HAS_HASHDJB2_LSX
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
diff --git a/libs/libyuv/source/row_mmi.cc b/libs/libyuv/source/row_mmi.cc
deleted file mode 100644
index d8726d0937..0000000000
--- a/libs/libyuv/source/row_mmi.cc
+++ /dev/null
@@ -1,6042 +0,0 @@
-/*
- * Copyright 2011 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-#include "libyuv/row.h"
-
-#include <string.h> // For memcpy and memset.
-
-#include "libyuv/basic_types.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-// This module is for Mips MMI.
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-void RGB24ToARGBRow_MMI(const uint8_t* src_rgb24,
- uint8_t* dst_argb,
- int width) {
- uint64_t src0, src1, dest;
- const uint64_t mask = 0xff000000ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gslwlc1 %[src0], 0x03(%[src_ptr]) \n\t"
- "gslwrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gslwlc1 %[src1], 0x06(%[src_ptr]) \n\t"
- "gslwrc1 %[src1], 0x03(%[src_ptr]) \n\t"
-
- "or %[src0], %[src0], %[mask] \n\t"
- "or %[src1], %[src1], %[mask] \n\t"
- "punpcklwd %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "gslwlc1 %[src0], 0x09(%[src_ptr]) \n\t"
- "gslwrc1 %[src0], 0x06(%[src_ptr]) \n\t"
- "gslwlc1 %[src1], 0x0c(%[src_ptr]) \n\t"
- "gslwrc1 %[src1], 0x09(%[src_ptr]) \n\t"
-
- "or %[src0], %[src0], %[mask] \n\t"
- "or %[src1], %[src1], %[mask] \n\t"
- "punpcklwd %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x0c \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_rgb24), [dst_ptr] "r"(dst_argb), [width] "r"(width),
- [mask] "f"(mask)
- : "memory");
-}
-
-void RAWToARGBRow_MMI(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
- uint64_t src0, src1, dest;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0xff000000ULL;
- const uint64_t mask2 = 0xc6;
-
- __asm__ volatile(
- "1: \n\t"
- "gslwlc1 %[src0], 0x03(%[src_ptr]) \n\t"
- "gslwrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gslwlc1 %[src1], 0x06(%[src_ptr]) \n\t"
- "gslwrc1 %[src1], 0x03(%[src_ptr]) \n\t"
-
- "or %[src0], %[src0], %[mask1] \n\t"
- "punpcklbh %[src0], %[src0], %[mask0] \n\t"
- "pshufh %[src0], %[src0], %[mask2] \n\t"
- "or %[src1], %[src1], %[mask1] \n\t"
- "punpcklbh %[src1], %[src1], %[mask0] \n\t"
- "pshufh %[src1], %[src1], %[mask2] \n\t"
- "packushb %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "gslwlc1 %[src0], 0x09(%[src_ptr]) \n\t"
- "gslwrc1 %[src0], 0x06(%[src_ptr]) \n\t"
- "gslwlc1 %[src1], 0x0c(%[src_ptr]) \n\t"
- "gslwrc1 %[src1], 0x09(%[src_ptr]) \n\t"
-
- "or %[src0], %[src0], %[mask1] \n\t"
- "punpcklbh %[src0], %[src0], %[mask0] \n\t"
- "pshufh %[src0], %[src0], %[mask2] \n\t"
- "or %[src1], %[src1], %[mask1] \n\t"
- "punpcklbh %[src1], %[src1], %[mask0] \n\t"
- "pshufh %[src1], %[src1], %[mask2] \n\t"
- "packushb %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x0c \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_raw), [dst_ptr] "r"(dst_argb), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [mask2] "f"(mask2), [width] "r"(width)
- : "memory");
-}
-
-void RAWToRGB24Row_MMI(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
- uint64_t src0, src1;
- uint64_t ftmp[4];
- uint64_t mask0 = 0xc6;
- uint64_t mask1 = 0x6c;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_raw]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_raw]) \n\t"
- "gslwrc1 %[src1], 0x08(%[src_raw]) \n\t"
- "gslwlc1 %[src1], 0x0b(%[src_raw]) \n\t"
-
- "punpcklbh %[ftmp0], %[src0], %[zero] \n\t"
- "pshufh %[ftmp0], %[ftmp0], %[mask0] \n\t"
- "punpckhbh %[ftmp1], %[src0], %[zero] \n\t"
- "punpcklbh %[src1], %[src1], %[zero] \n\t"
- "pextrh %[ftmp2], %[ftmp0], %[three] \n\t"
- "pextrh %[ftmp3], %[ftmp1], %[one] \n\t"
- "pinsrh_3 %[ftmp0], %[ftmp0], %[ftmp3] \n\t"
- "pextrh %[ftmp3], %[ftmp1], %[two] \n\t"
- "pinsrh_1 %[ftmp1], %[ftmp1], %[ftmp2] \n\t"
- "pshufh %[src1], %[src1], %[mask1] \n\t"
- "pextrh %[ftmp2], %[src1], %[zero] \n\t"
- "pinsrh_2 %[ftmp1], %[ftmp1], %[ftmp2] \n\t"
- "pinsrh_0 %[src1], %[src1], %[ftmp3] \n\t"
- "packushb %[ftmp0], %[ftmp0], %[ftmp1] \n\t"
- "packushb %[src1], %[src1], %[zero] \n\t"
-
- "gssdrc1 %[ftmp0], 0x00(%[dst_rgb24]) \n\t"
- "gssdlc1 %[ftmp0], 0x07(%[dst_rgb24]) \n\t"
- "gsswrc1 %[src1], 0x08(%[dst_rgb24]) \n\t"
- "gsswlc1 %[src1], 0x0b(%[dst_rgb24]) \n\t"
-
- "daddiu %[src_raw], %[src_raw], 0x0c \n\t"
- "daddiu %[dst_rgb24], %[dst_rgb24], 0x0c \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [ftmp0] "=&f"(ftmp[0]),
- [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2]), [ftmp3] "=&f"(ftmp[3])
- : [src_raw] "r"(src_raw), [dst_rgb24] "r"(dst_rgb24), [width] "r"(width),
- [mask0] "f"(mask0), [mask1] "f"(mask1), [zero] "f"(0x00),
- [one] "f"(0x01), [two] "f"(0x02), [three] "f"(0x03)
- : "memory");
-}
-
-void RGB565ToARGBRow_MMI(const uint8_t* src_rgb565,
- uint8_t* dst_argb,
- int width) {
- uint64_t ftmp[5];
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0007000700070007;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb565]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psrlh %[r], %[src1], %[three] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[two] \n\t"
- "psrlh %[src1], %[g], %[four] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "packushb %[b], %[b], %[r] \n\t"
- "packushb %[g], %[g], %[c1] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x00(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x07(%[dst_argb]) \n\t"
- "punpckhhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x08(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x0f(%[dst_argb]) \n\t"
- "daddiu %[src_rgb565], %[src_rgb565], 0x08 \n\t"
- "daddiu %[dst_argb], %[dst_argb], 0x10 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b] "=&f"(ftmp[2]),
- [g] "=&f"(ftmp[3]), [r] "=&f"(ftmp[4])
- : [src_rgb565] "r"(src_rgb565), [dst_argb] "r"(dst_argb),
- [width] "r"(width), [c0] "f"(c0), [c1] "f"(c1), [c2] "f"(c2),
- [eight] "f"(0x08), [five] "f"(0x05), [three] "f"(0x03), [two] "f"(0x02),
- [four] "f"(0x04)
- : "memory");
-}
-
-void ARGB1555ToARGBRow_MMI(const uint8_t* src_argb1555,
- uint8_t* dst_argb,
- int width) {
- uint64_t ftmp[6];
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0003000300030003;
- uint64_t c3 = 0x007c007c007c007c;
- uint64_t c4 = 0x0001000100010001;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb1555]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "and %[r], %[src1], %[c3] \n\t"
- "psrlh %[r], %[r], %[two] \n\t"
- "psrlh %[a], %[src1], %[seven] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[three] \n\t"
- "psrlh %[src1], %[g], %[two] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "xor %[a], %[a], %[c1] \n\t"
- "paddb %[a], %[a], %[c4] \n\t"
- "packushb %[b], %[b], %[r] \n\t"
- "packushb %[g], %[g], %[a] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x00(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x07(%[dst_argb]) \n\t"
- "punpckhhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x08(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x0f(%[dst_argb]) \n\t"
- "daddiu %[src_argb1555], %[src_argb1555], 0x08 \n\t"
- "daddiu %[dst_argb], %[dst_argb], 0x10 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b] "=&f"(ftmp[2]),
- [g] "=&f"(ftmp[3]), [r] "=&f"(ftmp[4]), [a] "=&f"(ftmp[5])
- : [src_argb1555] "r"(src_argb1555), [dst_argb] "r"(dst_argb),
- [width] "r"(width), [c0] "f"(c0), [c1] "f"(c1), [c2] "f"(c2),
- [c3] "f"(c3), [c4] "f"(c4), [eight] "f"(0x08), [five] "f"(0x05),
- [three] "f"(0x03), [two] "f"(0x02), [seven] "f"(0x07)
- : "memory");
-}
-
-void ARGB4444ToARGBRow_MMI(const uint8_t* src_argb4444,
- uint8_t* dst_argb,
- int width) {
- uint64_t ftmp[6];
- uint64_t c0 = 0x000f000f000f000f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb4444]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g], %[src0], %[four] \n\t"
- "and %[r], %[src1], %[c0] \n\t"
- "psrlh %[a], %[src1], %[four] \n\t"
- "psllh %[src0], %[b], %[four] \n\t"
- "or %[b], %[src0], %[b] \n\t"
- "psllh %[src0], %[g], %[four] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psllh %[src0], %[r], %[four] \n\t"
- "or %[r], %[src0], %[r] \n\t"
- "psllh %[src0], %[a], %[four] \n\t"
- "or %[a], %[src0], %[a] \n\t"
- "packushb %[b], %[b], %[r] \n\t"
- "packushb %[g], %[g], %[a] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x00(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x07(%[dst_argb]) \n\t"
- "punpckhhw %[r], %[src0], %[src1] \n\t"
- "gssdrc1 %[r], 0x08(%[dst_argb]) \n\t"
- "gssdlc1 %[r], 0x0f(%[dst_argb]) \n\t"
- "daddiu %[src_argb4444], %[src_argb4444], 0x08 \n\t"
- "daddiu %[dst_argb], %[dst_argb], 0x10 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b] "=&f"(ftmp[2]),
- [g] "=&f"(ftmp[3]), [r] "=&f"(ftmp[4]), [a] "=&f"(ftmp[5])
- : [src_argb4444] "r"(src_argb4444), [dst_argb] "r"(dst_argb),
- [width] "r"(width), [c0] "f"(c0), [c1] "f"(c1), [eight] "f"(0x08),
- [four] "f"(0x04)
- : "memory");
-}
-
-void ARGBToRGB24Row_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- uint64_t src;
-
- __asm__ volatile(
- "1: \n\t"
- "gslwlc1 %[src], 0x03(%[src_ptr]) \n\t"
- "gslwrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "gsswlc1 %[src], 0x03(%[dst_ptr]) \n\t"
- "gsswrc1 %[src], 0x00(%[dst_ptr]) \n\t"
-
- "gslwlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gslwrc1 %[src], 0x04(%[src_ptr]) \n\t"
- "gsswlc1 %[src], 0x06(%[dst_ptr]) \n\t"
- "gsswrc1 %[src], 0x03(%[dst_ptr]) \n\t"
-
- "gslwlc1 %[src], 0x0b(%[src_ptr]) \n\t"
- "gslwrc1 %[src], 0x08(%[src_ptr]) \n\t"
- "gsswlc1 %[src], 0x09(%[dst_ptr]) \n\t"
- "gsswrc1 %[src], 0x06(%[dst_ptr]) \n\t"
-
- "gslwlc1 %[src], 0x0f(%[src_ptr]) \n\t"
- "gslwrc1 %[src], 0x0c(%[src_ptr]) \n\t"
- "gsswlc1 %[src], 0x0c(%[dst_ptr]) \n\t"
- "gsswrc1 %[src], 0x09(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x0c \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_rgb), [width] "r"(width)
- : "memory");
-}
-
-void ARGBToRAWRow_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- uint64_t src0, src1;
- uint64_t ftmp[3];
- uint64_t mask0 = 0xc6;
- uint64_t mask1 = 0x18;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_argb]) \n\t"
-
- "punpcklbh %[ftmp0], %[src0], %[zero] \n\t"
- "pshufh %[ftmp0], %[ftmp0], %[mask0] \n\t"
- "punpckhbh %[ftmp1], %[src0], %[zero] \n\t"
- "punpcklbh %[ftmp2], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
-
- "pextrh %[src0], %[ftmp1], %[two] \n\t"
- "pinsrh_3 %[ftmp0], %[ftmp0], %[src0] \n\t"
- "pshufh %[ftmp1], %[ftmp1], %[one] \n\t"
-
- "pextrh %[src0], %[ftmp2], %[two] \n\t"
- "pinsrh_2 %[ftmp1], %[ftmp1], %[src0] \n\t"
- "pextrh %[src0], %[ftmp2], %[one] \n\t"
- "pinsrh_3 %[ftmp1], %[ftmp1], %[src0] \n\t"
- "pextrh %[src0], %[ftmp2], %[zero] \n\t"
- "pshufh %[src1], %[src1], %[mask1] \n\t"
- "pinsrh_0 %[src1], %[src1], %[src0] \n\t"
- "packushb %[ftmp0], %[ftmp0], %[ftmp1] \n\t"
- "packushb %[src1], %[src1], %[zero] \n\t"
-
- "gssdrc1 %[ftmp0], 0x00(%[dst_rgb]) \n\t"
- "gssdlc1 %[ftmp0], 0x07(%[dst_rgb]) \n\t"
- "gsswrc1 %[src1], 0x08(%[dst_rgb]) \n\t"
- "gsswlc1 %[src1], 0x0b(%[dst_rgb]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x10 \n\t"
- "daddiu %[dst_rgb], %[dst_rgb], 0x0c \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [ftmp0] "=&f"(ftmp[0]),
- [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2])
- : [src_argb] "r"(src_argb), [dst_rgb] "r"(dst_rgb), [width] "r"(width),
- [mask0] "f"(mask0), [mask1] "f"(mask1), [zero] "f"(0x00),
- [one] "f"(0x01), [two] "f"(0x02)
- : "memory");
-}
-
-void ARGBToRGB565Row_MMI(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- uint64_t src0, src1;
- uint64_t ftmp[3];
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_argb]) \n\t"
-
- "punpcklbh %[b], %[src0], %[src1] \n\t"
- "punpckhbh %[g], %[src0], %[src1] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklbh %[b], %[src0], %[zero] \n\t"
- "punpckhbh %[g], %[src0], %[zero] \n\t"
- "punpcklbh %[r], %[src1], %[zero] \n\t"
-
- "psrlh %[b], %[b], %[three] \n\t"
- "psrlh %[g], %[g], %[two] \n\t"
- "psrlh %[r], %[r], %[three] \n\t"
-
- "psllh %[g], %[g], %[five] \n\t"
- "psllh %[r], %[r], %[eleven] \n\t"
- "or %[b], %[b], %[g] \n\t"
- "or %[b], %[b], %[r] \n\t"
-
- "gssdrc1 %[b], 0x00(%[dst_rgb]) \n\t"
- "gssdlc1 %[b], 0x07(%[dst_rgb]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x10 \n\t"
- "daddiu %[dst_rgb], %[dst_rgb], 0x08 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [b] "=&f"(ftmp[0]),
- [g] "=&f"(ftmp[1]), [r] "=&f"(ftmp[2])
- : [src_argb] "r"(src_argb), [dst_rgb] "r"(dst_rgb), [width] "r"(width),
- [zero] "f"(0x00), [two] "f"(0x02), [three] "f"(0x03), [five] "f"(0x05),
- [eleven] "f"(0x0b)
- : "memory");
-}
-
-// dither4 is a row of 4 values from 4x4 dither matrix.
-// The 4x4 matrix contains values to increase RGB. When converting to
-// fewer bits (565) this provides an ordered dither.
-// The order in the 4x4 matrix in first byte is upper left.
-// The 4 values are passed as an int, then referenced as an array, so
-// endian will not affect order of the original matrix. But the dither4
-// will containing the first pixel in the lower byte for little endian
-// or the upper byte for big endian.
-void ARGBToRGB565DitherRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- const uint32_t dither4,
- int width) {
- uint64_t src0, src1;
- uint64_t ftmp[3];
- uint64_t c0 = 0x00ff00ff00ff00ff;
-
- __asm__ volatile(
- "punpcklbh %[dither], %[dither], %[zero] \n\t"
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_argb]) \n\t"
-
- "punpcklbh %[b], %[src0], %[src1] \n\t"
- "punpckhbh %[g], %[src0], %[src1] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklbh %[b], %[src0], %[zero] \n\t"
- "punpckhbh %[g], %[src0], %[zero] \n\t"
- "punpcklbh %[r], %[src1], %[zero] \n\t"
-
- "paddh %[b], %[b], %[dither] \n\t"
- "paddh %[g], %[g], %[dither] \n\t"
- "paddh %[r], %[r], %[dither] \n\t"
- "pcmpgth %[src0], %[b], %[c0] \n\t"
- "or %[src0], %[src0], %[b] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "pcmpgth %[src0], %[g], %[c0] \n\t"
- "or %[src0], %[src0], %[g] \n\t"
- "and %[g], %[src0], %[c0] \n\t"
- "pcmpgth %[src0], %[r], %[c0] \n\t"
- "or %[src0], %[src0], %[r] \n\t"
- "and %[r], %[src0], %[c0] \n\t"
-
- "psrlh %[b], %[b], %[three] \n\t"
- "psrlh %[g], %[g], %[two] \n\t"
- "psrlh %[r], %[r], %[three] \n\t"
-
- "psllh %[g], %[g], %[five] \n\t"
- "psllh %[r], %[r], %[eleven] \n\t"
- "or %[b], %[b], %[g] \n\t"
- "or %[b], %[b], %[r] \n\t"
-
- "gssdrc1 %[b], 0x00(%[dst_rgb]) \n\t"
- "gssdlc1 %[b], 0x07(%[dst_rgb]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x10 \n\t"
- "daddiu %[dst_rgb], %[dst_rgb], 0x08 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [b] "=&f"(ftmp[0]),
- [g] "=&f"(ftmp[1]), [r] "=&f"(ftmp[2])
- : [src_argb] "r"(src_argb), [dst_rgb] "r"(dst_rgb), [width] "r"(width),
- [dither] "f"(dither4), [c0] "f"(c0), [zero] "f"(0x00), [two] "f"(0x02),
- [three] "f"(0x03), [five] "f"(0x05), [eleven] "f"(0x0b)
- : "memory");
-}
-
-void ARGBToARGB1555Row_MMI(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- uint64_t src0, src1;
- uint64_t ftmp[4];
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_argb]) \n\t"
-
- "punpcklbh %[b], %[src0], %[src1] \n\t"
- "punpckhbh %[g], %[src0], %[src1] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklbh %[b], %[src0], %[zero] \n\t"
- "punpckhbh %[g], %[src0], %[zero] \n\t"
- "punpcklbh %[r], %[src1], %[zero] \n\t"
- "punpckhbh %[a], %[src1], %[zero] \n\t"
-
- "psrlh %[b], %[b], %[three] \n\t"
- "psrlh %[g], %[g], %[three] \n\t"
- "psrlh %[r], %[r], %[three] \n\t"
- "psrlh %[a], %[a], %[seven] \n\t"
-
- "psllh %[g], %[g], %[five] \n\t"
- "psllh %[r], %[r], %[ten] \n\t"
- "psllh %[a], %[a], %[fifteen] \n\t"
- "or %[b], %[b], %[g] \n\t"
- "or %[b], %[b], %[r] \n\t"
- "or %[b], %[b], %[a] \n\t"
-
- "gssdrc1 %[b], 0x00(%[dst_rgb]) \n\t"
- "gssdlc1 %[b], 0x07(%[dst_rgb]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x10 \n\t"
- "daddiu %[dst_rgb], %[dst_rgb], 0x08 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [b] "=&f"(ftmp[0]),
- [g] "=&f"(ftmp[1]), [r] "=&f"(ftmp[2]), [a] "=&f"(ftmp[3])
- : [src_argb] "r"(src_argb), [dst_rgb] "r"(dst_rgb), [width] "r"(width),
- [zero] "f"(0x00), [three] "f"(0x03), [five] "f"(0x05),
- [seven] "f"(0x07), [ten] "f"(0x0a), [fifteen] "f"(0x0f)
- : "memory");
-}
-
-void ARGBToARGB4444Row_MMI(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- uint64_t src0, src1;
- uint64_t ftmp[4];
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_argb]) \n\t"
-
- "punpcklbh %[b], %[src0], %[src1] \n\t"
- "punpckhbh %[g], %[src0], %[src1] \n\t"
- "punpcklbh %[src0], %[b], %[g] \n\t"
- "punpckhbh %[src1], %[b], %[g] \n\t"
- "punpcklbh %[b], %[src0], %[zero] \n\t"
- "punpckhbh %[g], %[src0], %[zero] \n\t"
- "punpcklbh %[r], %[src1], %[zero] \n\t"
- "punpckhbh %[a], %[src1], %[zero] \n\t"
-
- "psrlh %[b], %[b], %[four] \n\t"
- "psrlh %[g], %[g], %[four] \n\t"
- "psrlh %[r], %[r], %[four] \n\t"
- "psrlh %[a], %[a], %[four] \n\t"
-
- "psllh %[g], %[g], %[four] \n\t"
- "psllh %[r], %[r], %[eight] \n\t"
- "psllh %[a], %[a], %[twelve] \n\t"
- "or %[b], %[b], %[g] \n\t"
- "or %[b], %[b], %[r] \n\t"
- "or %[b], %[b], %[a] \n\t"
-
- "gssdrc1 %[b], 0x00(%[dst_rgb]) \n\t"
- "gssdlc1 %[b], 0x07(%[dst_rgb]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x10 \n\t"
- "daddiu %[dst_rgb], %[dst_rgb], 0x08 \n\t"
- "daddiu %[width], %[width], -0x04 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [b] "=&f"(ftmp[0]),
- [g] "=&f"(ftmp[1]), [r] "=&f"(ftmp[2]), [a] "=&f"(ftmp[3])
- : [src_argb] "r"(src_argb), [dst_rgb] "r"(dst_rgb), [width] "r"(width),
- [zero] "f"(0x00), [four] "f"(0x04), [eight] "f"(0x08),
- [twelve] "f"(0x0c)
- : "memory");
-}
-
-void ARGBToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0001004200810019;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x1f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x20 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void ARGBToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x0026004a00700002;
- const uint64_t mask_v = 0x00020070005e0012;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest0_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_u], %[dest0_u], %[value] \n\t"
- "pinsrh_3 %[dest0_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x10(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x17(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest1_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_u], %[dest1_u], %[value] \n\t"
- "pinsrh_3 %[dest1_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x20(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x27(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x20(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x27(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest2_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_u], %[dest2_u], %[value] \n\t"
- "pinsrh_3 %[dest2_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x28(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x28(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x30(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x37(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x30(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x37(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest3_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_u], %[dest3_u], %[value] \n\t"
- "pinsrh_3 %[dest3_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x38(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x3f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x38(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x3f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x40 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void BGRAToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0019008100420001;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x1f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x20 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void BGRAToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x00020070004a0026;
- const uint64_t mask_v = 0x0012005e00700002;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[dest0_u], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest0_u], %[dest0_u], %[value] \n\t"
- "pinsrh_0 %[dest0_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_0 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src1], %[src0] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src0], %[src1] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x10(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x17(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[dest1_u], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest1_u], %[dest1_u], %[value] \n\t"
- "pinsrh_0 %[dest1_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_0 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src1], %[src0] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src0], %[src1] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x20(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x27(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x20(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x27(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[dest2_u], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest2_u], %[dest2_u], %[value] \n\t"
- "pinsrh_0 %[dest2_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x28(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x28(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_0 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src1], %[src0] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src0], %[src1] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x30(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x37(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x30(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x37(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[dest3_u], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest3_u], %[dest3_u], %[value] \n\t"
- "pinsrh_0 %[dest3_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x38(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x3f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x38(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x3f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsrl %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_0 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src1], %[src0] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src0], %[src1] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x40 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void ABGRToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0001001900810042;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x1f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x20 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void ABGRToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x00020070004a0026;
- const uint64_t mask_v = 0x0012005e00700002;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest0_u], %[src0], %[value] \n\t"
- "dsll %[dest0_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_v], %[dest0_v], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src1], %[src0] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src0], %[src1] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x10(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x17(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest1_u], %[src0], %[value] \n\t"
- "dsll %[dest1_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_v], %[dest1_v], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src1], %[src0] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src0], %[src1] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x20(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x27(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x20(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x27(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest2_u], %[src0], %[value] \n\t"
- "dsll %[dest2_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_v], %[dest2_v], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x28(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x28(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src1], %[src0] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src0], %[src1] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x30(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x37(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x30(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x37(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest3_u], %[src0], %[value] \n\t"
- "dsll %[dest3_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_v], %[dest3_v], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x38(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x3f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x38(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x3f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src1], %[src0] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src0], %[src1] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x40 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void RGBAToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0042008100190001;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x1f(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x20 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void RGBAToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x0026004a00700002;
- const uint64_t mask_v = 0x00020070005e0012;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[dest0_u], %[src0], %[value] \n\t"
- "dsrl %[dest0_v], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest0_v], %[dest0_v], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[src_lo], %[src0], %[value] \n\t"
- "dsrl %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x10(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x17(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[dest1_u], %[src0], %[value] \n\t"
- "dsrl %[dest1_v], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest1_v], %[dest1_v], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[src_lo], %[src0], %[value] \n\t"
- "dsrl %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x20(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x27(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x20(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x27(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[dest2_u], %[src0], %[value] \n\t"
- "dsrl %[dest2_v], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest2_v], %[dest2_v], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x28(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x28(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[src_lo], %[src0], %[value] \n\t"
- "dsrl %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x30(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x37(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x30(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x37(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[dest3_u], %[src0], %[value] \n\t"
- "dsrl %[dest3_v], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[dest3_v], %[dest3_v], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x38(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x3f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x38(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x3f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_0 %[src_lo], %[src0], %[value] \n\t"
- "dsrl %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x40 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void RGB24ToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0001004200810019;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0d(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x06(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x13(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x0c(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x19(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x12(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x18 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void RGB24ToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x0026004a00700002;
- const uint64_t mask_v = 0x00020070005e0012;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest0_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_u], %[dest0_u], %[value] \n\t"
- "pinsrh_3 %[dest0_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x06(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0d(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x06(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0d(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x0c(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x13(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x0c(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x13(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest1_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_u], %[dest1_u], %[value] \n\t"
- "pinsrh_3 %[dest1_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x12(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x19(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x12(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x19(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest2_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_u], %[dest2_u], %[value] \n\t"
- "pinsrh_3 %[dest2_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x1e(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x25(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x1e(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x25(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x24(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2b(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x24(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2b(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[dest3_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_u], %[dest3_u], %[value] \n\t"
- "pinsrh_3 %[dest3_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x2a(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x31(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x2a(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x31(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x30 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void RAWToYRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest0, dest1, dest2, dest3;
- const uint64_t value = 0x1080;
- const uint64_t mask = 0x0001001900810042;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest0], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[dest0], %[src] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x0d(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x06(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[dest1], %[src] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x13(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x0c(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest2], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[dest2], %[src] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "gsldlc1 %[src], 0x19(%[src_argb0]) \n\t"
- "gsldrc1 %[src], 0x12(%[src_argb0]) \n\t"
- "punpcklbh %[src_lo], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "dsll %[src], %[src], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src], %[zero] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[dest3], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[dest3], %[src] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb0], %[src_argb0], 0x18 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3)
- : [src_argb0] "r"(src_argb0), [dst_y] "r"(dst_y), [width] "r"(width),
- [mask] "f"(mask), [value] "f"(value), [eight] "f"(0x08),
- [zero] "f"(0x00)
- : "memory");
-}
-
-void RAWToUVRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x00020070004a0026;
- const uint64_t mask_v = 0x0012005e00700002;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest0_u], %[src0], %[value] \n\t"
- "dsll %[dest0_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_v], %[dest0_v], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x06(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0d(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x06(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0d(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src1], %[src0] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src0], %[src1] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x0c(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x13(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x0c(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x13(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest1_u], %[src0], %[value] \n\t"
- "dsll %[dest1_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_v], %[dest1_v], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x12(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x19(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x12(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x19(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src1], %[src0] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src0], %[src1] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest2_u], %[src0], %[value] \n\t"
- "dsll %[dest2_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_v], %[dest2_v], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x1e(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x25(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x1e(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x25(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src1], %[src0] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src0], %[src1] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x24(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2b(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x24(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2b(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[dest3_u], %[src0], %[value] \n\t"
- "dsll %[dest3_v], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_v], %[dest3_v], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x2a(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x31(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x2a(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x31(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "dsll %[src0], %[src0], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "paddh %[src0], %[src_lo], %[src_hi] \n\t"
- "punpcklbh %[src_lo], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_lo] \n\t"
- "dsll %[src1], %[src1], %[eight] \n\t"
- "punpckhbh %[src_hi], %[src1], %[zero] \n\t"
- "paddh %[src0], %[src0], %[src_hi] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "pinsrh_3 %[src_lo], %[src0], %[value] \n\t"
- "dsll %[src_hi], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src1], %[src0] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src0], %[src1] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x30 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void ARGBToYJRow_MMI(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- uint64_t src, src_hi, src_lo;
- uint64_t dest, dest0, dest1, dest2, dest3;
- uint64_t tmp0, tmp1;
- const uint64_t shift = 0x07;
- const uint64_t value = 0x0040;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x00010026004B000FULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask1] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask1] \n\t"
- "punpcklwd %[tmp0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[tmp1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest0], %[dest0], %[shift] \n\t"
-
- "gsldlc1 %[src], 0x0f(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask1] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask1] \n\t"
- "punpcklwd %[tmp0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[tmp1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest1], %[dest1], %[shift] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask1] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask1] \n\t"
- "punpcklwd %[tmp0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[tmp1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest2], %[dest2], %[shift] \n\t"
-
- "gsldlc1 %[src], 0x1f(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask1] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask1] \n\t"
- "punpcklwd %[tmp0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[tmp1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest3], %[dest3], %[shift] \n\t"
-
- "packsswh %[tmp0], %[dest0], %[dest1] \n\t"
- "packsswh %[tmp1], %[dest2], %[dest3] \n\t"
- "packushb %[dest], %[tmp0], %[tmp1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x20 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest), [src_hi] "=&f"(src_hi),
- [src_lo] "=&f"(src_lo), [dest0] "=&f"(dest0), [dest1] "=&f"(dest1),
- [dest2] "=&f"(dest2), [dest3] "=&f"(dest3), [tmp0] "=&f"(tmp0),
- [tmp1] "=&f"(tmp1)
- : [src_ptr] "r"(src_argb0), [dst_ptr] "r"(dst_y), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [shift] "f"(shift), [value] "f"(value),
- [width] "r"(width)
- : "memory");
-}
-
-void ARGBToUVJRow_MMI(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src_rgb1;
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x002b0054007f0002;
- const uint64_t mask_v = 0x0002007f006b0014;
-
- __asm__ volatile(
- "1: \n\t"
- "daddu %[src_rgb1], %[src_rgb0], %[src_stride_rgb] \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[dest0_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_u], %[dest0_u], %[value] \n\t"
- "pinsrh_3 %[dest0_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x10(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x17(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[dest1_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_u], %[dest1_u], %[value] \n\t"
- "pinsrh_3 %[dest1_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x20(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x27(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x20(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x27(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[dest2_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_u], %[dest2_u], %[value] \n\t"
- "pinsrh_3 %[dest2_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x28(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x2f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x28(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x2f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x30(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x37(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x30(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x37(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[dest3_u], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_u], %[dest3_u], %[value] \n\t"
- "pinsrh_3 %[dest3_v], %[src0], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
-
- "gsldrc1 %[src0], 0x38(%[src_rgb0]) \n\t"
- "gsldlc1 %[src0], 0x3f(%[src_rgb0]) \n\t"
- "gsldrc1 %[src1], 0x38(%[src_rgb1]) \n\t"
- "gsldlc1 %[src1], 0x3f(%[src_rgb1]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "punpcklbh %[src0], %[src1], %[zero] \n\t"
- "punpckhbh %[src1], %[src1], %[zero] \n\t"
- "pavgh %[src0], %[src_lo], %[src0] \n\t"
- "pavgh %[src1], %[src_hi], %[src1] \n\t"
- "pavgh %[src0], %[src0], %[src1] \n\t"
- "dsll %[src_lo], %[src0], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src0], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb0], %[src_rgb0], 0x40 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src_rgb1] "=&r"(src_rgb1), [src0] "=&f"(ftmp[0]),
- [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]), [src_hi] "=&f"(ftmp[3]),
- [dest0_u] "=&f"(ftmp[4]), [dest0_v] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest1_v] "=&f"(ftmp[7]),
- [dest2_u] "=&f"(ftmp[8]), [dest2_v] "=&f"(ftmp[9]),
- [dest3_u] "=&f"(ftmp[10]), [dest3_v] "=&f"(ftmp[11])
- : [src_rgb0] "r"(src_rgb0), [src_stride_rgb] "r"(src_stride_rgb),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [mask_u] "f"(mask_u), [mask_v] "f"(mask_v), [value] "f"(value),
- [zero] "f"(0x00), [eight] "f"(0x08), [two] "f"(0x02),
- [sixteen] "f"(0x10)
- : "memory");
-}
-
-void RGB565ToYRow_MMI(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
- uint64_t ftmp[11];
- const uint64_t value = 0x1080108010801080;
- const uint64_t mask = 0x0001004200810019;
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0007000700070007;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb565]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psrlh %[r], %[src1], %[three] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[two] \n\t"
- "psrlh %[src1], %[g], %[four] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[src0], %[src1] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[src0], %[src1] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb565]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psrlh %[r], %[src1], %[three] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[two] \n\t"
- "psrlh %[src1], %[g], %[four] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[src0], %[src1] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[src0], %[src1] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_rgb565], %[src_rgb565], 0x10 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddiu %[width], %[width], -0x08 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]),
- [src_hi] "=&f"(ftmp[3]), [b] "=&f"(ftmp[4]), [g] "=&f"(ftmp[5]),
- [r] "=&f"(ftmp[6]), [dest0] "=&f"(ftmp[7]), [dest1] "=&f"(ftmp[8]),
- [dest2] "=&f"(ftmp[9]), [dest3] "=&f"(ftmp[10])
- : [src_rgb565] "r"(src_rgb565), [dst_y] "r"(dst_y), [value] "f"(value),
- [width] "r"(width), [c0] "f"(c0), [c1] "f"(c1), [c2] "f"(c2),
- [mask] "f"(mask), [eight] "f"(0x08), [five] "f"(0x05),
- [three] "f"(0x03), [two] "f"(0x02), [four] "f"(0x04)
- : "memory");
-}
-
-void ARGB1555ToYRow_MMI(const uint8_t* src_argb1555,
- uint8_t* dst_y,
- int width) {
- uint64_t ftmp[11];
- const uint64_t value = 0x1080108010801080;
- const uint64_t mask = 0x0001004200810019;
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0003000300030003;
- uint64_t c3 = 0x007c007c007c007c;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb1555]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "and %[r], %[src1], %[c3] \n\t"
- "psrlh %[r], %[r], %[two] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[three] \n\t"
- "psrlh %[src1], %[g], %[two] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[src0], %[src1] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[src0], %[src1] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_argb1555]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g], %[src1], %[c2] \n\t"
- "psllh %[g], %[g], %[three] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "and %[r], %[src1], %[c3] \n\t"
- "psrlh %[r], %[r], %[two] \n\t"
- "psllh %[src0], %[b], %[three] \n\t"
- "psrlh %[src1], %[b], %[two] \n\t"
- "or %[b], %[src0], %[src1] \n\t"
- "psllh %[src0], %[g], %[three] \n\t"
- "psrlh %[src1], %[g], %[two] \n\t"
- "or %[g], %[src0], %[src1] \n\t"
- "psllh %[src0], %[r], %[three] \n\t"
- "psrlh %[src1], %[r], %[two] \n\t"
- "or %[r], %[src0], %[src1] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[src0], %[src1] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[src0], %[src1] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb1555], %[src_argb1555], 0x10 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddiu %[width], %[width], -0x08 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]),
- [src_hi] "=&f"(ftmp[3]), [b] "=&f"(ftmp[4]), [g] "=&f"(ftmp[5]),
- [r] "=&f"(ftmp[6]), [dest0] "=&f"(ftmp[7]), [dest1] "=&f"(ftmp[8]),
- [dest2] "=&f"(ftmp[9]), [dest3] "=&f"(ftmp[10])
- : [src_argb1555] "r"(src_argb1555), [dst_y] "r"(dst_y),
- [width] "r"(width), [value] "f"(value), [mask] "f"(mask), [c0] "f"(c0),
- [c1] "f"(c1), [c2] "f"(c2), [c3] "f"(c3), [eight] "f"(0x08),
- [five] "f"(0x05), [three] "f"(0x03), [two] "f"(0x02), [seven] "f"(0x07)
- : "memory");
-}
-
-void ARGB4444ToYRow_MMI(const uint8_t* src_argb4444,
- uint8_t* dst_y,
- int width) {
- uint64_t ftmp[11];
- uint64_t value = 0x1080108010801080;
- uint64_t mask = 0x0001004200810019;
- uint64_t c0 = 0x000f000f000f000f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb4444]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g], %[src0], %[four] \n\t"
- "and %[r], %[src1], %[c0] \n\t"
- "psllh %[src0], %[b], %[four] \n\t"
- "or %[b], %[src0], %[b] \n\t"
- "psllh %[src0], %[g], %[four] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psllh %[src0], %[r], %[four] \n\t"
- "or %[r], %[src0], %[r] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest0], %[src0], %[src1] \n\t"
- "psrlw %[dest0], %[dest0], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest1], %[src0], %[src1] \n\t"
- "psrlw %[dest1], %[dest1], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_argb4444]) \n\t"
- "psrlh %[src1], %[src0], %[eight] \n\t"
- "and %[b], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g], %[src0], %[four] \n\t"
- "and %[r], %[src1], %[c0] \n\t"
- "psllh %[src0], %[b], %[four] \n\t"
- "or %[b], %[src0], %[b] \n\t"
- "psllh %[src0], %[g], %[four] \n\t"
- "or %[g], %[src0], %[g] \n\t"
- "psllh %[src0], %[r], %[four] \n\t"
- "or %[r], %[src0], %[r] \n\t"
- "punpcklhw %[src0], %[b], %[r] \n\t"
- "punpcklhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest2], %[src0], %[src1] \n\t"
- "psrlw %[dest2], %[dest2], %[eight] \n\t"
-
- "punpckhhw %[src0], %[b], %[r] \n\t"
- "punpckhhw %[src1], %[g], %[value] \n\t"
- "punpcklhw %[src_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[src_hi], %[src0], %[src1] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask] \n\t"
- "punpcklwd %[src0], %[src_lo], %[src_hi] \n\t"
- "punpckhwd %[src1], %[src_lo], %[src_hi] \n\t"
- "paddw %[dest3], %[src0], %[src1] \n\t"
- "psrlw %[dest3], %[dest3], %[eight] \n\t"
-
- "packsswh %[src_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[src_hi], %[dest2], %[dest3] \n\t"
- "packushb %[dest0], %[src_lo], %[src_hi] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_y]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_y]) \n\t"
-
- "daddiu %[src_argb4444], %[src_argb4444], 0x10 \n\t"
- "daddiu %[dst_y], %[dst_y], 0x08 \n\t"
- "daddiu %[width], %[width], -0x08 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]),
- [src_hi] "=&f"(ftmp[3]), [b] "=&f"(ftmp[4]), [g] "=&f"(ftmp[5]),
- [r] "=&f"(ftmp[6]), [dest0] "=&f"(ftmp[7]), [dest1] "=&f"(ftmp[8]),
- [dest2] "=&f"(ftmp[9]), [dest3] "=&f"(ftmp[10])
- : [src_argb4444] "r"(src_argb4444), [dst_y] "r"(dst_y),
- [width] "r"(width), [value] "f"(value), [mask] "f"(mask), [c0] "f"(c0),
- [c1] "f"(c1), [eight] "f"(0x08), [four] "f"(0x04)
- : "memory");
-}
-
-void RGB565ToUVRow_MMI(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t ftmp[13];
- uint64_t value = 0x2020202020202020;
- uint64_t mask_u = 0x0026004a00700002;
- uint64_t mask_v = 0x00020070005e0012;
- uint64_t mask = 0x93;
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0007000700070007;
- __asm__ volatile(
- "daddu %[next_rgb565], %[src_rgb565], %[next_rgb565] \n\t"
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_rgb565]) \n\t"
- "gsldrc1 %[src1], 0x00(%[next_rgb565]) \n\t"
- "gsldlc1 %[src1], 0x07(%[next_rgb565]) \n\t"
- "psrlh %[dest0_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest0_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "psrlh %[r0], %[dest0_u], %[three] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest0_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest0_v], %[src0], %[c2] \n\t"
- "psllh %[dest0_v], %[dest0_v], %[three] \n\t"
- "or %[dest0_v], %[src1], %[dest0_v] \n\t"
- "psrlh %[src0], %[src0], %[three] \n\t"
- "paddh %[b0], %[b0], %[dest0_u] \n\t"
- "paddh %[g0], %[g0], %[dest0_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest0_u], %[dest0_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest0_u], %[dest0_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest0_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest0_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[b0] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[g0] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_rgb565]) \n\t"
- "gsldrc1 %[src1], 0x08(%[next_rgb565]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[next_rgb565]) \n\t"
- "psrlh %[dest1_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest1_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "psrlh %[r0], %[dest1_u], %[three] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest1_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest1_v], %[src0], %[c2] \n\t"
- "psllh %[dest1_v], %[dest1_v], %[three] \n\t"
- "or %[dest1_v], %[src1], %[dest1_v] \n\t"
- "psrlh %[src0], %[src0], %[three] \n\t"
- "paddh %[b0], %[b0], %[dest1_u] \n\t"
- "paddh %[g0], %[g0], %[dest1_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest1_u], %[dest1_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest1_u], %[dest1_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest1_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest1_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[b0] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[g0] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_rgb565]) \n\t"
- "gsldrc1 %[src1], 0x10(%[next_rgb565]) \n\t"
- "gsldlc1 %[src1], 0x17(%[next_rgb565]) \n\t"
- "psrlh %[dest2_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest2_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "psrlh %[r0], %[dest2_u], %[three] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest2_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest2_v], %[src0], %[c2] \n\t"
- "psllh %[dest2_v], %[dest2_v], %[three] \n\t"
- "or %[dest2_v], %[src1], %[dest2_v] \n\t"
- "psrlh %[src0], %[src0], %[three] \n\t"
- "paddh %[b0], %[b0], %[dest2_u] \n\t"
- "paddh %[g0], %[g0], %[dest2_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest2_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest2_u], %[dest2_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest2_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest2_u], %[dest2_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest2_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest2_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[b0] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[g0] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_rgb565]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_rgb565]) \n\t"
- "gsldrc1 %[src1], 0x18(%[next_rgb565]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[next_rgb565]) \n\t"
- "psrlh %[dest3_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest3_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "psrlh %[r0], %[dest3_u], %[three] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest3_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest3_v], %[src0], %[c2] \n\t"
- "psllh %[dest3_v], %[dest3_v], %[three] \n\t"
- "or %[dest3_v], %[src1], %[dest3_v] \n\t"
- "psrlh %[src0], %[src0], %[three] \n\t"
- "paddh %[b0], %[b0], %[dest3_u] \n\t"
- "paddh %[g0], %[g0], %[dest3_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest3_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest3_u], %[dest3_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest3_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest3_u], %[dest3_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest3_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest3_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[b0] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[g0] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_rgb565], %[src_rgb565], 0x20 \n\t"
- "daddiu %[next_rgb565], %[next_rgb565], 0x20 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddiu %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b0] "=&f"(ftmp[2]),
- [g0] "=&f"(ftmp[3]), [r0] "=&f"(ftmp[4]), [dest0_u] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest2_u] "=&f"(ftmp[7]),
- [dest3_u] "=&f"(ftmp[8]), [dest0_v] "=&f"(ftmp[9]),
- [dest1_v] "=&f"(ftmp[10]), [dest2_v] "=&f"(ftmp[11]),
- [dest3_v] "=&f"(ftmp[12])
- : [src_rgb565] "r"(src_rgb565), [next_rgb565] "r"(src_stride_rgb565),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [value] "f"(value), [c0] "f"(c0), [c1] "f"(c1), [c2] "f"(c2),
- [mask] "f"(mask), [mask_u] "f"(mask_u), [mask_v] "f"(mask_v),
- [eight] "f"(0x08), [six] "f"(0x06), [five] "f"(0x05), [three] "f"(0x03),
- [one] "f"(0x01)
- : "memory");
-}
-
-void ARGB1555ToUVRow_MMI(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t ftmp[11];
- uint64_t value = 0x2020202020202020;
- uint64_t mask_u = 0x0026004a00700002;
- uint64_t mask_v = 0x00020070005e0012;
- uint64_t mask = 0x93;
- uint64_t c0 = 0x001f001f001f001f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t c2 = 0x0003000300030003;
- uint64_t c3 = 0x007c007c007c007c;
- __asm__ volatile(
- "daddu %[next_argb1555], %[src_argb1555], %[next_argb1555] \n\t"
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb1555]) \n\t"
- "gsldrc1 %[src1], 0x00(%[next_argb1555]) \n\t"
- "gsldlc1 %[src1], 0x07(%[next_argb1555]) \n\t"
- "psrlh %[dest0_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest0_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "and %[r0], %[dest0_u], %[c3] \n\t"
- "psrlh %[r0], %[r0], %[two] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest0_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest0_v], %[src0], %[c2] \n\t"
- "psllh %[dest0_v], %[dest0_v], %[three] \n\t"
- "or %[dest0_v], %[src1], %[dest0_v] \n\t"
- "and %[src0], %[src0], %[c3] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "paddh %[b0], %[b0], %[dest0_u] \n\t"
- "paddh %[g0], %[g0], %[dest0_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest0_u], %[dest0_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[six] \n\t"
- "psllh %[g0], %[g0], %[one] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest0_u], %[dest0_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest0_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest0_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[b0] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[g0] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_argb1555]) \n\t"
- "gsldrc1 %[src1], 0x08(%[next_argb1555]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[next_argb1555]) \n\t"
- "psrlh %[dest1_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest1_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "and %[r0], %[dest1_u], %[c3] \n\t"
- "psrlh %[r0], %[r0], %[two] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest1_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest1_v], %[src0], %[c2] \n\t"
- "psllh %[dest1_v], %[dest1_v], %[three] \n\t"
- "or %[dest1_v], %[src1], %[dest1_v] \n\t"
- "and %[src0], %[src0], %[c3] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "paddh %[b0], %[b0], %[dest1_u] \n\t"
- "paddh %[g0], %[g0], %[dest1_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest1_u], %[dest1_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[six] \n\t"
- "psllh %[g0], %[g0], %[one] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest1_u], %[dest1_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest1_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest1_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[b0] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[g0] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "packsswh %[dest0_u], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[dest1_u], %[dest0_v], %[dest1_v] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_argb1555]) \n\t"
- "gsldrc1 %[src1], 0x10(%[next_argb1555]) \n\t"
- "gsldlc1 %[src1], 0x17(%[next_argb1555]) \n\t"
- "psrlh %[dest2_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest2_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "and %[r0], %[dest2_u], %[c3] \n\t"
- "psrlh %[r0], %[r0], %[two] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest2_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest0_v], %[src0], %[c2] \n\t"
- "psllh %[dest0_v], %[dest0_v], %[three] \n\t"
- "or %[dest0_v], %[src1], %[dest0_v] \n\t"
- "and %[src0], %[src0], %[c3] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "paddh %[b0], %[b0], %[dest2_u] \n\t"
- "paddh %[g0], %[g0], %[dest0_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest2_u], %[dest0_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[six] \n\t"
- "psllh %[g0], %[g0], %[one] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest2_u], %[dest0_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest0_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest2_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[b0] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[g0] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_argb1555]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_argb1555]) \n\t"
- "gsldrc1 %[src1], 0x18(%[next_argb1555]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[next_argb1555]) \n\t"
- "psrlh %[dest3_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[src0], %[src0], %[five] \n\t"
- "and %[g0], %[dest3_u], %[c2] \n\t"
- "psllh %[g0], %[g0], %[three] \n\t"
- "or %[g0], %[src0], %[g0] \n\t"
- "and %[r0], %[dest3_u], %[c3] \n\t"
- "psrlh %[r0], %[r0], %[two] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest3_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[src1], %[src1], %[five] \n\t"
- "and %[dest1_v], %[src0], %[c2] \n\t"
- "psllh %[dest1_v], %[dest1_v], %[three] \n\t"
- "or %[dest1_v], %[src1], %[dest1_v] \n\t"
- "and %[src0], %[src0], %[c3] \n\t"
- "psrlh %[src0], %[src0], %[two] \n\t"
- "paddh %[b0], %[b0], %[dest3_u] \n\t"
- "paddh %[g0], %[g0], %[dest1_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest3_u], %[dest1_v] \n\t"
- "psrlh %[b0], %[src0], %[six] \n\t"
- "psllh %[r0], %[src0], %[one] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[six] \n\t"
- "psllh %[g0], %[g0], %[one] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest3_u], %[dest1_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest1_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest3_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[b0] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[g0] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[dest0_u], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
- "packsswh %[src1], %[dest0_v], %[dest1_v] \n\t"
- "packushb %[dest0_v], %[dest1_u], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_argb1555], %[src_argb1555], 0x20 \n\t"
- "daddiu %[next_argb1555], %[next_argb1555], 0x20 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddiu %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b0] "=&f"(ftmp[2]),
- [g0] "=&f"(ftmp[3]), [r0] "=&f"(ftmp[4]), [dest0_u] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest2_u] "=&f"(ftmp[7]),
- [dest3_u] "=&f"(ftmp[8]), [dest0_v] "=&f"(ftmp[9]),
- [dest1_v] "=&f"(ftmp[10])
- : [src_argb1555] "r"(src_argb1555),
- [next_argb1555] "r"(src_stride_argb1555), [dst_u] "r"(dst_u),
- [dst_v] "r"(dst_v), [width] "r"(width), [value] "f"(value),
- [c0] "f"(c0), [c1] "f"(c1), [c2] "f"(c2), [c3] "f"(c3),
- [mask] "f"(mask), [mask_u] "f"(mask_u), [mask_v] "f"(mask_v),
- [eight] "f"(0x08), [six] "f"(0x06), [five] "f"(0x05), [three] "f"(0x03),
- [two] "f"(0x02), [one] "f"(0x01)
- : "memory");
-}
-
-void ARGB4444ToUVRow_MMI(const uint8_t* src_argb4444,
- int src_stride_argb4444,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t ftmp[13];
- uint64_t value = 0x2020202020202020;
- uint64_t mask_u = 0x0026004a00700002;
- uint64_t mask_v = 0x00020070005e0012;
- uint64_t mask = 0x93;
- uint64_t c0 = 0x000f000f000f000f;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- __asm__ volatile(
- "daddu %[next_argb4444], %[src_argb4444], %[next_argb4444] \n\t"
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb4444]) \n\t"
- "gsldrc1 %[src1], 0x00(%[next_argb4444]) \n\t"
- "gsldlc1 %[src1], 0x07(%[next_argb4444]) \n\t"
- "psrlh %[dest0_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g0], %[src0], %[four] \n\t"
- "and %[r0], %[dest0_u], %[c0] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest0_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[dest0_v], %[src1], %[four] \n\t"
- "and %[src0], %[src0], %[c0] \n\t"
- "paddh %[b0], %[b0], %[dest0_u] \n\t"
- "paddh %[g0], %[g0], %[dest0_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest0_u], %[dest0_v] \n\t"
- "psrlh %[b0], %[src0], %[four] \n\t"
- "psllh %[r0], %[src0], %[two] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[four] \n\t"
- "psllh %[g0], %[g0], %[two] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest0_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest0_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest0_u], %[dest0_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest0_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest0_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[b0] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[g0] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_argb4444]) \n\t"
- "gsldrc1 %[src1], 0x08(%[next_argb4444]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[next_argb4444]) \n\t"
- "psrlh %[dest1_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g0], %[src0], %[four] \n\t"
- "and %[r0], %[dest1_u], %[c0] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest1_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[dest1_v], %[src1], %[four] \n\t"
- "and %[src0], %[src0], %[c0] \n\t"
- "paddh %[b0], %[b0], %[dest1_u] \n\t"
- "paddh %[g0], %[g0], %[dest1_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest1_u], %[dest1_v] \n\t"
- "psrlh %[b0], %[src0], %[four] \n\t"
- "psllh %[r0], %[src0], %[two] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[four] \n\t"
- "psllh %[g0], %[g0], %[two] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest1_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest1_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest1_u], %[dest1_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest1_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest1_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[b0] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[g0] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_argb4444]) \n\t"
- "gsldrc1 %[src1], 0x10(%[next_argb4444]) \n\t"
- "gsldlc1 %[src1], 0x17(%[next_argb4444]) \n\t"
- "psrlh %[dest2_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g0], %[src0], %[four] \n\t"
- "and %[r0], %[dest2_u], %[c0] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest2_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[dest2_v], %[src1], %[four] \n\t"
- "and %[src0], %[src0], %[c0] \n\t"
- "paddh %[b0], %[b0], %[dest2_u] \n\t"
- "paddh %[g0], %[g0], %[dest2_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest2_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest2_u], %[dest2_v] \n\t"
- "psrlh %[b0], %[src0], %[four] \n\t"
- "psllh %[r0], %[src0], %[two] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[four] \n\t"
- "psllh %[g0], %[g0], %[two] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest2_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest2_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest2_u], %[dest2_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest2_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest2_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[b0] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[g0] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_argb4444]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_argb4444]) \n\t"
- "gsldrc1 %[src1], 0x18(%[next_argb4444]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[next_argb4444]) \n\t"
- "psrlh %[dest3_u], %[src0], %[eight] \n\t"
- "and %[b0], %[src0], %[c0] \n\t"
- "and %[src0], %[src0], %[c1] \n\t"
- "psrlh %[g0], %[src0], %[four] \n\t"
- "and %[r0], %[dest3_u], %[c0] \n\t"
- "psrlh %[src0], %[src1], %[eight] \n\t"
- "and %[dest3_u], %[src1], %[c0] \n\t"
- "and %[src1], %[src1], %[c1] \n\t"
- "psrlh %[dest3_v], %[src1], %[four] \n\t"
- "and %[src0], %[src0], %[c0] \n\t"
- "paddh %[b0], %[b0], %[dest3_u] \n\t"
- "paddh %[g0], %[g0], %[dest3_v] \n\t"
- "paddh %[r0], %[r0], %[src0] \n\t"
- "punpcklhw %[src0], %[b0], %[r0] \n\t"
- "punpckhhw %[src1], %[b0], %[r0] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest3_v], %[src0], %[src1] \n\t"
- "paddh %[src0], %[dest3_u], %[dest3_v] \n\t"
- "psrlh %[b0], %[src0], %[four] \n\t"
- "psllh %[r0], %[src0], %[two] \n\t"
- "or %[b0], %[b0], %[r0] \n\t"
- "psrlh %[r0], %[g0], %[four] \n\t"
- "psllh %[g0], %[g0], %[two] \n\t"
- "or %[g0], %[g0], %[r0] \n\t"
- "punpcklhw %[src0], %[g0], %[value] \n\t"
- "punpckhhw %[src1], %[g0], %[value] \n\t"
- "punpcklwd %[dest3_u], %[src0], %[src1] \n\t"
- "punpckhwd %[dest3_v], %[src0], %[src1] \n\t"
- "paddh %[g0], %[dest3_u], %[dest3_v] \n\t"
- "punpcklhw %[src0], %[b0], %[g0] \n\t"
- "punpckhhw %[src1], %[b0], %[g0] \n\t"
-
- "pmaddhw %[dest3_v], %[src0], %[mask_v] \n\t"
- "pshufh %[dest3_u], %[src0], %[mask] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[g0], %[src1], %[mask_v] \n\t"
- "pshufh %[b0], %[src1], %[mask] \n\t"
- "pmaddhw %[b0], %[b0], %[mask_u] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[b0] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[b0] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[g0] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[g0] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_argb4444], %[src_argb4444], 0x20 \n\t"
- "daddiu %[next_argb4444], %[next_argb4444], 0x20 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddiu %[width], %[width], -0x10 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [b0] "=&f"(ftmp[2]),
- [g0] "=&f"(ftmp[3]), [r0] "=&f"(ftmp[4]), [dest0_u] "=&f"(ftmp[5]),
- [dest1_u] "=&f"(ftmp[6]), [dest2_u] "=&f"(ftmp[7]),
- [dest3_u] "=&f"(ftmp[8]), [dest0_v] "=&f"(ftmp[9]),
- [dest1_v] "=&f"(ftmp[10]), [dest2_v] "=&f"(ftmp[11]),
- [dest3_v] "=&f"(ftmp[12])
- : [src_argb4444] "r"(src_argb4444),
- [next_argb4444] "r"(src_stride_argb4444), [dst_u] "r"(dst_u),
- [dst_v] "r"(dst_v), [width] "r"(width), [value] "f"(value),
- [c0] "f"(c0), [c1] "f"(c1), [mask] "f"(mask), [mask_u] "f"(mask_u),
- [mask_v] "f"(mask_v), [eight] "f"(0x08), [four] "f"(0x04),
- [two] "f"(0x02)
- : "memory");
-}
-
-void ARGBToUV444Row_MMI(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t ftmp[12];
- const uint64_t value = 0x4040;
- const uint64_t mask_u = 0x0026004a00700002;
- const uint64_t mask_v = 0x00020070005e0012;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_argb]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "dsll %[dest0_u], %[src_lo], %[sixteen] \n\t"
- "pinsrh_0 %[dest0_u], %[dest0_u], %[value] \n\t"
- "pinsrh_3 %[dest0_v], %[src_lo], %[value] \n\t"
- "pmaddhw %[dest0_u], %[dest0_u], %[mask_u] \n\t"
- "pmaddhw %[dest0_v], %[dest0_v], %[mask_v] \n\t"
-
- "dsll %[src_lo], %[src_hi], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest0_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest0_u], %[src_lo] \n\t"
- "psubw %[dest0_u], %[src0], %[src1] \n\t"
- "psraw %[dest0_u], %[dest0_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest0_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest0_v], %[src_hi] \n\t"
- "psubw %[dest0_v], %[src1], %[src0] \n\t"
- "psraw %[dest0_v], %[dest0_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x08(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x0f(%[src_argb]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "dsll %[dest1_u], %[src_lo], %[sixteen] \n\t"
- "pinsrh_0 %[dest1_u], %[dest1_u], %[value] \n\t"
- "pinsrh_3 %[dest1_v], %[src_lo], %[value] \n\t"
- "pmaddhw %[dest1_u], %[dest1_u], %[mask_u] \n\t"
- "pmaddhw %[dest1_v], %[dest1_v], %[mask_v] \n\t"
- "dsll %[src_lo], %[src_hi], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest1_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest1_u], %[src_lo] \n\t"
- "psubw %[dest1_u], %[src0], %[src1] \n\t"
- "psraw %[dest1_u], %[dest1_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest1_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest1_v], %[src_hi] \n\t"
- "psubw %[dest1_v], %[src1], %[src0] \n\t"
- "psraw %[dest1_v], %[dest1_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_argb]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "dsll %[dest2_u], %[src_lo], %[sixteen] \n\t"
- "pinsrh_0 %[dest2_u], %[dest2_u], %[value] \n\t"
- "pinsrh_3 %[dest2_v], %[src_lo], %[value] \n\t"
- "pmaddhw %[dest2_u], %[dest2_u], %[mask_u] \n\t"
- "pmaddhw %[dest2_v], %[dest2_v], %[mask_v] \n\t"
- "dsll %[src_lo], %[src_hi], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest2_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest2_u], %[src_lo] \n\t"
- "psubw %[dest2_u], %[src0], %[src1] \n\t"
- "psraw %[dest2_u], %[dest2_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest2_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest2_v], %[src_hi] \n\t"
- "psubw %[dest2_v], %[src1], %[src0] \n\t"
- "psraw %[dest2_v], %[dest2_v], %[eight] \n\t"
-
- "gsldrc1 %[src0], 0x18(%[src_argb]) \n\t"
- "gsldlc1 %[src0], 0x1f(%[src_argb]) \n\t"
- "punpcklbh %[src_lo], %[src0], %[zero] \n\t"
- "punpckhbh %[src_hi], %[src0], %[zero] \n\t"
- "dsll %[dest3_u], %[src_lo], %[sixteen] \n\t"
- "pinsrh_0 %[dest3_u], %[dest3_u], %[value] \n\t"
- "pinsrh_3 %[dest3_v], %[src_lo], %[value] \n\t"
- "pmaddhw %[dest3_u], %[dest3_u], %[mask_u] \n\t"
- "pmaddhw %[dest3_v], %[dest3_v], %[mask_v] \n\t"
- "dsll %[src_lo], %[src_hi], %[sixteen] \n\t"
- "pinsrh_0 %[src_lo], %[src_lo], %[value] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[value] \n\t"
- "pmaddhw %[src_lo], %[src_lo], %[mask_u] \n\t"
- "pmaddhw %[src_hi], %[src_hi], %[mask_v] \n\t"
-
- "punpcklwd %[src0], %[dest3_u], %[src_lo] \n\t"
- "punpckhwd %[src1], %[dest3_u], %[src_lo] \n\t"
- "psubw %[dest3_u], %[src0], %[src1] \n\t"
- "psraw %[dest3_u], %[dest3_u], %[eight] \n\t"
- "punpcklwd %[src0], %[dest3_v], %[src_hi] \n\t"
- "punpckhwd %[src1], %[dest3_v], %[src_hi] \n\t"
- "psubw %[dest3_v], %[src1], %[src0] \n\t"
- "psraw %[dest3_v], %[dest3_v], %[eight] \n\t"
-
- "packsswh %[src0], %[dest0_u], %[dest1_u] \n\t"
- "packsswh %[src1], %[dest2_u], %[dest3_u] \n\t"
- "packushb %[dest0_u], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_u], 0x07(%[dst_u]) \n\t"
- "gssdrc1 %[dest0_u], 0x00(%[dst_u]) \n\t"
-
- "packsswh %[src0], %[dest0_v], %[dest1_v] \n\t"
- "packsswh %[src1], %[dest2_v], %[dest3_v] \n\t"
- "packushb %[dest0_v], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest0_v], 0x07(%[dst_v]) \n\t"
- "gssdrc1 %[dest0_v], 0x00(%[dst_v]) \n\t"
-
- "daddiu %[src_argb], %[src_argb], 0x20 \n\t"
- "daddiu %[dst_u], %[dst_u], 0x08 \n\t"
- "daddiu %[dst_v], %[dst_v], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bgtz %[width], 1b \n\t"
- : [src0] "=&f"(ftmp[0]), [src1] "=&f"(ftmp[1]), [src_lo] "=&f"(ftmp[2]),
- [src_hi] "=&f"(ftmp[3]), [dest0_u] "=&f"(ftmp[4]),
- [dest0_v] "=&f"(ftmp[5]), [dest1_u] "=&f"(ftmp[6]),
- [dest1_v] "=&f"(ftmp[7]), [dest2_u] "=&f"(ftmp[8]),
- [dest2_v] "=&f"(ftmp[9]), [dest3_u] "=&f"(ftmp[10]),
- [dest3_v] "=&f"(ftmp[11])
- : [src_argb] "r"(src_argb), [dst_u] "r"(dst_u), [dst_v] "r"(dst_v),
- [width] "r"(width), [mask_u] "f"(mask_u), [mask_v] "f"(mask_v),
- [value] "f"(value), [zero] "f"(0x00), [sixteen] "f"(0x10),
- [eight] "f"(0x08)
- : "memory");
-}
-
-void ARGBGrayRow_MMI(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
- uint64_t src, src_lo, src_hi, src37, dest, dest_lo, dest_hi;
- uint64_t tmp0, tmp1;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x01;
- const uint64_t mask2 = 0x00400026004B000FULL;
- const uint64_t mask3 = 0xFF000000FF000000ULL;
- const uint64_t mask4 = ~mask3;
- const uint64_t shift = 0x07;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
-
- "and %[src37], %[src], %[mask3] \n\t"
-
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_lo], %[src_lo], %[mask1] \n\t"
- "pmaddhw %[dest_lo], %[src_lo], %[mask2] \n\t"
- "punpcklwd %[tmp0], %[dest_lo], %[dest_lo] \n\t"
- "punpckhwd %[tmp1], %[dest_lo], %[dest_lo] \n\t"
- "paddw %[dest_lo], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest_lo], %[dest_lo], %[shift] \n\t"
- "packsswh %[dest_lo], %[dest_lo], %[dest_lo] \n\t"
-
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
- "pinsrh_3 %[src_hi], %[src_hi], %[mask1] \n\t"
- "pmaddhw %[dest_hi], %[src_hi], %[mask2] \n\t"
- "punpcklwd %[tmp0], %[dest_hi], %[dest_hi] \n\t"
- "punpckhwd %[tmp1], %[dest_hi], %[dest_hi] \n\t"
- "paddw %[dest_hi], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest_hi], %[dest_hi], %[shift] \n\t"
- "packsswh %[dest_hi], %[dest_hi], %[dest_hi] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "and %[dest], %[dest], %[mask4] \n\t"
- "or %[dest], %[dest], %[src37] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo), [tmp0] "=&f"(tmp0),
- [tmp1] "=&f"(tmp1), [src] "=&f"(src), [dest] "=&f"(dest),
- [src37] "=&f"(src37)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [width] "r"(width),
- [shift] "f"(shift), [mask0] "f"(mask0), [mask1] "f"(mask1),
- [mask2] "f"(mask2), [mask3] "f"(mask3), [mask4] "f"(mask4)
- : "memory");
-}
-
-// Convert a row of image to Sepia tone.
-void ARGBSepiaRow_MMI(uint8_t* dst_argb, int width) {
- uint64_t dest, dest_lo, dest_hi, dest37, dest0, dest1, dest2;
- uint64_t tmp0, tmp1;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x002300440011ULL;
- const uint64_t mask2 = 0x002D00580016ULL;
- const uint64_t mask3 = 0x003200620018ULL;
- const uint64_t mask4 = 0xFF000000FF000000ULL;
- const uint64_t shift = 0x07;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "and %[dest37], %[dest], %[mask4] \n\t"
-
- "punpcklbh %[dest_lo], %[dest], %[mask0] \n\t"
- "pmaddhw %[dest0], %[dest_lo], %[mask1] \n\t"
- "pmaddhw %[dest1], %[dest_lo], %[mask2] \n\t"
- "pmaddhw %[dest2], %[dest_lo], %[mask3] \n\t"
- "punpcklwd %[tmp0], %[dest0], %[dest1] \n\t"
- "punpckhwd %[tmp1], %[dest0], %[dest1] \n\t"
- "paddw %[dest0], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest0], %[dest0], %[shift] \n\t"
- "punpcklwd %[tmp0], %[dest2], %[mask0] \n\t"
- "punpckhwd %[tmp1], %[dest2], %[mask0] \n\t"
- "paddw %[dest1], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest1], %[dest1], %[shift] \n\t"
- "packsswh %[dest_lo], %[dest0], %[dest1] \n\t"
-
- "punpckhbh %[dest_hi], %[dest], %[mask0] \n\t"
- "pmaddhw %[dest0], %[dest_hi], %[mask1] \n\t"
- "pmaddhw %[dest1], %[dest_hi], %[mask2] \n\t"
- "pmaddhw %[dest2], %[dest_hi], %[mask3] \n\t"
- "punpcklwd %[tmp0], %[dest0], %[dest1] \n\t"
- "punpckhwd %[tmp1], %[dest0], %[dest1] \n\t"
- "paddw %[dest0], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest0], %[dest0], %[shift] \n\t"
- "punpcklwd %[tmp0], %[dest2], %[mask0] \n\t"
- "punpckhwd %[tmp1], %[dest2], %[mask0] \n\t"
- "paddw %[dest1], %[tmp0], %[tmp1] \n\t"
- "psrlw %[dest1], %[dest1], %[shift] \n\t"
- "packsswh %[dest_hi], %[dest0], %[dest1] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "or %[dest], %[dest], %[dest37] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest37] "=&f"(dest37), [tmp0] "=&f"(tmp0), [tmp1] "=&f"(tmp1),
- [dest] "=&f"(dest)
- : [dst_ptr] "r"(dst_argb), [width] "r"(width), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [mask2] "f"(mask2), [mask3] "f"(mask3),
- [mask4] "f"(mask4), [shift] "f"(shift)
- : "memory");
-}
-
-// Apply color matrix to a row of image. Matrix is signed.
-// TODO(fbarchard): Consider adding rounding (+32).
-void ARGBColorMatrixRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width) {
- uint64_t src, src_hi, src_lo, dest, dest_lo, dest_hi, dest0, dest1, dest2,
- dest3;
- uint64_t matrix, matrix_hi, matrix_lo;
- uint64_t tmp0, tmp1;
- const uint64_t shift0 = 0x06;
- const uint64_t shift1 = 0x08;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x08;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
-
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
-
- "gsldlc1 %[matrix], 0x07(%[matrix_ptr]) \n\t"
- "gsldrc1 %[matrix], 0x00(%[matrix_ptr]) \n\t"
- "punpcklbh %[matrix_lo], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "psrah %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "punpckhbh %[matrix_hi], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "psrah %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "pmaddhw %[dest_lo], %[src_lo], %[matrix_lo] \n\t"
- "pmaddhw %[dest_hi], %[src_lo], %[matrix_hi] \n\t"
- "punpcklwd %[tmp0], %[dest_lo], %[dest_hi] \n\t"
- "punpckhwd %[tmp1], %[dest_lo], %[dest_hi] \n\t"
- "paddw %[dest0], %[tmp0], %[tmp1] \n\t"
- "psraw %[dest0], %[dest0], %[shift0] \n\t"
-
- "gsldlc1 %[matrix], 0x0f(%[matrix_ptr]) \n\t"
- "gsldrc1 %[matrix], 0x08(%[matrix_ptr]) \n\t"
- "punpcklbh %[matrix_lo], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "psrah %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "punpckhbh %[matrix_hi], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "psrah %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "pmaddhw %[dest_lo], %[src_lo], %[matrix_lo] \n\t"
- "pmaddhw %[dest_hi], %[src_lo], %[matrix_hi] \n\t"
- "punpcklwd %[tmp0], %[dest_lo], %[dest_hi] \n\t"
- "punpckhwd %[tmp1], %[dest_lo], %[dest_hi] \n\t"
- "paddw %[dest1], %[tmp0], %[tmp1] \n\t"
- "psraw %[dest1], %[dest1], %[shift0] \n\t"
-
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
-
- "gsldlc1 %[matrix], 0x07(%[matrix_ptr]) \n\t"
- "gsldrc1 %[matrix], 0x00(%[matrix_ptr]) \n\t"
- "punpcklbh %[matrix_lo], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "psrah %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "punpckhbh %[matrix_hi], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "psrah %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "pmaddhw %[dest_lo], %[src_hi], %[matrix_lo] \n\t"
- "pmaddhw %[dest_hi], %[src_hi], %[matrix_hi] \n\t"
- "punpcklwd %[tmp0], %[dest_lo], %[dest_hi] \n\t"
- "punpckhwd %[tmp1], %[dest_lo], %[dest_hi] \n\t"
- "paddw %[dest2], %[tmp0], %[tmp1] \n\t"
- "psraw %[dest2], %[dest2], %[shift0] \n\t"
-
- "gsldlc1 %[matrix], 0x0f(%[matrix_ptr]) \n\t"
- "gsldrc1 %[matrix], 0x08(%[matrix_ptr]) \n\t"
- "punpcklbh %[matrix_lo], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "psrah %[matrix_lo], %[matrix_lo], %[shift1] \n\t"
- "punpckhbh %[matrix_hi], %[matrix], %[mask0] \n\t"
- "psllh %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "psrah %[matrix_hi], %[matrix_hi], %[shift1] \n\t"
- "pmaddhw %[dest_lo], %[src_hi], %[matrix_lo] \n\t"
- "pmaddhw %[dest_hi], %[src_hi], %[matrix_hi] \n\t"
- "punpcklwd %[tmp0], %[dest_lo], %[dest_hi] \n\t"
- "punpckhwd %[tmp1], %[dest_lo], %[dest_hi] \n\t"
- "paddw %[dest3], %[tmp0], %[tmp1] \n\t"
- "psraw %[dest3], %[dest3], %[shift0] \n\t"
-
- "packsswh %[tmp0], %[dest0], %[dest1] \n\t"
- "packsswh %[tmp1], %[dest2], %[dest3] \n\t"
- "packushb %[dest], %[tmp0], %[tmp1] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3), [src] "=&f"(src), [dest] "=&f"(dest),
- [tmp0] "=&f"(tmp0), [tmp1] "=&f"(tmp1), [matrix_hi] "=&f"(matrix_hi),
- [matrix_lo] "=&f"(matrix_lo), [matrix] "=&f"(matrix)
- : [src_ptr] "r"(src_argb), [matrix_ptr] "r"(matrix_argb),
- [dst_ptr] "r"(dst_argb), [width] "r"(width), [shift0] "f"(shift0),
- [shift1] "f"(shift1), [mask0] "f"(mask0), [mask1] "f"(mask1)
- : "memory");
-}
-
-void ARGBShadeRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- uint32_t value) {
- uint64_t src, src_hi, src_lo, dest, dest_lo, dest_hi;
- const uint64_t shift = 0x08;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[src] \n\t"
- "punpckhbh %[src_hi], %[src], %[src] \n\t"
-
- "punpcklbh %[value], %[value], %[value] \n\t"
-
- "pmulhuh %[dest_lo], %[src_lo], %[value] \n\t"
- "psrlh %[dest_lo], %[dest_lo], %[shift] \n\t"
- "pmulhuh %[dest_hi], %[src_hi], %[value] \n\t"
- "psrlh %[dest_hi], %[dest_hi], %[shift] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo), [src] "=&f"(src),
- [dest] "=&f"(dest)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [width] "r"(width),
- [value] "f"(value), [shift] "f"(shift)
- : "memory");
-}
-
-void ARGBMultiplyRow_MMI(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- uint64_t src0, src0_hi, src0_lo, src1, src1_hi, src1_lo;
- uint64_t dest, dest_lo, dest_hi;
- const uint64_t mask = 0x0;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 0x07(%[src0_ptr]) \n\t"
- "gsldrc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "punpcklbh %[src0_lo], %[src0], %[src0] \n\t"
- "punpckhbh %[src0_hi], %[src0], %[src0] \n\t"
-
- "gsldlc1 %[src1], 0x07(%[src1_ptr]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "punpcklbh %[src1_lo], %[src1], %[mask] \n\t"
- "punpckhbh %[src1_hi], %[src1], %[mask] \n\t"
-
- "pmulhuh %[dest_lo], %[src0_lo], %[src1_lo] \n\t"
- "pmulhuh %[dest_hi], %[src0_hi], %[src1_hi] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t"
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0_hi] "=&f"(src0_hi), [src0_lo] "=&f"(src0_lo),
- [src1_hi] "=&f"(src1_hi), [src1_lo] "=&f"(src1_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo), [src0] "=&f"(src0),
- [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src_argb0), [src1_ptr] "r"(src_argb1),
- [dst_ptr] "r"(dst_argb), [width] "r"(width), [mask] "f"(mask)
- : "memory");
-}
-
-void ARGBAddRow_MMI(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- uint64_t src0, src1, dest;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 0x07(%[src0_ptr]) \n\t"
- "gsldrc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src1_ptr]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "paddusb %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t"
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src_argb0), [src1_ptr] "r"(src_argb1),
- [dst_ptr] "r"(dst_argb), [width] "r"(width)
- : "memory");
-}
-
-void ARGBSubtractRow_MMI(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- uint64_t src0, src1, dest;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 0x07(%[src0_ptr]) \n\t"
- "gsldrc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src1_ptr]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "psubusb %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t"
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src_argb0), [src1_ptr] "r"(src_argb1),
- [dst_ptr] "r"(dst_argb), [width] "r"(width)
- : "memory");
-}
-
-// Sobel functions which mimics SSSE3.
-void SobelXRow_MMI(const uint8_t* src_y0,
- const uint8_t* src_y1,
- const uint8_t* src_y2,
- uint8_t* dst_sobelx,
- int width) {
- uint64_t y00 = 0, y10 = 0, y20 = 0;
- uint64_t y02 = 0, y12 = 0, y22 = 0;
- uint64_t zero = 0x0;
- uint64_t sobel = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[y00], 0x07(%[src_y0]) \n\t" // a=src_y0[i]
- "gsldrc1 %[y00], 0x00(%[src_y0]) \n\t"
- "gsldlc1 %[y02], 0x09(%[src_y0]) \n\t" // a_sub=src_y0[i+2]
- "gsldrc1 %[y02], 0x02(%[src_y0]) \n\t"
-
- "gsldlc1 %[y10], 0x07(%[src_y1]) \n\t" // b=src_y1[i]
- "gsldrc1 %[y10], 0x00(%[src_y1]) \n\t"
- "gsldlc1 %[y12], 0x09(%[src_y1]) \n\t" // b_sub=src_y1[i+2]
- "gsldrc1 %[y12], 0x02(%[src_y1]) \n\t"
-
- "gsldlc1 %[y20], 0x07(%[src_y2]) \n\t" // c=src_y2[i]
- "gsldrc1 %[y20], 0x00(%[src_y2]) \n\t"
- "gsldlc1 %[y22], 0x09(%[src_y2]) \n\t" // c_sub=src_y2[i+2]
- "gsldrc1 %[y22], 0x02(%[src_y2]) \n\t"
-
- "punpcklbh %[y00], %[y00], %[zero] \n\t"
- "punpcklbh %[y10], %[y10], %[zero] \n\t"
- "punpcklbh %[y20], %[y20], %[zero] \n\t"
-
- "punpcklbh %[y02], %[y02], %[zero] \n\t"
- "punpcklbh %[y12], %[y12], %[zero] \n\t"
- "punpcklbh %[y22], %[y22], %[zero] \n\t"
-
- "paddh %[y00], %[y00], %[y10] \n\t" // a+b
- "paddh %[y20], %[y20], %[y10] \n\t" // c+b
- "paddh %[y00], %[y00], %[y20] \n\t" // a+2b+c
-
- "paddh %[y02], %[y02], %[y12] \n\t" // a_sub+b_sub
- "paddh %[y22], %[y22], %[y12] \n\t" // c_sub+b_sub
- "paddh %[y02], %[y02], %[y22] \n\t" // a_sub+2b_sub+c_sub
-
- "pmaxsh %[y10], %[y00], %[y02] \n\t"
- "pminsh %[y20], %[y00], %[y02] \n\t"
- "psubh %[sobel], %[y10], %[y20] \n\t" // Abs
-
- "gsldlc1 %[y00], 0x0B(%[src_y0]) \n\t"
- "gsldrc1 %[y00], 0x04(%[src_y0]) \n\t"
- "gsldlc1 %[y02], 0x0D(%[src_y0]) \n\t"
- "gsldrc1 %[y02], 0x06(%[src_y0]) \n\t"
-
- "gsldlc1 %[y10], 0x0B(%[src_y1]) \n\t"
- "gsldrc1 %[y10], 0x04(%[src_y1]) \n\t"
- "gsldlc1 %[y12], 0x0D(%[src_y1]) \n\t"
- "gsldrc1 %[y12], 0x06(%[src_y1]) \n\t"
-
- "gsldlc1 %[y20], 0x0B(%[src_y2]) \n\t"
- "gsldrc1 %[y20], 0x04(%[src_y2]) \n\t"
- "gsldlc1 %[y22], 0x0D(%[src_y2]) \n\t"
- "gsldrc1 %[y22], 0x06(%[src_y2]) \n\t"
-
- "punpcklbh %[y00], %[y00], %[zero] \n\t"
- "punpcklbh %[y10], %[y10], %[zero] \n\t"
- "punpcklbh %[y20], %[y20], %[zero] \n\t"
-
- "punpcklbh %[y02], %[y02], %[zero] \n\t"
- "punpcklbh %[y12], %[y12], %[zero] \n\t"
- "punpcklbh %[y22], %[y22], %[zero] \n\t"
-
- "paddh %[y00], %[y00], %[y10] \n\t"
- "paddh %[y20], %[y20], %[y10] \n\t"
- "paddh %[y00], %[y00], %[y20] \n\t"
-
- "paddh %[y02], %[y02], %[y12] \n\t"
- "paddh %[y22], %[y22], %[y12] \n\t"
- "paddh %[y02], %[y02], %[y22] \n\t"
-
- "pmaxsh %[y10], %[y00], %[y02] \n\t"
- "pminsh %[y20], %[y00], %[y02] \n\t"
- "psubh %[y00], %[y10], %[y20] \n\t"
-
- "packushb %[sobel], %[sobel], %[y00] \n\t" // clamp255
- "gssdrc1 %[sobel], 0(%[dst_sobelx]) \n\t"
- "gssdlc1 %[sobel], 7(%[dst_sobelx]) \n\t"
-
- "daddiu %[src_y0], %[src_y0], 8 \n\t"
- "daddiu %[src_y1], %[src_y1], 8 \n\t"
- "daddiu %[src_y2], %[src_y2], 8 \n\t"
- "daddiu %[dst_sobelx], %[dst_sobelx], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [sobel] "=&f"(sobel), [y00] "=&f"(y00), [y10] "=&f"(y10),
- [y20] "=&f"(y20), [y02] "=&f"(y02), [y12] "=&f"(y12), [y22] "=&f"(y22)
- : [src_y0] "r"(src_y0), [src_y1] "r"(src_y1), [src_y2] "r"(src_y2),
- [dst_sobelx] "r"(dst_sobelx), [width] "r"(width), [zero] "f"(zero)
- : "memory");
-}
-
-void SobelYRow_MMI(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
- int width) {
- uint64_t y00 = 0, y01 = 0, y02 = 0;
- uint64_t y10 = 0, y11 = 0, y12 = 0;
- uint64_t zero = 0x0;
- uint64_t sobel = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[y00], 0x07(%[src_y0]) \n\t" // a=src_y0[i]
- "gsldrc1 %[y00], 0x00(%[src_y0]) \n\t"
- "gsldlc1 %[y01], 0x08(%[src_y0]) \n\t" // b=src_y0[i+1]
- "gsldrc1 %[y01], 0x01(%[src_y0]) \n\t"
- "gsldlc1 %[y02], 0x09(%[src_y0]) \n\t" // c=src_y0[i+2]
- "gsldrc1 %[y02], 0x02(%[src_y0]) \n\t"
-
- "gsldlc1 %[y10], 0x07(%[src_y1]) \n\t" // a_sub=src_y1[i]
- "gsldrc1 %[y10], 0x00(%[src_y1]) \n\t"
- "gsldlc1 %[y11], 0x08(%[src_y1]) \n\t" // b_sub=src_y1[i+1]
- "gsldrc1 %[y11], 0x01(%[src_y1]) \n\t"
- "gsldlc1 %[y12], 0x09(%[src_y1]) \n\t" // c_sub=src_y1[i+2]
- "gsldrc1 %[y12], 0x02(%[src_y1]) \n\t"
-
- "punpcklbh %[y00], %[y00], %[zero] \n\t"
- "punpcklbh %[y01], %[y01], %[zero] \n\t"
- "punpcklbh %[y02], %[y02], %[zero] \n\t"
-
- "punpcklbh %[y10], %[y10], %[zero] \n\t"
- "punpcklbh %[y11], %[y11], %[zero] \n\t"
- "punpcklbh %[y12], %[y12], %[zero] \n\t"
-
- "paddh %[y00], %[y00], %[y01] \n\t" // a+b
- "paddh %[y02], %[y02], %[y01] \n\t" // c+b
- "paddh %[y00], %[y00], %[y02] \n\t" // a+2b+c
-
- "paddh %[y10], %[y10], %[y11] \n\t" // a_sub+b_sub
- "paddh %[y12], %[y12], %[y11] \n\t" // c_sub+b_sub
- "paddh %[y10], %[y10], %[y12] \n\t" // a_sub+2b_sub+c_sub
-
- "pmaxsh %[y02], %[y00], %[y10] \n\t"
- "pminsh %[y12], %[y00], %[y10] \n\t"
- "psubh %[sobel], %[y02], %[y12] \n\t" // Abs
-
- "gsldlc1 %[y00], 0x0B(%[src_y0]) \n\t"
- "gsldrc1 %[y00], 0x04(%[src_y0]) \n\t"
- "gsldlc1 %[y01], 0x0C(%[src_y0]) \n\t"
- "gsldrc1 %[y01], 0x05(%[src_y0]) \n\t"
- "gsldlc1 %[y02], 0x0D(%[src_y0]) \n\t"
- "gsldrc1 %[y02], 0x06(%[src_y0]) \n\t"
-
- "gsldlc1 %[y10], 0x0B(%[src_y1]) \n\t"
- "gsldrc1 %[y10], 0x04(%[src_y1]) \n\t"
- "gsldlc1 %[y11], 0x0C(%[src_y1]) \n\t"
- "gsldrc1 %[y11], 0x05(%[src_y1]) \n\t"
- "gsldlc1 %[y12], 0x0D(%[src_y1]) \n\t"
- "gsldrc1 %[y12], 0x06(%[src_y1]) \n\t"
-
- "punpcklbh %[y00], %[y00], %[zero] \n\t"
- "punpcklbh %[y01], %[y01], %[zero] \n\t"
- "punpcklbh %[y02], %[y02], %[zero] \n\t"
-
- "punpcklbh %[y10], %[y10], %[zero] \n\t"
- "punpcklbh %[y11], %[y11], %[zero] \n\t"
- "punpcklbh %[y12], %[y12], %[zero] \n\t"
-
- "paddh %[y00], %[y00], %[y01] \n\t"
- "paddh %[y02], %[y02], %[y01] \n\t"
- "paddh %[y00], %[y00], %[y02] \n\t"
-
- "paddh %[y10], %[y10], %[y11] \n\t"
- "paddh %[y12], %[y12], %[y11] \n\t"
- "paddh %[y10], %[y10], %[y12] \n\t"
-
- "pmaxsh %[y02], %[y00], %[y10] \n\t"
- "pminsh %[y12], %[y00], %[y10] \n\t"
- "psubh %[y00], %[y02], %[y12] \n\t"
-
- "packushb %[sobel], %[sobel], %[y00] \n\t" // clamp255
- "gssdrc1 %[sobel], 0(%[dst_sobely]) \n\t"
- "gssdlc1 %[sobel], 7(%[dst_sobely]) \n\t"
-
- "daddiu %[src_y0], %[src_y0], 8 \n\t"
- "daddiu %[src_y1], %[src_y1], 8 \n\t"
- "daddiu %[dst_sobely], %[dst_sobely], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [sobel] "=&f"(sobel), [y00] "=&f"(y00), [y01] "=&f"(y01),
- [y02] "=&f"(y02), [y10] "=&f"(y10), [y11] "=&f"(y11), [y12] "=&f"(y12)
- : [src_y0] "r"(src_y0), [src_y1] "r"(src_y1),
- [dst_sobely] "r"(dst_sobely), [width] "r"(width), [zero] "f"(zero)
- : "memory");
-}
-
-void SobelRow_MMI(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- double temp[3];
- uint64_t c1 = 0xff000000ff000000;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[t0], 0x07(%[src_sobelx]) \n\t" // a=src_sobelx[i]
- "gsldrc1 %[t0], 0x00(%[src_sobelx]) \n\t"
- "gsldlc1 %[t1], 0x07(%[src_sobely]) \n\t" // b=src_sobely[i]
- "gsldrc1 %[t1], 0x00(%[src_sobely]) \n\t"
- // s7 s6 s5 s4 s3 s2 s1 s0 = a+b
- "paddusb %[t2] , %[t0], %[t1] \n\t"
-
- // s3 s2 s1 s0->s3 s3 s2 s2 s1 s1 s0 s0
- "punpcklbh %[t0], %[t2], %[t2] \n\t"
-
- // s1 s1 s0 s0->s1 s2 s1 s1 s0 s0 s0 s0
- "punpcklbh %[t1], %[t0], %[t0] \n\t"
- "or %[t1], %[t1], %[c1] \n\t"
- // 255 s1 s1 s1 s55 s0 s0 s0
- "gssdrc1 %[t1], 0x00(%[dst_argb]) \n\t"
- "gssdlc1 %[t1], 0x07(%[dst_argb]) \n\t"
-
- // s3 s3 s2 s2->s3 s3 s3 s3 s2 s2 s2 s2
- "punpckhbh %[t1], %[t0], %[t0] \n\t"
- "or %[t1], %[t1], %[c1] \n\t"
- // 255 s3 s3 s3 255 s2 s2 s2
- "gssdrc1 %[t1], 0x08(%[dst_argb]) \n\t"
- "gssdlc1 %[t1], 0x0f(%[dst_argb]) \n\t"
-
- // s7 s6 s5 s4->s7 s7 s6 s6 s5 s5 s4 s4
- "punpckhbh %[t0], %[t2], %[t2] \n\t"
-
- // s5 s5 s4 s4->s5 s5 s5 s5 s4 s4 s4 s4
- "punpcklbh %[t1], %[t0], %[t0] \n\t"
- "or %[t1], %[t1], %[c1] \n\t"
- "gssdrc1 %[t1], 0x10(%[dst_argb]) \n\t"
- "gssdlc1 %[t1], 0x17(%[dst_argb]) \n\t"
-
- // s7 s7 s6 s6->s7 s7 s7 s7 s6 s6 s6 s6
- "punpckhbh %[t1], %[t0], %[t0] \n\t"
- "or %[t1], %[t1], %[c1] \n\t"
- "gssdrc1 %[t1], 0x18(%[dst_argb]) \n\t"
- "gssdlc1 %[t1], 0x1f(%[dst_argb]) \n\t"
-
- "daddiu %[dst_argb], %[dst_argb], 32 \n\t"
- "daddiu %[src_sobelx], %[src_sobelx], 8 \n\t"
- "daddiu %[src_sobely], %[src_sobely], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [t2] "=&f"(temp[2])
- : [src_sobelx] "r"(src_sobelx), [src_sobely] "r"(src_sobely),
- [dst_argb] "r"(dst_argb), [width] "r"(width), [c1] "f"(c1)
- : "memory");
-}
-
-void SobelToPlaneRow_MMI(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_y,
- int width) {
- uint64_t tr = 0;
- uint64_t tb = 0;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[tr], 0x0(%[src_sobelx]) \n\t"
- "gsldlc1 %[tr], 0x7(%[src_sobelx]) \n\t" // r=src_sobelx[i]
- "gsldrc1 %[tb], 0x0(%[src_sobely]) \n\t"
- "gsldlc1 %[tb], 0x7(%[src_sobely]) \n\t" // b=src_sobely[i]
- "paddusb %[tr], %[tr], %[tb] \n\t" // g
- "gssdrc1 %[tr], 0x0(%[dst_y]) \n\t"
- "gssdlc1 %[tr], 0x7(%[dst_y]) \n\t"
-
- "daddiu %[dst_y], %[dst_y], 8 \n\t"
- "daddiu %[src_sobelx], %[src_sobelx], 8 \n\t"
- "daddiu %[src_sobely], %[src_sobely], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [tr] "=&f"(tr), [tb] "=&f"(tb)
- : [src_sobelx] "r"(src_sobelx), [src_sobely] "r"(src_sobely),
- [dst_y] "r"(dst_y), [width] "r"(width)
- : "memory");
-}
-
-void SobelXYRow_MMI(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- uint64_t temp[3];
- uint64_t result = 0;
- uint64_t gb = 0;
- uint64_t cr = 0;
- uint64_t c1 = 0xffffffffffffffff;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[tr], 0x07(%[src_sobelx]) \n\t" // r=src_sobelx[i]
- "gsldrc1 %[tr], 0x00(%[src_sobelx]) \n\t"
- "gsldlc1 %[tb], 0x07(%[src_sobely]) \n\t" // b=src_sobely[i]
- "gsldrc1 %[tb], 0x00(%[src_sobely]) \n\t"
- "paddusb %[tg] , %[tr], %[tb] \n\t" // g
-
- // g3 b3 g2 b2 g1 b1 g0 b0
- "punpcklbh %[gb], %[tb], %[tg] \n\t"
- // c3 r3 r2 r2 c1 r1 c0 r0
- "punpcklbh %[cr], %[tr], %[c1] \n\t"
- // c1 r1 g1 b1 c0 r0 g0 b0
- "punpcklhw %[result], %[gb], %[cr] \n\t"
- "gssdrc1 %[result], 0x00(%[dst_argb]) \n\t"
- "gssdlc1 %[result], 0x07(%[dst_argb]) \n\t"
- // c3 r3 g3 b3 c2 r2 g2 b2
- "punpckhhw %[result], %[gb], %[cr] \n\t"
- "gssdrc1 %[result], 0x08(%[dst_argb]) \n\t"
- "gssdlc1 %[result], 0x0f(%[dst_argb]) \n\t"
-
- // g7 b7 g6 b6 g5 b5 g4 b4
- "punpckhbh %[gb], %[tb], %[tg] \n\t"
- // c7 r7 c6 r6 c5 r5 c4 r4
- "punpckhbh %[cr], %[tr], %[c1] \n\t"
- // c5 r5 g5 b5 c4 r4 g4 b4
- "punpcklhw %[result], %[gb], %[cr] \n\t"
- "gssdrc1 %[result], 0x10(%[dst_argb]) \n\t"
- "gssdlc1 %[result], 0x17(%[dst_argb]) \n\t"
- // c7 r7 g7 b7 c6 r6 g6 b6
- "punpckhhw %[result], %[gb], %[cr] \n\t"
- "gssdrc1 %[result], 0x18(%[dst_argb]) \n\t"
- "gssdlc1 %[result], 0x1f(%[dst_argb]) \n\t"
-
- "daddiu %[dst_argb], %[dst_argb], 32 \n\t"
- "daddiu %[src_sobelx], %[src_sobelx], 8 \n\t"
- "daddiu %[src_sobely], %[src_sobely], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [tr] "=&f"(temp[0]), [tb] "=&f"(temp[1]), [tg] "=&f"(temp[2]),
- [gb] "=&f"(gb), [cr] "=&f"(cr), [result] "=&f"(result)
- : [src_sobelx] "r"(src_sobelx), [src_sobely] "r"(src_sobely),
- [dst_argb] "r"(dst_argb), [width] "r"(width), [c1] "f"(c1)
- : "memory");
-}
-
-void J400ToARGBRow_MMI(const uint8_t* src_y, uint8_t* dst_argb, int width) {
- // Copy a Y to RGB.
- uint64_t src, dest;
- const uint64_t mask0 = 0x00ffffff00ffffffULL;
- const uint64_t mask1 = ~mask0;
-
- __asm__ volatile(
- "1: \n\t"
- "gslwlc1 %[src], 0x03(%[src_ptr]) \n\t"
- "gslwrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src], %[src], %[src] \n\t"
- "punpcklhw %[dest], %[src], %[src] \n\t"
- "and %[dest], %[dest], %[mask0] \n\t"
- "or %[dest], %[dest], %[mask1] \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
-
- "punpckhhw %[dest], %[src], %[src] \n\t"
- "and %[dest], %[dest], %[mask0] \n\t"
- "or %[dest], %[dest], %[mask1] \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x04 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_y), [dst_ptr] "r"(dst_argb), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [width] "r"(width)
- : "memory");
-}
-
-void I400ToARGBRow_MMI(const uint8_t* src_y, uint8_t* rgb_buf, int width) {
- uint64_t src, src_lo, src_hi, dest, dest_lo, dest_hi;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x55;
- const uint64_t mask2 = 0xAA;
- const uint64_t mask3 = 0xFF;
- const uint64_t mask4 = 0x4A354A354A354A35ULL;
- const uint64_t mask5 = 0x0488048804880488ULL;
- const uint64_t shift0 = 0x08;
- const uint64_t shift1 = 0x06;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t"
-
- "pshufh %[src], %[src_lo], %[mask0] \n\t"
- "psllh %[dest_lo], %[src], %[shift0] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[src] \n\t"
- "pmulhuh %[dest_lo], %[dest_lo], %[mask4] \n\t"
- "psubh %[dest_lo], %[dest_lo], %[mask5] \n\t"
- "psrah %[dest_lo], %[dest_lo], %[shift1] \n\t"
- "pinsrh_3 %[dest_lo], %[dest_lo], %[mask3] \n\t"
- "pshufh %[src], %[src_lo], %[mask1] \n\t"
- "psllh %[dest_hi], %[src], %[shift0] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[src] \n\t"
- "pmulhuh %[dest_hi], %[dest_hi], %[mask4] \n\t"
- "psubh %[dest_hi], %[dest_hi], %[mask5] \n\t"
- "psrah %[dest_hi], %[dest_hi], %[shift1] \n\t"
- "pinsrh_3 %[dest_hi], %[dest_hi], %[mask3] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "pshufh %[src], %[src_lo], %[mask2] \n\t"
- "psllh %[dest_lo], %[src], %[shift0] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[src] \n\t"
- "pmulhuh %[dest_lo], %[dest_lo], %[mask4] \n\t"
- "psubh %[dest_lo], %[dest_lo], %[mask5] \n\t"
- "psrah %[dest_lo], %[dest_lo], %[shift1] \n\t"
- "pinsrh_3 %[dest_lo], %[dest_lo], %[mask3] \n\t"
- "pshufh %[src], %[src_lo], %[mask3] \n\t"
- "psllh %[dest_hi], %[src], %[shift0] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[src] \n\t"
- "pmulhuh %[dest_hi], %[dest_hi], %[mask4] \n\t"
- "psubh %[dest_hi], %[dest_hi], %[mask5] \n\t"
- "psrah %[dest_hi], %[dest_hi], %[shift1] \n\t"
- "pinsrh_3 %[dest_hi], %[dest_hi], %[mask3] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "pshufh %[src], %[src_hi], %[mask0] \n\t"
- "psllh %[dest_lo], %[src], %[shift0] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[src] \n\t"
- "pmulhuh %[dest_lo], %[dest_lo], %[mask4] \n\t"
- "psubh %[dest_lo], %[dest_lo], %[mask5] \n\t"
- "psrah %[dest_lo], %[dest_lo], %[shift1] \n\t"
- "pinsrh_3 %[dest_lo], %[dest_lo], %[mask3] \n\t"
- "pshufh %[src], %[src_hi], %[mask1] \n\t"
- "psllh %[dest_hi], %[src], %[shift0] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[src] \n\t"
- "pmulhuh %[dest_hi], %[dest_hi], %[mask4] \n\t"
- "psubh %[dest_hi], %[dest_hi], %[mask5] \n\t"
- "psrah %[dest_hi], %[dest_hi], %[shift1] \n\t"
- "pinsrh_3 %[dest_hi], %[dest_hi], %[mask3] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x17(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x10(%[dst_ptr]) \n\t"
-
- "pshufh %[src], %[src_hi], %[mask2] \n\t"
- "psllh %[dest_lo], %[src], %[shift0] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[src] \n\t"
- "pmulhuh %[dest_lo], %[dest_lo], %[mask4] \n\t"
- "psubh %[dest_lo], %[dest_lo], %[mask5] \n\t"
- "psrah %[dest_lo], %[dest_lo], %[shift1] \n\t"
- "pinsrh_3 %[dest_lo], %[dest_lo], %[mask3] \n\t"
- "pshufh %[src], %[src_hi], %[mask3] \n\t"
- "psllh %[dest_hi], %[src], %[shift0] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[src] \n\t"
- "pmulhuh %[dest_hi], %[dest_hi], %[mask4] \n\t"
- "psubh %[dest_hi], %[dest_hi], %[mask5] \n\t"
- "psrah %[dest_hi], %[dest_hi], %[shift1] \n\t"
- "pinsrh_3 %[dest_hi], %[dest_hi], %[mask3] \n\t"
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x1f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x18(%[dst_ptr]) \n\t"
-
- "daddi %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x20 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest), [src_hi] "=&f"(src_hi),
- [src_lo] "=&f"(src_lo), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo)
- : [src_ptr] "r"(src_y), [dst_ptr] "r"(rgb_buf), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [mask2] "f"(mask2), [mask3] "f"(mask3),
- [mask4] "f"(mask4), [mask5] "f"(mask5), [shift0] "f"(shift0),
- [shift1] "f"(shift1), [width] "r"(width)
- : "memory");
-}
-
-void MirrorRow_MMI(const uint8_t* src, uint8_t* dst, int width) {
- uint64_t source, src0, src1, dest;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x1b;
-
- src += width - 1;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[source], 0(%[src_ptr]) \n\t"
- "gsldrc1 %[source], -7(%[src_ptr]) \n\t"
- "punpcklbh %[src0], %[source], %[mask0] \n\t"
- "pshufh %[src0], %[src0], %[mask1] \n\t"
- "punpckhbh %[src1], %[source], %[mask0] \n\t"
- "pshufh %[src1], %[src1], %[mask1] \n\t"
- "packushb %[dest], %[src1], %[src0] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddi %[src_ptr], %[src_ptr], -0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [source] "=&f"(source), [dest] "=&f"(dest), [src0] "=&f"(src0),
- [src1] "=&f"(src1)
- : [src_ptr] "r"(src), [dst_ptr] "r"(dst), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [width] "r"(width)
- : "memory");
-}
-
-void MirrorUVRow_MMI(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t src0, src1, dest0, dest1;
- const uint64_t mask0 = 0x00ff00ff00ff00ffULL;
- const uint64_t mask1 = 0x1b;
- const uint64_t shift = 0x08;
-
- src_uv += (width - 1) << 1;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 1(%[src_ptr]) \n\t"
- "gsldrc1 %[src0], -6(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], -7(%[src_ptr]) \n\t"
- "gsldrc1 %[src1], -14(%[src_ptr]) \n\t"
-
- "and %[dest0], %[src0], %[mask0] \n\t"
- "pshufh %[dest0], %[dest0], %[mask1] \n\t"
- "and %[dest1], %[src1], %[mask0] \n\t"
- "pshufh %[dest1], %[dest1], %[mask1] \n\t"
- "packushb %[dest0], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dstu_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dstu_ptr]) \n\t"
-
- "psrlh %[dest0], %[src0], %[shift] \n\t"
- "pshufh %[dest0], %[dest0], %[mask1] \n\t"
- "psrlh %[dest1], %[src1], %[shift] \n\t"
- "pshufh %[dest1], %[dest1], %[mask1] \n\t"
- "packushb %[dest0], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dstv_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dstv_ptr]) \n\t"
-
- "daddi %[src_ptr], %[src_ptr], -0x10 \n\t"
- "daddiu %[dstu_ptr], %[dstu_ptr], 0x08 \n\t"
- "daddiu %[dstv_ptr], %[dstv_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [src0] "=&f"(src0),
- [src1] "=&f"(src1)
- : [src_ptr] "r"(src_uv), [dstu_ptr] "r"(dst_u), [dstv_ptr] "r"(dst_v),
- [width] "r"(width), [mask0] "f"(mask0), [mask1] "f"(mask1),
- [shift] "f"(shift)
- : "memory");
-}
-
-void ARGBMirrorRow_MMI(const uint8_t* src, uint8_t* dst, int width) {
- src += (width - 1) * 4;
- uint64_t temp = 0x0;
- uint64_t shuff = 0x4e; // 01 00 11 10
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[temp], 3(%[src]) \n\t"
- "gsldrc1 %[temp], -4(%[src]) \n\t"
- "pshufh %[temp], %[temp], %[shuff] \n\t"
- "gssdrc1 %[temp], 0x0(%[dst]) \n\t"
- "gssdlc1 %[temp], 0x7(%[dst]) \n\t"
-
- "daddiu %[src], %[src], -0x08 \n\t"
- "daddiu %[dst], %[dst], 0x08 \n\t"
- "daddiu %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [temp] "=&f"(temp)
- : [src] "r"(src), [dst] "r"(dst), [width] "r"(width), [shuff] "f"(shuff)
- : "memory");
-}
-
-void SplitUVRow_MMI(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t c0 = 0x00ff00ff00ff00ff;
- uint64_t temp[4];
- uint64_t shift = 0x08;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_uv]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_uv]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_uv]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_uv]) \n\t"
-
- "and %[t2], %[t0], %[c0] \n\t"
- "and %[t3], %[t1], %[c0] \n\t"
- "packushb %[t2], %[t2], %[t3] \n\t"
- "gssdrc1 %[t2], 0x0(%[dst_u]) \n\t"
- "gssdlc1 %[t2], 0x7(%[dst_u]) \n\t"
-
- "psrlh %[t2], %[t0], %[shift] \n\t"
- "psrlh %[t3], %[t1], %[shift] \n\t"
- "packushb %[t2], %[t2], %[t3] \n\t"
- "gssdrc1 %[t2], 0x0(%[dst_v]) \n\t"
- "gssdlc1 %[t2], 0x7(%[dst_v]) \n\t"
-
- "daddiu %[src_uv], %[src_uv], 16 \n\t"
- "daddiu %[dst_u], %[dst_u], 8 \n\t"
- "daddiu %[dst_v], %[dst_v], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [t2] "=&f"(temp[2]),
- [t3] "=&f"(temp[3])
- : [src_uv] "r"(src_uv), [dst_u] "r"(dst_u), [dst_v] "r"(dst_v),
- [width] "r"(width), [c0] "f"(c0), [shift] "f"(shift)
- : "memory");
-}
-
-void MergeUVRow_MMI(const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uv,
- int width) {
- uint64_t temp[3];
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x0(%[src_u]) \n\t"
- "gsldlc1 %[t0], 0x7(%[src_u]) \n\t"
- "gsldrc1 %[t1], 0x0(%[src_v]) \n\t"
- "gsldlc1 %[t1], 0x7(%[src_v]) \n\t"
- "punpcklbh %[t2], %[t0], %[t1] \n\t"
- "gssdrc1 %[t2], 0x0(%[dst_uv]) \n\t"
- "gssdlc1 %[t2], 0x7(%[dst_uv]) \n\t"
- "punpckhbh %[t2], %[t0], %[t1] \n\t"
- "gssdrc1 %[t2], 0x8(%[dst_uv]) \n\t"
- "gssdlc1 %[t2], 0xf(%[dst_uv]) \n\t"
-
- "daddiu %[src_u], %[src_u], 8 \n\t"
- "daddiu %[src_v], %[src_v], 8 \n\t"
- "daddiu %[dst_uv], %[dst_uv], 16 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [t2] "=&f"(temp[2])
- : [dst_uv] "r"(dst_uv), [src_u] "r"(src_u), [src_v] "r"(src_v),
- [width] "r"(width)
- : "memory");
-}
-
-void SplitRGBRow_MMI(const uint8_t* src_rgb,
- uint8_t* dst_r,
- uint8_t* dst_g,
- uint8_t* dst_b,
- int width) {
- uint64_t src[4];
- uint64_t dest_hi, dest_lo, dest;
-
- __asm__ volatile(
- "1: \n\t"
- "gslwlc1 %[src0], 0x03(%[src_ptr]) \n\t"
- "gslwrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gslwlc1 %[src1], 0x06(%[src_ptr]) \n\t"
- "gslwrc1 %[src1], 0x03(%[src_ptr]) \n\t"
- "punpcklbh %[dest_lo], %[src0], %[src1] \n\t"
- "gslwlc1 %[src2], 0x09(%[src_ptr]) \n\t"
- "gslwrc1 %[src2], 0x06(%[src_ptr]) \n\t"
- "gslwlc1 %[src3], 0x0c(%[src_ptr]) \n\t"
- "gslwrc1 %[src3], 0x09(%[src_ptr]) \n\t"
- "punpcklbh %[dest_hi], %[src2], %[src3] \n\t"
-
- "punpcklhw %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gsswlc1 %[dest], 0x03(%[dstr_ptr]) \n\t"
- "gsswrc1 %[dest], 0x00(%[dstr_ptr]) \n\t"
- "punpckhwd %[dest], %[dest], %[dest] \n\t"
- "gsswlc1 %[dest], 0x03(%[dstg_ptr]) \n\t"
- "gsswrc1 %[dest], 0x00(%[dstg_ptr]) \n\t"
- "punpckhhw %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gsswlc1 %[dest], 0x03(%[dstb_ptr]) \n\t"
- "gsswrc1 %[dest], 0x00(%[dstb_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x0c \n\t"
- "daddiu %[dstr_ptr], %[dstr_ptr], 0x04 \n\t"
- "daddiu %[dstg_ptr], %[dstg_ptr], 0x04 \n\t"
- "daddiu %[dstb_ptr], %[dstb_ptr], 0x04 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src[0]), [src1] "=&f"(src[1]), [src2] "=&f"(src[2]),
- [src3] "=&f"(src[3]), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_rgb), [dstr_ptr] "r"(dst_r), [dstg_ptr] "r"(dst_g),
- [dstb_ptr] "r"(dst_b), [width] "r"(width)
- : "memory");
-}
-
-void MergeRGBRow_MMI(const uint8_t* src_r,
- const uint8_t* src_g,
- const uint8_t* src_b,
- uint8_t* dst_rgb,
- int width) {
- uint64_t srcr, srcg, srcb, dest;
- uint64_t srcrg_hi, srcrg_lo, srcbz_hi, srcbz_lo;
- const uint64_t temp = 0x0;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[srcr], 0x07(%[srcr_ptr]) \n\t"
- "gsldrc1 %[srcr], 0x00(%[srcr_ptr]) \n\t"
- "gsldlc1 %[srcg], 0x07(%[srcg_ptr]) \n\t"
- "gsldrc1 %[srcg], 0x00(%[srcg_ptr]) \n\t"
- "punpcklbh %[srcrg_lo], %[srcr], %[srcg] \n\t"
- "punpckhbh %[srcrg_hi], %[srcr], %[srcg] \n\t"
-
- "gsldlc1 %[srcb], 0x07(%[srcb_ptr]) \n\t"
- "gsldrc1 %[srcb], 0x00(%[srcb_ptr]) \n\t"
- "punpcklbh %[srcbz_lo], %[srcb], %[temp] \n\t"
- "punpckhbh %[srcbz_hi], %[srcb], %[temp] \n\t"
-
- "punpcklhw %[dest], %[srcrg_lo], %[srcbz_lo] \n\t"
- "gsswlc1 %[dest], 0x03(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
- "punpckhwd %[dest], %[dest], %[dest] \n\t"
- "gsswlc1 %[dest], 0x06(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x03(%[dst_ptr]) \n\t"
- "punpckhhw %[dest], %[srcrg_lo], %[srcbz_lo] \n\t"
- "gsswlc1 %[dest], 0x09(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x06(%[dst_ptr]) \n\t"
- "punpckhwd %[dest], %[dest], %[dest] \n\t"
- "gsswlc1 %[dest], 0x0c(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x09(%[dst_ptr]) \n\t"
- "punpcklhw %[dest], %[srcrg_hi], %[srcbz_hi] \n\t"
- "gsswlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x0c(%[dst_ptr]) \n\t"
- "punpckhwd %[dest], %[dest], %[dest] \n\t"
- "gsswlc1 %[dest], 0x12(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "punpckhhw %[dest], %[srcrg_hi], %[srcbz_hi] \n\t"
- "gsswlc1 %[dest], 0x15(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x12(%[dst_ptr]) \n\t"
- "punpckhwd %[dest], %[dest], %[dest] \n\t"
- "gsswlc1 %[dest], 0x18(%[dst_ptr]) \n\t"
- "gsswrc1 %[dest], 0x15(%[dst_ptr]) \n\t"
-
- "daddiu %[srcr_ptr], %[srcr_ptr], 0x08 \n\t"
- "daddiu %[srcg_ptr], %[srcg_ptr], 0x08 \n\t"
- "daddiu %[srcb_ptr], %[srcb_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x18 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [srcr] "=&f"(srcr), [srcg] "=&f"(srcg), [srcb] "=&f"(srcb),
- [dest] "=&f"(dest), [srcrg_hi] "=&f"(srcrg_hi),
- [srcrg_lo] "=&f"(srcrg_lo), [srcbz_hi] "=&f"(srcbz_hi),
- [srcbz_lo] "=&f"(srcbz_lo)
- : [srcr_ptr] "r"(src_r), [srcg_ptr] "r"(src_g), [srcb_ptr] "r"(src_b),
- [dst_ptr] "r"(dst_rgb), [width] "r"(width), [temp] "f"(temp)
- : "memory");
-}
-
-// Filter 2 rows of YUY2 UV's (422) into U and V (420).
-void YUY2ToUVRow_MMI(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t c0 = 0xff00ff00ff00ff00;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t temp[3];
- uint64_t data[4];
- uint64_t shift = 0x08;
- uint64_t src_stride = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_yuy2]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_yuy2]) \n\t"
- "daddu %[src_stride], %[src_yuy2], %[src_stride_yuy2] \n\t"
- "gsldrc1 %[t1], 0x00(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x07(%[src_stride]) \n\t"
- "pavgb %[t0], %[t0], %[t1] \n\t"
-
- "gsldrc1 %[t2], 0x08(%[src_yuy2]) \n\t"
- "gsldlc1 %[t2], 0x0f(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_stride]) \n\t"
- "pavgb %[t1], %[t2], %[t1] \n\t"
-
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "psrlh %[t0], %[t0], %[shift] \n\t"
- "psrlh %[t1], %[t1], %[shift] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d0], %[t0], %[c1] \n\t"
- "psrlh %[d1], %[t1], %[shift] \n\t"
-
- "gsldrc1 %[t0], 0x10(%[src_yuy2]) \n\t"
- "gsldlc1 %[t0], 0x17(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x10(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x17(%[src_stride]) \n\t"
- "pavgb %[t0], %[t0], %[t1] \n\t"
-
- "gsldrc1 %[t2], 0x18(%[src_yuy2]) \n\t"
- "gsldlc1 %[t2], 0x1f(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x18(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x1f(%[src_stride]) \n\t"
- "pavgb %[t1], %[t2], %[t1] \n\t"
-
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "psrlh %[t0], %[t0], %[shift] \n\t"
- "psrlh %[t1], %[t1], %[shift] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d2], %[t0], %[c1] \n\t"
- "psrlh %[d3], %[t1], %[shift] \n\t"
-
- "packushb %[d0], %[d0], %[d2] \n\t"
- "packushb %[d1], %[d1], %[d3] \n\t"
- "gssdrc1 %[d0], 0x0(%[dst_u]) \n\t"
- "gssdlc1 %[d0], 0x7(%[dst_u]) \n\t"
- "gssdrc1 %[d1], 0x0(%[dst_v]) \n\t"
- "gssdlc1 %[d1], 0x7(%[dst_v]) \n\t"
- "daddiu %[src_yuy2], %[src_yuy2], 32 \n\t"
- "daddiu %[dst_u], %[dst_u], 8 \n\t"
- "daddiu %[dst_v], %[dst_v], 8 \n\t"
- "daddiu %[width], %[width], -16 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [t2] "=&f"(temp[2]),
- [d0] "=&f"(data[0]), [d1] "=&f"(data[1]), [d2] "=&f"(data[2]),
- [d3] "=&f"(data[3]), [src_stride] "=&r"(src_stride)
- : [src_yuy2] "r"(src_yuy2), [src_stride_yuy2] "r"(src_stride_yuy2),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [c0] "f"(c0), [c1] "f"(c1), [shift] "f"(shift)
- : "memory");
-}
-
-// Copy row of YUY2 UV's (422) into U and V (422).
-void YUY2ToUV422Row_MMI(const uint8_t* src_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- uint64_t c0 = 0xff00ff00ff00ff00;
- uint64_t c1 = 0x00ff00ff00ff00ff;
- uint64_t temp[2];
- uint64_t data[4];
- uint64_t shift = 0x08;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_yuy2]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_yuy2]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_yuy2]) \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "psrlh %[t0], %[t0], %[shift] \n\t"
- "psrlh %[t1], %[t1], %[shift] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d0], %[t0], %[c1] \n\t"
- "psrlh %[d1], %[t1], %[shift] \n\t"
-
- "gsldrc1 %[t0], 0x10(%[src_yuy2]) \n\t"
- "gsldlc1 %[t0], 0x17(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x18(%[src_yuy2]) \n\t"
- "gsldlc1 %[t1], 0x1f(%[src_yuy2]) \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "psrlh %[t0], %[t0], %[shift] \n\t"
- "psrlh %[t1], %[t1], %[shift] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d2], %[t0], %[c1] \n\t"
- "psrlh %[d3], %[t1], %[shift] \n\t"
-
- "packushb %[d0], %[d0], %[d2] \n\t"
- "packushb %[d1], %[d1], %[d3] \n\t"
- "gssdrc1 %[d0], 0x0(%[dst_u]) \n\t"
- "gssdlc1 %[d0], 0x7(%[dst_u]) \n\t"
- "gssdrc1 %[d1], 0x0(%[dst_v]) \n\t"
- "gssdlc1 %[d1], 0x7(%[dst_v]) \n\t"
- "daddiu %[src_yuy2], %[src_yuy2], 32 \n\t"
- "daddiu %[dst_u], %[dst_u], 8 \n\t"
- "daddiu %[dst_v], %[dst_v], 8 \n\t"
- "daddiu %[width], %[width], -16 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [d0] "=&f"(data[0]),
- [d1] "=&f"(data[1]), [d2] "=&f"(data[2]), [d3] "=&f"(data[3])
- : [src_yuy2] "r"(src_yuy2), [dst_u] "r"(dst_u), [dst_v] "r"(dst_v),
- [width] "r"(width), [c0] "f"(c0), [c1] "f"(c1), [shift] "f"(shift)
- : "memory");
-}
-
-// Copy row of YUY2 Y's (422) into Y (420/422).
-void YUY2ToYRow_MMI(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
- uint64_t c0 = 0x00ff00ff00ff00ff;
- uint64_t temp[2];
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_yuy2]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_yuy2]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_yuy2]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_yuy2]) \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "gssdrc1 %[t0], 0x0(%[dst_y]) \n\t"
- "gssdlc1 %[t0], 0x7(%[dst_y]) \n\t"
- "daddiu %[src_yuy2], %[src_yuy2], 16 \n\t"
- "daddiu %[dst_y], %[dst_y], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1])
- : [src_yuy2] "r"(src_yuy2), [dst_y] "r"(dst_y), [width] "r"(width),
- [c0] "f"(c0)
- : "memory");
-}
-
-// Filter 2 rows of UYVY UV's (422) into U and V (420).
-void UYVYToUVRow_MMI(const uint8_t* src_uyvy,
- int src_stride_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- // Output a row of UV values.
- uint64_t c0 = 0x00ff00ff00ff00ff;
- uint64_t temp[3];
- uint64_t data[4];
- uint64_t shift = 0x08;
- uint64_t src_stride = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_uyvy]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_uyvy]) \n\t"
- "daddu %[src_stride], %[src_uyvy], %[src_stride_uyvy] \n\t"
- "gsldrc1 %[t1], 0x00(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x07(%[src_stride]) \n\t"
- "pavgb %[t0], %[t0], %[t1] \n\t"
-
- "gsldrc1 %[t2], 0x08(%[src_uyvy]) \n\t"
- "gsldlc1 %[t2], 0x0f(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_stride]) \n\t"
- "pavgb %[t1], %[t2], %[t1] \n\t"
-
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d0], %[t0], %[c0] \n\t"
- "psrlh %[d1], %[t1], %[shift] \n\t"
-
- "gsldrc1 %[t0], 0x10(%[src_uyvy]) \n\t"
- "gsldlc1 %[t0], 0x17(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x10(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x17(%[src_stride]) \n\t"
- "pavgb %[t0], %[t0], %[t1] \n\t"
-
- "gsldrc1 %[t2], 0x18(%[src_uyvy]) \n\t"
- "gsldlc1 %[t2], 0x1f(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x18(%[src_stride]) \n\t"
- "gsldlc1 %[t1], 0x1f(%[src_stride]) \n\t"
- "pavgb %[t1], %[t2], %[t1] \n\t"
-
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d2], %[t0], %[c0] \n\t"
- "psrlh %[d3], %[t1], %[shift] \n\t"
-
- "packushb %[d0], %[d0], %[d2] \n\t"
- "packushb %[d1], %[d1], %[d3] \n\t"
- "gssdrc1 %[d0], 0x0(%[dst_u]) \n\t"
- "gssdlc1 %[d0], 0x7(%[dst_u]) \n\t"
- "gssdrc1 %[d1], 0x0(%[dst_v]) \n\t"
- "gssdlc1 %[d1], 0x7(%[dst_v]) \n\t"
- "daddiu %[src_uyvy], %[src_uyvy], 32 \n\t"
- "daddiu %[dst_u], %[dst_u], 8 \n\t"
- "daddiu %[dst_v], %[dst_v], 8 \n\t"
- "daddiu %[width], %[width], -16 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [t2] "=&f"(temp[2]),
- [d0] "=&f"(data[0]), [d1] "=&f"(data[1]), [d2] "=&f"(data[2]),
- [d3] "=&f"(data[3]), [src_stride] "=&r"(src_stride)
- : [src_uyvy] "r"(src_uyvy), [src_stride_uyvy] "r"(src_stride_uyvy),
- [dst_u] "r"(dst_u), [dst_v] "r"(dst_v), [width] "r"(width),
- [c0] "f"(c0), [shift] "f"(shift)
- : "memory");
-}
-
-// Copy row of UYVY UV's (422) into U and V (422).
-void UYVYToUV422Row_MMI(const uint8_t* src_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- // Output a row of UV values.
- uint64_t c0 = 0x00ff00ff00ff00ff;
- uint64_t temp[2];
- uint64_t data[4];
- uint64_t shift = 0x08;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_uyvy]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_uyvy]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_uyvy]) \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d0], %[t0], %[c0] \n\t"
- "psrlh %[d1], %[t1], %[shift] \n\t"
-
- "gsldrc1 %[t0], 0x10(%[src_uyvy]) \n\t"
- "gsldlc1 %[t0], 0x17(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x18(%[src_uyvy]) \n\t"
- "gsldlc1 %[t1], 0x1f(%[src_uyvy]) \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "mov.s %[t1], %[t0] \n\t"
- "and %[d2], %[t0], %[c0] \n\t"
- "psrlh %[d3], %[t1], %[shift] \n\t"
-
- "packushb %[d0], %[d0], %[d2] \n\t"
- "packushb %[d1], %[d1], %[d3] \n\t"
- "gssdrc1 %[d0], 0x0(%[dst_u]) \n\t"
- "gssdlc1 %[d0], 0x7(%[dst_u]) \n\t"
- "gssdrc1 %[d1], 0x0(%[dst_v]) \n\t"
- "gssdlc1 %[d1], 0x7(%[dst_v]) \n\t"
- "daddiu %[src_uyvy], %[src_uyvy], 32 \n\t"
- "daddiu %[dst_u], %[dst_u], 8 \n\t"
- "daddiu %[dst_v], %[dst_v], 8 \n\t"
- "daddiu %[width], %[width], -16 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1]), [d0] "=&f"(data[0]),
- [d1] "=&f"(data[1]), [d2] "=&f"(data[2]), [d3] "=&f"(data[3])
- : [src_uyvy] "r"(src_uyvy), [dst_u] "r"(dst_u), [dst_v] "r"(dst_v),
- [width] "r"(width), [c0] "f"(c0), [shift] "f"(shift)
- : "memory");
-}
-
-// Copy row of UYVY Y's (422) into Y (420/422).
-void UYVYToYRow_MMI(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
- // Output a row of Y values.
- uint64_t c0 = 0x00ff00ff00ff00ff;
- uint64_t shift = 0x08;
- uint64_t temp[2];
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[t0], 0x00(%[src_uyvy]) \n\t"
- "gsldlc1 %[t0], 0x07(%[src_uyvy]) \n\t"
- "gsldrc1 %[t1], 0x08(%[src_uyvy]) \n\t"
- "gsldlc1 %[t1], 0x0f(%[src_uyvy]) \n\t"
- "dsrl %[t0], %[t0], %[shift] \n\t"
- "dsrl %[t1], %[t1], %[shift] \n\t"
- "and %[t0], %[t0], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "and %[t1], %[t1], %[c0] \n\t"
- "packushb %[t0], %[t0], %[t1] \n\t"
- "gssdrc1 %[t0], 0x0(%[dst_y]) \n\t"
- "gssdlc1 %[t0], 0x7(%[dst_y]) \n\t"
- "daddiu %[src_uyvy], %[src_uyvy], 16 \n\t"
- "daddiu %[dst_y], %[dst_y], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp[0]), [t1] "=&f"(temp[1])
- : [src_uyvy] "r"(src_uyvy), [dst_y] "r"(dst_y), [width] "r"(width),
- [c0] "f"(c0), [shift] "f"(shift)
- : "memory");
-}
-
-// Blend src_argb0 over src_argb1 and store to dst_argb.
-// dst_argb may be src_argb0 or src_argb1.
-// This code mimics the SSSE3 version for better testability.
-void ARGBBlendRow_MMI(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- uint64_t src0, src1, dest, alpha, src0_hi, src0_lo, src1_hi, src1_lo, dest_hi,
- dest_lo;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x00FFFFFF00FFFFFFULL;
- const uint64_t mask2 = 0x00FF00FF00FF00FFULL;
- const uint64_t mask3 = 0xFF;
- const uint64_t mask4 = ~mask1;
- const uint64_t shift = 0x08;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 0x07(%[src0_ptr]) \n\t"
- "gsldrc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "punpcklbh %[src0_lo], %[src0], %[mask0] \n\t"
-
- "gsldlc1 %[src1], 0x07(%[src1_ptr]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "punpcklbh %[src1_lo], %[src1], %[mask0] \n\t"
-
- "psubush %[alpha], %[mask2], %[src0_lo] \n\t"
- "pshufh %[alpha], %[alpha], %[mask3] \n\t"
- "pmullh %[dest_lo], %[src1_lo], %[alpha] \n\t"
- "psrlh %[dest_lo], %[dest_lo], %[shift] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[src0_lo] \n\t"
-
- "punpckhbh %[src0_hi], %[src0], %[mask0] \n\t"
- "punpckhbh %[src1_hi], %[src1], %[mask0] \n\t"
-
- "psubush %[alpha], %[mask2], %[src0_hi] \n\t"
- "pshufh %[alpha], %[alpha], %[mask3] \n\t"
- "pmullh %[dest_hi], %[src1_hi], %[alpha] \n\t"
- "psrlh %[dest_hi], %[dest_hi], %[shift] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[src0_hi] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[dest], %[mask4] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t"
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [alpha] "=&f"(alpha),
- [dest] "=&f"(dest), [src0_hi] "=&f"(src0_hi), [src0_lo] "=&f"(src0_lo),
- [src1_hi] "=&f"(src1_hi), [src1_lo] "=&f"(src1_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo)
- : [src0_ptr] "r"(src_argb0), [src1_ptr] "r"(src_argb1),
- [dst_ptr] "r"(dst_argb), [mask0] "f"(mask0), [mask1] "f"(mask1),
- [mask2] "f"(mask2), [mask3] "f"(mask3), [mask4] "f"(mask4),
- [shift] "f"(shift), [width] "r"(width)
- : "memory");
-}
-
-void BlendPlaneRow_MMI(const uint8_t* src0,
- const uint8_t* src1,
- const uint8_t* alpha,
- uint8_t* dst,
- int width) {
- uint64_t source0, source1, dest, alph;
- uint64_t src0_hi, src0_lo, src1_hi, src1_lo, alpha_hi, alpha_lo, dest_hi,
- dest_lo;
- uint64_t alpha_rev, alpha_rev_lo, alpha_rev_hi;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0xFFFFFFFFFFFFFFFFULL;
- const uint64_t mask2 = 0x00FF00FF00FF00FFULL;
- const uint64_t shift = 0x08;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src0], 0x07(%[src0_ptr]) \n\t"
- "gsldrc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "punpcklbh %[src0_lo], %[src0], %[mask0] \n\t"
- "punpckhbh %[src0_hi], %[src0], %[mask0] \n\t"
-
- "gsldlc1 %[src1], 0x07(%[src1_ptr]) \n\t"
- "gsldrc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "punpcklbh %[src1_lo], %[src1], %[mask0] \n\t"
- "punpckhbh %[src1_hi], %[src1], %[mask0] \n\t"
-
- "gsldlc1 %[alpha], 0x07(%[alpha_ptr]) \n\t"
- "gsldrc1 %[alpha], 0x00(%[alpha_ptr]) \n\t"
- "psubusb %[alpha_r], %[mask1], %[alpha] \n\t"
- "punpcklbh %[alpha_lo], %[alpha], %[mask0] \n\t"
- "punpckhbh %[alpha_hi], %[alpha], %[mask0] \n\t"
- "punpcklbh %[alpha_rlo], %[alpha_r], %[mask0] \n\t"
- "punpckhbh %[alpha_rhi], %[alpha_r], %[mask0] \n\t"
-
- "pmullh %[dest_lo], %[src0_lo], %[alpha_lo] \n\t"
- "pmullh %[dest], %[src1_lo], %[alpha_rlo] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[dest] \n\t"
- "paddush %[dest_lo], %[dest_lo], %[mask2] \n\t"
- "psrlh %[dest_lo], %[dest_lo], %[shift] \n\t"
-
- "pmullh %[dest_hi], %[src0_hi], %[alpha_hi] \n\t"
- "pmullh %[dest], %[src1_hi], %[alpha_rhi] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[dest] \n\t"
- "paddush %[dest_hi], %[dest_hi], %[mask2] \n\t"
- "psrlh %[dest_hi], %[dest_hi], %[shift] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t"
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[alpha_ptr], %[alpha_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(source0), [src1] "=&f"(source1), [alpha] "=&f"(alph),
- [dest] "=&f"(dest), [src0_hi] "=&f"(src0_hi), [src0_lo] "=&f"(src0_lo),
- [src1_hi] "=&f"(src1_hi), [src1_lo] "=&f"(src1_lo),
- [alpha_hi] "=&f"(alpha_hi), [alpha_lo] "=&f"(alpha_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [alpha_rlo] "=&f"(alpha_rev_lo), [alpha_rhi] "=&f"(alpha_rev_hi),
- [alpha_r] "=&f"(alpha_rev)
- : [src0_ptr] "r"(src0), [src1_ptr] "r"(src1), [alpha_ptr] "r"(alpha),
- [dst_ptr] "r"(dst), [mask0] "f"(mask0), [mask1] "f"(mask1),
- [mask2] "f"(mask2), [shift] "f"(shift), [width] "r"(width)
- : "memory");
-}
-
-// Multiply source RGB by alpha and store to destination.
-// This code mimics the SSSE3 version for better testability.
-void ARGBAttenuateRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- uint64_t src, src_hi, src_lo, dest, dest_hi, dest_lo, alpha;
- const uint64_t mask0 = 0xFF;
- const uint64_t mask1 = 0xFF000000FF000000ULL;
- const uint64_t mask2 = ~mask1;
- const uint64_t shift = 0x08;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[src] \n\t"
- "punpckhbh %[src_hi], %[src], %[src] \n\t"
-
- "pshufh %[alpha], %[src_lo], %[mask0] \n\t"
- "pmulhuh %[dest_lo], %[alpha], %[src_lo] \n\t"
- "psrlh %[dest_lo], %[dest_lo], %[shift] \n\t"
- "pshufh %[alpha], %[src_hi], %[mask0] \n\t"
- "pmulhuh %[dest_hi], %[alpha], %[src_hi] \n\t"
- "psrlh %[dest_hi], %[dest_hi], %[shift] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "and %[dest], %[dest], %[mask2] \n\t"
- "and %[src], %[src], %[mask1] \n\t"
- "or %[dest], %[dest], %[src] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest), [src_hi] "=&f"(src_hi),
- [src_lo] "=&f"(src_lo), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [alpha] "=&f"(alpha)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [mask2] "f"(mask2), [shift] "f"(shift),
- [width] "r"(width)
- : "memory");
-}
-
-void ComputeCumulativeSumRow_MMI(const uint8_t* row,
- int32_t* cumsum,
- const int32_t* previous_cumsum,
- int width) {
- int64_t row_sum[2] = {0, 0};
- uint64_t src, dest0, dest1, presrc0, presrc1, dest;
- const uint64_t mask = 0x0;
-
- __asm__ volatile(
- "xor %[row_sum0], %[row_sum0], %[row_sum0] \n\t"
- "xor %[row_sum1], %[row_sum1], %[row_sum1] \n\t"
-
- "1: \n\t"
- "gslwlc1 %[src], 0x03(%[row_ptr]) \n\t"
- "gslwrc1 %[src], 0x00(%[row_ptr]) \n\t"
-
- "punpcklbh %[src], %[src], %[mask] \n\t"
- "punpcklhw %[dest0], %[src], %[mask] \n\t"
- "punpckhhw %[dest1], %[src], %[mask] \n\t"
-
- "paddw %[row_sum0], %[row_sum0], %[dest0] \n\t"
- "paddw %[row_sum1], %[row_sum1], %[dest1] \n\t"
-
- "gsldlc1 %[presrc0], 0x07(%[pre_ptr]) \n\t"
- "gsldrc1 %[presrc0], 0x00(%[pre_ptr]) \n\t"
- "gsldlc1 %[presrc1], 0x0f(%[pre_ptr]) \n\t"
- "gsldrc1 %[presrc1], 0x08(%[pre_ptr]) \n\t"
-
- "paddw %[dest0], %[row_sum0], %[presrc0] \n\t"
- "paddw %[dest1], %[row_sum1], %[presrc1] \n\t"
-
- "gssdlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
- "gssdlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[row_ptr], %[row_ptr], 0x04 \n\t"
- "daddiu %[pre_ptr], %[pre_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x01 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest), [dest0] "=&f"(dest0),
- [dest1] "=&f"(dest1), [row_sum0] "+&f"(row_sum[0]),
- [row_sum1] "+&f"(row_sum[1]), [presrc0] "=&f"(presrc0),
- [presrc1] "=&f"(presrc1)
- : [row_ptr] "r"(row), [pre_ptr] "r"(previous_cumsum),
- [dst_ptr] "r"(cumsum), [width] "r"(width), [mask] "f"(mask)
- : "memory");
-}
-
-// C version 2x2 -> 2x1.
-void InterpolateRow_MMI(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int width,
- int source_y_fraction) {
- if (source_y_fraction == 0) {
- __asm__ volatile(
- "1: \n\t"
- "ld $t0, 0x0(%[src_ptr]) \n\t"
- "sd $t0, 0x0(%[dst_ptr]) \n\t"
- "daddiu %[src_ptr], %[src_ptr], 8 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- :
- : [dst_ptr] "r"(dst_ptr), [src_ptr] "r"(src_ptr), [width] "r"(width)
- : "memory");
- return;
- }
- if (source_y_fraction == 128) {
- uint64_t uv = 0x0;
- uint64_t uv_stride = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[uv], 0x0(%[src_ptr]) \n\t"
- "gsldlc1 %[uv], 0x7(%[src_ptr]) \n\t"
- "daddu $t0, %[src_ptr], %[stride] \n\t"
- "gsldrc1 %[uv_stride], 0x0($t0) \n\t"
- "gsldlc1 %[uv_stride], 0x7($t0) \n\t"
-
- "pavgb %[uv], %[uv], %[uv_stride] \n\t"
- "gssdrc1 %[uv], 0x0(%[dst_ptr]) \n\t"
- "gssdlc1 %[uv], 0x7(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 8 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [uv] "=&f"(uv), [uv_stride] "=&f"(uv_stride)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst_ptr), [width] "r"(width),
- [stride] "r"((int64_t)src_stride)
- : "memory");
- return;
- }
- const uint8_t* src_ptr1 = src_ptr + src_stride;
- uint64_t temp;
- uint64_t data[4];
- uint64_t zero = 0x0;
- uint64_t c0 = 0x0080008000800080;
- uint64_t fy0 = 0x0100010001000100;
- uint64_t shift = 0x8;
- __asm__ volatile(
- "pshufh %[fy1], %[fy1], %[zero] \n\t"
- "psubh %[fy0], %[fy0], %[fy1] \n\t"
- "1: \n\t"
- "gsldrc1 %[t0], 0x0(%[src_ptr]) \n\t"
- "gsldlc1 %[t0], 0x7(%[src_ptr]) \n\t"
- "punpcklbh %[d0], %[t0], %[zero] \n\t"
- "punpckhbh %[d1], %[t0], %[zero] \n\t"
- "gsldrc1 %[t0], 0x0(%[src_ptr1]) \n\t"
- "gsldlc1 %[t0], 0x7(%[src_ptr1]) \n\t"
- "punpcklbh %[d2], %[t0], %[zero] \n\t"
- "punpckhbh %[d3], %[t0], %[zero] \n\t"
-
- "pmullh %[d0], %[d0], %[fy0] \n\t"
- "pmullh %[d2], %[d2], %[fy1] \n\t"
- "paddh %[d0], %[d0], %[d2] \n\t"
- "paddh %[d0], %[d0], %[c0] \n\t"
- "psrlh %[d0], %[d0], %[shift] \n\t"
-
- "pmullh %[d1], %[d1], %[fy0] \n\t"
- "pmullh %[d3], %[d3], %[fy1] \n\t"
- "paddh %[d1], %[d1], %[d3] \n\t"
- "paddh %[d1], %[d1], %[c0] \n\t"
- "psrlh %[d1], %[d1], %[shift] \n\t"
-
- "packushb %[d0], %[d0], %[d1] \n\t"
- "gssdrc1 %[d0], 0x0(%[dst_ptr]) \n\t"
- "gssdlc1 %[d0], 0x7(%[dst_ptr]) \n\t"
- "daddiu %[src_ptr], %[src_ptr], 8 \n\t"
- "daddiu %[src_ptr1], %[src_ptr1], 8 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 8 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [t0] "=&f"(temp), [d0] "=&f"(data[0]), [d1] "=&f"(data[1]),
- [d2] "=&f"(data[2]), [d3] "=&f"(data[3])
- : [src_ptr] "r"(src_ptr), [src_ptr1] "r"(src_ptr1),
- [dst_ptr] "r"(dst_ptr), [width] "r"(width),
- [fy1] "f"(source_y_fraction), [fy0] "f"(fy0), [c0] "f"(c0),
- [shift] "f"(shift), [zero] "f"(zero)
- : "memory");
-}
-
-// Use first 4 shuffler values to reorder ARGB channels.
-void ARGBShuffleRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
- uint64_t source, dest0, dest1, dest;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = (shuffler[0] & 0x03) | ((shuffler[1] & 0x03) << 2) |
- ((shuffler[2] & 0x03) << 4) |
- ((shuffler[3] & 0x03) << 6);
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
-
- "punpcklbh %[dest0], %[src], %[mask0] \n\t"
- "pshufh %[dest0], %[dest0], %[mask1] \n\t"
- "punpckhbh %[dest1], %[src], %[mask0] \n\t"
- "pshufh %[dest1], %[dest1], %[mask1] \n\t"
- "packushb %[dest], %[dest0], %[dest1] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(source), [dest] "=&f"(dest), [dest0] "=&f"(dest0),
- [dest1] "=&f"(dest1)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [width] "r"(width)
- : "memory");
-}
-
-void I422ToYUY2Row_MMI(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_frame,
- int width) {
- uint64_t temp[3];
- uint64_t vu = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[ty], 0x7(%[src_y]) \n\t" // r=src_sobelx[i]
- "gsldrc1 %[ty], 0x0(%[src_y]) \n\t" // r=src_sobelx[i]
- "gslwlc1 %[tu], 0x3(%[src_u]) \n\t" // b=src_sobely[i]
- "gslwrc1 %[tu], 0x0(%[src_u]) \n\t" // b=src_sobely[i]
- "gslwlc1 %[tv], 0x3(%[src_v]) \n\t" // b=src_sobely[i]
- "gslwrc1 %[tv], 0x0(%[src_v]) \n\t" // b=src_sobely[i]
- "punpcklbh %[vu], %[tu], %[tv] \n\t" // g
- "punpcklbh %[tu], %[ty], %[vu] \n\t" // g
- "gssdlc1 %[tu], 0x7(%[dst_frame]) \n\t"
- "gssdrc1 %[tu], 0x0(%[dst_frame]) \n\t"
- "punpckhbh %[tu], %[ty], %[vu] \n\t" // g
- "gssdlc1 %[tu], 0x0F(%[dst_frame]) \n\t"
- "gssdrc1 %[tu], 0x08(%[dst_frame]) \n\t"
- "daddiu %[src_y], %[src_y], 8 \n\t"
- "daddiu %[src_u], %[src_u], 4 \n\t"
- "daddiu %[src_v], %[src_v], 4 \n\t"
- "daddiu %[dst_frame], %[dst_frame], 16 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [ty] "=&f"(temp[1]), [tu] "=&f"(temp[1]), [tv] "=&f"(temp[1]),
- [vu] "=&f"(vu)
- : [src_y] "r"(src_y), [src_u] "r"(src_u), [src_v] "r"(src_v),
- [dst_frame] "r"(dst_frame), [width] "r"(width)
- : "memory");
-}
-
-void I422ToUYVYRow_MMI(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_frame,
- int width) {
- uint64_t temp[3];
- uint64_t vu = 0x0;
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[ty], 0x7(%[src_y]) \n\t" // r=src_sobelx[i]
- "gsldrc1 %[ty], 0x0(%[src_y]) \n\t" // r=src_sobelx[i]
- "gslwlc1 %[tu], 0x3(%[src_u]) \n\t" // b=src_sobely[i]
- "gslwrc1 %[tu], 0x0(%[src_u]) \n\t" // b=src_sobely[i]
- "gslwlc1 %[tv], 0x3(%[src_v]) \n\t" // b=src_sobely[i]
- "gslwrc1 %[tv], 0x0(%[src_v]) \n\t" // b=src_sobely[i]
- "punpcklbh %[vu], %[tu], %[tv] \n\t" // g
- "punpcklbh %[tu], %[vu], %[ty] \n\t" // g
- "gssdlc1 %[tu], 0x7(%[dst_frame]) \n\t"
- "gssdrc1 %[tu], 0x0(%[dst_frame]) \n\t"
- "punpckhbh %[tu], %[vu], %[ty] \n\t" // g
- "gssdlc1 %[tu], 0x0F(%[dst_frame]) \n\t"
- "gssdrc1 %[tu], 0x08(%[dst_frame]) \n\t"
- "daddiu %[src_y], %[src_y], 8 \n\t"
- "daddiu %[src_u], %[src_u], 4 \n\t"
- "daddiu %[src_v], %[src_v], 4 \n\t"
- "daddiu %[dst_frame], %[dst_frame], 16 \n\t"
- "daddiu %[width], %[width], -8 \n\t"
- "bgtz %[width], 1b \n\t"
- "nop \n\t"
- : [ty] "=&f"(temp[1]), [tu] "=&f"(temp[1]), [tv] "=&f"(temp[1]),
- [vu] "=&f"(vu)
- : [src_y] "r"(src_y), [src_u] "r"(src_u), [src_v] "r"(src_v),
- [dst_frame] "r"(dst_frame), [width] "r"(width)
- : "memory");
-}
-
-void ARGBCopyAlphaRow_MMI(const uint8_t* src, uint8_t* dst, int width) {
- uint64_t source, dest;
- const uint64_t mask0 = 0xff000000ff000000ULL;
- const uint64_t mask1 = ~mask0;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "and %[src], %[src], %[mask0] \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[src], %[dest] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(source), [dest] "=&f"(dest)
- : [src_ptr] "r"(src), [dst_ptr] "r"(dst), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [width] "r"(width)
- : "memory");
-}
-
-void ARGBExtractAlphaRow_MMI(const uint8_t* src_argb,
- uint8_t* dst_a,
- int width) {
- uint64_t src, dest0, dest1, dest_lo, dest_hi, dest;
- const uint64_t mask = 0xff000000ff000000ULL;
- const uint64_t shift = 0x18;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "and %[dest0], %[src], %[mask] \n\t"
- "psrlw %[dest0], %[dest0], %[shift] \n\t"
- "gsldlc1 %[src], 0x0f(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x08(%[src_ptr]) \n\t"
- "and %[dest1], %[src], %[mask] \n\t"
- "psrlw %[dest1], %[dest1], %[shift] \n\t"
- "packsswh %[dest_lo], %[dest0], %[dest1] \n\t"
-
- "gsldlc1 %[src], 0x17(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x10(%[src_ptr]) \n\t"
- "and %[dest0], %[src], %[mask] \n\t"
- "psrlw %[dest0], %[dest0], %[shift] \n\t"
- "gsldlc1 %[src], 0x1f(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x18(%[src_ptr]) \n\t"
- "and %[dest1], %[src], %[mask] \n\t"
- "psrlw %[dest1], %[dest1], %[shift] \n\t"
- "packsswh %[dest_hi], %[dest0], %[dest1] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x20 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest), [dest0] "=&f"(dest0),
- [dest1] "=&f"(dest1), [dest_lo] "=&f"(dest_lo), [dest_hi] "=&f"(dest_hi)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_a), [mask] "f"(mask),
- [shift] "f"(shift), [width] "r"(width)
- : "memory");
-}
-
-void ARGBCopyYToAlphaRow_MMI(const uint8_t* src, uint8_t* dst, int width) {
- uint64_t source, dest0, dest1, dest;
- const uint64_t mask0 = 0x0;
- const uint64_t mask1 = 0x00ffffff00ffffffULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
-
- "punpcklbh %[dest0], %[mask0], %[src] \n\t"
- "punpcklhw %[dest1], %[mask0], %[dest0] \n\t"
- "gsldlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[dest], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
- "punpckhhw %[dest1], %[mask0], %[dest0] \n\t"
- "gsldlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[dest], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "punpckhbh %[dest0], %[mask0], %[src] \n\t"
- "punpcklhw %[dest1], %[mask0], %[dest0] \n\t"
- "gsldlc1 %[dest], 0x17(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x10(%[dst_ptr]) \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[dest], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x17(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x10(%[dst_ptr]) \n\t"
- "punpckhhw %[dest1], %[mask0], %[dest0] \n\t"
- "gsldlc1 %[dest], 0x1f(%[dst_ptr]) \n\t"
- "gsldrc1 %[dest], 0x18(%[dst_ptr]) \n\t"
- "and %[dest], %[dest], %[mask1] \n\t"
- "or %[dest], %[dest], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x1f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x18(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x20 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(source), [dest] "=&f"(dest), [dest0] "=&f"(dest0),
- [dest1] "=&f"(dest1)
- : [src_ptr] "r"(src), [dst_ptr] "r"(dst), [mask0] "f"(mask0),
- [mask1] "f"(mask1), [width] "r"(width)
- : "memory");
-}
-
-#endif // !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
diff --git a/libs/libyuv/source/row_msa.cc b/libs/libyuv/source/row_msa.cc
deleted file mode 100644
index 5c0239a37f..0000000000
--- a/libs/libyuv/source/row_msa.cc
+++ /dev/null
@@ -1,3512 +0,0 @@
-/*
- * Copyright 2016 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <string.h>
-
-#include "libyuv/row.h"
-
-// This module is for GCC MSA
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#include "libyuv/macros_msa.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-#define ALPHA_VAL (-1)
-
-// Fill YUV -> RGB conversion constants into vectors
-#define YUVTORGB_SETUP(yuvconst, ub, vr, ug, vg, bb, bg, br, yg) \
- { \
- ub = __msa_fill_w(yuvconst->kUVToB[0]); \
- vr = __msa_fill_w(yuvconst->kUVToR[1]); \
- ug = __msa_fill_w(yuvconst->kUVToG[0]); \
- vg = __msa_fill_w(yuvconst->kUVToG[1]); \
- bb = __msa_fill_w(yuvconst->kUVBiasB[0]); \
- bg = __msa_fill_w(yuvconst->kUVBiasG[0]); \
- br = __msa_fill_w(yuvconst->kUVBiasR[0]); \
- yg = __msa_fill_w(yuvconst->kYToRgb[0]); \
- }
-
-// Load YUV 422 pixel data
-#define READYUV422(psrc_y, psrc_u, psrc_v, out_y, out_u, out_v) \
- { \
- uint64_t y_m; \
- uint32_t u_m, v_m; \
- v4i32 zero_m = {0}; \
- y_m = LD(psrc_y); \
- u_m = LW(psrc_u); \
- v_m = LW(psrc_v); \
- out_y = (v16u8)__msa_insert_d((v2i64)zero_m, 0, (int64_t)y_m); \
- out_u = (v16u8)__msa_insert_w(zero_m, 0, (int32_t)u_m); \
- out_v = (v16u8)__msa_insert_w(zero_m, 0, (int32_t)v_m); \
- }
-
-// Clip input vector elements between 0 to 255
-#define CLIP_0TO255(in0, in1, in2, in3, in4, in5) \
- { \
- v4i32 max_m = __msa_ldi_w(0xFF); \
- \
- in0 = __msa_maxi_s_w(in0, 0); \
- in1 = __msa_maxi_s_w(in1, 0); \
- in2 = __msa_maxi_s_w(in2, 0); \
- in3 = __msa_maxi_s_w(in3, 0); \
- in4 = __msa_maxi_s_w(in4, 0); \
- in5 = __msa_maxi_s_w(in5, 0); \
- in0 = __msa_min_s_w(max_m, in0); \
- in1 = __msa_min_s_w(max_m, in1); \
- in2 = __msa_min_s_w(max_m, in2); \
- in3 = __msa_min_s_w(max_m, in3); \
- in4 = __msa_min_s_w(max_m, in4); \
- in5 = __msa_min_s_w(max_m, in5); \
- }
-
-// Convert 8 pixels of YUV 420 to RGB.
-#define YUVTORGB(in_y, in_uv, ubvr, ugvg, bb, bg, br, yg, out_b, out_g, out_r) \
- { \
- v8i16 vec0_m, vec1_m; \
- v4i32 reg0_m, reg1_m, reg2_m, reg3_m, reg4_m; \
- v4i32 reg5_m, reg6_m, reg7_m; \
- v16i8 zero_m = {0}; \
- \
- vec0_m = (v8i16)__msa_ilvr_b((v16i8)in_y, (v16i8)in_y); \
- vec1_m = (v8i16)__msa_ilvr_b((v16i8)zero_m, (v16i8)in_uv); \
- reg0_m = (v4i32)__msa_ilvr_h((v8i16)zero_m, (v8i16)vec0_m); \
- reg1_m = (v4i32)__msa_ilvl_h((v8i16)zero_m, (v8i16)vec0_m); \
- reg2_m = (v4i32)__msa_ilvr_h((v8i16)zero_m, (v8i16)vec1_m); \
- reg3_m = (v4i32)__msa_ilvl_h((v8i16)zero_m, (v8i16)vec1_m); \
- reg0_m *= yg; \
- reg1_m *= yg; \
- reg2_m *= ubvr; \
- reg3_m *= ubvr; \
- reg0_m = __msa_srai_w(reg0_m, 16); \
- reg1_m = __msa_srai_w(reg1_m, 16); \
- reg4_m = __msa_dotp_s_w((v8i16)vec1_m, (v8i16)ugvg); \
- reg5_m = __msa_ilvev_w(reg2_m, reg2_m); \
- reg6_m = __msa_ilvev_w(reg3_m, reg3_m); \
- reg7_m = __msa_ilvr_w(reg4_m, reg4_m); \
- reg2_m = __msa_ilvod_w(reg2_m, reg2_m); \
- reg3_m = __msa_ilvod_w(reg3_m, reg3_m); \
- reg4_m = __msa_ilvl_w(reg4_m, reg4_m); \
- reg5_m = reg0_m - reg5_m; \
- reg6_m = reg1_m - reg6_m; \
- reg2_m = reg0_m - reg2_m; \
- reg3_m = reg1_m - reg3_m; \
- reg7_m = reg0_m - reg7_m; \
- reg4_m = reg1_m - reg4_m; \
- reg5_m += bb; \
- reg6_m += bb; \
- reg7_m += bg; \
- reg4_m += bg; \
- reg2_m += br; \
- reg3_m += br; \
- reg5_m = __msa_srai_w(reg5_m, 6); \
- reg6_m = __msa_srai_w(reg6_m, 6); \
- reg7_m = __msa_srai_w(reg7_m, 6); \
- reg4_m = __msa_srai_w(reg4_m, 6); \
- reg2_m = __msa_srai_w(reg2_m, 6); \
- reg3_m = __msa_srai_w(reg3_m, 6); \
- CLIP_0TO255(reg5_m, reg6_m, reg7_m, reg4_m, reg2_m, reg3_m); \
- out_b = __msa_pckev_h((v8i16)reg6_m, (v8i16)reg5_m); \
- out_g = __msa_pckev_h((v8i16)reg4_m, (v8i16)reg7_m); \
- out_r = __msa_pckev_h((v8i16)reg3_m, (v8i16)reg2_m); \
- }
-
-// Pack and Store 8 ARGB values.
-#define STOREARGB(in0, in1, in2, in3, pdst_argb) \
- { \
- v8i16 vec0_m, vec1_m; \
- v16u8 dst0_m, dst1_m; \
- vec0_m = (v8i16)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \
- vec1_m = (v8i16)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \
- dst0_m = (v16u8)__msa_ilvr_h(vec1_m, vec0_m); \
- dst1_m = (v16u8)__msa_ilvl_h(vec1_m, vec0_m); \
- ST_UB2(dst0_m, dst1_m, pdst_argb, 16); \
- }
-
-// Takes ARGB input and calculates Y.
-#define ARGBTOY(argb0, argb1, argb2, argb3, const0, const1, const2, shift, \
- y_out) \
- { \
- v16u8 vec0_m, vec1_m, vec2_m, vec3_m; \
- v8u16 reg0_m, reg1_m; \
- \
- vec0_m = (v16u8)__msa_pckev_h((v8i16)argb1, (v8i16)argb0); \
- vec1_m = (v16u8)__msa_pckev_h((v8i16)argb3, (v8i16)argb2); \
- vec2_m = (v16u8)__msa_pckod_h((v8i16)argb1, (v8i16)argb0); \
- vec3_m = (v16u8)__msa_pckod_h((v8i16)argb3, (v8i16)argb2); \
- reg0_m = __msa_dotp_u_h(vec0_m, const0); \
- reg1_m = __msa_dotp_u_h(vec1_m, const0); \
- reg0_m = __msa_dpadd_u_h(reg0_m, vec2_m, const1); \
- reg1_m = __msa_dpadd_u_h(reg1_m, vec3_m, const1); \
- reg0_m += const2; \
- reg1_m += const2; \
- reg0_m = (v8u16)__msa_srai_h((v8i16)reg0_m, shift); \
- reg1_m = (v8u16)__msa_srai_h((v8i16)reg1_m, shift); \
- y_out = (v16u8)__msa_pckev_b((v16i8)reg1_m, (v16i8)reg0_m); \
- }
-
-// Loads current and next row of ARGB input and averages it to calculate U and V
-#define READ_ARGB(s_ptr, t_ptr, argb0, argb1, argb2, argb3) \
- { \
- v16u8 src0_m, src1_m, src2_m, src3_m, src4_m, src5_m, src6_m, src7_m; \
- v16u8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \
- v16u8 vec8_m, vec9_m; \
- v8u16 reg0_m, reg1_m, reg2_m, reg3_m, reg4_m, reg5_m, reg6_m, reg7_m; \
- v8u16 reg8_m, reg9_m; \
- \
- src0_m = (v16u8)__msa_ld_b((void*)s, 0); \
- src1_m = (v16u8)__msa_ld_b((void*)s, 16); \
- src2_m = (v16u8)__msa_ld_b((void*)s, 32); \
- src3_m = (v16u8)__msa_ld_b((void*)s, 48); \
- src4_m = (v16u8)__msa_ld_b((void*)t, 0); \
- src5_m = (v16u8)__msa_ld_b((void*)t, 16); \
- src6_m = (v16u8)__msa_ld_b((void*)t, 32); \
- src7_m = (v16u8)__msa_ld_b((void*)t, 48); \
- vec0_m = (v16u8)__msa_ilvr_b((v16i8)src0_m, (v16i8)src4_m); \
- vec1_m = (v16u8)__msa_ilvr_b((v16i8)src1_m, (v16i8)src5_m); \
- vec2_m = (v16u8)__msa_ilvr_b((v16i8)src2_m, (v16i8)src6_m); \
- vec3_m = (v16u8)__msa_ilvr_b((v16i8)src3_m, (v16i8)src7_m); \
- vec4_m = (v16u8)__msa_ilvl_b((v16i8)src0_m, (v16i8)src4_m); \
- vec5_m = (v16u8)__msa_ilvl_b((v16i8)src1_m, (v16i8)src5_m); \
- vec6_m = (v16u8)__msa_ilvl_b((v16i8)src2_m, (v16i8)src6_m); \
- vec7_m = (v16u8)__msa_ilvl_b((v16i8)src3_m, (v16i8)src7_m); \
- reg0_m = __msa_hadd_u_h(vec0_m, vec0_m); \
- reg1_m = __msa_hadd_u_h(vec1_m, vec1_m); \
- reg2_m = __msa_hadd_u_h(vec2_m, vec2_m); \
- reg3_m = __msa_hadd_u_h(vec3_m, vec3_m); \
- reg4_m = __msa_hadd_u_h(vec4_m, vec4_m); \
- reg5_m = __msa_hadd_u_h(vec5_m, vec5_m); \
- reg6_m = __msa_hadd_u_h(vec6_m, vec6_m); \
- reg7_m = __msa_hadd_u_h(vec7_m, vec7_m); \
- reg8_m = (v8u16)__msa_pckev_d((v2i64)reg4_m, (v2i64)reg0_m); \
- reg9_m = (v8u16)__msa_pckev_d((v2i64)reg5_m, (v2i64)reg1_m); \
- reg8_m += (v8u16)__msa_pckod_d((v2i64)reg4_m, (v2i64)reg0_m); \
- reg9_m += (v8u16)__msa_pckod_d((v2i64)reg5_m, (v2i64)reg1_m); \
- reg0_m = (v8u16)__msa_pckev_d((v2i64)reg6_m, (v2i64)reg2_m); \
- reg1_m = (v8u16)__msa_pckev_d((v2i64)reg7_m, (v2i64)reg3_m); \
- reg0_m += (v8u16)__msa_pckod_d((v2i64)reg6_m, (v2i64)reg2_m); \
- reg1_m += (v8u16)__msa_pckod_d((v2i64)reg7_m, (v2i64)reg3_m); \
- reg8_m = (v8u16)__msa_srai_h((v8i16)reg8_m, 2); \
- reg9_m = (v8u16)__msa_srai_h((v8i16)reg9_m, 2); \
- reg0_m = (v8u16)__msa_srai_h((v8i16)reg0_m, 2); \
- reg1_m = (v8u16)__msa_srai_h((v8i16)reg1_m, 2); \
- argb0 = (v16u8)__msa_pckev_b((v16i8)reg9_m, (v16i8)reg8_m); \
- argb1 = (v16u8)__msa_pckev_b((v16i8)reg1_m, (v16i8)reg0_m); \
- src0_m = (v16u8)__msa_ld_b((void*)s, 64); \
- src1_m = (v16u8)__msa_ld_b((void*)s, 80); \
- src2_m = (v16u8)__msa_ld_b((void*)s, 96); \
- src3_m = (v16u8)__msa_ld_b((void*)s, 112); \
- src4_m = (v16u8)__msa_ld_b((void*)t, 64); \
- src5_m = (v16u8)__msa_ld_b((void*)t, 80); \
- src6_m = (v16u8)__msa_ld_b((void*)t, 96); \
- src7_m = (v16u8)__msa_ld_b((void*)t, 112); \
- vec2_m = (v16u8)__msa_ilvr_b((v16i8)src0_m, (v16i8)src4_m); \
- vec3_m = (v16u8)__msa_ilvr_b((v16i8)src1_m, (v16i8)src5_m); \
- vec4_m = (v16u8)__msa_ilvr_b((v16i8)src2_m, (v16i8)src6_m); \
- vec5_m = (v16u8)__msa_ilvr_b((v16i8)src3_m, (v16i8)src7_m); \
- vec6_m = (v16u8)__msa_ilvl_b((v16i8)src0_m, (v16i8)src4_m); \
- vec7_m = (v16u8)__msa_ilvl_b((v16i8)src1_m, (v16i8)src5_m); \
- vec8_m = (v16u8)__msa_ilvl_b((v16i8)src2_m, (v16i8)src6_m); \
- vec9_m = (v16u8)__msa_ilvl_b((v16i8)src3_m, (v16i8)src7_m); \
- reg0_m = __msa_hadd_u_h(vec2_m, vec2_m); \
- reg1_m = __msa_hadd_u_h(vec3_m, vec3_m); \
- reg2_m = __msa_hadd_u_h(vec4_m, vec4_m); \
- reg3_m = __msa_hadd_u_h(vec5_m, vec5_m); \
- reg4_m = __msa_hadd_u_h(vec6_m, vec6_m); \
- reg5_m = __msa_hadd_u_h(vec7_m, vec7_m); \
- reg6_m = __msa_hadd_u_h(vec8_m, vec8_m); \
- reg7_m = __msa_hadd_u_h(vec9_m, vec9_m); \
- reg8_m = (v8u16)__msa_pckev_d((v2i64)reg4_m, (v2i64)reg0_m); \
- reg9_m = (v8u16)__msa_pckev_d((v2i64)reg5_m, (v2i64)reg1_m); \
- reg8_m += (v8u16)__msa_pckod_d((v2i64)reg4_m, (v2i64)reg0_m); \
- reg9_m += (v8u16)__msa_pckod_d((v2i64)reg5_m, (v2i64)reg1_m); \
- reg0_m = (v8u16)__msa_pckev_d((v2i64)reg6_m, (v2i64)reg2_m); \
- reg1_m = (v8u16)__msa_pckev_d((v2i64)reg7_m, (v2i64)reg3_m); \
- reg0_m += (v8u16)__msa_pckod_d((v2i64)reg6_m, (v2i64)reg2_m); \
- reg1_m += (v8u16)__msa_pckod_d((v2i64)reg7_m, (v2i64)reg3_m); \
- reg8_m = (v8u16)__msa_srai_h((v8i16)reg8_m, 2); \
- reg9_m = (v8u16)__msa_srai_h((v8i16)reg9_m, 2); \
- reg0_m = (v8u16)__msa_srai_h((v8i16)reg0_m, 2); \
- reg1_m = (v8u16)__msa_srai_h((v8i16)reg1_m, 2); \
- argb2 = (v16u8)__msa_pckev_b((v16i8)reg9_m, (v16i8)reg8_m); \
- argb3 = (v16u8)__msa_pckev_b((v16i8)reg1_m, (v16i8)reg0_m); \
- }
-
-// Takes ARGB input and calculates U and V.
-#define ARGBTOUV(argb0, argb1, argb2, argb3, const0, const1, const2, const3, \
- shf0, shf1, shf2, shf3, v_out, u_out) \
- { \
- v16u8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \
- v8u16 reg0_m, reg1_m, reg2_m, reg3_m; \
- \
- vec0_m = (v16u8)__msa_vshf_b(shf0, (v16i8)argb1, (v16i8)argb0); \
- vec1_m = (v16u8)__msa_vshf_b(shf0, (v16i8)argb3, (v16i8)argb2); \
- vec2_m = (v16u8)__msa_vshf_b(shf1, (v16i8)argb1, (v16i8)argb0); \
- vec3_m = (v16u8)__msa_vshf_b(shf1, (v16i8)argb3, (v16i8)argb2); \
- vec4_m = (v16u8)__msa_vshf_b(shf2, (v16i8)argb1, (v16i8)argb0); \
- vec5_m = (v16u8)__msa_vshf_b(shf2, (v16i8)argb3, (v16i8)argb2); \
- vec6_m = (v16u8)__msa_vshf_b(shf3, (v16i8)argb1, (v16i8)argb0); \
- vec7_m = (v16u8)__msa_vshf_b(shf3, (v16i8)argb3, (v16i8)argb2); \
- reg0_m = __msa_dotp_u_h(vec0_m, const1); \
- reg1_m = __msa_dotp_u_h(vec1_m, const1); \
- reg2_m = __msa_dotp_u_h(vec4_m, const1); \
- reg3_m = __msa_dotp_u_h(vec5_m, const1); \
- reg0_m += const3; \
- reg1_m += const3; \
- reg2_m += const3; \
- reg3_m += const3; \
- reg0_m -= __msa_dotp_u_h(vec2_m, const0); \
- reg1_m -= __msa_dotp_u_h(vec3_m, const0); \
- reg2_m -= __msa_dotp_u_h(vec6_m, const2); \
- reg3_m -= __msa_dotp_u_h(vec7_m, const2); \
- v_out = (v16u8)__msa_pckod_b((v16i8)reg1_m, (v16i8)reg0_m); \
- u_out = (v16u8)__msa_pckod_b((v16i8)reg3_m, (v16i8)reg2_m); \
- }
-
-// Load I444 pixel data
-#define READI444(psrc_y, psrc_u, psrc_v, out_y, out_u, out_v) \
- { \
- uint64_t y_m, u_m, v_m; \
- v2i64 zero_m = {0}; \
- y_m = LD(psrc_y); \
- u_m = LD(psrc_u); \
- v_m = LD(psrc_v); \
- out_y = (v16u8)__msa_insert_d(zero_m, 0, (int64_t)y_m); \
- out_u = (v16u8)__msa_insert_d(zero_m, 0, (int64_t)u_m); \
- out_v = (v16u8)__msa_insert_d(zero_m, 0, (int64_t)v_m); \
- }
-
-void MirrorRow_MSA(const uint8_t* src, uint8_t* dst, int width) {
- int x;
- v16u8 src0, src1, src2, src3;
- v16u8 dst0, dst1, dst2, dst3;
- v16i8 shuffler = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
- src += width - 64;
-
- for (x = 0; x < width; x += 64) {
- LD_UB4(src, 16, src3, src2, src1, src0);
- VSHF_B2_UB(src3, src3, src2, src2, shuffler, shuffler, dst3, dst2);
- VSHF_B2_UB(src1, src1, src0, src0, shuffler, shuffler, dst1, dst0);
- ST_UB4(dst0, dst1, dst2, dst3, dst, 16);
- dst += 64;
- src -= 64;
- }
-}
-
-void ARGBMirrorRow_MSA(const uint8_t* src, uint8_t* dst, int width) {
- int x;
- v16u8 src0, src1, src2, src3;
- v16u8 dst0, dst1, dst2, dst3;
- v16i8 shuffler = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
- src += width * 4 - 64;
-
- for (x = 0; x < width; x += 16) {
- LD_UB4(src, 16, src3, src2, src1, src0);
- VSHF_B2_UB(src3, src3, src2, src2, shuffler, shuffler, dst3, dst2);
- VSHF_B2_UB(src1, src1, src0, src0, shuffler, shuffler, dst1, dst0);
- ST_UB4(dst0, dst1, dst2, dst3, dst, 16);
- dst += 64;
- src -= 64;
- }
-}
-
-void I422ToYUY2Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_yuy2,
- int width) {
- int x;
- v16u8 src_u0, src_v0, src_y0, src_y1, vec_uv0, vec_uv1;
- v16u8 dst_yuy2_0, dst_yuy2_1, dst_yuy2_2, dst_yuy2_3;
-
- for (x = 0; x < width; x += 32) {
- src_u0 = LD_UB(src_u);
- src_v0 = LD_UB(src_v);
- LD_UB2(src_y, 16, src_y0, src_y1);
- ILVRL_B2_UB(src_v0, src_u0, vec_uv0, vec_uv1);
- ILVRL_B2_UB(vec_uv0, src_y0, dst_yuy2_0, dst_yuy2_1);
- ILVRL_B2_UB(vec_uv1, src_y1, dst_yuy2_2, dst_yuy2_3);
- ST_UB4(dst_yuy2_0, dst_yuy2_1, dst_yuy2_2, dst_yuy2_3, dst_yuy2, 16);
- src_u += 16;
- src_v += 16;
- src_y += 32;
- dst_yuy2 += 64;
- }
-}
-
-void I422ToUYVYRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uyvy,
- int width) {
- int x;
- v16u8 src_u0, src_v0, src_y0, src_y1, vec_uv0, vec_uv1;
- v16u8 dst_uyvy0, dst_uyvy1, dst_uyvy2, dst_uyvy3;
-
- for (x = 0; x < width; x += 32) {
- src_u0 = LD_UB(src_u);
- src_v0 = LD_UB(src_v);
- LD_UB2(src_y, 16, src_y0, src_y1);
- ILVRL_B2_UB(src_v0, src_u0, vec_uv0, vec_uv1);
- ILVRL_B2_UB(src_y0, vec_uv0, dst_uyvy0, dst_uyvy1);
- ILVRL_B2_UB(src_y1, vec_uv1, dst_uyvy2, dst_uyvy3);
- ST_UB4(dst_uyvy0, dst_uyvy1, dst_uyvy2, dst_uyvy3, dst_uyvy, 16);
- src_u += 16;
- src_v += 16;
- src_y += 32;
- dst_uyvy += 64;
- }
-}
-
-void I422ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- STOREARGB(vec0, vec1, vec2, alpha, dst_argb);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- dst_argb += 32;
- }
-}
-
-void I422ToRGBARow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- STOREARGB(alpha, vec0, vec1, vec2, dst_argb);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- dst_argb += 32;
- }
-}
-
-void I422AlphaToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- const uint8_t* src_a,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- int64_t data_a;
- v16u8 src0, src1, src2, src3;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v4i32 zero = {0};
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- data_a = LD(src_a);
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- src3 = (v16u8)__msa_insert_d((v2i64)zero, 0, data_a);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- src3 = (v16u8)__msa_ilvr_b((v16i8)src3, (v16i8)src3);
- STOREARGB(vec0, vec1, vec2, src3, dst_argb);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- src_a += 8;
- dst_argb += 32;
- }
-}
-
-void I422ToRGB24Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int32_t width) {
- int x;
- int64_t data_u, data_v;
- v16u8 src0, src1, src2, src3, src4, dst0, dst1, dst2;
- v8i16 vec0, vec1, vec2, vec3, vec4, vec5;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 reg0, reg1, reg2, reg3;
- v2i64 zero = {0};
- v16i8 shuffler0 = {0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, 8, 9, 20, 10};
- v16i8 shuffler1 = {0, 21, 1, 2, 22, 3, 4, 23, 5, 6, 24, 7, 8, 25, 9, 10};
- v16i8 shuffler2 = {26, 6, 7, 27, 8, 9, 28, 10,
- 11, 29, 12, 13, 30, 14, 15, 31};
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_y, 0);
- data_u = LD(src_u);
- data_v = LD(src_v);
- src1 = (v16u8)__msa_insert_d(zero, 0, data_u);
- src2 = (v16u8)__msa_insert_d(zero, 0, data_v);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- src3 = (v16u8)__msa_sldi_b((v16i8)src0, (v16i8)src0, 8);
- src4 = (v16u8)__msa_sldi_b((v16i8)src1, (v16i8)src1, 8);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- YUVTORGB(src3, src4, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec3, vec4, vec5);
- reg0 = (v16u8)__msa_ilvev_b((v16i8)vec1, (v16i8)vec0);
- reg2 = (v16u8)__msa_ilvev_b((v16i8)vec4, (v16i8)vec3);
- reg3 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec2);
- reg1 = (v16u8)__msa_sldi_b((v16i8)reg2, (v16i8)reg0, 11);
- dst0 = (v16u8)__msa_vshf_b(shuffler0, (v16i8)reg3, (v16i8)reg0);
- dst1 = (v16u8)__msa_vshf_b(shuffler1, (v16i8)reg3, (v16i8)reg1);
- dst2 = (v16u8)__msa_vshf_b(shuffler2, (v16i8)reg3, (v16i8)reg2);
- ST_UB2(dst0, dst1, dst_argb, 16);
- ST_UB(dst2, (dst_argb + 32));
- src_y += 16;
- src_u += 8;
- src_v += 8;
- dst_argb += 48;
- }
-}
-
-// TODO(fbarchard): Consider AND instead of shift to isolate 5 upper bits of R.
-void I422ToRGB565Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2, dst0;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec2, vec1);
- vec0 = __msa_srai_h(vec0, 3);
- vec1 = __msa_srai_h(vec1, 3);
- vec2 = __msa_srai_h(vec2, 2);
- vec1 = __msa_slli_h(vec1, 11);
- vec2 = __msa_slli_h(vec2, 5);
- vec0 |= vec1;
- dst0 = (v16u8)(vec2 | vec0);
- ST_UB(dst0, dst_rgb565);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- dst_rgb565 += 16;
- }
-}
-
-// TODO(fbarchard): Consider AND instead of shift to isolate 4 upper bits of G.
-void I422ToARGB4444Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb4444,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2, dst0;
- v8i16 vec0, vec1, vec2;
- v8u16 reg0, reg1, reg2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v8u16 const_0xF000 = (v8u16)__msa_fill_h(0xF000);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- reg0 = (v8u16)__msa_srai_h(vec0, 4);
- reg1 = (v8u16)__msa_srai_h(vec1, 4);
- reg2 = (v8u16)__msa_srai_h(vec2, 4);
- reg1 = (v8u16)__msa_slli_h((v8i16)reg1, 4);
- reg2 = (v8u16)__msa_slli_h((v8i16)reg2, 8);
- reg1 |= const_0xF000;
- reg0 |= reg2;
- dst0 = (v16u8)(reg1 | reg0);
- ST_UB(dst0, dst_argb4444);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- dst_argb4444 += 16;
- }
-}
-
-void I422ToARGB1555Row_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb1555,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2, dst0;
- v8i16 vec0, vec1, vec2;
- v8u16 reg0, reg1, reg2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v8u16 const_0x8000 = (v8u16)__msa_fill_h(0x8000);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- READYUV422(src_y, src_u, src_v, src0, src1, src2);
- src1 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- reg0 = (v8u16)__msa_srai_h(vec0, 3);
- reg1 = (v8u16)__msa_srai_h(vec1, 3);
- reg2 = (v8u16)__msa_srai_h(vec2, 3);
- reg1 = (v8u16)__msa_slli_h((v8i16)reg1, 5);
- reg2 = (v8u16)__msa_slli_h((v8i16)reg2, 10);
- reg1 |= const_0x8000;
- reg0 |= reg2;
- dst0 = (v16u8)(reg1 | reg0);
- ST_UB(dst0, dst_argb1555);
- src_y += 8;
- src_u += 4;
- src_v += 4;
- dst_argb1555 += 16;
- }
-}
-
-void YUY2ToYRow_MSA(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_yuy2, 16, src0, src1, src2, src3);
- dst0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_y, 16);
- src_yuy2 += 64;
- dst_y += 32;
- }
-}
-
-void YUY2ToUVRow_MSA(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_yuy2_next = src_yuy2 + src_stride_yuy2;
- int x;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v16u8 vec0, vec1, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_yuy2, 16, src0, src1, src2, src3);
- LD_UB4(src_yuy2_next, 16, src4, src5, src6, src7);
- src0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- src1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- src2 = (v16u8)__msa_pckod_b((v16i8)src5, (v16i8)src4);
- src3 = (v16u8)__msa_pckod_b((v16i8)src7, (v16i8)src6);
- vec0 = __msa_aver_u_b(src0, src2);
- vec1 = __msa_aver_u_b(src1, src3);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_yuy2 += 64;
- src_yuy2_next += 64;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void YUY2ToUV422Row_MSA(const uint8_t* src_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_yuy2, 16, src0, src1, src2, src3);
- src0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- src1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_yuy2 += 64;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void UYVYToYRow_MSA(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_uyvy, 16, src0, src1, src2, src3);
- dst0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_y, 16);
- src_uyvy += 64;
- dst_y += 32;
- }
-}
-
-void UYVYToUVRow_MSA(const uint8_t* src_uyvy,
- int src_stride_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_uyvy_next = src_uyvy + src_stride_uyvy;
- int x;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v16u8 vec0, vec1, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_uyvy, 16, src0, src1, src2, src3);
- LD_UB4(src_uyvy_next, 16, src4, src5, src6, src7);
- src0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- src1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- src2 = (v16u8)__msa_pckev_b((v16i8)src5, (v16i8)src4);
- src3 = (v16u8)__msa_pckev_b((v16i8)src7, (v16i8)src6);
- vec0 = __msa_aver_u_b(src0, src2);
- vec1 = __msa_aver_u_b(src1, src3);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_uyvy += 64;
- src_uyvy_next += 64;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void UYVYToUV422Row_MSA(const uint8_t* src_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- LD_UB4(src_uyvy, 16, src0, src1, src2, src3);
- src0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- src1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_uyvy += 64;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void ARGBToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, vec0, vec1, vec2, vec3, dst0;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5;
- v16i8 zero = {0};
- v8u16 const_0x19 = (v8u16)__msa_ldi_h(0x19);
- v8u16 const_0x81 = (v8u16)__msa_ldi_h(0x81);
- v8u16 const_0x42 = (v8u16)__msa_ldi_h(0x42);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 48);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- vec2 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- vec3 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- reg0 = (v8u16)__msa_ilvev_b(zero, (v16i8)vec0);
- reg1 = (v8u16)__msa_ilvev_b(zero, (v16i8)vec1);
- reg2 = (v8u16)__msa_ilvev_b(zero, (v16i8)vec2);
- reg3 = (v8u16)__msa_ilvev_b(zero, (v16i8)vec3);
- reg4 = (v8u16)__msa_ilvod_b(zero, (v16i8)vec0);
- reg5 = (v8u16)__msa_ilvod_b(zero, (v16i8)vec1);
- reg0 *= const_0x19;
- reg1 *= const_0x19;
- reg2 *= const_0x81;
- reg3 *= const_0x81;
- reg4 *= const_0x42;
- reg5 *= const_0x42;
- reg0 += reg2;
- reg1 += reg3;
- reg0 += reg4;
- reg1 += reg5;
- reg0 += const_0x1080;
- reg1 += const_0x1080;
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 8);
- reg1 = (v8u16)__msa_srai_h((v8i16)reg1, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- ST_UB(dst0, dst_y);
- src_argb0 += 64;
- dst_y += 16;
- }
-}
-
-void ARGBToUVRow_MSA(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* src_argb0_next = src_argb0 + src_stride_argb;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8, reg9;
- v16u8 dst0, dst1;
- v8u16 const_0x70 = (v8u16)__msa_ldi_h(0x70);
- v8u16 const_0x4A = (v8u16)__msa_ldi_h(0x4A);
- v8u16 const_0x26 = (v8u16)__msa_ldi_h(0x26);
- v8u16 const_0x5E = (v8u16)__msa_ldi_h(0x5E);
- v8u16 const_0x12 = (v8u16)__msa_ldi_h(0x12);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
-
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 48);
- src4 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 64);
- src5 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 80);
- src6 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 96);
- src7 = (v16u8)__msa_ld_b((v16u8*)src_argb0, 112);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- vec2 = (v16u8)__msa_pckev_b((v16i8)src5, (v16i8)src4);
- vec3 = (v16u8)__msa_pckev_b((v16i8)src7, (v16i8)src6);
- vec4 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- vec5 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- vec6 = (v16u8)__msa_pckod_b((v16i8)src5, (v16i8)src4);
- vec7 = (v16u8)__msa_pckod_b((v16i8)src7, (v16i8)src6);
- vec8 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- vec9 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- vec4 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- vec5 = (v16u8)__msa_pckev_b((v16i8)vec7, (v16i8)vec6);
- vec0 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- vec1 = (v16u8)__msa_pckod_b((v16i8)vec3, (v16i8)vec2);
- reg0 = __msa_hadd_u_h(vec8, vec8);
- reg1 = __msa_hadd_u_h(vec9, vec9);
- reg2 = __msa_hadd_u_h(vec4, vec4);
- reg3 = __msa_hadd_u_h(vec5, vec5);
- reg4 = __msa_hadd_u_h(vec0, vec0);
- reg5 = __msa_hadd_u_h(vec1, vec1);
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 16);
- src2 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 32);
- src3 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 48);
- src4 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 64);
- src5 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 80);
- src6 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 96);
- src7 = (v16u8)__msa_ld_b((v16u8*)src_argb0_next, 112);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- vec2 = (v16u8)__msa_pckev_b((v16i8)src5, (v16i8)src4);
- vec3 = (v16u8)__msa_pckev_b((v16i8)src7, (v16i8)src6);
- vec4 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- vec5 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- vec6 = (v16u8)__msa_pckod_b((v16i8)src5, (v16i8)src4);
- vec7 = (v16u8)__msa_pckod_b((v16i8)src7, (v16i8)src6);
- vec8 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- vec9 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- vec4 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- vec5 = (v16u8)__msa_pckev_b((v16i8)vec7, (v16i8)vec6);
- vec0 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- vec1 = (v16u8)__msa_pckod_b((v16i8)vec3, (v16i8)vec2);
- reg0 += __msa_hadd_u_h(vec8, vec8);
- reg1 += __msa_hadd_u_h(vec9, vec9);
- reg2 += __msa_hadd_u_h(vec4, vec4);
- reg3 += __msa_hadd_u_h(vec5, vec5);
- reg4 += __msa_hadd_u_h(vec0, vec0);
- reg5 += __msa_hadd_u_h(vec1, vec1);
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 2);
- reg1 = (v8u16)__msa_srai_h((v8i16)reg1, 2);
- reg2 = (v8u16)__msa_srai_h((v8i16)reg2, 2);
- reg3 = (v8u16)__msa_srai_h((v8i16)reg3, 2);
- reg4 = (v8u16)__msa_srai_h((v8i16)reg4, 2);
- reg5 = (v8u16)__msa_srai_h((v8i16)reg5, 2);
- reg6 = reg0 * const_0x70;
- reg7 = reg1 * const_0x70;
- reg8 = reg2 * const_0x4A;
- reg9 = reg3 * const_0x4A;
- reg6 += const_0x8080;
- reg7 += const_0x8080;
- reg8 += reg4 * const_0x26;
- reg9 += reg5 * const_0x26;
- reg0 *= const_0x12;
- reg1 *= const_0x12;
- reg2 *= const_0x5E;
- reg3 *= const_0x5E;
- reg4 *= const_0x70;
- reg5 *= const_0x70;
- reg2 += reg0;
- reg3 += reg1;
- reg4 += const_0x8080;
- reg5 += const_0x8080;
- reg6 -= reg8;
- reg7 -= reg9;
- reg4 -= reg2;
- reg5 -= reg3;
- reg6 = (v8u16)__msa_srai_h((v8i16)reg6, 8);
- reg7 = (v8u16)__msa_srai_h((v8i16)reg7, 8);
- reg4 = (v8u16)__msa_srai_h((v8i16)reg4, 8);
- reg5 = (v8u16)__msa_srai_h((v8i16)reg5, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg7, (v16i8)reg6);
- dst1 = (v16u8)__msa_pckev_b((v16i8)reg5, (v16i8)reg4);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_argb0 += 128;
- src_argb0_next += 128;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void ARGBToRGB24Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2;
- v16i8 shuffler0 = {0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
- v16i8 shuffler1 = {5, 6, 8, 9, 10, 12, 13, 14,
- 16, 17, 18, 20, 21, 22, 24, 25};
- v16i8 shuffler2 = {10, 12, 13, 14, 16, 17, 18, 20,
- 21, 22, 24, 25, 26, 28, 29, 30};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb, 48);
- dst0 = (v16u8)__msa_vshf_b(shuffler0, (v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(shuffler1, (v16i8)src2, (v16i8)src1);
- dst2 = (v16u8)__msa_vshf_b(shuffler2, (v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_rgb, 16);
- ST_UB(dst2, (dst_rgb + 32));
- src_argb += 64;
- dst_rgb += 48;
- }
-}
-
-void ARGBToRAWRow_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2;
- v16i8 shuffler0 = {2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16, 22};
- v16i8 shuffler1 = {5, 4, 10, 9, 8, 14, 13, 12,
- 18, 17, 16, 22, 21, 20, 26, 25};
- v16i8 shuffler2 = {8, 14, 13, 12, 18, 17, 16, 22,
- 21, 20, 26, 25, 24, 30, 29, 28};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb, 48);
- dst0 = (v16u8)__msa_vshf_b(shuffler0, (v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(shuffler1, (v16i8)src2, (v16i8)src1);
- dst2 = (v16u8)__msa_vshf_b(shuffler2, (v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_rgb, 16);
- ST_UB(dst2, (dst_rgb + 32));
- src_argb += 64;
- dst_rgb += 48;
- }
-}
-
-void ARGBToRGB565Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
- int x;
- v16u8 src0, src1, dst0;
- v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec0 = (v16u8)__msa_srai_b((v16i8)src0, 3);
- vec1 = (v16u8)__msa_slli_b((v16i8)src0, 3);
- vec2 = (v16u8)__msa_srai_b((v16i8)src0, 5);
- vec4 = (v16u8)__msa_srai_b((v16i8)src1, 3);
- vec5 = (v16u8)__msa_slli_b((v16i8)src1, 3);
- vec6 = (v16u8)__msa_srai_b((v16i8)src1, 5);
- vec1 = (v16u8)__msa_sldi_b(zero, (v16i8)vec1, 1);
- vec2 = (v16u8)__msa_sldi_b(zero, (v16i8)vec2, 1);
- vec5 = (v16u8)__msa_sldi_b(zero, (v16i8)vec5, 1);
- vec6 = (v16u8)__msa_sldi_b(zero, (v16i8)vec6, 1);
- vec3 = (v16u8)__msa_sldi_b(zero, (v16i8)src0, 2);
- vec7 = (v16u8)__msa_sldi_b(zero, (v16i8)src1, 2);
- vec0 = __msa_binsli_b(vec0, vec1, 2);
- vec1 = __msa_binsli_b(vec2, vec3, 4);
- vec4 = __msa_binsli_b(vec4, vec5, 2);
- vec5 = __msa_binsli_b(vec6, vec7, 4);
- vec0 = (v16u8)__msa_ilvev_b((v16i8)vec1, (v16i8)vec0);
- vec4 = (v16u8)__msa_ilvev_b((v16i8)vec5, (v16i8)vec4);
- dst0 = (v16u8)__msa_pckev_h((v8i16)vec4, (v8i16)vec0);
- ST_UB(dst0, dst_rgb);
- src_argb += 32;
- dst_rgb += 16;
- }
-}
-
-void ARGBToARGB1555Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- int x;
- v16u8 src0, src1, dst0;
- v16u8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec0 = (v16u8)__msa_srai_b((v16i8)src0, 3);
- vec1 = (v16u8)__msa_slli_b((v16i8)src0, 2);
- vec2 = (v16u8)__msa_srai_b((v16i8)vec0, 3);
- vec1 = (v16u8)__msa_sldi_b(zero, (v16i8)vec1, 1);
- vec2 = (v16u8)__msa_sldi_b(zero, (v16i8)vec2, 1);
- vec3 = (v16u8)__msa_srai_b((v16i8)src0, 1);
- vec5 = (v16u8)__msa_srai_b((v16i8)src1, 3);
- vec6 = (v16u8)__msa_slli_b((v16i8)src1, 2);
- vec7 = (v16u8)__msa_srai_b((v16i8)vec5, 3);
- vec6 = (v16u8)__msa_sldi_b(zero, (v16i8)vec6, 1);
- vec7 = (v16u8)__msa_sldi_b(zero, (v16i8)vec7, 1);
- vec8 = (v16u8)__msa_srai_b((v16i8)src1, 1);
- vec3 = (v16u8)__msa_sldi_b(zero, (v16i8)vec3, 2);
- vec8 = (v16u8)__msa_sldi_b(zero, (v16i8)vec8, 2);
- vec4 = (v16u8)__msa_sldi_b(zero, (v16i8)src0, 3);
- vec9 = (v16u8)__msa_sldi_b(zero, (v16i8)src1, 3);
- vec0 = __msa_binsli_b(vec0, vec1, 2);
- vec5 = __msa_binsli_b(vec5, vec6, 2);
- vec1 = __msa_binsli_b(vec2, vec3, 5);
- vec6 = __msa_binsli_b(vec7, vec8, 5);
- vec1 = __msa_binsli_b(vec1, vec4, 0);
- vec6 = __msa_binsli_b(vec6, vec9, 0);
- vec0 = (v16u8)__msa_ilvev_b((v16i8)vec1, (v16i8)vec0);
- vec1 = (v16u8)__msa_ilvev_b((v16i8)vec6, (v16i8)vec5);
- dst0 = (v16u8)__msa_pckev_h((v8i16)vec1, (v8i16)vec0);
- ST_UB(dst0, dst_rgb);
- src_argb += 32;
- dst_rgb += 16;
- }
-}
-
-void ARGBToARGB4444Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- int x;
- v16u8 src0, src1;
- v16u8 vec0, vec1;
- v16u8 dst0;
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec0 = (v16u8)__msa_srai_b((v16i8)src0, 4);
- vec1 = (v16u8)__msa_srai_b((v16i8)src1, 4);
- src0 = (v16u8)__msa_sldi_b(zero, (v16i8)src0, 1);
- src1 = (v16u8)__msa_sldi_b(zero, (v16i8)src1, 1);
- vec0 = __msa_binsli_b(vec0, src0, 3);
- vec1 = __msa_binsli_b(vec1, src1, 3);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_rgb);
- src_argb += 32;
- dst_rgb += 16;
- }
-}
-
-void ARGBToUV444Row_MSA(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int32_t width) {
- int32_t x;
- v16u8 src0, src1, src2, src3, reg0, reg1, reg2, reg3, dst0, dst1;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v8u16 vec8, vec9, vec10, vec11;
- v8u16 const_112 = (v8u16)__msa_ldi_h(112);
- v8u16 const_74 = (v8u16)__msa_ldi_h(74);
- v8u16 const_38 = (v8u16)__msa_ldi_h(38);
- v8u16 const_94 = (v8u16)__msa_ldi_h(94);
- v8u16 const_18 = (v8u16)__msa_ldi_h(18);
- v8u16 const_32896 = (v8u16)__msa_fill_h(32896);
- v16i8 zero = {0};
-
- for (x = width; x > 0; x -= 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb, 48);
- reg0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- reg1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- reg2 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- reg3 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- src0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- src1 = (v16u8)__msa_pckev_b((v16i8)reg3, (v16i8)reg2);
- src2 = (v16u8)__msa_pckod_b((v16i8)reg1, (v16i8)reg0);
- vec0 = (v8u16)__msa_ilvr_b(zero, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b(zero, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b(zero, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b(zero, (v16i8)src1);
- vec4 = (v8u16)__msa_ilvr_b(zero, (v16i8)src2);
- vec5 = (v8u16)__msa_ilvl_b(zero, (v16i8)src2);
- vec10 = vec0 * const_18;
- vec11 = vec1 * const_18;
- vec8 = vec2 * const_94;
- vec9 = vec3 * const_94;
- vec6 = vec4 * const_112;
- vec7 = vec5 * const_112;
- vec0 *= const_112;
- vec1 *= const_112;
- vec2 *= const_74;
- vec3 *= const_74;
- vec4 *= const_38;
- vec5 *= const_38;
- vec8 += vec10;
- vec9 += vec11;
- vec6 += const_32896;
- vec7 += const_32896;
- vec0 += const_32896;
- vec1 += const_32896;
- vec2 += vec4;
- vec3 += vec5;
- vec0 -= vec2;
- vec1 -= vec3;
- vec6 -= vec8;
- vec7 -= vec9;
- vec0 = (v8u16)__msa_srai_h((v8i16)vec0, 8);
- vec1 = (v8u16)__msa_srai_h((v8i16)vec1, 8);
- vec6 = (v8u16)__msa_srai_h((v8i16)vec6, 8);
- vec7 = (v8u16)__msa_srai_h((v8i16)vec7, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec7, (v16i8)vec6);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- src_argb += 64;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void ARGBMultiplyRow_MSA(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, dst0;
- v8u16 vec0, vec1, vec2, vec3;
- v4u32 reg0, reg1, reg2, reg3;
- v8i16 zero = {0};
-
- for (x = 0; x < width; x += 4) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb1, 0);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src0, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)zero, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)zero, (v16i8)src1);
- reg0 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec0);
- reg1 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec0);
- reg2 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec1);
- reg3 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec1);
- reg0 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec2);
- reg1 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec2);
- reg2 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec3);
- reg3 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec3);
- reg0 = (v4u32)__msa_srai_w((v4i32)reg0, 16);
- reg1 = (v4u32)__msa_srai_w((v4i32)reg1, 16);
- reg2 = (v4u32)__msa_srai_w((v4i32)reg2, 16);
- reg3 = (v4u32)__msa_srai_w((v4i32)reg3, 16);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_argb);
- src_argb0 += 16;
- src_argb1 += 16;
- dst_argb += 16;
- }
-}
-
-void ARGBAddRow_MSA(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb1, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_argb1, 16);
- dst0 = __msa_adds_u_b(src0, src2);
- dst1 = __msa_adds_u_b(src1, src3);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb0 += 32;
- src_argb1 += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBSubtractRow_MSA(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb1, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_argb1, 16);
- dst0 = __msa_subs_u_b(src0, src2);
- dst1 = __msa_subs_u_b(src1, src3);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb0 += 32;
- src_argb1 += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBAttenuateRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, dst0, dst1;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
- v4u32 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
- v8i16 zero = {0};
- v16u8 mask = {0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src0, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src1, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src1, (v16i8)src1);
- vec4 = (v8u16)__msa_fill_h(vec0[3]);
- vec5 = (v8u16)__msa_fill_h(vec0[7]);
- vec6 = (v8u16)__msa_fill_h(vec1[3]);
- vec7 = (v8u16)__msa_fill_h(vec1[7]);
- vec4 = (v8u16)__msa_pckev_d((v2i64)vec5, (v2i64)vec4);
- vec5 = (v8u16)__msa_pckev_d((v2i64)vec7, (v2i64)vec6);
- vec6 = (v8u16)__msa_fill_h(vec2[3]);
- vec7 = (v8u16)__msa_fill_h(vec2[7]);
- vec8 = (v8u16)__msa_fill_h(vec3[3]);
- vec9 = (v8u16)__msa_fill_h(vec3[7]);
- vec6 = (v8u16)__msa_pckev_d((v2i64)vec7, (v2i64)vec6);
- vec7 = (v8u16)__msa_pckev_d((v2i64)vec9, (v2i64)vec8);
- reg0 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec4);
- reg1 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec4);
- reg2 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec5);
- reg3 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec5);
- reg4 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec6);
- reg5 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec6);
- reg6 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec7);
- reg7 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec7);
- reg0 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec0);
- reg1 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec0);
- reg2 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec1);
- reg3 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec1);
- reg4 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec2);
- reg5 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec2);
- reg6 *= (v4u32)__msa_ilvr_h(zero, (v8i16)vec3);
- reg7 *= (v4u32)__msa_ilvl_h(zero, (v8i16)vec3);
- reg0 = (v4u32)__msa_srai_w((v4i32)reg0, 24);
- reg1 = (v4u32)__msa_srai_w((v4i32)reg1, 24);
- reg2 = (v4u32)__msa_srai_w((v4i32)reg2, 24);
- reg3 = (v4u32)__msa_srai_w((v4i32)reg3, 24);
- reg4 = (v4u32)__msa_srai_w((v4i32)reg4, 24);
- reg5 = (v4u32)__msa_srai_w((v4i32)reg5, 24);
- reg6 = (v4u32)__msa_srai_w((v4i32)reg6, 24);
- reg7 = (v4u32)__msa_srai_w((v4i32)reg7, 24);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec2 = (v8u16)__msa_pckev_h((v8i16)reg5, (v8i16)reg4);
- vec3 = (v8u16)__msa_pckev_h((v8i16)reg7, (v8i16)reg6);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- dst0 = __msa_bmnz_v(dst0, src0, mask);
- dst1 = __msa_bmnz_v(dst1, src1, mask);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBToRGB565DitherRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- uint32_t dither4,
- int width) {
- int x;
- v16u8 src0, src1, dst0, vec0, vec1;
- v8i16 vec_d0;
- v8i16 reg0, reg1, reg2;
- v16i8 zero = {0};
- v8i16 max = __msa_ldi_h(0xFF);
-
- vec_d0 = (v8i16)__msa_fill_w(dither4);
- vec_d0 = (v8i16)__msa_ilvr_b(zero, (v16i8)vec_d0);
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- reg0 = (v8i16)__msa_ilvev_b(zero, (v16i8)vec0);
- reg1 = (v8i16)__msa_ilvev_b(zero, (v16i8)vec1);
- reg2 = (v8i16)__msa_ilvod_b(zero, (v16i8)vec0);
- reg0 += vec_d0;
- reg1 += vec_d0;
- reg2 += vec_d0;
- reg0 = __msa_maxi_s_h((v8i16)reg0, 0);
- reg1 = __msa_maxi_s_h((v8i16)reg1, 0);
- reg2 = __msa_maxi_s_h((v8i16)reg2, 0);
- reg0 = __msa_min_s_h((v8i16)max, (v8i16)reg0);
- reg1 = __msa_min_s_h((v8i16)max, (v8i16)reg1);
- reg2 = __msa_min_s_h((v8i16)max, (v8i16)reg2);
- reg0 = __msa_srai_h(reg0, 3);
- reg2 = __msa_srai_h(reg2, 3);
- reg1 = __msa_srai_h(reg1, 2);
- reg2 = __msa_slli_h(reg2, 11);
- reg1 = __msa_slli_h(reg1, 5);
- reg0 |= reg1;
- dst0 = (v16u8)(reg0 | reg2);
- ST_UB(dst0, dst_rgb);
- src_argb += 32;
- dst_rgb += 16;
- }
-}
-
-void ARGBShuffleRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
- int x;
- v16u8 src0, src1, dst0, dst1;
- v16i8 vec0;
- v16i8 shuffler_vec = {0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12};
- int32_t val = LW((int32_t*)shuffler);
-
- vec0 = (v16i8)__msa_fill_w(val);
- shuffler_vec += vec0;
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb, 16);
- dst0 = (v16u8)__msa_vshf_b(shuffler_vec, (v16i8)src0, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(shuffler_vec, (v16i8)src1, (v16i8)src1);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBShadeRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- uint32_t value) {
- int x;
- v16u8 src0, dst0;
- v8u16 vec0, vec1;
- v4u32 reg0, reg1, reg2, reg3, rgba_scale;
- v8i16 zero = {0};
-
- rgba_scale[0] = value;
- rgba_scale = (v4u32)__msa_ilvr_b((v16i8)rgba_scale, (v16i8)rgba_scale);
- rgba_scale = (v4u32)__msa_ilvr_h(zero, (v8i16)rgba_scale);
-
- for (x = 0; x < width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb, 0);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src0, (v16i8)src0);
- reg0 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec0);
- reg1 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec0);
- reg2 = (v4u32)__msa_ilvr_h(zero, (v8i16)vec1);
- reg3 = (v4u32)__msa_ilvl_h(zero, (v8i16)vec1);
- reg0 *= rgba_scale;
- reg1 *= rgba_scale;
- reg2 *= rgba_scale;
- reg3 *= rgba_scale;
- reg0 = (v4u32)__msa_srai_w((v4i32)reg0, 24);
- reg1 = (v4u32)__msa_srai_w((v4i32)reg1, 24);
- reg2 = (v4u32)__msa_srai_w((v4i32)reg2, 24);
- reg3 = (v4u32)__msa_srai_w((v4i32)reg3, 24);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_argb);
- src_argb += 16;
- dst_argb += 16;
- }
-}
-
-void ARGBGrayRow_MSA(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
- int x;
- v16u8 src0, src1, vec0, vec1, dst0, dst1;
- v8u16 reg0;
- v16u8 const_0x26 = (v16u8)__msa_ldi_h(0x26);
- v16u8 const_0x4B0F = (v16u8)__msa_fill_h(0x4B0F);
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb, 16);
- vec0 = (v16u8)__msa_pckev_h((v8i16)src1, (v8i16)src0);
- vec1 = (v16u8)__msa_pckod_h((v8i16)src1, (v8i16)src0);
- reg0 = __msa_dotp_u_h(vec0, const_0x4B0F);
- reg0 = __msa_dpadd_u_h(reg0, vec1, const_0x26);
- reg0 = (v8u16)__msa_srari_h((v8i16)reg0, 7);
- vec0 = (v16u8)__msa_ilvev_b((v16i8)reg0, (v16i8)reg0);
- vec1 = (v16u8)__msa_ilvod_b((v16i8)vec1, (v16i8)vec0);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)vec1, (v16i8)vec0);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBSepiaRow_MSA(uint8_t* dst_argb, int width) {
- int x;
- v16u8 src0, src1, dst0, dst1, vec0, vec1, vec2, vec3, vec4, vec5;
- v8u16 reg0, reg1, reg2;
- v16u8 const_0x4411 = (v16u8)__msa_fill_h(0x4411);
- v16u8 const_0x23 = (v16u8)__msa_ldi_h(0x23);
- v16u8 const_0x5816 = (v16u8)__msa_fill_h(0x5816);
- v16u8 const_0x2D = (v16u8)__msa_ldi_h(0x2D);
- v16u8 const_0x6218 = (v16u8)__msa_fill_h(0x6218);
- v16u8 const_0x32 = (v16u8)__msa_ldi_h(0x32);
- v8u16 const_0xFF = (v8u16)__msa_ldi_h(0xFF);
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((v16u8*)dst_argb, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)dst_argb, 16);
- vec0 = (v16u8)__msa_pckev_h((v8i16)src1, (v8i16)src0);
- vec1 = (v16u8)__msa_pckod_h((v8i16)src1, (v8i16)src0);
- vec3 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec1);
- reg0 = (v8u16)__msa_dotp_u_h(vec0, const_0x4411);
- reg1 = (v8u16)__msa_dotp_u_h(vec0, const_0x5816);
- reg2 = (v8u16)__msa_dotp_u_h(vec0, const_0x6218);
- reg0 = (v8u16)__msa_dpadd_u_h(reg0, vec1, const_0x23);
- reg1 = (v8u16)__msa_dpadd_u_h(reg1, vec1, const_0x2D);
- reg2 = (v8u16)__msa_dpadd_u_h(reg2, vec1, const_0x32);
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 7);
- reg1 = (v8u16)__msa_srai_h((v8i16)reg1, 7);
- reg2 = (v8u16)__msa_srai_h((v8i16)reg2, 7);
- reg1 = (v8u16)__msa_min_u_h((v8u16)reg1, const_0xFF);
- reg2 = (v8u16)__msa_min_u_h((v8u16)reg2, const_0xFF);
- vec0 = (v16u8)__msa_pckev_b((v16i8)reg0, (v16i8)reg0);
- vec1 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg1);
- vec2 = (v16u8)__msa_pckev_b((v16i8)reg2, (v16i8)reg2);
- vec4 = (v16u8)__msa_ilvr_b((v16i8)vec2, (v16i8)vec0);
- vec5 = (v16u8)__msa_ilvr_b((v16i8)vec3, (v16i8)vec1);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)vec5, (v16i8)vec4);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)vec5, (v16i8)vec4);
- ST_UB2(dst0, dst1, dst_argb, 16);
- dst_argb += 32;
- }
-}
-
-void ARGB4444ToARGBRow_MSA(const uint8_t* src_argb4444,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1;
- v8u16 vec0, vec1, vec2, vec3;
- v16u8 dst0, dst1, dst2, dst3;
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((v16u8*)src_argb4444, 0);
- src1 = (v16u8)__msa_ld_b((v16u8*)src_argb4444, 16);
- vec0 = (v8u16)__msa_andi_b(src0, 0x0F);
- vec1 = (v8u16)__msa_andi_b(src1, 0x0F);
- vec2 = (v8u16)__msa_andi_b(src0, 0xF0);
- vec3 = (v8u16)__msa_andi_b(src1, 0xF0);
- vec0 |= (v8u16)__msa_slli_b((v16i8)vec0, 4);
- vec1 |= (v8u16)__msa_slli_b((v16i8)vec1, 4);
- vec2 |= (v8u16)__msa_srli_b((v16i8)vec2, 4);
- vec3 |= (v8u16)__msa_srli_b((v16i8)vec3, 4);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)vec2, (v16i8)vec0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)vec2, (v16i8)vec0);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)vec3, (v16i8)vec1);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)vec3, (v16i8)vec1);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_argb4444 += 32;
- dst_argb += 64;
- }
-}
-
-void ARGB1555ToARGBRow_MSA(const uint8_t* src_argb1555,
- uint8_t* dst_argb,
- int width) {
- int x;
- v8u16 src0, src1;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5;
- v16u8 reg0, reg1, reg2, reg3, reg4, reg5, reg6;
- v16u8 dst0, dst1, dst2, dst3;
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_h((void*)src_argb1555, 0);
- src1 = (v8u16)__msa_ld_h((void*)src_argb1555, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src1 & const_0x1F;
- src0 = (v8u16)__msa_srli_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srli_h((v8i16)src1, 5);
- vec2 = src0 & const_0x1F;
- vec3 = src1 & const_0x1F;
- src0 = (v8u16)__msa_srli_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srli_h((v8i16)src1, 5);
- vec4 = src0 & const_0x1F;
- vec5 = src1 & const_0x1F;
- src0 = (v8u16)__msa_srli_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srli_h((v8i16)src1, 5);
- reg0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- reg1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- reg2 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- reg3 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- reg4 = (v16u8)__msa_slli_b((v16i8)reg0, 3);
- reg5 = (v16u8)__msa_slli_b((v16i8)reg1, 3);
- reg6 = (v16u8)__msa_slli_b((v16i8)reg2, 3);
- reg4 |= (v16u8)__msa_srai_b((v16i8)reg0, 2);
- reg5 |= (v16u8)__msa_srai_b((v16i8)reg1, 2);
- reg6 |= (v16u8)__msa_srai_b((v16i8)reg2, 2);
- reg3 = -reg3;
- reg0 = (v16u8)__msa_ilvr_b((v16i8)reg6, (v16i8)reg4);
- reg1 = (v16u8)__msa_ilvl_b((v16i8)reg6, (v16i8)reg4);
- reg2 = (v16u8)__msa_ilvr_b((v16i8)reg3, (v16i8)reg5);
- reg3 = (v16u8)__msa_ilvl_b((v16i8)reg3, (v16i8)reg5);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)reg2, (v16i8)reg0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)reg2, (v16i8)reg0);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)reg3, (v16i8)reg1);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)reg3, (v16i8)reg1);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_argb1555 += 32;
- dst_argb += 64;
- }
-}
-
-void RGB565ToARGBRow_MSA(const uint8_t* src_rgb565,
- uint8_t* dst_argb,
- int width) {
- int x;
- v8u16 src0, src1, vec0, vec1, vec2, vec3, vec4, vec5;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5;
- v16u8 res0, res1, res2, res3, dst0, dst1, dst2, dst3;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
- v8u16 const_0x7E0 = (v8u16)__msa_fill_h(0x7E0);
- v8u16 const_0xF800 = (v8u16)__msa_fill_h(0xF800);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_h((void*)src_rgb565, 0);
- src1 = (v8u16)__msa_ld_h((void*)src_rgb565, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src0 & const_0x7E0;
- vec2 = src0 & const_0xF800;
- vec3 = src1 & const_0x1F;
- vec4 = src1 & const_0x7E0;
- vec5 = src1 & const_0xF800;
- reg0 = (v8u16)__msa_slli_h((v8i16)vec0, 3);
- reg1 = (v8u16)__msa_srli_h((v8i16)vec1, 3);
- reg2 = (v8u16)__msa_srli_h((v8i16)vec2, 8);
- reg3 = (v8u16)__msa_slli_h((v8i16)vec3, 3);
- reg4 = (v8u16)__msa_srli_h((v8i16)vec4, 3);
- reg5 = (v8u16)__msa_srli_h((v8i16)vec5, 8);
- reg0 |= (v8u16)__msa_srli_h((v8i16)vec0, 2);
- reg1 |= (v8u16)__msa_srli_h((v8i16)vec1, 9);
- reg2 |= (v8u16)__msa_srli_h((v8i16)vec2, 13);
- reg3 |= (v8u16)__msa_srli_h((v8i16)vec3, 2);
- reg4 |= (v8u16)__msa_srli_h((v8i16)vec4, 9);
- reg5 |= (v8u16)__msa_srli_h((v8i16)vec5, 13);
- res0 = (v16u8)__msa_ilvev_b((v16i8)reg2, (v16i8)reg0);
- res1 = (v16u8)__msa_ilvev_b((v16i8)alpha, (v16i8)reg1);
- res2 = (v16u8)__msa_ilvev_b((v16i8)reg5, (v16i8)reg3);
- res3 = (v16u8)__msa_ilvev_b((v16i8)alpha, (v16i8)reg4);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)res1, (v16i8)res0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)res1, (v16i8)res0);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)res3, (v16i8)res2);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)res3, (v16i8)res2);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_rgb565 += 32;
- dst_argb += 64;
- }
-}
-
-void RGB24ToARGBRow_MSA(const uint8_t* src_rgb24,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, src2;
- v16u8 vec0, vec1, vec2;
- v16u8 dst0, dst1, dst2, dst3;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v16i8 shuffler = {0, 1, 2, 16, 3, 4, 5, 17, 6, 7, 8, 18, 9, 10, 11, 19};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_rgb24, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_rgb24, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_rgb24, 32);
- vec0 = (v16u8)__msa_sldi_b((v16i8)src1, (v16i8)src0, 12);
- vec1 = (v16u8)__msa_sldi_b((v16i8)src2, (v16i8)src1, 8);
- vec2 = (v16u8)__msa_sldi_b((v16i8)src2, (v16i8)src2, 4);
- dst0 = (v16u8)__msa_vshf_b(shuffler, (v16i8)alpha, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(shuffler, (v16i8)alpha, (v16i8)vec0);
- dst2 = (v16u8)__msa_vshf_b(shuffler, (v16i8)alpha, (v16i8)vec1);
- dst3 = (v16u8)__msa_vshf_b(shuffler, (v16i8)alpha, (v16i8)vec2);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_rgb24 += 48;
- dst_argb += 64;
- }
-}
-
-void RAWToARGBRow_MSA(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
- int x;
- v16u8 src0, src1, src2;
- v16u8 vec0, vec1, vec2;
- v16u8 dst0, dst1, dst2, dst3;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v16i8 mask = {2, 1, 0, 16, 5, 4, 3, 17, 8, 7, 6, 18, 11, 10, 9, 19};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_raw, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_raw, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_raw, 32);
- vec0 = (v16u8)__msa_sldi_b((v16i8)src1, (v16i8)src0, 12);
- vec1 = (v16u8)__msa_sldi_b((v16i8)src2, (v16i8)src1, 8);
- vec2 = (v16u8)__msa_sldi_b((v16i8)src2, (v16i8)src2, 4);
- dst0 = (v16u8)__msa_vshf_b(mask, (v16i8)alpha, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(mask, (v16i8)alpha, (v16i8)vec0);
- dst2 = (v16u8)__msa_vshf_b(mask, (v16i8)alpha, (v16i8)vec1);
- dst3 = (v16u8)__msa_vshf_b(mask, (v16i8)alpha, (v16i8)vec2);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_raw += 48;
- dst_argb += 64;
- }
-}
-
-void ARGB1555ToYRow_MSA(const uint8_t* src_argb1555,
- uint8_t* dst_y,
- int width) {
- int x;
- v8u16 src0, src1, vec0, vec1, vec2, vec3, vec4, vec5;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5;
- v16u8 dst0;
- v8u16 const_0x19 = (v8u16)__msa_ldi_h(0x19);
- v8u16 const_0x81 = (v8u16)__msa_ldi_h(0x81);
- v8u16 const_0x42 = (v8u16)__msa_ldi_h(0x42);
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_b((void*)src_argb1555, 0);
- src1 = (v8u16)__msa_ld_b((void*)src_argb1555, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src1 & const_0x1F;
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 5);
- vec2 = src0 & const_0x1F;
- vec3 = src1 & const_0x1F;
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 5);
- vec4 = src0 & const_0x1F;
- vec5 = src1 & const_0x1F;
- reg0 = (v8u16)__msa_slli_h((v8i16)vec0, 3);
- reg1 = (v8u16)__msa_slli_h((v8i16)vec1, 3);
- reg0 |= (v8u16)__msa_srai_h((v8i16)vec0, 2);
- reg1 |= (v8u16)__msa_srai_h((v8i16)vec1, 2);
- reg2 = (v8u16)__msa_slli_h((v8i16)vec2, 3);
- reg3 = (v8u16)__msa_slli_h((v8i16)vec3, 3);
- reg2 |= (v8u16)__msa_srai_h((v8i16)vec2, 2);
- reg3 |= (v8u16)__msa_srai_h((v8i16)vec3, 2);
- reg4 = (v8u16)__msa_slli_h((v8i16)vec4, 3);
- reg5 = (v8u16)__msa_slli_h((v8i16)vec5, 3);
- reg4 |= (v8u16)__msa_srai_h((v8i16)vec4, 2);
- reg5 |= (v8u16)__msa_srai_h((v8i16)vec5, 2);
- reg0 *= const_0x19;
- reg1 *= const_0x19;
- reg2 *= const_0x81;
- reg3 *= const_0x81;
- reg4 *= const_0x42;
- reg5 *= const_0x42;
- reg0 += reg2;
- reg1 += reg3;
- reg0 += reg4;
- reg1 += reg5;
- reg0 += const_0x1080;
- reg1 += const_0x1080;
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 8);
- reg1 = (v8u16)__msa_srai_h((v8i16)reg1, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- ST_UB(dst0, dst_y);
- src_argb1555 += 32;
- dst_y += 16;
- }
-}
-
-void RGB565ToYRow_MSA(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
- int x;
- v8u16 src0, src1, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5;
- v4u32 res0, res1, res2, res3;
- v16u8 dst0;
- v4u32 const_0x810019 = (v4u32)__msa_fill_w(0x810019);
- v4u32 const_0x010042 = (v4u32)__msa_fill_w(0x010042);
- v8i16 const_0x1080 = __msa_fill_h(0x1080);
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
- v8u16 const_0x7E0 = (v8u16)__msa_fill_h(0x7E0);
- v8u16 const_0xF800 = (v8u16)__msa_fill_h(0xF800);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_b((void*)src_rgb565, 0);
- src1 = (v8u16)__msa_ld_b((void*)src_rgb565, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src0 & const_0x7E0;
- vec2 = src0 & const_0xF800;
- vec3 = src1 & const_0x1F;
- vec4 = src1 & const_0x7E0;
- vec5 = src1 & const_0xF800;
- reg0 = (v8u16)__msa_slli_h((v8i16)vec0, 3);
- reg1 = (v8u16)__msa_srli_h((v8i16)vec1, 3);
- reg2 = (v8u16)__msa_srli_h((v8i16)vec2, 8);
- reg3 = (v8u16)__msa_slli_h((v8i16)vec3, 3);
- reg4 = (v8u16)__msa_srli_h((v8i16)vec4, 3);
- reg5 = (v8u16)__msa_srli_h((v8i16)vec5, 8);
- reg0 |= (v8u16)__msa_srli_h((v8i16)vec0, 2);
- reg1 |= (v8u16)__msa_srli_h((v8i16)vec1, 9);
- reg2 |= (v8u16)__msa_srli_h((v8i16)vec2, 13);
- reg3 |= (v8u16)__msa_srli_h((v8i16)vec3, 2);
- reg4 |= (v8u16)__msa_srli_h((v8i16)vec4, 9);
- reg5 |= (v8u16)__msa_srli_h((v8i16)vec5, 13);
- vec0 = (v8u16)__msa_ilvr_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_ilvl_h((v8i16)reg1, (v8i16)reg0);
- vec2 = (v8u16)__msa_ilvr_h((v8i16)reg4, (v8i16)reg3);
- vec3 = (v8u16)__msa_ilvl_h((v8i16)reg4, (v8i16)reg3);
- vec4 = (v8u16)__msa_ilvr_h(const_0x1080, (v8i16)reg2);
- vec5 = (v8u16)__msa_ilvl_h(const_0x1080, (v8i16)reg2);
- vec6 = (v8u16)__msa_ilvr_h(const_0x1080, (v8i16)reg5);
- vec7 = (v8u16)__msa_ilvl_h(const_0x1080, (v8i16)reg5);
- res0 = __msa_dotp_u_w(vec0, (v8u16)const_0x810019);
- res1 = __msa_dotp_u_w(vec1, (v8u16)const_0x810019);
- res2 = __msa_dotp_u_w(vec2, (v8u16)const_0x810019);
- res3 = __msa_dotp_u_w(vec3, (v8u16)const_0x810019);
- res0 = __msa_dpadd_u_w(res0, vec4, (v8u16)const_0x010042);
- res1 = __msa_dpadd_u_w(res1, vec5, (v8u16)const_0x010042);
- res2 = __msa_dpadd_u_w(res2, vec6, (v8u16)const_0x010042);
- res3 = __msa_dpadd_u_w(res3, vec7, (v8u16)const_0x010042);
- res0 = (v4u32)__msa_srai_w((v4i32)res0, 8);
- res1 = (v4u32)__msa_srai_w((v4i32)res1, 8);
- res2 = (v4u32)__msa_srai_w((v4i32)res2, 8);
- res3 = (v4u32)__msa_srai_w((v4i32)res3, 8);
- vec0 = (v8u16)__msa_pckev_h((v8i16)res1, (v8i16)res0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)res3, (v8i16)res2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_y);
- src_rgb565 += 32;
- dst_y += 16;
- }
-}
-
-void RGB24ToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, reg0, reg1, reg2, reg3, dst0;
- v8u16 vec0, vec1, vec2, vec3;
- v8u16 const_0x8119 = (v8u16)__msa_fill_h(0x8119);
- v8u16 const_0x42 = (v8u16)__msa_fill_h(0x42);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
- v16i8 mask0 = {0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 12};
- v16i8 mask1 = {12, 13, 14, 15, 15, 16, 17, 18,
- 18, 19, 20, 21, 21, 22, 23, 24};
- v16i8 mask2 = {8, 9, 10, 11, 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20};
- v16i8 mask3 = {4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15, 16};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- reg0 = (v16u8)__msa_vshf_b(mask0, zero, (v16i8)src0);
- reg1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- reg2 = (v16u8)__msa_vshf_b(mask2, (v16i8)src2, (v16i8)src1);
- reg3 = (v16u8)__msa_vshf_b(mask3, zero, (v16i8)src2);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec2 = (v8u16)__msa_pckod_h((v8i16)reg1, (v8i16)reg0);
- vec3 = (v8u16)__msa_pckod_h((v8i16)reg3, (v8i16)reg2);
- vec0 = __msa_dotp_u_h((v16u8)vec0, (v16u8)const_0x8119);
- vec1 = __msa_dotp_u_h((v16u8)vec1, (v16u8)const_0x8119);
- vec0 = __msa_dpadd_u_h(vec0, (v16u8)vec2, (v16u8)const_0x42);
- vec1 = __msa_dpadd_u_h(vec1, (v16u8)vec3, (v16u8)const_0x42);
- vec0 += const_0x1080;
- vec1 += const_0x1080;
- vec0 = (v8u16)__msa_srai_h((v8i16)vec0, 8);
- vec1 = (v8u16)__msa_srai_h((v8i16)vec1, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_y);
- src_argb0 += 48;
- dst_y += 16;
- }
-}
-
-void RAWToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, reg0, reg1, reg2, reg3, dst0;
- v8u16 vec0, vec1, vec2, vec3;
- v8u16 const_0x8142 = (v8u16)__msa_fill_h(0x8142);
- v8u16 const_0x19 = (v8u16)__msa_fill_h(0x19);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
- v16i8 mask0 = {0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 12};
- v16i8 mask1 = {12, 13, 14, 15, 15, 16, 17, 18,
- 18, 19, 20, 21, 21, 22, 23, 24};
- v16i8 mask2 = {8, 9, 10, 11, 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20};
- v16i8 mask3 = {4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15, 16};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- reg0 = (v16u8)__msa_vshf_b(mask0, zero, (v16i8)src0);
- reg1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- reg2 = (v16u8)__msa_vshf_b(mask2, (v16i8)src2, (v16i8)src1);
- reg3 = (v16u8)__msa_vshf_b(mask3, zero, (v16i8)src2);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec2 = (v8u16)__msa_pckod_h((v8i16)reg1, (v8i16)reg0);
- vec3 = (v8u16)__msa_pckod_h((v8i16)reg3, (v8i16)reg2);
- vec0 = __msa_dotp_u_h((v16u8)vec0, (v16u8)const_0x8142);
- vec1 = __msa_dotp_u_h((v16u8)vec1, (v16u8)const_0x8142);
- vec0 = __msa_dpadd_u_h(vec0, (v16u8)vec2, (v16u8)const_0x19);
- vec1 = __msa_dpadd_u_h(vec1, (v16u8)vec3, (v16u8)const_0x19);
- vec0 += const_0x1080;
- vec1 += const_0x1080;
- vec0 = (v8u16)__msa_srai_h((v8i16)vec0, 8);
- vec1 = (v8u16)__msa_srai_h((v8i16)vec1, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_y);
- src_argb0 += 48;
- dst_y += 16;
- }
-}
-
-void ARGB1555ToUVRow_MSA(const uint8_t* src_argb1555,
- int src_stride_argb1555,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint16_t* s = (const uint16_t*)src_argb1555;
- const uint16_t* t = (const uint16_t*)(src_argb1555 + src_stride_argb1555);
- int64_t res0, res1;
- v8u16 src0, src1, src2, src3, reg0, reg1, reg2, reg3;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6;
- v16u8 dst0;
- v8u16 const_0x70 = (v8u16)__msa_ldi_h(0x70);
- v8u16 const_0x4A = (v8u16)__msa_ldi_h(0x4A);
- v8u16 const_0x26 = (v8u16)__msa_ldi_h(0x26);
- v8u16 const_0x5E = (v8u16)__msa_ldi_h(0x5E);
- v8u16 const_0x12 = (v8u16)__msa_ldi_h(0x12);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_b((void*)s, 0);
- src1 = (v8u16)__msa_ld_b((void*)s, 16);
- src2 = (v8u16)__msa_ld_b((void*)t, 0);
- src3 = (v8u16)__msa_ld_b((void*)t, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src1 & const_0x1F;
- vec0 += src2 & const_0x1F;
- vec1 += src3 & const_0x1F;
- vec0 = (v8u16)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 5);
- src2 = (v8u16)__msa_srai_h((v8i16)src2, 5);
- src3 = (v8u16)__msa_srai_h((v8i16)src3, 5);
- vec2 = src0 & const_0x1F;
- vec3 = src1 & const_0x1F;
- vec2 += src2 & const_0x1F;
- vec3 += src3 & const_0x1F;
- vec2 = (v8u16)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 5);
- src2 = (v8u16)__msa_srai_h((v8i16)src2, 5);
- src3 = (v8u16)__msa_srai_h((v8i16)src3, 5);
- vec4 = src0 & const_0x1F;
- vec5 = src1 & const_0x1F;
- vec4 += src2 & const_0x1F;
- vec5 += src3 & const_0x1F;
- vec4 = (v8u16)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- vec0 = __msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec2 = __msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec4 = __msa_hadd_u_h((v16u8)vec4, (v16u8)vec4);
- vec6 = (v8u16)__msa_slli_h((v8i16)vec0, 1);
- vec6 |= (v8u16)__msa_srai_h((v8i16)vec0, 6);
- vec0 = (v8u16)__msa_slli_h((v8i16)vec2, 1);
- vec0 |= (v8u16)__msa_srai_h((v8i16)vec2, 6);
- vec2 = (v8u16)__msa_slli_h((v8i16)vec4, 1);
- vec2 |= (v8u16)__msa_srai_h((v8i16)vec4, 6);
- reg0 = vec6 * const_0x70;
- reg1 = vec0 * const_0x4A;
- reg2 = vec2 * const_0x70;
- reg3 = vec0 * const_0x5E;
- reg0 += const_0x8080;
- reg1 += vec2 * const_0x26;
- reg2 += const_0x8080;
- reg3 += vec6 * const_0x12;
- reg0 -= reg1;
- reg2 -= reg3;
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 8);
- reg2 = (v8u16)__msa_srai_h((v8i16)reg2, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg2, (v16i8)reg0);
- res0 = __msa_copy_u_d((v2i64)dst0, 0);
- res1 = __msa_copy_u_d((v2i64)dst0, 1);
- SD(res0, dst_u);
- SD(res1, dst_v);
- s += 16;
- t += 16;
- dst_u += 8;
- dst_v += 8;
- }
-}
-
-void RGB565ToUVRow_MSA(const uint8_t* src_rgb565,
- int src_stride_rgb565,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint16_t* s = (const uint16_t*)src_rgb565;
- const uint16_t* t = (const uint16_t*)(src_rgb565 + src_stride_rgb565);
- int64_t res0, res1;
- v8u16 src0, src1, src2, src3, reg0, reg1, reg2, reg3;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5;
- v16u8 dst0;
- v8u16 const_0x70 = (v8u16)__msa_ldi_h(0x70);
- v8u16 const_0x4A = (v8u16)__msa_ldi_h(0x4A);
- v8u16 const_0x26 = (v8u16)__msa_ldi_h(0x26);
- v8u16 const_0x5E = (v8u16)__msa_ldi_h(0x5E);
- v8u16 const_0x12 = (v8u16)__msa_ldi_h(0x12);
- v8u16 const_32896 = (v8u16)__msa_fill_h(0x8080);
- v8u16 const_0x1F = (v8u16)__msa_ldi_h(0x1F);
- v8u16 const_0x3F = (v8u16)__msa_fill_h(0x3F);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v8u16)__msa_ld_b((void*)s, 0);
- src1 = (v8u16)__msa_ld_b((void*)s, 16);
- src2 = (v8u16)__msa_ld_b((void*)t, 0);
- src3 = (v8u16)__msa_ld_b((void*)t, 16);
- vec0 = src0 & const_0x1F;
- vec1 = src1 & const_0x1F;
- vec0 += src2 & const_0x1F;
- vec1 += src3 & const_0x1F;
- vec0 = (v8u16)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 5);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 5);
- src2 = (v8u16)__msa_srai_h((v8i16)src2, 5);
- src3 = (v8u16)__msa_srai_h((v8i16)src3, 5);
- vec2 = src0 & const_0x3F;
- vec3 = src1 & const_0x3F;
- vec2 += src2 & const_0x3F;
- vec3 += src3 & const_0x3F;
- vec1 = (v8u16)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- src0 = (v8u16)__msa_srai_h((v8i16)src0, 6);
- src1 = (v8u16)__msa_srai_h((v8i16)src1, 6);
- src2 = (v8u16)__msa_srai_h((v8i16)src2, 6);
- src3 = (v8u16)__msa_srai_h((v8i16)src3, 6);
- vec4 = src0 & const_0x1F;
- vec5 = src1 & const_0x1F;
- vec4 += src2 & const_0x1F;
- vec5 += src3 & const_0x1F;
- vec2 = (v8u16)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- vec0 = __msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = __msa_hadd_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = __msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = (v8u16)__msa_slli_h((v8i16)vec0, 1);
- vec3 |= (v8u16)__msa_srai_h((v8i16)vec0, 6);
- vec4 = (v8u16)__msa_slli_h((v8i16)vec2, 1);
- vec4 |= (v8u16)__msa_srai_h((v8i16)vec2, 6);
- reg0 = vec3 * const_0x70;
- reg1 = vec1 * const_0x4A;
- reg2 = vec4 * const_0x70;
- reg3 = vec1 * const_0x5E;
- reg0 += const_32896;
- reg1 += vec4 * const_0x26;
- reg2 += const_32896;
- reg3 += vec3 * const_0x12;
- reg0 -= reg1;
- reg2 -= reg3;
- reg0 = (v8u16)__msa_srai_h((v8i16)reg0, 8);
- reg2 = (v8u16)__msa_srai_h((v8i16)reg2, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg2, (v16i8)reg0);
- res0 = __msa_copy_u_d((v2i64)dst0, 0);
- res1 = __msa_copy_u_d((v2i64)dst0, 1);
- SD(res0, dst_u);
- SD(res1, dst_v);
- s += 16;
- t += 16;
- dst_u += 8;
- dst_v += 8;
- }
-}
-
-void RGB24ToUVRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- int64_t res0, res1;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v16u8 inp0, inp1, inp2, inp3, inp4, inp5;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v8i16 reg0, reg1, reg2, reg3;
- v16u8 dst0;
- v8u16 const_0x70 = (v8u16)__msa_fill_h(0x70);
- v8u16 const_0x4A = (v8u16)__msa_fill_h(0x4A);
- v8u16 const_0x26 = (v8u16)__msa_fill_h(0x26);
- v8u16 const_0x5E = (v8u16)__msa_fill_h(0x5E);
- v8u16 const_0x12 = (v8u16)__msa_fill_h(0x12);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
- v16i8 mask = {0, 1, 2, 16, 3, 4, 5, 17, 6, 7, 8, 18, 9, 10, 11, 19};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 16) {
- inp0 = (v16u8)__msa_ld_b((void*)s, 0);
- inp1 = (v16u8)__msa_ld_b((void*)s, 16);
- inp2 = (v16u8)__msa_ld_b((void*)s, 32);
- inp3 = (v16u8)__msa_ld_b((void*)t, 0);
- inp4 = (v16u8)__msa_ld_b((void*)t, 16);
- inp5 = (v16u8)__msa_ld_b((void*)t, 32);
- src1 = (v16u8)__msa_sldi_b((v16i8)inp1, (v16i8)inp0, 12);
- src5 = (v16u8)__msa_sldi_b((v16i8)inp4, (v16i8)inp3, 12);
- src2 = (v16u8)__msa_sldi_b((v16i8)inp2, (v16i8)inp1, 8);
- src6 = (v16u8)__msa_sldi_b((v16i8)inp5, (v16i8)inp4, 8);
- src3 = (v16u8)__msa_sldi_b((v16i8)inp2, (v16i8)inp2, 4);
- src7 = (v16u8)__msa_sldi_b((v16i8)inp5, (v16i8)inp5, 4);
- src0 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)inp0);
- src1 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src1);
- src2 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src2);
- src3 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src3);
- src4 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)inp3);
- src5 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src5);
- src6 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src6);
- src7 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src7);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src4, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src4, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src5, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src5, (v16i8)src1);
- vec4 = (v8u16)__msa_ilvr_b((v16i8)src6, (v16i8)src2);
- vec5 = (v8u16)__msa_ilvl_b((v16i8)src6, (v16i8)src2);
- vec6 = (v8u16)__msa_ilvr_b((v16i8)src7, (v16i8)src3);
- vec7 = (v8u16)__msa_ilvl_b((v16i8)src7, (v16i8)src3);
- vec0 = (v8u16)__msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = (v8u16)__msa_hadd_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = (v8u16)__msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = (v8u16)__msa_hadd_u_h((v16u8)vec3, (v16u8)vec3);
- vec4 = (v8u16)__msa_hadd_u_h((v16u8)vec4, (v16u8)vec4);
- vec5 = (v8u16)__msa_hadd_u_h((v16u8)vec5, (v16u8)vec5);
- vec6 = (v8u16)__msa_hadd_u_h((v16u8)vec6, (v16u8)vec6);
- vec7 = (v8u16)__msa_hadd_u_h((v16u8)vec7, (v16u8)vec7);
- reg0 = (v8i16)__msa_pckev_d((v2i64)vec1, (v2i64)vec0);
- reg1 = (v8i16)__msa_pckev_d((v2i64)vec3, (v2i64)vec2);
- reg2 = (v8i16)__msa_pckev_d((v2i64)vec5, (v2i64)vec4);
- reg3 = (v8i16)__msa_pckev_d((v2i64)vec7, (v2i64)vec6);
- reg0 += (v8i16)__msa_pckod_d((v2i64)vec1, (v2i64)vec0);
- reg1 += (v8i16)__msa_pckod_d((v2i64)vec3, (v2i64)vec2);
- reg2 += (v8i16)__msa_pckod_d((v2i64)vec5, (v2i64)vec4);
- reg3 += (v8i16)__msa_pckod_d((v2i64)vec7, (v2i64)vec6);
- reg0 = __msa_srai_h((v8i16)reg0, 2);
- reg1 = __msa_srai_h((v8i16)reg1, 2);
- reg2 = __msa_srai_h((v8i16)reg2, 2);
- reg3 = __msa_srai_h((v8i16)reg3, 2);
- vec4 = (v8u16)__msa_pckev_h(reg1, reg0);
- vec5 = (v8u16)__msa_pckev_h(reg3, reg2);
- vec6 = (v8u16)__msa_pckod_h(reg1, reg0);
- vec7 = (v8u16)__msa_pckod_h(reg3, reg2);
- vec0 = (v8u16)__msa_pckev_h((v8i16)vec5, (v8i16)vec4);
- vec1 = (v8u16)__msa_pckev_h((v8i16)vec7, (v8i16)vec6);
- vec2 = (v8u16)__msa_pckod_h((v8i16)vec5, (v8i16)vec4);
- vec3 = vec0 * const_0x70;
- vec4 = vec1 * const_0x4A;
- vec5 = vec2 * const_0x26;
- vec2 *= const_0x70;
- vec1 *= const_0x5E;
- vec0 *= const_0x12;
- reg0 = __msa_subv_h((v8i16)vec3, (v8i16)vec4);
- reg1 = __msa_subv_h((v8i16)const_0x8080, (v8i16)vec5);
- reg2 = __msa_subv_h((v8i16)vec2, (v8i16)vec1);
- reg3 = __msa_subv_h((v8i16)const_0x8080, (v8i16)vec0);
- reg0 += reg1;
- reg2 += reg3;
- reg0 = __msa_srai_h(reg0, 8);
- reg2 = __msa_srai_h(reg2, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg2, (v16i8)reg0);
- res0 = __msa_copy_u_d((v2i64)dst0, 0);
- res1 = __msa_copy_u_d((v2i64)dst0, 1);
- SD(res0, dst_u);
- SD(res1, dst_v);
- t += 48;
- s += 48;
- dst_u += 8;
- dst_v += 8;
- }
-}
-
-void RAWToUVRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- int64_t res0, res1;
- v16u8 inp0, inp1, inp2, inp3, inp4, inp5;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v8i16 reg0, reg1, reg2, reg3;
- v16u8 dst0;
- v8u16 const_0x70 = (v8u16)__msa_fill_h(0x70);
- v8u16 const_0x4A = (v8u16)__msa_fill_h(0x4A);
- v8u16 const_0x26 = (v8u16)__msa_fill_h(0x26);
- v8u16 const_0x5E = (v8u16)__msa_fill_h(0x5E);
- v8u16 const_0x12 = (v8u16)__msa_fill_h(0x12);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
- v16i8 mask = {0, 1, 2, 16, 3, 4, 5, 17, 6, 7, 8, 18, 9, 10, 11, 19};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 16) {
- inp0 = (v16u8)__msa_ld_b((void*)s, 0);
- inp1 = (v16u8)__msa_ld_b((void*)s, 16);
- inp2 = (v16u8)__msa_ld_b((void*)s, 32);
- inp3 = (v16u8)__msa_ld_b((void*)t, 0);
- inp4 = (v16u8)__msa_ld_b((void*)t, 16);
- inp5 = (v16u8)__msa_ld_b((void*)t, 32);
- src1 = (v16u8)__msa_sldi_b((v16i8)inp1, (v16i8)inp0, 12);
- src5 = (v16u8)__msa_sldi_b((v16i8)inp4, (v16i8)inp3, 12);
- src2 = (v16u8)__msa_sldi_b((v16i8)inp2, (v16i8)inp1, 8);
- src6 = (v16u8)__msa_sldi_b((v16i8)inp5, (v16i8)inp4, 8);
- src3 = (v16u8)__msa_sldi_b((v16i8)inp2, (v16i8)inp2, 4);
- src7 = (v16u8)__msa_sldi_b((v16i8)inp5, (v16i8)inp5, 4);
- src0 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)inp0);
- src1 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src1);
- src2 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src2);
- src3 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src3);
- src4 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)inp3);
- src5 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src5);
- src6 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src6);
- src7 = (v16u8)__msa_vshf_b(mask, (v16i8)zero, (v16i8)src7);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src4, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src4, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src5, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src5, (v16i8)src1);
- vec4 = (v8u16)__msa_ilvr_b((v16i8)src6, (v16i8)src2);
- vec5 = (v8u16)__msa_ilvl_b((v16i8)src6, (v16i8)src2);
- vec6 = (v8u16)__msa_ilvr_b((v16i8)src7, (v16i8)src3);
- vec7 = (v8u16)__msa_ilvl_b((v16i8)src7, (v16i8)src3);
- vec0 = (v8u16)__msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = (v8u16)__msa_hadd_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = (v8u16)__msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = (v8u16)__msa_hadd_u_h((v16u8)vec3, (v16u8)vec3);
- vec4 = (v8u16)__msa_hadd_u_h((v16u8)vec4, (v16u8)vec4);
- vec5 = (v8u16)__msa_hadd_u_h((v16u8)vec5, (v16u8)vec5);
- vec6 = (v8u16)__msa_hadd_u_h((v16u8)vec6, (v16u8)vec6);
- vec7 = (v8u16)__msa_hadd_u_h((v16u8)vec7, (v16u8)vec7);
- reg0 = (v8i16)__msa_pckev_d((v2i64)vec1, (v2i64)vec0);
- reg1 = (v8i16)__msa_pckev_d((v2i64)vec3, (v2i64)vec2);
- reg2 = (v8i16)__msa_pckev_d((v2i64)vec5, (v2i64)vec4);
- reg3 = (v8i16)__msa_pckev_d((v2i64)vec7, (v2i64)vec6);
- reg0 += (v8i16)__msa_pckod_d((v2i64)vec1, (v2i64)vec0);
- reg1 += (v8i16)__msa_pckod_d((v2i64)vec3, (v2i64)vec2);
- reg2 += (v8i16)__msa_pckod_d((v2i64)vec5, (v2i64)vec4);
- reg3 += (v8i16)__msa_pckod_d((v2i64)vec7, (v2i64)vec6);
- reg0 = __msa_srai_h(reg0, 2);
- reg1 = __msa_srai_h(reg1, 2);
- reg2 = __msa_srai_h(reg2, 2);
- reg3 = __msa_srai_h(reg3, 2);
- vec4 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec5 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec6 = (v8u16)__msa_pckod_h((v8i16)reg1, (v8i16)reg0);
- vec7 = (v8u16)__msa_pckod_h((v8i16)reg3, (v8i16)reg2);
- vec0 = (v8u16)__msa_pckod_h((v8i16)vec5, (v8i16)vec4);
- vec1 = (v8u16)__msa_pckev_h((v8i16)vec7, (v8i16)vec6);
- vec2 = (v8u16)__msa_pckev_h((v8i16)vec5, (v8i16)vec4);
- vec3 = vec0 * const_0x70;
- vec4 = vec1 * const_0x4A;
- vec5 = vec2 * const_0x26;
- vec2 *= const_0x70;
- vec1 *= const_0x5E;
- vec0 *= const_0x12;
- reg0 = __msa_subv_h((v8i16)vec3, (v8i16)vec4);
- reg1 = __msa_subv_h((v8i16)const_0x8080, (v8i16)vec5);
- reg2 = __msa_subv_h((v8i16)vec2, (v8i16)vec1);
- reg3 = __msa_subv_h((v8i16)const_0x8080, (v8i16)vec0);
- reg0 += reg1;
- reg2 += reg3;
- reg0 = __msa_srai_h(reg0, 8);
- reg2 = __msa_srai_h(reg2, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg2, (v16i8)reg0);
- res0 = __msa_copy_u_d((v2i64)dst0, 0);
- res1 = __msa_copy_u_d((v2i64)dst0, 1);
- SD(res0, dst_u);
- SD(res1, dst_v);
- t += 48;
- s += 48;
- dst_u += 8;
- dst_v += 8;
- }
-}
-
-void NV12ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- uint64_t val0, val1;
- v16u8 src0, src1, res0, res1, dst0, dst1;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 zero = {0};
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- val0 = LD(src_y);
- val1 = LD(src_uv);
- src0 = (v16u8)__msa_insert_d((v2i64)zero, 0, val0);
- src1 = (v16u8)__msa_insert_d((v2i64)zero, 0, val1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- res0 = (v16u8)__msa_ilvev_b((v16i8)vec2, (v16i8)vec0);
- res1 = (v16u8)__msa_ilvev_b((v16i8)alpha, (v16i8)vec1);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)res1, (v16i8)res0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)res1, (v16i8)res0);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_y += 8;
- src_uv += 8;
- dst_argb += 32;
- }
-}
-
-void NV12ToRGB565Row_MSA(const uint8_t* src_y,
- const uint8_t* src_uv,
- uint8_t* dst_rgb565,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- uint64_t val0, val1;
- v16u8 src0, src1, dst0;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 zero = {0};
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- val0 = LD(src_y);
- val1 = LD(src_uv);
- src0 = (v16u8)__msa_insert_d((v2i64)zero, 0, val0);
- src1 = (v16u8)__msa_insert_d((v2i64)zero, 0, val1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- vec0 = vec0 >> 3;
- vec1 = (vec1 >> 2) << 5;
- vec2 = (vec2 >> 3) << 11;
- dst0 = (v16u8)(vec0 | vec1 | vec2);
- ST_UB(dst0, dst_rgb565);
- src_y += 8;
- src_uv += 8;
- dst_rgb565 += 16;
- }
-}
-
-void NV21ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_vu,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- uint64_t val0, val1;
- v16u8 src0, src1, res0, res1, dst0, dst1;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v16u8 zero = {0};
- v16i8 shuffler = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- val0 = LD(src_y);
- val1 = LD(src_vu);
- src0 = (v16u8)__msa_insert_d((v2i64)zero, 0, val0);
- src1 = (v16u8)__msa_insert_d((v2i64)zero, 0, val1);
- src1 = (v16u8)__msa_vshf_b(shuffler, (v16i8)src1, (v16i8)src1);
- YUVTORGB(src0, src1, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- res0 = (v16u8)__msa_ilvev_b((v16i8)vec2, (v16i8)vec0);
- res1 = (v16u8)__msa_ilvev_b((v16i8)alpha, (v16i8)vec1);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)res1, (v16i8)res0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)res1, (v16i8)res0);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_y += 8;
- src_vu += 8;
- dst_argb += 32;
- }
-}
-
-void SobelRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, vec0, dst0, dst1, dst2, dst3;
- v16i8 mask0 = {0, 0, 0, 16, 1, 1, 1, 16, 2, 2, 2, 16, 3, 3, 3, 16};
- v16i8 const_0x4 = __msa_ldi_b(0x4);
- v16i8 mask1 = mask0 + const_0x4;
- v16i8 mask2 = mask1 + const_0x4;
- v16i8 mask3 = mask2 + const_0x4;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_sobelx, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_sobely, 0);
- vec0 = __msa_adds_u_b(src0, src1);
- dst0 = (v16u8)__msa_vshf_b(mask0, (v16i8)alpha, (v16i8)vec0);
- dst1 = (v16u8)__msa_vshf_b(mask1, (v16i8)alpha, (v16i8)vec0);
- dst2 = (v16u8)__msa_vshf_b(mask2, (v16i8)alpha, (v16i8)vec0);
- dst3 = (v16u8)__msa_vshf_b(mask3, (v16i8)alpha, (v16i8)vec0);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_sobelx += 16;
- src_sobely += 16;
- dst_argb += 64;
- }
-}
-
-void SobelToPlaneRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_y,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
-
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((void*)src_sobelx, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_sobelx, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_sobely, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_sobely, 16);
- dst0 = __msa_adds_u_b(src0, src2);
- dst1 = __msa_adds_u_b(src1, src3);
- ST_UB2(dst0, dst1, dst_y, 16);
- src_sobelx += 32;
- src_sobely += 32;
- dst_y += 32;
- }
-}
-
-void SobelXYRow_MSA(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, vec0, vec1, vec2;
- v16u8 reg0, reg1, dst0, dst1, dst2, dst3;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_sobelx, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_sobely, 0);
- vec0 = __msa_adds_u_b(src0, src1);
- vec1 = (v16u8)__msa_ilvr_b((v16i8)src0, (v16i8)src1);
- vec2 = (v16u8)__msa_ilvl_b((v16i8)src0, (v16i8)src1);
- reg0 = (v16u8)__msa_ilvr_b((v16i8)alpha, (v16i8)vec0);
- reg1 = (v16u8)__msa_ilvl_b((v16i8)alpha, (v16i8)vec0);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)reg0, (v16i8)vec1);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)reg0, (v16i8)vec1);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)reg1, (v16i8)vec2);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)reg1, (v16i8)vec2);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_sobelx += 16;
- src_sobely += 16;
- dst_argb += 64;
- }
-}
-
-void ARGBToYJRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0;
- v16u8 const_0x4B0F = (v16u8)__msa_fill_h(0x4B0F);
- v16u8 const_0x26 = (v16u8)__msa_fill_h(0x26);
- v8u16 const_0x40 = (v8u16)__msa_fill_h(0x40);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb0, 48);
- ARGBTOY(src0, src1, src2, src3, const_0x4B0F, const_0x26, const_0x40, 7,
- dst0);
- ST_UB(dst0, dst_y);
- src_argb0 += 64;
- dst_y += 16;
- }
-}
-
-void BGRAToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0;
- v16u8 const_0x4200 = (v16u8)__msa_fill_h(0x4200);
- v16u8 const_0x1981 = (v16u8)__msa_fill_h(0x1981);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb0, 48);
- ARGBTOY(src0, src1, src2, src3, const_0x4200, const_0x1981, const_0x1080, 8,
- dst0);
- ST_UB(dst0, dst_y);
- src_argb0 += 64;
- dst_y += 16;
- }
-}
-
-void ABGRToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0;
- v16u8 const_0x8142 = (v16u8)__msa_fill_h(0x8142);
- v16u8 const_0x19 = (v16u8)__msa_fill_h(0x19);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb0, 48);
- ARGBTOY(src0, src1, src2, src3, const_0x8142, const_0x19, const_0x1080, 8,
- dst0);
- ST_UB(dst0, dst_y);
- src_argb0 += 64;
- dst_y += 16;
- }
-}
-
-void RGBAToYRow_MSA(const uint8_t* src_argb0, uint8_t* dst_y, int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0;
- v16u8 const_0x1900 = (v16u8)__msa_fill_h(0x1900);
- v16u8 const_0x4281 = (v16u8)__msa_fill_h(0x4281);
- v8u16 const_0x1080 = (v8u16)__msa_fill_h(0x1080);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb0, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb0, 48);
- ARGBTOY(src0, src1, src2, src3, const_0x1900, const_0x4281, const_0x1080, 8,
- dst0);
- ST_UB(dst0, dst_y);
- src_argb0 += 64;
- dst_y += 16;
- }
-}
-
-void ARGBToUVJRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
- v16u8 vec0, vec1, vec2, vec3;
- v16u8 dst0, dst1;
- v16i8 shuffler0 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
- v16i8 shuffler1 = {2, 3, 6, 7, 10, 11, 14, 15,
- 18, 19, 22, 23, 26, 27, 30, 31};
- v16i8 shuffler2 = {0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23, 24, 27, 28, 31};
- v16i8 shuffler3 = {1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30};
- v16u8 const_0x7F = (v16u8)__msa_fill_h(0x7F);
- v16u8 const_0x6B14 = (v16u8)__msa_fill_h(0x6B14);
- v16u8 const_0x2B54 = (v16u8)__msa_fill_h(0x2B54);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
-
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((void*)s, 0);
- src1 = (v16u8)__msa_ld_b((void*)s, 16);
- src2 = (v16u8)__msa_ld_b((void*)s, 32);
- src3 = (v16u8)__msa_ld_b((void*)s, 48);
- src4 = (v16u8)__msa_ld_b((void*)t, 0);
- src5 = (v16u8)__msa_ld_b((void*)t, 16);
- src6 = (v16u8)__msa_ld_b((void*)t, 32);
- src7 = (v16u8)__msa_ld_b((void*)t, 48);
- src0 = __msa_aver_u_b(src0, src4);
- src1 = __msa_aver_u_b(src1, src5);
- src2 = __msa_aver_u_b(src2, src6);
- src3 = __msa_aver_u_b(src3, src7);
- src4 = (v16u8)__msa_pckev_w((v4i32)src1, (v4i32)src0);
- src5 = (v16u8)__msa_pckev_w((v4i32)src3, (v4i32)src2);
- src6 = (v16u8)__msa_pckod_w((v4i32)src1, (v4i32)src0);
- src7 = (v16u8)__msa_pckod_w((v4i32)src3, (v4i32)src2);
- vec0 = __msa_aver_u_b(src4, src6);
- vec1 = __msa_aver_u_b(src5, src7);
- src0 = (v16u8)__msa_ld_b((void*)s, 64);
- src1 = (v16u8)__msa_ld_b((void*)s, 80);
- src2 = (v16u8)__msa_ld_b((void*)s, 96);
- src3 = (v16u8)__msa_ld_b((void*)s, 112);
- src4 = (v16u8)__msa_ld_b((void*)t, 64);
- src5 = (v16u8)__msa_ld_b((void*)t, 80);
- src6 = (v16u8)__msa_ld_b((void*)t, 96);
- src7 = (v16u8)__msa_ld_b((void*)t, 112);
- src0 = __msa_aver_u_b(src0, src4);
- src1 = __msa_aver_u_b(src1, src5);
- src2 = __msa_aver_u_b(src2, src6);
- src3 = __msa_aver_u_b(src3, src7);
- src4 = (v16u8)__msa_pckev_w((v4i32)src1, (v4i32)src0);
- src5 = (v16u8)__msa_pckev_w((v4i32)src3, (v4i32)src2);
- src6 = (v16u8)__msa_pckod_w((v4i32)src1, (v4i32)src0);
- src7 = (v16u8)__msa_pckod_w((v4i32)src3, (v4i32)src2);
- vec2 = __msa_aver_u_b(src4, src6);
- vec3 = __msa_aver_u_b(src5, src7);
- ARGBTOUV(vec0, vec1, vec2, vec3, const_0x6B14, const_0x7F, const_0x2B54,
- const_0x8080, shuffler1, shuffler0, shuffler2, shuffler3, dst0,
- dst1);
- ST_UB(dst0, dst_v);
- ST_UB(dst1, dst_u);
- s += 128;
- t += 128;
- dst_v += 16;
- dst_u += 16;
- }
-}
-
-void BGRAToUVRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- v16u8 dst0, dst1, vec0, vec1, vec2, vec3;
- v16i8 shuffler0 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
- v16i8 shuffler1 = {2, 3, 6, 7, 10, 11, 14, 15,
- 18, 19, 22, 23, 26, 27, 30, 31};
- v16i8 shuffler2 = {0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23, 24, 27, 28, 31};
- v16i8 shuffler3 = {2, 1, 6, 5, 10, 9, 14, 13, 18, 17, 22, 21, 26, 25, 30, 29};
- v16u8 const_0x125E = (v16u8)__msa_fill_h(0x125E);
- v16u8 const_0x7000 = (v16u8)__msa_fill_h(0x7000);
- v16u8 const_0x264A = (v16u8)__msa_fill_h(0x264A);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
-
- for (x = 0; x < width; x += 32) {
- READ_ARGB(s, t, vec0, vec1, vec2, vec3);
- ARGBTOUV(vec0, vec1, vec2, vec3, const_0x125E, const_0x7000, const_0x264A,
- const_0x8080, shuffler0, shuffler1, shuffler2, shuffler3, dst0,
- dst1);
- ST_UB(dst0, dst_v);
- ST_UB(dst1, dst_u);
- s += 128;
- t += 128;
- dst_v += 16;
- dst_u += 16;
- }
-}
-
-void ABGRToUVRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- v16u8 src0, src1, src2, src3;
- v16u8 dst0, dst1;
- v16i8 shuffler0 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
- v16i8 shuffler1 = {2, 3, 6, 7, 10, 11, 14, 15,
- 18, 19, 22, 23, 26, 27, 30, 31};
- v16i8 shuffler2 = {0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23, 24, 27, 28, 31};
- v16i8 shuffler3 = {1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30};
- v16u8 const_0x4A26 = (v16u8)__msa_fill_h(0x4A26);
- v16u8 const_0x0070 = (v16u8)__msa_fill_h(0x0070);
- v16u8 const_0x125E = (v16u8)__msa_fill_h(0x125E);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
-
- for (x = 0; x < width; x += 32) {
- READ_ARGB(s, t, src0, src1, src2, src3);
- ARGBTOUV(src0, src1, src2, src3, const_0x4A26, const_0x0070, const_0x125E,
- const_0x8080, shuffler1, shuffler0, shuffler2, shuffler3, dst0,
- dst1);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- s += 128;
- t += 128;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void RGBAToUVRow_MSA(const uint8_t* src_rgb0,
- int src_stride_rgb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- const uint8_t* s = src_rgb0;
- const uint8_t* t = src_rgb0 + src_stride_rgb;
- v16u8 dst0, dst1, vec0, vec1, vec2, vec3;
- v16i8 shuffler0 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
- v16i8 shuffler1 = {2, 3, 6, 7, 10, 11, 14, 15,
- 18, 19, 22, 23, 26, 27, 30, 31};
- v16i8 shuffler2 = {0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23, 24, 27, 28, 31};
- v16i8 shuffler3 = {2, 1, 6, 5, 10, 9, 14, 13, 18, 17, 22, 21, 26, 25, 30, 29};
- v16u8 const_0x125E = (v16u8)__msa_fill_h(0x264A);
- v16u8 const_0x7000 = (v16u8)__msa_fill_h(0x7000);
- v16u8 const_0x264A = (v16u8)__msa_fill_h(0x125E);
- v8u16 const_0x8080 = (v8u16)__msa_fill_h(0x8080);
-
- for (x = 0; x < width; x += 32) {
- READ_ARGB(s, t, vec0, vec1, vec2, vec3);
- ARGBTOUV(vec0, vec1, vec2, vec3, const_0x125E, const_0x7000, const_0x264A,
- const_0x8080, shuffler0, shuffler1, shuffler2, shuffler3, dst0,
- dst1);
- ST_UB(dst0, dst_u);
- ST_UB(dst1, dst_v);
- s += 128;
- t += 128;
- dst_u += 16;
- dst_v += 16;
- }
-}
-
-void I444ToARGBRow_MSA(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2, dst0, dst1;
- v8u16 vec0, vec1, vec2;
- v4i32 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8, reg9;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v8i16 zero = {0};
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
-
- for (x = 0; x < width; x += 8) {
- READI444(src_y, src_u, src_v, src0, src1, src2);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- reg0 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec0);
- reg1 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec0);
- reg0 *= vec_yg;
- reg1 *= vec_yg;
- reg0 = __msa_srai_w(reg0, 16);
- reg1 = __msa_srai_w(reg1, 16);
- reg4 = reg0 + vec_br;
- reg5 = reg1 + vec_br;
- reg2 = reg0 + vec_bg;
- reg3 = reg1 + vec_bg;
- reg0 += vec_bb;
- reg1 += vec_bb;
- vec0 = (v8u16)__msa_ilvr_b((v16i8)zero, (v16i8)src1);
- vec1 = (v8u16)__msa_ilvr_b((v16i8)zero, (v16i8)src2);
- reg6 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec0);
- reg7 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec0);
- reg8 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec1);
- reg9 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec1);
- reg0 -= reg6 * vec_ub;
- reg1 -= reg7 * vec_ub;
- reg2 -= reg6 * vec_ug;
- reg3 -= reg7 * vec_ug;
- reg4 -= reg8 * vec_vr;
- reg5 -= reg9 * vec_vr;
- reg2 -= reg8 * vec_vg;
- reg3 -= reg9 * vec_vg;
- reg0 = __msa_srai_w(reg0, 6);
- reg1 = __msa_srai_w(reg1, 6);
- reg2 = __msa_srai_w(reg2, 6);
- reg3 = __msa_srai_w(reg3, 6);
- reg4 = __msa_srai_w(reg4, 6);
- reg5 = __msa_srai_w(reg5, 6);
- CLIP_0TO255(reg0, reg1, reg2, reg3, reg4, reg5);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec2 = (v8u16)__msa_pckev_h((v8i16)reg5, (v8i16)reg4);
- vec0 = (v8u16)__msa_ilvev_b((v16i8)vec1, (v16i8)vec0);
- vec1 = (v8u16)__msa_ilvev_b((v16i8)alpha, (v16i8)vec2);
- dst0 = (v16u8)__msa_ilvr_h((v8i16)vec1, (v8i16)vec0);
- dst1 = (v16u8)__msa_ilvl_h((v8i16)vec1, (v8i16)vec0);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_y += 8;
- src_u += 8;
- src_v += 8;
- dst_argb += 32;
- }
-}
-
-void I400ToARGBRow_MSA(const uint8_t* src_y, uint8_t* dst_argb, int width) {
- int x;
- v16u8 src0, res0, res1, res2, res3, res4, dst0, dst1, dst2, dst3;
- v8i16 vec0, vec1;
- v4i32 reg0, reg1, reg2, reg3;
- v4i32 vec_yg = __msa_fill_w(0x4A35);
- v8i16 vec_ygb = __msa_fill_h(0xFB78);
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
- v8i16 max = __msa_ldi_h(0xFF);
- v8i16 zero = {0};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_y, 0);
- vec0 = (v8i16)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- vec1 = (v8i16)__msa_ilvl_b((v16i8)src0, (v16i8)src0);
- reg0 = (v4i32)__msa_ilvr_h(zero, vec0);
- reg1 = (v4i32)__msa_ilvl_h(zero, vec0);
- reg2 = (v4i32)__msa_ilvr_h(zero, vec1);
- reg3 = (v4i32)__msa_ilvl_h(zero, vec1);
- reg0 *= vec_yg;
- reg1 *= vec_yg;
- reg2 *= vec_yg;
- reg3 *= vec_yg;
- reg0 = __msa_srai_w(reg0, 16);
- reg1 = __msa_srai_w(reg1, 16);
- reg2 = __msa_srai_w(reg2, 16);
- reg3 = __msa_srai_w(reg3, 16);
- vec0 = (v8i16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8i16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- vec0 += vec_ygb;
- vec1 += vec_ygb;
- vec0 = __msa_srai_h(vec0, 6);
- vec1 = __msa_srai_h(vec1, 6);
- vec0 = __msa_maxi_s_h(vec0, 0);
- vec1 = __msa_maxi_s_h(vec1, 0);
- vec0 = __msa_min_s_h(max, vec0);
- vec1 = __msa_min_s_h(max, vec1);
- res0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- res1 = (v16u8)__msa_ilvr_b((v16i8)res0, (v16i8)res0);
- res2 = (v16u8)__msa_ilvl_b((v16i8)res0, (v16i8)res0);
- res3 = (v16u8)__msa_ilvr_b((v16i8)alpha, (v16i8)res0);
- res4 = (v16u8)__msa_ilvl_b((v16i8)alpha, (v16i8)res0);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)res3, (v16i8)res1);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)res3, (v16i8)res1);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)res4, (v16i8)res2);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)res4, (v16i8)res2);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_y += 16;
- dst_argb += 64;
- }
-}
-
-void J400ToARGBRow_MSA(const uint8_t* src_y, uint8_t* dst_argb, int width) {
- int x;
- v16u8 src0, vec0, vec1, vec2, vec3, dst0, dst1, dst2, dst3;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_y, 0);
- vec0 = (v16u8)__msa_ilvr_b((v16i8)src0, (v16i8)src0);
- vec1 = (v16u8)__msa_ilvl_b((v16i8)src0, (v16i8)src0);
- vec2 = (v16u8)__msa_ilvr_b((v16i8)alpha, (v16i8)src0);
- vec3 = (v16u8)__msa_ilvl_b((v16i8)alpha, (v16i8)src0);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)vec2, (v16i8)vec0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)vec2, (v16i8)vec0);
- dst2 = (v16u8)__msa_ilvr_b((v16i8)vec3, (v16i8)vec1);
- dst3 = (v16u8)__msa_ilvl_b((v16i8)vec3, (v16i8)vec1);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- src_y += 16;
- dst_argb += 64;
- }
-}
-
-void YUY2ToARGBRow_MSA(const uint8_t* src_yuy2,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_yuy2, 0);
- src1 = (v16u8)__msa_pckev_b((v16i8)src0, (v16i8)src0);
- src2 = (v16u8)__msa_pckod_b((v16i8)src0, (v16i8)src0);
- YUVTORGB(src1, src2, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- STOREARGB(vec0, vec1, vec2, alpha, dst_argb);
- src_yuy2 += 16;
- dst_argb += 32;
- }
-}
-
-void UYVYToARGBRow_MSA(const uint8_t* src_uyvy,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- int x;
- v16u8 src0, src1, src2;
- v8i16 vec0, vec1, vec2;
- v4i32 vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg, vec_br, vec_yg;
- v4i32 vec_ubvr, vec_ugvg;
- v16u8 alpha = (v16u8)__msa_ldi_b(ALPHA_VAL);
-
- YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_bb, vec_bg,
- vec_br, vec_yg);
- vec_ubvr = __msa_ilvr_w(vec_vr, vec_ub);
- vec_ugvg = (v4i32)__msa_ilvev_h((v8i16)vec_vg, (v8i16)vec_ug);
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_uyvy, 0);
- src1 = (v16u8)__msa_pckod_b((v16i8)src0, (v16i8)src0);
- src2 = (v16u8)__msa_pckev_b((v16i8)src0, (v16i8)src0);
- YUVTORGB(src1, src2, vec_ubvr, vec_ugvg, vec_bb, vec_bg, vec_br, vec_yg,
- vec0, vec1, vec2);
- STOREARGB(vec0, vec1, vec2, alpha, dst_argb);
- src_uyvy += 16;
- dst_argb += 32;
- }
-}
-
-void InterpolateRow_MSA(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int width,
- int32_t source_y_fraction) {
- int32_t y1_fraction = source_y_fraction;
- int32_t y0_fraction = 256 - y1_fraction;
- uint16_t y_fractions;
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
- v8u16 vec0, vec1, vec2, vec3, y_frac;
-
- if (0 == y1_fraction) {
- memcpy(dst_ptr, src_ptr, width);
- return;
- }
-
- if (128 == y1_fraction) {
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((void*)s, 0);
- src1 = (v16u8)__msa_ld_b((void*)s, 16);
- src2 = (v16u8)__msa_ld_b((void*)t, 0);
- src3 = (v16u8)__msa_ld_b((void*)t, 16);
- dst0 = __msa_aver_u_b(src0, src2);
- dst1 = __msa_aver_u_b(src1, src3);
- ST_UB2(dst0, dst1, dst_ptr, 16);
- s += 32;
- t += 32;
- dst_ptr += 32;
- }
- return;
- }
-
- y_fractions = (uint16_t)(y0_fraction + (y1_fraction << 8));
- y_frac = (v8u16)__msa_fill_h(y_fractions);
-
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((void*)s, 0);
- src1 = (v16u8)__msa_ld_b((void*)s, 16);
- src2 = (v16u8)__msa_ld_b((void*)t, 0);
- src3 = (v16u8)__msa_ld_b((void*)t, 16);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- vec0 = (v8u16)__msa_dotp_u_h((v16u8)vec0, (v16u8)y_frac);
- vec1 = (v8u16)__msa_dotp_u_h((v16u8)vec1, (v16u8)y_frac);
- vec2 = (v8u16)__msa_dotp_u_h((v16u8)vec2, (v16u8)y_frac);
- vec3 = (v8u16)__msa_dotp_u_h((v16u8)vec3, (v16u8)y_frac);
- vec0 = (v8u16)__msa_srari_h((v8i16)vec0, 8);
- vec1 = (v8u16)__msa_srari_h((v8i16)vec1, 8);
- vec2 = (v8u16)__msa_srari_h((v8i16)vec2, 8);
- vec3 = (v8u16)__msa_srari_h((v8i16)vec3, 8);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- ST_UB2(dst0, dst1, dst_ptr, 16);
- s += 32;
- t += 32;
- dst_ptr += 32;
- }
-}
-
-void ARGBSetRow_MSA(uint8_t* dst_argb, uint32_t v32, int width) {
- int x;
- v4i32 dst0 = __builtin_msa_fill_w(v32);
-
- for (x = 0; x < width; x += 4) {
- ST_UB(dst0, dst_argb);
- dst_argb += 16;
- }
-}
-
-void RAWToRGB24Row_MSA(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
- int x;
- v16u8 src0, src1, src2, src3, src4, dst0, dst1, dst2;
- v16i8 shuffler0 = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, 17};
- v16i8 shuffler1 = {8, 7, 12, 11, 10, 15, 14, 13,
- 18, 17, 16, 21, 20, 19, 24, 23};
- v16i8 shuffler2 = {14, 19, 18, 17, 22, 21, 20, 25,
- 24, 23, 28, 27, 26, 31, 30, 29};
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_raw, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_raw, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_raw, 32);
- src3 = (v16u8)__msa_sldi_b((v16i8)src1, (v16i8)src0, 8);
- src4 = (v16u8)__msa_sldi_b((v16i8)src2, (v16i8)src1, 8);
- dst0 = (v16u8)__msa_vshf_b(shuffler0, (v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(shuffler1, (v16i8)src4, (v16i8)src3);
- dst2 = (v16u8)__msa_vshf_b(shuffler2, (v16i8)src2, (v16i8)src1);
- ST_UB2(dst0, dst1, dst_rgb24, 16);
- ST_UB(dst2, (dst_rgb24 + 32));
- src_raw += 48;
- dst_rgb24 += 48;
- }
-}
-
-void MergeUVRow_MSA(const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uv,
- int width) {
- int x;
- v16u8 src0, src1, dst0, dst1;
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_u, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_v, 0);
- dst0 = (v16u8)__msa_ilvr_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_ilvl_b((v16i8)src1, (v16i8)src0);
- ST_UB2(dst0, dst1, dst_uv, 16);
- src_u += 16;
- src_v += 16;
- dst_uv += 32;
- }
-}
-
-void ARGBExtractAlphaRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_a,
- int width) {
- int i;
- v16u8 src0, src1, src2, src3, vec0, vec1, dst0;
-
- for (i = 0; i < width; i += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_argb, 48);
- vec0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- dst0 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_a);
- src_argb += 64;
- dst_a += 16;
- }
-}
-
-void ARGBBlendRow_MSA(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v8u16 vec8, vec9, vec10, vec11, vec12, vec13;
- v8u16 const_256 = (v8u16)__msa_ldi_h(256);
- v16u8 const_255 = (v16u8)__msa_ldi_b(255);
- v16u8 mask = {0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)src_argb0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_argb0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_argb1, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_argb1, 16);
- vec0 = (v8u16)__msa_ilvr_b(zero, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b(zero, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b(zero, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b(zero, (v16i8)src1);
- vec4 = (v8u16)__msa_ilvr_b(zero, (v16i8)src2);
- vec5 = (v8u16)__msa_ilvl_b(zero, (v16i8)src2);
- vec6 = (v8u16)__msa_ilvr_b(zero, (v16i8)src3);
- vec7 = (v8u16)__msa_ilvl_b(zero, (v16i8)src3);
- vec8 = (v8u16)__msa_fill_h(vec0[3]);
- vec9 = (v8u16)__msa_fill_h(vec0[7]);
- vec10 = (v8u16)__msa_fill_h(vec1[3]);
- vec11 = (v8u16)__msa_fill_h(vec1[7]);
- vec8 = (v8u16)__msa_pckev_d((v2i64)vec9, (v2i64)vec8);
- vec9 = (v8u16)__msa_pckev_d((v2i64)vec11, (v2i64)vec10);
- vec10 = (v8u16)__msa_fill_h(vec2[3]);
- vec11 = (v8u16)__msa_fill_h(vec2[7]);
- vec12 = (v8u16)__msa_fill_h(vec3[3]);
- vec13 = (v8u16)__msa_fill_h(vec3[7]);
- vec10 = (v8u16)__msa_pckev_d((v2i64)vec11, (v2i64)vec10);
- vec11 = (v8u16)__msa_pckev_d((v2i64)vec13, (v2i64)vec12);
- vec8 = const_256 - vec8;
- vec9 = const_256 - vec9;
- vec10 = const_256 - vec10;
- vec11 = const_256 - vec11;
- vec8 *= vec4;
- vec9 *= vec5;
- vec10 *= vec6;
- vec11 *= vec7;
- vec8 = (v8u16)__msa_srai_h((v8i16)vec8, 8);
- vec9 = (v8u16)__msa_srai_h((v8i16)vec9, 8);
- vec10 = (v8u16)__msa_srai_h((v8i16)vec10, 8);
- vec11 = (v8u16)__msa_srai_h((v8i16)vec11, 8);
- vec0 += vec8;
- vec1 += vec9;
- vec2 += vec10;
- vec3 += vec11;
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- dst0 = __msa_bmnz_v(dst0, const_255, mask);
- dst1 = __msa_bmnz_v(dst1, const_255, mask);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb0 += 32;
- src_argb1 += 32;
- dst_argb += 32;
- }
-}
-
-void ARGBQuantizeRow_MSA(uint8_t* dst_argb,
- int scale,
- int interval_size,
- int interval_offset,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
- v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
- v4i32 tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
- v4i32 vec_scale = __msa_fill_w(scale);
- v16u8 vec_int_sz = (v16u8)__msa_fill_b(interval_size);
- v16u8 vec_int_ofst = (v16u8)__msa_fill_b(interval_offset);
- v16i8 mask = {0, 1, 2, 19, 4, 5, 6, 23, 8, 9, 10, 27, 12, 13, 14, 31};
- v16i8 zero = {0};
-
- for (x = 0; x < width; x += 8) {
- src0 = (v16u8)__msa_ld_b((void*)dst_argb, 0);
- src1 = (v16u8)__msa_ld_b((void*)dst_argb, 16);
- src2 = (v16u8)__msa_ld_b((void*)dst_argb, 32);
- src3 = (v16u8)__msa_ld_b((void*)dst_argb, 48);
- vec0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0);
- vec1 = (v8i16)__msa_ilvl_b(zero, (v16i8)src0);
- vec2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1);
- vec3 = (v8i16)__msa_ilvl_b(zero, (v16i8)src1);
- vec4 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2);
- vec5 = (v8i16)__msa_ilvl_b(zero, (v16i8)src2);
- vec6 = (v8i16)__msa_ilvr_b(zero, (v16i8)src3);
- vec7 = (v8i16)__msa_ilvl_b(zero, (v16i8)src3);
- tmp0 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec0);
- tmp1 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec0);
- tmp2 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec1);
- tmp3 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec1);
- tmp4 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec2);
- tmp5 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec2);
- tmp6 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec3);
- tmp7 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec3);
- tmp8 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec4);
- tmp9 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec4);
- tmp10 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec5);
- tmp11 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec5);
- tmp12 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec6);
- tmp13 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec6);
- tmp14 = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)vec7);
- tmp15 = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)vec7);
- tmp0 *= vec_scale;
- tmp1 *= vec_scale;
- tmp2 *= vec_scale;
- tmp3 *= vec_scale;
- tmp4 *= vec_scale;
- tmp5 *= vec_scale;
- tmp6 *= vec_scale;
- tmp7 *= vec_scale;
- tmp8 *= vec_scale;
- tmp9 *= vec_scale;
- tmp10 *= vec_scale;
- tmp11 *= vec_scale;
- tmp12 *= vec_scale;
- tmp13 *= vec_scale;
- tmp14 *= vec_scale;
- tmp15 *= vec_scale;
- tmp0 >>= 16;
- tmp1 >>= 16;
- tmp2 >>= 16;
- tmp3 >>= 16;
- tmp4 >>= 16;
- tmp5 >>= 16;
- tmp6 >>= 16;
- tmp7 >>= 16;
- tmp8 >>= 16;
- tmp9 >>= 16;
- tmp10 >>= 16;
- tmp11 >>= 16;
- tmp12 >>= 16;
- tmp13 >>= 16;
- tmp14 >>= 16;
- tmp15 >>= 16;
- vec0 = (v8i16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec1 = (v8i16)__msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- vec2 = (v8i16)__msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
- vec3 = (v8i16)__msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
- vec4 = (v8i16)__msa_pckev_h((v8i16)tmp9, (v8i16)tmp8);
- vec5 = (v8i16)__msa_pckev_h((v8i16)tmp11, (v8i16)tmp10);
- vec6 = (v8i16)__msa_pckev_h((v8i16)tmp13, (v8i16)tmp12);
- vec7 = (v8i16)__msa_pckev_h((v8i16)tmp15, (v8i16)tmp14);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- dst2 = (v16u8)__msa_pckev_b((v16i8)vec5, (v16i8)vec4);
- dst3 = (v16u8)__msa_pckev_b((v16i8)vec7, (v16i8)vec6);
- dst0 *= vec_int_sz;
- dst1 *= vec_int_sz;
- dst2 *= vec_int_sz;
- dst3 *= vec_int_sz;
- dst0 += vec_int_ofst;
- dst1 += vec_int_ofst;
- dst2 += vec_int_ofst;
- dst3 += vec_int_ofst;
- dst0 = (v16u8)__msa_vshf_b(mask, (v16i8)src0, (v16i8)dst0);
- dst1 = (v16u8)__msa_vshf_b(mask, (v16i8)src1, (v16i8)dst1);
- dst2 = (v16u8)__msa_vshf_b(mask, (v16i8)src2, (v16i8)dst2);
- dst3 = (v16u8)__msa_vshf_b(mask, (v16i8)src3, (v16i8)dst3);
- ST_UB4(dst0, dst1, dst2, dst3, dst_argb, 16);
- dst_argb += 64;
- }
-}
-
-void ARGBColorMatrixRow_MSA(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width) {
- int32_t x;
- v16i8 src0;
- v16u8 src1, src2, dst0, dst1;
- v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
- v8i16 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
- v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
- v4i32 tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
- v16i8 zero = {0};
- v8i16 max = __msa_ldi_h(255);
-
- src0 = __msa_ld_b((void*)matrix_argb, 0);
- vec0 = (v8i16)__msa_ilvr_b(zero, src0);
- vec1 = (v8i16)__msa_ilvl_b(zero, src0);
-
- for (x = 0; x < width; x += 8) {
- src1 = (v16u8)__msa_ld_b((void*)src_argb, 0);
- src2 = (v16u8)__msa_ld_b((void*)src_argb, 16);
- vec2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1);
- vec3 = (v8i16)__msa_ilvl_b(zero, (v16i8)src1);
- vec4 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2);
- vec5 = (v8i16)__msa_ilvl_b(zero, (v16i8)src2);
- vec6 = (v8i16)__msa_pckod_d((v2i64)vec2, (v2i64)vec2);
- vec7 = (v8i16)__msa_pckod_d((v2i64)vec3, (v2i64)vec3);
- vec8 = (v8i16)__msa_pckod_d((v2i64)vec4, (v2i64)vec4);
- vec9 = (v8i16)__msa_pckod_d((v2i64)vec5, (v2i64)vec5);
- vec2 = (v8i16)__msa_pckev_d((v2i64)vec2, (v2i64)vec2);
- vec3 = (v8i16)__msa_pckev_d((v2i64)vec3, (v2i64)vec3);
- vec4 = (v8i16)__msa_pckev_d((v2i64)vec4, (v2i64)vec4);
- vec5 = (v8i16)__msa_pckev_d((v2i64)vec5, (v2i64)vec5);
- vec10 = vec2 * vec0;
- vec11 = vec2 * vec1;
- vec12 = vec6 * vec0;
- vec13 = vec6 * vec1;
- tmp0 = __msa_hadd_s_w(vec10, vec10);
- tmp1 = __msa_hadd_s_w(vec11, vec11);
- tmp2 = __msa_hadd_s_w(vec12, vec12);
- tmp3 = __msa_hadd_s_w(vec13, vec13);
- vec14 = vec3 * vec0;
- vec15 = vec3 * vec1;
- vec16 = vec7 * vec0;
- vec17 = vec7 * vec1;
- tmp4 = __msa_hadd_s_w(vec14, vec14);
- tmp5 = __msa_hadd_s_w(vec15, vec15);
- tmp6 = __msa_hadd_s_w(vec16, vec16);
- tmp7 = __msa_hadd_s_w(vec17, vec17);
- vec10 = __msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec11 = __msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- vec12 = __msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
- vec13 = __msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
- tmp0 = __msa_hadd_s_w(vec10, vec10);
- tmp1 = __msa_hadd_s_w(vec11, vec11);
- tmp2 = __msa_hadd_s_w(vec12, vec12);
- tmp3 = __msa_hadd_s_w(vec13, vec13);
- tmp0 = __msa_srai_w(tmp0, 6);
- tmp1 = __msa_srai_w(tmp1, 6);
- tmp2 = __msa_srai_w(tmp2, 6);
- tmp3 = __msa_srai_w(tmp3, 6);
- vec2 = vec4 * vec0;
- vec6 = vec4 * vec1;
- vec3 = vec8 * vec0;
- vec7 = vec8 * vec1;
- tmp8 = __msa_hadd_s_w(vec2, vec2);
- tmp9 = __msa_hadd_s_w(vec6, vec6);
- tmp10 = __msa_hadd_s_w(vec3, vec3);
- tmp11 = __msa_hadd_s_w(vec7, vec7);
- vec4 = vec5 * vec0;
- vec8 = vec5 * vec1;
- vec5 = vec9 * vec0;
- vec9 = vec9 * vec1;
- tmp12 = __msa_hadd_s_w(vec4, vec4);
- tmp13 = __msa_hadd_s_w(vec8, vec8);
- tmp14 = __msa_hadd_s_w(vec5, vec5);
- tmp15 = __msa_hadd_s_w(vec9, vec9);
- vec14 = __msa_pckev_h((v8i16)tmp9, (v8i16)tmp8);
- vec15 = __msa_pckev_h((v8i16)tmp11, (v8i16)tmp10);
- vec16 = __msa_pckev_h((v8i16)tmp13, (v8i16)tmp12);
- vec17 = __msa_pckev_h((v8i16)tmp15, (v8i16)tmp14);
- tmp4 = __msa_hadd_s_w(vec14, vec14);
- tmp5 = __msa_hadd_s_w(vec15, vec15);
- tmp6 = __msa_hadd_s_w(vec16, vec16);
- tmp7 = __msa_hadd_s_w(vec17, vec17);
- tmp4 = __msa_srai_w(tmp4, 6);
- tmp5 = __msa_srai_w(tmp5, 6);
- tmp6 = __msa_srai_w(tmp6, 6);
- tmp7 = __msa_srai_w(tmp7, 6);
- vec10 = __msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec11 = __msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- vec12 = __msa_pckev_h((v8i16)tmp5, (v8i16)tmp4);
- vec13 = __msa_pckev_h((v8i16)tmp7, (v8i16)tmp6);
- vec10 = __msa_maxi_s_h(vec10, 0);
- vec11 = __msa_maxi_s_h(vec11, 0);
- vec12 = __msa_maxi_s_h(vec12, 0);
- vec13 = __msa_maxi_s_h(vec13, 0);
- vec10 = __msa_min_s_h(vec10, max);
- vec11 = __msa_min_s_h(vec11, max);
- vec12 = __msa_min_s_h(vec12, max);
- vec13 = __msa_min_s_h(vec13, max);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec11, (v16i8)vec10);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec13, (v16i8)vec12);
- ST_UB2(dst0, dst1, dst_argb, 16);
- src_argb += 32;
- dst_argb += 32;
- }
-}
-
-void SplitUVRow_MSA(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
-
- for (x = 0; x < width; x += 32) {
- src0 = (v16u8)__msa_ld_b((void*)src_uv, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_uv, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_uv, 32);
- src3 = (v16u8)__msa_ld_b((void*)src_uv, 48);
- dst0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- dst2 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- dst3 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_u, 16);
- ST_UB2(dst2, dst3, dst_v, 16);
- src_uv += 64;
- dst_u += 32;
- dst_v += 32;
- }
-}
-
-void SetRow_MSA(uint8_t* dst, uint8_t v8, int width) {
- int x;
- v16u8 dst0 = (v16u8)__msa_fill_b(v8);
-
- for (x = 0; x < width; x += 16) {
- ST_UB(dst0, dst);
- dst += 16;
- }
-}
-
-void MirrorUVRow_MSA(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- int x;
- v16u8 src0, src1, src2, src3;
- v16u8 dst0, dst1, dst2, dst3;
- v16i8 mask0 = {30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0};
- v16i8 mask1 = {31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1};
-
- src_uv += (2 * width);
-
- for (x = 0; x < width; x += 32) {
- src_uv -= 64;
- src2 = (v16u8)__msa_ld_b((void*)src_uv, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_uv, 16);
- src0 = (v16u8)__msa_ld_b((void*)src_uv, 32);
- src1 = (v16u8)__msa_ld_b((void*)src_uv, 48);
- dst0 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
- dst2 = (v16u8)__msa_vshf_b(mask0, (v16i8)src1, (v16i8)src0);
- dst3 = (v16u8)__msa_vshf_b(mask0, (v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst_v, 16);
- ST_UB2(dst2, dst3, dst_u, 16);
- dst_u += 32;
- dst_v += 32;
- }
-}
-
-void SobelXRow_MSA(const uint8_t* src_y0,
- const uint8_t* src_y1,
- const uint8_t* src_y2,
- uint8_t* dst_sobelx,
- int32_t width) {
- int x;
- v16u8 src0, src1, src2, src3, src4, src5, dst0;
- v8i16 vec0, vec1, vec2, vec3, vec4, vec5;
- v16i8 mask0 = {0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7, 9};
- v16i8 tmp = __msa_ldi_b(8);
- v16i8 mask1 = mask0 + tmp;
- v8i16 zero = {0};
- v8i16 max = __msa_ldi_h(255);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_y0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_y0, 16);
- src2 = (v16u8)__msa_ld_b((void*)src_y1, 0);
- src3 = (v16u8)__msa_ld_b((void*)src_y1, 16);
- src4 = (v16u8)__msa_ld_b((void*)src_y2, 0);
- src5 = (v16u8)__msa_ld_b((void*)src_y2, 16);
- vec0 = (v8i16)__msa_vshf_b(mask0, (v16i8)src1, (v16i8)src0);
- vec1 = (v8i16)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- vec2 = (v8i16)__msa_vshf_b(mask0, (v16i8)src3, (v16i8)src2);
- vec3 = (v8i16)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
- vec4 = (v8i16)__msa_vshf_b(mask0, (v16i8)src5, (v16i8)src4);
- vec5 = (v8i16)__msa_vshf_b(mask1, (v16i8)src5, (v16i8)src4);
- vec0 = (v8i16)__msa_hsub_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = (v8i16)__msa_hsub_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = (v8i16)__msa_hsub_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = (v8i16)__msa_hsub_u_h((v16u8)vec3, (v16u8)vec3);
- vec4 = (v8i16)__msa_hsub_u_h((v16u8)vec4, (v16u8)vec4);
- vec5 = (v8i16)__msa_hsub_u_h((v16u8)vec5, (v16u8)vec5);
- vec0 += vec2;
- vec1 += vec3;
- vec4 += vec2;
- vec5 += vec3;
- vec0 += vec4;
- vec1 += vec5;
- vec0 = __msa_add_a_h(zero, vec0);
- vec1 = __msa_add_a_h(zero, vec1);
- vec0 = __msa_maxi_s_h(vec0, 0);
- vec1 = __msa_maxi_s_h(vec1, 0);
- vec0 = __msa_min_s_h(max, vec0);
- vec1 = __msa_min_s_h(max, vec1);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_sobelx);
- src_y0 += 16;
- src_y1 += 16;
- src_y2 += 16;
- dst_sobelx += 16;
- }
-}
-
-void SobelYRow_MSA(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
- int32_t width) {
- int x;
- v16u8 src0, src1, dst0;
- v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6;
- v8i16 zero = {0};
- v8i16 max = __msa_ldi_h(255);
-
- for (x = 0; x < width; x += 16) {
- src0 = (v16u8)__msa_ld_b((void*)src_y0, 0);
- src1 = (v16u8)__msa_ld_b((void*)src_y1, 0);
- vec0 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)src0);
- vec1 = (v8i16)__msa_ilvl_b((v16i8)zero, (v16i8)src0);
- vec2 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)src1);
- vec3 = (v8i16)__msa_ilvl_b((v16i8)zero, (v16i8)src1);
- vec0 -= vec2;
- vec1 -= vec3;
- vec6[0] = src_y0[16] - src_y1[16];
- vec6[1] = src_y0[17] - src_y1[17];
- vec2 = (v8i16)__msa_sldi_b((v16i8)vec1, (v16i8)vec0, 2);
- vec3 = (v8i16)__msa_sldi_b((v16i8)vec6, (v16i8)vec1, 2);
- vec4 = (v8i16)__msa_sldi_b((v16i8)vec1, (v16i8)vec0, 4);
- vec5 = (v8i16)__msa_sldi_b((v16i8)vec6, (v16i8)vec1, 4);
- vec0 += vec2;
- vec1 += vec3;
- vec4 += vec2;
- vec5 += vec3;
- vec0 += vec4;
- vec1 += vec5;
- vec0 = __msa_add_a_h(zero, vec0);
- vec1 = __msa_add_a_h(zero, vec1);
- vec0 = __msa_maxi_s_h(vec0, 0);
- vec1 = __msa_maxi_s_h(vec1, 0);
- vec0 = __msa_min_s_h(max, vec0);
- vec1 = __msa_min_s_h(max, vec1);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst_sobely);
- src_y0 += 16;
- src_y1 += 16;
- dst_sobely += 16;
- }
-}
-
-void HalfFloatRow_MSA(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- int i;
- v8u16 src0, src1, src2, src3, dst0, dst1, dst2, dst3;
- v4u32 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v4f32 fvec0, fvec1, fvec2, fvec3, fvec4, fvec5, fvec6, fvec7;
- v4f32 mult_vec;
- v8i16 zero = {0};
- mult_vec[0] = 1.9259299444e-34f * scale;
- mult_vec = (v4f32)__msa_splati_w((v4i32)mult_vec, 0);
-
- for (i = 0; i < width; i += 32) {
- src0 = (v8u16)__msa_ld_h((void*)src, 0);
- src1 = (v8u16)__msa_ld_h((void*)src, 16);
- src2 = (v8u16)__msa_ld_h((void*)src, 32);
- src3 = (v8u16)__msa_ld_h((void*)src, 48);
- vec0 = (v4u32)__msa_ilvr_h(zero, (v8i16)src0);
- vec1 = (v4u32)__msa_ilvl_h(zero, (v8i16)src0);
- vec2 = (v4u32)__msa_ilvr_h(zero, (v8i16)src1);
- vec3 = (v4u32)__msa_ilvl_h(zero, (v8i16)src1);
- vec4 = (v4u32)__msa_ilvr_h(zero, (v8i16)src2);
- vec5 = (v4u32)__msa_ilvl_h(zero, (v8i16)src2);
- vec6 = (v4u32)__msa_ilvr_h(zero, (v8i16)src3);
- vec7 = (v4u32)__msa_ilvl_h(zero, (v8i16)src3);
- fvec0 = __msa_ffint_u_w(vec0);
- fvec1 = __msa_ffint_u_w(vec1);
- fvec2 = __msa_ffint_u_w(vec2);
- fvec3 = __msa_ffint_u_w(vec3);
- fvec4 = __msa_ffint_u_w(vec4);
- fvec5 = __msa_ffint_u_w(vec5);
- fvec6 = __msa_ffint_u_w(vec6);
- fvec7 = __msa_ffint_u_w(vec7);
- fvec0 *= mult_vec;
- fvec1 *= mult_vec;
- fvec2 *= mult_vec;
- fvec3 *= mult_vec;
- fvec4 *= mult_vec;
- fvec5 *= mult_vec;
- fvec6 *= mult_vec;
- fvec7 *= mult_vec;
- vec0 = ((v4u32)fvec0) >> 13;
- vec1 = ((v4u32)fvec1) >> 13;
- vec2 = ((v4u32)fvec2) >> 13;
- vec3 = ((v4u32)fvec3) >> 13;
- vec4 = ((v4u32)fvec4) >> 13;
- vec5 = ((v4u32)fvec5) >> 13;
- vec6 = ((v4u32)fvec6) >> 13;
- vec7 = ((v4u32)fvec7) >> 13;
- dst0 = (v8u16)__msa_pckev_h((v8i16)vec1, (v8i16)vec0);
- dst1 = (v8u16)__msa_pckev_h((v8i16)vec3, (v8i16)vec2);
- dst2 = (v8u16)__msa_pckev_h((v8i16)vec5, (v8i16)vec4);
- dst3 = (v8u16)__msa_pckev_h((v8i16)vec7, (v8i16)vec6);
- ST_UH2(dst0, dst1, dst, 8);
- ST_UH2(dst2, dst3, dst + 16, 8);
- src += 32;
- dst += 32;
- }
-}
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
-
-#endif // !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
diff --git a/libs/libyuv/source/row_neon.cc b/libs/libyuv/source/row_neon.cc
index a9dc4e46b7..ae4782d6b2 100644
--- a/libs/libyuv/source/row_neon.cc
+++ b/libs/libyuv/source/row_neon.cc
@@ -8,103 +8,141 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/row.h"
+#include <string.h>
-#include <stdio.h>
+#include "libyuv/convert_from_argb.h" // For ArgbConstants
+#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC Neon
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
+// d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are
+// reserved.
+
+// q0: Y uint16x8_t
+// d2: U uint8x8_t
+// d3: V uint8x8_t
+
// Read 8 Y, 4 U and 4 V from 422
#define READYUV422 \
- "vld1.8 {d0}, [%0]! \n" \
- "vld1.32 {d2[0]}, [%1]! \n" \
- "vld1.32 {d2[1]}, [%2]! \n"
+ "vld1.8 {d0}, [%[src_y]]! \n" \
+ "vld1.32 {d2[0]}, [%[src_u]]! \n" \
+ "vld1.32 {d2[1]}, [%[src_v]]! \n" \
+ "vmov.u8 d1, d0 \n" \
+ "vmovl.u8 q1, d2 \n" \
+ "vzip.u8 d0, d1 \n" \
+ "vsli.u16 q1, q1, #8 \n"
// Read 8 Y, 8 U and 8 V from 444
#define READYUV444 \
- "vld1.8 {d0}, [%0]! \n" \
- "vld1.8 {d2}, [%1]! \n" \
- "vld1.8 {d3}, [%2]! \n" \
- "vpaddl.u8 q1, q1 \n" \
- "vrshrn.u16 d2, q1, #1 \n"
+ "vld1.8 {d0}, [%[src_y]]! \n" \
+ "vld1.8 {d2}, [%[src_u]]! \n" \
+ "vmovl.u8 q0, d0 \n" \
+ "vld1.8 {d3}, [%[src_v]]! \n" \
+ "vsli.u16 q0, q0, #8 \n"
// Read 8 Y, and set 4 U and 4 V to 128
#define READYUV400 \
- "vld1.8 {d0}, [%0]! \n" \
- "vmov.u8 d2, #128 \n"
+ "vld1.8 {d0}, [%[src_y]]! \n" \
+ "vmov.u8 q1, #128 \n" \
+ "vmovl.u8 q0, d0 \n" \
+ "vsli.u16 q0, q0, #8 \n"
// Read 8 Y and 4 UV from NV12
-#define READNV12 \
- "vld1.8 {d0}, [%0]! \n" \
- "vld1.8 {d2}, [%1]! \n" \
- "vmov.u8 d3, d2 \n" /* split odd/even uv apart */ \
- "vuzp.u8 d2, d3 \n" \
- "vtrn.u32 d2, d3 \n"
+#define READNV12 \
+ "vld1.8 {d0}, [%[src_y]]! \n" \
+ "vld1.8 {d2}, [%[src_uv]]! \n" \
+ "vmov.u8 d1, d0 \n" \
+ "vmov.u8 d3, d2 \n" \
+ "vzip.u8 d0, d1 \n" \
+ "vsli.u16 d2, d2, #8 \n" /* Duplicate low byte (U) */ \
+ "vsri.u16 d3, d3, #8 \n" /* Duplicate high byte (V) */
// Read 8 Y and 4 VU from NV21
#define READNV21 \
- "vld1.8 {d0}, [%0]! \n" \
- "vld1.8 {d2}, [%1]! \n" \
- "vmov.u8 d3, d2 \n" /* split odd/even uv apart */ \
- "vuzp.u8 d3, d2 \n" \
- "vtrn.u32 d2, d3 \n"
+ "vld1.8 {d0}, [%[src_y]]! \n" \
+ "vld1.8 {d2}, [%[src_vu]]! \n" \
+ "vmov.u8 d1, d0 \n" \
+ "vmov.u8 d3, d2 \n" \
+ "vzip.u8 d0, d1 \n" \
+ "vsri.u16 d2, d2, #8 \n" /* Duplicate high byte (U) */ \
+ "vsli.u16 d3, d3, #8 \n" /* Duplicate low byte (V) */
// Read 8 YUY2
#define READYUY2 \
- "vld2.8 {d0, d2}, [%0]! \n" \
+ "vld2.8 {d0, d2}, [%[src_yuy2]]! \n" \
+ "vmovl.u8 q0, d0 \n" \
"vmov.u8 d3, d2 \n" \
- "vuzp.u8 d2, d3 \n" \
- "vtrn.u32 d2, d3 \n"
+ "vsli.u16 q0, q0, #8 \n" \
+ "vsli.u16 d2, d2, #8 \n" \
+ "vsri.u16 d3, d3, #8 \n"
// Read 8 UYVY
#define READUYVY \
- "vld2.8 {d2, d3}, [%0]! \n" \
- "vmov.u8 d0, d3 \n" \
+ "vld2.8 {d2, d3}, [%[src_uyvy]]! \n" \
+ "vmovl.u8 q0, d3 \n" \
"vmov.u8 d3, d2 \n" \
- "vuzp.u8 d2, d3 \n" \
- "vtrn.u32 d2, d3 \n"
-
-#define YUVTORGB_SETUP \
- "vld1.8 {d24}, [%[kUVToRB]] \n" \
- "vld1.8 {d25}, [%[kUVToG]] \n" \
- "vld1.16 {d26[], d27[]}, [%[kUVBiasBGR]]! \n" \
- "vld1.16 {d8[], d9[]}, [%[kUVBiasBGR]]! \n" \
- "vld1.16 {d28[], d29[]}, [%[kUVBiasBGR]] \n" \
- "vld1.32 {d30[], d31[]}, [%[kYToRgb]] \n"
-
-#define YUVTORGB \
- "vmull.u8 q8, d2, d24 \n" /* u/v B/R component */ \
- "vmull.u8 q9, d2, d25 \n" /* u/v G component */ \
- "vmovl.u8 q0, d0 \n" /* Y */ \
- "vmovl.s16 q10, d1 \n" \
- "vmovl.s16 q0, d0 \n" \
- "vmul.s32 q10, q10, q15 \n" \
- "vmul.s32 q0, q0, q15 \n" \
- "vqshrun.s32 d0, q0, #16 \n" \
- "vqshrun.s32 d1, q10, #16 \n" /* Y */ \
- "vadd.s16 d18, d19 \n" \
- "vshll.u16 q1, d16, #16 \n" /* Replicate u * UB */ \
- "vshll.u16 q10, d17, #16 \n" /* Replicate v * VR */ \
- "vshll.u16 q3, d18, #16 \n" /* Replicate (v*VG + u*UG)*/ \
- "vaddw.u16 q1, q1, d16 \n" \
- "vaddw.u16 q10, q10, d17 \n" \
- "vaddw.u16 q3, q3, d18 \n" \
- "vqadd.s16 q8, q0, q13 \n" /* B */ \
- "vqadd.s16 q9, q0, q14 \n" /* R */ \
- "vqadd.s16 q0, q0, q4 \n" /* G */ \
- "vqadd.s16 q8, q8, q1 \n" /* B */ \
- "vqadd.s16 q9, q9, q10 \n" /* R */ \
- "vqsub.s16 q0, q0, q3 \n" /* G */ \
- "vqshrun.s16 d20, q8, #6 \n" /* B */ \
- "vqshrun.s16 d22, q9, #6 \n" /* R */ \
- "vqshrun.s16 d21, q0, #6 \n" /* G */
+ "vsli.u16 q0, q0, #8 \n" \
+ "vsli.u16 d2, d2, #8 \n" \
+ "vsri.u16 d3, d3, #8 \n"
+
+// TODO: Use single register for kUVCoeff and multiply by lane
+#define YUVTORGB_SETUP \
+ "vld1.16 {d31}, [%[kRGBCoeffBias]] \n" \
+ "vld4.8 {d26[], d27[], d28[], d29[]}, [%[kUVCoeff]] \n" \
+ "vdup.u16 q10, d31[1] \n" \
+ "vdup.u16 q11, d31[2] \n" \
+ "vdup.u16 q12, d31[3] \n" \
+ "vdup.u16 d31, d31[0] \n"
+
+// q0: B uint16x8_t
+// q1: G uint16x8_t
+// q2: R uint16x8_t
+
+// Convert from YUV to 2.14 fixed point RGB
+#define YUVTORGB \
+ "vmull.u16 q2, d1, d31 \n" \
+ "vmull.u8 q8, d3, d29 \n" /* DGV */ \
+ "vmull.u16 q0, d0, d31 \n" \
+ "vmlal.u8 q8, d2, d28 \n" /* DG */ \
+ "vqshrn.u32 d0, q0, #16 \n" \
+ "vqshrn.u32 d1, q2, #16 \n" /* Y */ \
+ "vmull.u8 q9, d2, d26 \n" /* DB */ \
+ "vmull.u8 q2, d3, d27 \n" /* DR */ \
+ "vadd.u16 q4, q0, q11 \n" /* G */ \
+ "vadd.u16 q2, q0, q2 \n" /* R */ \
+ "vadd.u16 q0, q0, q9 \n" /* B */ \
+ "vqsub.u16 q1, q4, q8 \n" /* G */ \
+ "vqsub.u16 q0, q0, q10 \n" /* B */ \
+ "vqsub.u16 q2, q2, q12 \n" /* R */
+
+// Convert from 2.14 fixed point RGB To 8 bit RGB
+#define RGBTORGB8 \
+ "vqshrn.u16 d4, q2, #6 \n" /* R */ \
+ "vqshrn.u16 d2, q1, #6 \n" /* G */ \
+ "vqshrn.u16 d0, q0, #6 \n" /* B */
+
+#define YUVTORGB_REGS \
+ "q0", "q1", "q2", "q4", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "d31"
+
+#define STORERGBA \
+ "vmov.u8 d1, d0 \n" \
+ "vmov.u8 d3, d4 \n" \
+ "vmov.u8 d0, d6 \n" \
+ "vst4.8 {d0, d1, d2, d3}, [%[dst_rgba]]! \n"
void I444ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -114,22 +152,43 @@ void I444ToARGBRow_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READYUV444 YUVTORGB
- "subs %4, %4, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%3]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
+}
+
+void I444ToRGB24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
void I422ToARGBRow_NEON(const uint8_t* src_y,
@@ -140,22 +199,46 @@ void I422ToARGBRow_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- "subs %4, %4, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%3]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
+}
+
+void I444AlphaToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vld1.8 {d6}, [%[src_a]]! \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
@@ -167,23 +250,21 @@ void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- "subs %5, %5, #8 \n"
- "vld1.8 {d23}, [%3]! \n"
- "vst4.8 {d20, d21, d22, d23}, [%4]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(src_a), // %3
- "+r"(dst_argb), // %4
- "+r"(width) // %5
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vld1.8 {d6}, [%[src_a]]! \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void I422ToRGBARow_NEON(const uint8_t* src_y,
@@ -194,22 +275,22 @@ void I422ToRGBARow_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- "subs %4, %4, #8 \n"
- "vmov.u8 d19, #255 \n" // YUVTORGB modified d19
- "vst4.8 {d19, d20, d21, d22}, [%3]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgba), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READYUV422 //
+ "subs %[width], %[width], #8 \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ STORERGBA //
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void I422ToRGB24Row_NEON(const uint8_t* src_y,
@@ -220,29 +301,27 @@ void I422ToRGB24Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- "subs %4, %4, #8 \n"
- "vst3.8 {d20, d21, d22}, [%3]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgb24), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
#define ARGBTORGB565 \
- "vshll.u8 q0, d22, #8 \n" /* R */ \
- "vshll.u8 q8, d21, #8 \n" /* G */ \
- "vshll.u8 q9, d20, #8 \n" /* B */ \
- "vsri.16 q0, q8, #5 \n" /* RG */ \
- "vsri.16 q0, q9, #11 \n" /* RGB */
+ "vshll.u8 q2, d4, #8 \n" /* R */ \
+ "vshll.u8 q1, d2, #8 \n" /* G */ \
+ "vshll.u8 q0, d0, #8 \n" /* B */ \
+ "vsri.16 q2, q1, #5 \n" /* RG */ \
+ "vsri.16 q2, q0, #11 \n" /* RGB */
void I422ToRGB565Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -252,31 +331,29 @@ void I422ToRGB565Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- "subs %4, %4, #8 \n" ARGBTORGB565
- "vst1.8 {q0}, [%3]! \n" // store 8 pixels RGB565.
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgb565), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "1: \n" //
+ READYUV422 "subs %[width], %[width], #8 \n" YUVTORGB
+ RGBTORGB8 ARGBTORGB565
+ "vst1.8 {q2}, [%[dst_rgb565]]! \n" // store 8 pixels RGB565.
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb565] "+r"(dst_rgb565), // %[dst_rgb565]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
#define ARGBTOARGB1555 \
- "vshll.u8 q0, d23, #8 \n" /* A */ \
- "vshll.u8 q8, d22, #8 \n" /* R */ \
- "vshll.u8 q9, d21, #8 \n" /* G */ \
- "vshll.u8 q10, d20, #8 \n" /* B */ \
- "vsri.16 q0, q8, #1 \n" /* AR */ \
- "vsri.16 q0, q9, #6 \n" /* ARG */ \
- "vsri.16 q0, q10, #11 \n" /* ARGB */
+ "vshll.u8 q3, d6, #8 \n" /* A */ \
+ "vshll.u8 q2, d4, #8 \n" /* R */ \
+ "vshll.u8 q1, d2, #8 \n" /* G */ \
+ "vshll.u8 q0, d0, #8 \n" /* B */ \
+ "vsri.16 q3, q2, #1 \n" /* AR */ \
+ "vsri.16 q3, q1, #6 \n" /* ARG */ \
+ "vsri.16 q3, q0, #11 \n" /* ARGB */
void I422ToARGB1555Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -286,31 +363,29 @@ void I422ToARGB1555Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- "subs %4, %4, #8 \n"
- "vmov.u8 d23, #255 \n" ARGBTOARGB1555
- "vst1.8 {q0}, [%3]! \n" // store 8 pixels
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb1555), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ "vmov.u8 d6, #0xff \n" ARGBTOARGB1555
+ "vst1.8 {q3}, [%[dst_argb1555]]! \n" // store 8 pixels RGB1555.
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb1555] "+r"(dst_argb1555), // %[dst_argb1555]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "q3");
}
#define ARGBTOARGB4444 \
- "vshr.u8 d20, d20, #4 \n" /* B */ \
- "vbic.32 d21, d21, d4 \n" /* G */ \
- "vshr.u8 d22, d22, #4 \n" /* R */ \
- "vbic.32 d23, d23, d4 \n" /* A */ \
- "vorr d0, d20, d21 \n" /* BG */ \
- "vorr d1, d22, d23 \n" /* RA */ \
+ "vshr.u8 d0, d0, #4 \n" /* B */ \
+ "vbic.32 d2, d2, d7 \n" /* G */ \
+ "vshr.u8 d4, d4, #4 \n" /* R */ \
+ "vbic.32 d6, d6, d7 \n" /* A */ \
+ "vorr d0, d0, d2 \n" /* BG */ \
+ "vorr d1, d4, d6 \n" /* RA */ \
"vzip.u8 d0, d1 \n" /* BGRA */
void I422ToARGB4444Row_NEON(const uint8_t* src_y,
@@ -321,56 +396,53 @@ void I422ToARGB4444Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "vmov.u8 d4, #0x0f \n" // vbic bits to clear
- "1: \n"
+ "vmov.u8 d6, #255 \n"
+ "vmov.u8 d7, #0x0f \n" // vbic bits to clear
+ "1: \n" //
+ READYUV422 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n" ARGBTOARGB4444
+ "vst1.8 {q0}, [%[dst_argb4444]]! \n" // store 8 pixels
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb4444] "+r"(dst_argb4444), // %[dst_argb4444]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "q3");
+}
- READYUV422 YUVTORGB
- "subs %4, %4, #8 \n"
- "vmov.u8 d23, #255 \n" ARGBTOARGB4444
- "vst1.8 {q0}, [%3]! \n" // store 8 pixels
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb4444), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
-}
-
-void I400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+void I400ToARGBRow_NEON(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
asm volatile(
YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READYUV400 YUVTORGB
- "subs %2, %2, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB] "r"(&kYuvI601Constants.kUVToRB),
- [kUVToG] "r"(&kYuvI601Constants.kUVToG),
- [kUVBiasBGR] "r"(&kYuvI601Constants.kUVBiasBGR),
- [kYToRgb] "r"(&kYuvI601Constants.kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READYUV400 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
- "vmov.u8 d23, #255 \n"
- "1: \n"
- "vld1.8 {d20}, [%0]! \n"
- "vmov d21, d20 \n"
- "vmov d22, d20 \n"
- "subs %2, %2, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%1]! \n"
- "bgt 1b \n"
+ "vmov.u8 d23, #255 \n"
+ "1: \n"
+ "vld1.8 {d20}, [%0]! \n"
+ "subs %2, %2, #8 \n"
+ "vmov d21, d20 \n"
+ "vmov d22, d20 \n"
+ "vst4.8 {d20, d21, d22, d23}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -383,22 +455,21 @@ void NV12ToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READNV12 YUVTORGB
- "subs %3, %3, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%2]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
- "q10", "q11", "q12", "q13", "q14", "q15");
+ asm volatile(
+ YUVTORGB_SETUP
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void NV21ToARGBRow_NEON(const uint8_t* src_y,
@@ -406,22 +477,21 @@ void NV21ToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READNV21 YUVTORGB
- "subs %3, %3, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%2]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_vu), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
- "q10", "q11", "q12", "q13", "q14", "q15");
+ asm volatile(
+ YUVTORGB_SETUP
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READNV21 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_vu] "+r"(src_vu), // %[src_vu]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void NV12ToRGB24Row_NEON(const uint8_t* src_y,
@@ -430,25 +500,18 @@ void NV12ToRGB24Row_NEON(const uint8_t* src_y,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile(
-
- YUVTORGB_SETUP
-
- "1: \n"
-
- READNV12 YUVTORGB
- "subs %3, %3, #8 \n"
- "vst3.8 {d20, d21, d22}, [%2]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_rgb24), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ YUVTORGB_SETUP "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
void NV21ToRGB24Row_NEON(const uint8_t* src_y,
@@ -457,25 +520,18 @@ void NV21ToRGB24Row_NEON(const uint8_t* src_y,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile(
-
- YUVTORGB_SETUP
-
- "1: \n"
-
- READNV21 YUVTORGB
- "subs %3, %3, #8 \n"
- "vst3.8 {d20, d21, d22}, [%2]! \n"
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_vu), // %1
- "+r"(dst_rgb24), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ YUVTORGB_SETUP "1: \n" //
+ READNV21 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_vu] "+r"(src_vu), // %[src_vu]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
void NV12ToRGB565Row_NEON(const uint8_t* src_y,
@@ -485,62 +541,58 @@ void NV12ToRGB565Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READNV12 YUVTORGB
- "subs %3, %3, #8 \n" ARGBTORGB565
- "vst1.8 {q0}, [%2]! \n" // store 8 pixels RGB565.
- "bgt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_rgb565), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9", "q10", "q11",
- "q12", "q13", "q14", "q15");
+ "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n" ARGBTORGB565
+ "vst1.8 {q2}, [%[dst_rgb565]]! \n" // store 8 pixels RGB565.
+ "bgt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb565] "+r"(dst_rgb565), // %[dst_rgb565]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
void YUY2ToARGBRow_NEON(const uint8_t* src_yuy2,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READYUY2 YUVTORGB
- "subs %2, %2, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src_yuy2), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
- "q10", "q11", "q12", "q13", "q14", "q15");
+ asm volatile(
+ YUVTORGB_SETUP
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READYUY2 YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
void UYVYToARGBRow_NEON(const uint8_t* src_uyvy,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(YUVTORGB_SETUP
- "vmov.u8 d23, #255 \n"
- "1: \n" READUYVY YUVTORGB
- "subs %2, %2, #8 \n"
- "vst4.8 {d20, d21, d22, d23}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src_uyvy), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q8", "q9",
- "q10", "q11", "q12", "q13", "q14", "q15");
+ asm volatile(
+ YUVTORGB_SETUP
+ "vmov.u8 d6, #255 \n"
+ "1: \n" //
+ READUYVY YUVTORGB RGBTORGB8
+ "subs %[width], %[width], #8 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
+ "bgt 1b \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "d6");
}
// Reads 16 pairs of UV and write even values to dst_u and odd to dst_v.
@@ -549,12 +601,12 @@ void SplitUVRow_NEON(const uint8_t* src_uv,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "vld2.8 {q0, q1}, [%0]! \n" // load 16 pairs of UV
- "subs %3, %3, #16 \n" // 16 processed per loop
- "vst1.8 {q0}, [%1]! \n" // store U
- "vst1.8 {q1}, [%2]! \n" // store V
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {q0, q1}, [%0]! \n" // load 16 pairs of UV
+ "subs %3, %3, #16 \n" // 16 processed per loop
+ "vst1.8 {q0}, [%1]! \n" // store U
+ "vst1.8 {q1}, [%2]! \n" // store V
+ "bgt 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -564,18 +616,179 @@ void SplitUVRow_NEON(const uint8_t* src_uv,
);
}
+// Reads 16 byte Y's from tile and writes out 16 Y's.
+// MM21 Y tiles are 16x32 so src_tile_stride = 512 bytes
+// MM21 UV tiles are 8x16 so src_tile_stride = 256 bytes
+// width measured in bytes so 8 UV = 16.
+void DetileRow_NEON(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q0}, [%0], %3 \n" // load 16 bytes
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "pld [%0, #1792] \n"
+ "vst1.8 {q0}, [%1]! \n" // store 16 bytes
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride) // %3
+ : "cc", "memory", "q0" // Clobber List
+ );
+}
+
+// Reads 16 byte Y's of 16 bits from tile and writes out 16 Y's.
+void DetileRow_16_NEON(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld1.16 {q0, q1}, [%0], %3 \n" // load 16 pixels
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "pld [%0, #3584] \n"
+ "vst1.16 {q0, q1}, [%1]! \n" // store 16 pixels
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride * 2) // %3
+ : "cc", "memory", "q0", "q1" // Clobber List
+ );
+}
+
+// Read 16 bytes of UV, detile, and write 8 bytes of U and 8 bytes of V.
+void DetileSplitUVRow_NEON(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld2.8 {d0, d1}, [%0], %4 \n"
+ "subs %3, %3, #16 \n"
+ "pld [%0, #1792] \n"
+ "vst1.8 {d0}, [%1]! \n"
+ "vst1.8 {d1}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(src_tile_stride) // %4
+ : "cc", "memory", "d0", "d1" // Clobber List
+ );
+}
+
+#if defined(LIBYUV_USE_ST2)
+// Read 16 Y, 8 UV, and write 8 YUYV.
+void DetileToYUY2_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q0}, [%0], %4 \n" // Load 16 Y
+ "pld [%0, #1792] \n"
+ "vld1.8 {q1}, [%1], %5 \n" // Load 8 UV
+ "pld [%1, #1792] \n"
+ "subs %3, %3, #16 \n"
+ "vst2.8 {q0, q1}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_uv), // %1
+ "+r"(dst_yuy2), // %2
+ "+r"(width) // %3
+ : "r"(src_y_tile_stride), // %4
+ "r"(src_uv_tile_stride) // %5
+ : "cc", "memory", "d0", "d1", "d2", "d3" // Clobber list
+ );
+}
+#else
+// Read 16 Y, 8 UV, and write 8 YUYV.
+void DetileToYUY2_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q0}, [%0], %4 \n" // Load 16 Y
+ "vld1.8 {q1}, [%1], %5 \n" // Load 8 UV
+ "subs %3, %3, #16 \n"
+ "pld [%0, #1792] \n"
+ "vzip.8 q0, q1 \n"
+ "pld [%1, #1792] \n"
+ "vst1.8 {q0, q1}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_uv), // %1
+ "+r"(dst_yuy2), // %2
+ "+r"(width) // %3
+ : "r"(src_y_tile_stride), // %4
+ "r"(src_uv_tile_stride) // %5
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber list
+ );
+}
+#endif
+
+void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size) {
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q14}, [%0]! \n" // Load lower bits.
+ "vld1.8 {q9}, [%0]! \n" // Load upper bits row
+ // by row.
+ "vld1.8 {q11}, [%0]! \n"
+ "vld1.8 {q13}, [%0]! \n"
+ "vld1.8 {q15}, [%0]! \n"
+ "subs %2, %2, #80 \n"
+ "vshl.u8 q8, q14, #6 \n" // Shift lower bit data
+ // appropriately.
+ "vshl.u8 q10, q14, #4 \n"
+ "vshl.u8 q12, q14, #2 \n"
+ "vzip.u8 q8, q9 \n" // Interleave upper and
+ // lower bits.
+ "vzip.u8 q10, q11 \n"
+ "vzip.u8 q12, q13 \n"
+ "vzip.u8 q14, q15 \n"
+ "vsri.u16 q8, q8, #10 \n" // Copy upper 6 bits
+ // into lower 6 bits for
+ // better accuracy in
+ // conversions.
+ "vsri.u16 q9, q9, #10 \n"
+ "vsri.u16 q10, q10, #10 \n"
+ "vsri.u16 q11, q11, #10 \n"
+ "vsri.u16 q12, q12, #10 \n"
+ "vsri.u16 q13, q13, #10 \n"
+ "vsri.u16 q14, q14, #10 \n"
+ "vsri.u16 q15, q15, #10 \n"
+ "vstmia %1!, {q8-q15} \n" // Store pixel block (64
+ // pixels).
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(size) // %2
+ :
+ : "cc", "memory", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
+}
+
// Reads 16 U's and V's and writes out 16 pairs of UV.
void MergeUVRow_NEON(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load U
- "vld1.8 {q1}, [%1]! \n" // load V
- "subs %3, %3, #16 \n" // 16 processed per loop
- "vst2.8 {q0, q1}, [%2]! \n" // store 16 pairs of UV
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load U
+ "vld1.8 {q1}, [%1]! \n" // load V
+ "subs %3, %3, #16 \n" // 16 processed per loop
+ "vst2.8 {q0, q1}, [%2]! \n" // store 16 pairs of UV
+ "bgt 1b \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -592,21 +805,21 @@ void SplitRGBRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
- "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB
- "vld3.8 {d1, d3, d5}, [%0]! \n" // next 8 RGB
- "subs %4, %4, #16 \n" // 16 processed per loop
- "vst1.8 {q0}, [%1]! \n" // store R
- "vst1.8 {q1}, [%2]! \n" // store G
- "vst1.8 {q2}, [%3]! \n" // store B
- "bgt 1b \n"
+ "1: \n"
+ "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB
+ "vld3.8 {d1, d3, d5}, [%0]! \n" // next 8 RGB
+ "subs %4, %4, #16 \n" // 16 processed per loop
+ "vst1.8 {q0}, [%1]! \n" // store R
+ "vst1.8 {q1}, [%2]! \n" // store G
+ "vst1.8 {q2}, [%3]! \n" // store B
+ "bgt 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
"+r"(dst_b), // %3
"+r"(width) // %4
: // Input registers
- : "cc", "memory", "d0", "d1", "d2" // Clobber List
+ : "cc", "memory", "q0", "q1", "q2" // Clobber List
);
}
@@ -617,14 +830,14 @@ void MergeRGBRow_NEON(const uint8_t* src_r,
uint8_t* dst_rgb,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load R
- "vld1.8 {q1}, [%1]! \n" // load G
- "vld1.8 {q2}, [%2]! \n" // load B
- "subs %4, %4, #16 \n" // 16 processed per loop
- "vst3.8 {d0, d2, d4}, [%3]! \n" // store 8 RGB
- "vst3.8 {d1, d3, d5}, [%3]! \n" // next 8 RGB
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load R
+ "vld1.8 {q1}, [%1]! \n" // load G
+ "vld1.8 {q2}, [%2]! \n" // load B
+ "subs %4, %4, #16 \n" // 16 processed per loop
+ "vst3.8 {d0, d2, d4}, [%3]! \n" // store 8 RGB
+ "vst3.8 {d1, d3, d5}, [%3]! \n" // next 8 RGB
+ "bgt 1b \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -635,14 +848,341 @@ void MergeRGBRow_NEON(const uint8_t* src_r,
);
}
+// Reads 16 packed ARGB and write to planar dst_r, dst_g, dst_b, dst_a.
+void SplitARGBRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // next 8 ARGB
+ "subs %5, %5, #16 \n" // 16 processed per loop
+ "vst1.8 {q0}, [%3]! \n" // store B
+ "vst1.8 {q1}, [%2]! \n" // store G
+ "vst1.8 {q2}, [%1]! \n" // store R
+ "vst1.8 {q3}, [%4]! \n" // store A
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(dst_a), // %4
+ "+r"(width) // %5
+ : // Input registers
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List
+ );
+}
+
+// Reads 16 planar R's, G's and B's and writes out 16 packed ARGB at a time
+void MergeARGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q2}, [%0]! \n" // load R
+ "vld1.8 {q1}, [%1]! \n" // load G
+ "vld1.8 {q0}, [%2]! \n" // load B
+ "vld1.8 {q3}, [%3]! \n" // load A
+ "subs %5, %5, #16 \n" // 16 processed per loop
+ "vst4.8 {d0, d2, d4, d6}, [%4]! \n" // store 8 ARGB
+ "vst4.8 {d1, d3, d5, d7}, [%4]! \n" // next 8 ARGB
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
+ : // Input registers
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List
+ );
+}
+
+// Reads 16 packed ARGB and write to planar dst_r, dst_g, dst_b.
+void SplitXRGBRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // next 8 ARGB
+ "subs %4, %4, #16 \n" // 16 processed per loop
+ "vst1.8 {q0}, [%3]! \n" // store B
+ "vst1.8 {q1}, [%2]! \n" // store G
+ "vst1.8 {q2}, [%1]! \n" // store R
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : // Input registers
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List
+ );
+}
+
+// Reads 16 planar R's, G's, B's and A's and writes out 16 packed ARGB at a time
+void MergeXRGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vmov.u8 q3, #255 \n" // load A(255)
+ "1: \n"
+ "vld1.8 {q2}, [%0]! \n" // load R
+ "vld1.8 {q1}, [%1]! \n" // load G
+ "vld1.8 {q0}, [%2]! \n" // load B
+ "subs %4, %4, #16 \n" // 16 processed per loop
+ "vst4.8 {d0, d2, d4, d6}, [%3]! \n" // store 8 ARGB
+ "vst4.8 {d1, d3, d5, d7}, [%3]! \n" // next 8 ARGB
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+r"(width) // %4
+ : // Input registers
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List
+ );
+}
+
+void MergeXR30Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ int shift = 10 - depth;
+ asm volatile(
+ "vmov.u32 q14, #1023 \n"
+ "vdup.32 q15, %5 \n"
+ "1: \n"
+ "vld1.16 {d4}, [%2]! \n" // B
+ "vld1.16 {d2}, [%1]! \n" // G
+ "vld1.16 {d0}, [%0]! \n" // R
+ "subs %4, %4, #4 \n"
+ "vmovl.u16 q2, d4 \n" // B
+ "vmovl.u16 q1, d2 \n" // G
+ "vmovl.u16 q0, d0 \n" // R
+ "vshl.u32 q2, q2, q15 \n" // 000B
+ "vshl.u32 q1, q1, q15 \n"
+ "vshl.u32 q0, q0, q15 \n"
+ "vmin.u32 q2, q2, q14 \n"
+ "vmin.u32 q1, q1, q14 \n"
+ "vmin.u32 q0, q0, q14 \n"
+ "vsli.u32 q2, q1, #10 \n" // 00GB
+ "vsli.u32 q2, q0, #20 \n" // 0RGB
+ "vorr.u32 q2, #0xc0000000 \n" // ARGB (AR30)
+ "vst1.8 {q2}, [%3]! \n"
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar30), // %3
+ "+r"(width) // %4
+ : "r"(shift) // %5
+ : "memory", "cc", "q0", "q1", "q2", "q14", "q15");
+}
+
+void MergeXR30Row_10_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int /* depth */,
+ int width) {
+ asm volatile(
+ "vmov.u32 q14, #1023 \n"
+ "1: \n"
+ "vld1.16 {d4}, [%2]! \n" // B
+ "vld1.16 {d2}, [%1]! \n" // G
+ "vld1.16 {d0}, [%0]! \n" // R
+ "subs %4, %4, #4 \n"
+ "vmovl.u16 q2, d4 \n" // 000B
+ "vmovl.u16 q1, d2 \n" // G
+ "vmovl.u16 q0, d0 \n" // R
+ "vmin.u32 q2, q2, q14 \n"
+ "vmin.u32 q1, q1, q14 \n"
+ "vmin.u32 q0, q0, q14 \n"
+ "vsli.u32 q2, q1, #10 \n" // 00GB
+ "vsli.u32 q2, q0, #20 \n" // 0RGB
+ "vorr.u32 q2, #0xc0000000 \n" // ARGB (AR30)
+ "vst1.8 {q2}, [%3]! \n"
+ "bgt 1b \n"
+ "3: \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar30), // %3
+ "+r"(width) // %4
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q14");
+}
+
+void MergeAR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ int mask = (1 << depth) - 1;
+ asm volatile(
+
+ "vdup.u16 q15, %6 \n"
+ "vdup.u16 q14, %7 \n"
+ "1: \n"
+ "vld1.16 {q2}, [%0]! \n" // R
+ "vld1.16 {q1}, [%1]! \n" // G
+ "vld1.16 {q0}, [%2]! \n" // B
+ "vld1.16 {q3}, [%3]! \n" // A
+ "subs %5, %5, #8 \n"
+ "vmin.u16 q2, q2, q14 \n"
+ "vmin.u16 q1, q1, q14 \n"
+ "vmin.u16 q0, q0, q14 \n"
+ "vmin.u16 q3, q3, q14 \n"
+ "vshl.u16 q2, q2, q15 \n"
+ "vshl.u16 q1, q1, q15 \n"
+ "vshl.u16 q0, q0, q15 \n"
+ "vshl.u16 q3, q3, q15 \n"
+ "vst4.16 {d0, d2, d4, d6}, [%4]! \n"
+ "vst4.16 {d1, d3, d5, d7}, [%4]! \n"
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_ar64), // %4
+ "+r"(width) // %5
+ : "r"(shift), // %6
+ "r"(mask) // %7
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q14", "q15");
+}
+
+void MergeXR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ int mask = (1 << depth) - 1;
+ asm volatile(
+
+ "vmov.u8 q3, #0xff \n" // A (0xffff)
+ "vdup.u16 q15, %5 \n"
+ "vdup.u16 q14, %6 \n"
+ "1: \n"
+ "vld1.16 {q2}, [%0]! \n" // R
+ "vld1.16 {q1}, [%1]! \n" // G
+ "vld1.16 {q0}, [%2]! \n" // B
+ "subs %4, %4, #8 \n"
+ "vmin.u16 q2, q2, q14 \n"
+ "vmin.u16 q1, q1, q14 \n"
+ "vmin.u16 q0, q0, q14 \n"
+ "vshl.u16 q2, q2, q15 \n"
+ "vshl.u16 q1, q1, q15 \n"
+ "vshl.u16 q0, q0, q15 \n"
+ "vst4.16 {d0, d2, d4, d6}, [%3]! \n"
+ "vst4.16 {d1, d3, d5, d7}, [%3]! \n"
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar64), // %3
+ "+r"(width) // %4
+ : "r"(shift), // %5
+ "r"(mask) // %6
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q14", "q15");
+}
+
+void MergeARGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ int shift = 8 - depth;
+ asm volatile(
+
+ "vdup.16 q15, %6 \n"
+ "1: \n"
+ "vld1.16 {q2}, [%0]! \n" // R
+ "vld1.16 {q1}, [%1]! \n" // G
+ "vld1.16 {q0}, [%2]! \n" // B
+ "vld1.16 {q3}, [%3]! \n" // A
+ "subs %5, %5, #8 \n"
+ "vshl.u16 q2, q2, q15 \n"
+ "vshl.u16 q1, q1, q15 \n"
+ "vshl.u16 q0, q0, q15 \n"
+ "vshl.u16 q3, q3, q15 \n"
+ "vqmovn.u16 d0, q0 \n"
+ "vqmovn.u16 d1, q1 \n"
+ "vqmovn.u16 d2, q2 \n"
+ "vqmovn.u16 d3, q3 \n"
+ "vst4.8 {d0, d1, d2, d3}, [%4]! \n"
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
+ : "r"(shift) // %6
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q15");
+}
+
+void MergeXRGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ int shift = 8 - depth;
+ asm volatile(
+
+ "vdup.16 q15, %5 \n"
+ "vmov.u8 d6, #0xff \n" // A (0xff)
+ "1: \n"
+ "vld1.16 {q2}, [%0]! \n" // R
+ "vld1.16 {q1}, [%1]! \n" // G
+ "vld1.16 {q0}, [%2]! \n" // B
+ "subs %4, %4, #8 \n"
+ "vshl.u16 q2, q2, q15 \n"
+ "vshl.u16 q1, q1, q15 \n"
+ "vshl.u16 q0, q0, q15 \n"
+ "vqmovn.u16 d5, q2 \n"
+ "vqmovn.u16 d4, q1 \n"
+ "vqmovn.u16 d3, q0 \n"
+ "vst4.u8 {d3, d4, d5, d6}, [%3]! \n"
+ "bgt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+r"(width) // %4
+ : "r"(shift) // %5
+ : "memory", "cc", "q0", "q1", "q2", "d6", "q15");
+}
+
// Copy multiple of 32. vld4.8 allow unaligned and is fastest on a15.
void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "1: \n"
- "vld1.8 {d0, d1, d2, d3}, [%0]! \n" // load 32
- "subs %2, %2, #32 \n" // 32 processed per loop
- "vst1.8 {d0, d1, d2, d3}, [%1]! \n" // store 32
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d0, d1, d2, d3}, [%0]! \n" // load 32
+ "subs %2, %2, #32 \n" // 32 processed per loop
+ "vst1.8 {d0, d1, d2, d3}, [%1]! \n" // store 32
+ "bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2 // Output registers
@@ -654,11 +1194,11 @@ void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
// SetRow writes 'width' bytes using an 8 bit value repeated.
void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
asm volatile(
- "vdup.8 q0, %2 \n" // duplicate 16 bytes
- "1: \n"
- "subs %1, %1, #16 \n" // 16 bytes per loop
- "vst1.8 {q0}, [%0]! \n" // store
- "bgt 1b \n"
+ "vdup.8 q0, %2 \n" // duplicate 16 bytes
+ "1: \n"
+ "subs %1, %1, #16 \n" // 16 bytes per loop
+ "vst1.8 {q0}, [%0]! \n" // store
+ "bgt 1b \n"
: "+r"(dst), // %0
"+r"(width) // %1
: "r"(v8) // %2
@@ -668,11 +1208,11 @@ void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
// ARGBSetRow writes 'width' pixels using an 32 bit value repeated.
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width) {
asm volatile(
- "vdup.u32 q0, %2 \n" // duplicate 4 ints
- "1: \n"
- "subs %1, %1, #4 \n" // 4 pixels per loop
- "vst1.8 {q0}, [%0]! \n" // store
- "bgt 1b \n"
+ "vdup.u32 q0, %2 \n" // duplicate 4 ints
+ "1: \n"
+ "subs %1, %1, #4 \n" // 4 pixels per loop
+ "vst1.8 {q0}, [%0]! \n" // store
+ "bgt 1b \n"
: "+r"(dst), // %0
"+r"(width) // %1
: "r"(v32) // %2
@@ -682,41 +1222,62 @@ void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width) {
void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
// Start at end of source row.
- "mov r3, #-16 \n"
- "add %0, %0, %2 \n"
- "sub %0, #16 \n"
-
- "1: \n"
- "vld1.8 {q0}, [%0], r3 \n" // src -= 16
- "subs %2, #16 \n" // 16 pixels per loop.
- "vrev64.8 q0, q0 \n"
- "vst1.8 {d1}, [%1]! \n" // dst += 16
- "vst1.8 {d0}, [%1]! \n"
- "bgt 1b \n"
+ "add %0, %0, %2 \n"
+ "sub %0, %0, #32 \n" // 32 bytes per loop
+
+ "1: \n"
+ "vld1.8 {q1, q2}, [%0], %3 \n" // src -= 32
+ "subs %2, #32 \n" // 32 pixels per loop.
+ "vrev64.8 q0, q2 \n"
+ "vrev64.8 q1, q1 \n"
+ "vswp d0, d1 \n"
+ "vswp d2, d3 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // dst += 32
+ "bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
- :
- : "cc", "memory", "r3", "q0");
+ : "r"(-32) // %3
+ : "cc", "memory", "q0", "q1", "q2");
}
-void MirrorUVRow_NEON(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
asm volatile(
// Start at end of source row.
- "mov r12, #-16 \n"
- "add %0, %0, %3, lsl #1 \n"
- "sub %0, #16 \n"
+ "mov r12, #-16 \n"
+ "add %0, %0, %2, lsl #1 \n"
+ "sub %0, #16 \n"
+
+ "1: \n"
+ "vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16
+ "subs %2, #8 \n" // 8 pixels per loop.
+ "vrev64.8 q0, q0 \n"
+ "vst2.8 {d0, d1}, [%1]! \n" // dst += 16
+ "bgt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "r12", "q0");
+}
- "1: \n"
- "vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16
- "subs %3, #8 \n" // 8 pixels per loop.
- "vrev64.8 q0, q0 \n"
- "vst1.8 {d0}, [%1]! \n" // dst += 8
- "vst1.8 {d1}, [%2]! \n"
- "bgt 1b \n"
+void MirrorSplitUVRow_NEON(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ // Start at end of source row.
+ "mov r12, #-16 \n"
+ "add %0, %0, %3, lsl #1 \n"
+ "sub %0, #16 \n"
+
+ "1: \n"
+ "vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16
+ "subs %3, #8 \n" // 8 pixels per loop.
+ "vrev64.8 q0, q0 \n"
+ "vst1.8 {d0}, [%1]! \n" // dst += 8
+ "vst1.8 {d1}, [%2]! \n"
+ "bgt 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -725,37 +1286,57 @@ void MirrorUVRow_NEON(const uint8_t* src_uv,
: "cc", "memory", "r12", "q0");
}
-void ARGBMirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
+void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
asm volatile(
- // Start at end of source row.
- "mov r3, #-16 \n"
- "add %0, %0, %2, lsl #2 \n"
- "sub %0, #16 \n"
+ "add %0, %0, %2, lsl #2 \n"
+ "sub %0, #32 \n"
+
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0], %3 \n" // src -= 32
+ "subs %2, #8 \n" // 8 pixels per loop.
+ "vrev64.8 d0, d0 \n"
+ "vrev64.8 d1, d1 \n"
+ "vrev64.8 d2, d2 \n"
+ "vrev64.8 d3, d3 \n"
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // dst += 32
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(-32) // %3
+ : "cc", "memory", "d0", "d1", "d2", "d3");
+}
- "1: \n"
- "vld1.8 {q0}, [%0], r3 \n" // src -= 16
- "subs %2, #4 \n" // 4 pixels per loop.
- "vrev64.32 q0, q0 \n"
- "vst1.8 {d1}, [%1]! \n" // dst += 16
- "vst1.8 {d0}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "r3", "q0");
+void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ src_rgb24 += width * 3 - 24;
+ asm volatile(
+ "1: \n"
+ "vld3.8 {d0, d1, d2}, [%0], %3 \n" // src -= 24
+ "subs %2, #8 \n" // 8 pixels per loop.
+ "vrev64.8 d0, d0 \n"
+ "vrev64.8 d1, d1 \n"
+ "vrev64.8 d2, d2 \n"
+ "vst3.8 {d0, d1, d2}, [%1]! \n" // dst += 24
+ "bgt 1b \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(width) // %2
+ : "r"(-24) // %3
+ : "cc", "memory", "d0", "d1", "d2");
}
void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d4, #255 \n" // Alpha
- "1: \n"
- "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RGB24.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vmov.u8 d4, #255 \n" // Alpha
+ "1: \n"
+ "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RGB24.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_rgb24), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -766,13 +1347,13 @@ void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
- "vmov.u8 d4, #255 \n" // Alpha
- "1: \n"
- "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vswp.u8 d1, d3 \n" // swap R, B
- "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vmov.u8 d4, #255 \n" // Alpha
+ "1: \n"
+ "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vswp.u8 d1, d3 \n" // swap R, B
+ "vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_raw), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -781,20 +1362,37 @@ void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
);
}
+void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ asm volatile(
+ "vmov.u8 d0, #255 \n" // Alpha
+ "1: \n"
+ "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vswp.u8 d1, d3 \n" // swap R, B
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of RGBA.
+ "bgt 1b \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_rgba), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "d0", "d1", "d2", "d3" // Clobber List
+ );
+}
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
asm volatile(
- "1: \n"
- "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vswp.u8 d1, d3 \n" // swap R, B
- "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of
- // RGB24.
- "bgt 1b \n"
+ "1: \n"
+ "vld3.8 {d0, d2, d4}, [%0]! \n" // load 16 pixels of RAW.
+ "vld3.8 {d1, d3, d5}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 processed per loop.
+ "vswp.u8 q0, q2 \n" // swap R, B
+ "vst3.8 {d0, d2, d4}, [%1]! \n" // store 16 pixels of
+ "vst3.8 {d1, d3, d5}, [%1]! \n" // RGB24.
+ "bgt 1b \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
:
- : "cc", "memory", "d1", "d2", "d3" // Clobber List
+ : "cc", "memory", "q0", "q1", "q2" // Clobber List
);
}
@@ -814,13 +1412,13 @@ void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d3, #255 \n" // Alpha
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
RGB565TOARGB
- "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -860,13 +1458,13 @@ void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d3, #255 \n" // Alpha
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGB1555TOARGB
- "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -889,13 +1487,13 @@ void ARGB4444ToARGBRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d3, #255 \n" // Alpha
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGB4444TOARGB
- "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -908,28 +1506,29 @@ void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of
- // RGB24.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 16 pixels of ARGB.
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 processed per loop.
+ "vst3.8 {d0, d2, d4}, [%1]! \n" // store 16 RGB24 pixels.
+ "vst3.8 {d1, d3, d5}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
:
- : "cc", "memory", "d1", "d2", "d3", "d4" // Clobber List
+ : "cc", "memory", "q0", "q1", "q2", "q3" // Clobber List
);
}
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vswp.u8 d1, d3 \n" // swap R, B
- "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of RAW.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vswp.u8 d1, d3 \n" // swap R, B
+ "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of RAW.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_raw), // %1
"+r"(width) // %2
@@ -940,11 +1539,11 @@ void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2.
- "subs %2, %2, #16 \n" // 16 processed per loop.
- "vst1.8 {q0}, [%1]! \n" // store 16 pixels of Y.
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2.
+ "subs %2, %2, #16 \n" // 16 processed per loop.
+ "vst1.8 {q0}, [%1]! \n" // store 16 pixels of Y.
+ "bgt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -955,11 +1554,11 @@ void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
void UYVYToYRow_NEON(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of UYVY.
- "subs %2, %2, #16 \n" // 16 processed per loop.
- "vst1.8 {q1}, [%1]! \n" // store 16 pixels of Y.
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of UYVY.
+ "subs %2, %2, #16 \n" // 16 processed per loop.
+ "vst1.8 {q1}, [%1]! \n" // store 16 pixels of Y.
+ "bgt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -973,12 +1572,12 @@ void YUY2ToUV422Row_NEON(const uint8_t* src_yuy2,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
- "subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
- "vst1.8 {d1}, [%1]! \n" // store 8 U.
- "vst1.8 {d3}, [%2]! \n" // store 8 V.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
+ "subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
+ "vst1.8 {d1}, [%1]! \n" // store 8 U.
+ "vst1.8 {d3}, [%2]! \n" // store 8 V.
+ "bgt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -993,12 +1592,12 @@ void UYVYToUV422Row_NEON(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
- "subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
- "vst1.8 {d0}, [%1]! \n" // store 8 U.
- "vst1.8 {d2}, [%2]! \n" // store 8 V.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
+ "subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
+ "vst1.8 {d0}, [%1]! \n" // store 8 U.
+ "vst1.8 {d2}, [%2]! \n" // store 8 V.
+ "bgt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -1014,16 +1613,16 @@ void YUY2ToUVRow_NEON(const uint8_t* src_yuy2,
uint8_t* dst_v,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // stride + src_yuy2
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
- "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
- "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row YUY2.
- "vrhadd.u8 d1, d1, d5 \n" // average rows of U
- "vrhadd.u8 d3, d3, d7 \n" // average rows of V
- "vst1.8 {d1}, [%2]! \n" // store 8 U.
- "vst1.8 {d3}, [%3]! \n" // store 8 V.
- "bgt 1b \n"
+ "add %1, %0, %1 \n" // stride + src_yuy2
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row YUY2.
+ "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
+ "vrhadd.u8 d1, d1, d5 \n" // average rows of U
+ "vrhadd.u8 d3, d3, d7 \n" // average rows of V
+ "vst1.8 {d1}, [%2]! \n" // store 8 U.
+ "vst1.8 {d3}, [%3]! \n" // store 8 V.
+ "bgt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(stride_yuy2), // %1
"+r"(dst_u), // %2
@@ -1041,16 +1640,16 @@ void UYVYToUVRow_NEON(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // stride + src_uyvy
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
- "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
- "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row UYVY.
- "vrhadd.u8 d0, d0, d4 \n" // average rows of U
- "vrhadd.u8 d2, d2, d6 \n" // average rows of V
- "vst1.8 {d0}, [%2]! \n" // store 8 U.
- "vst1.8 {d2}, [%3]! \n" // store 8 V.
- "bgt 1b \n"
+ "add %1, %0, %1 \n" // stride + src_uyvy
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row UYVY.
+ "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
+ "vrhadd.u8 d0, d0, d4 \n" // average rows of U
+ "vrhadd.u8 d2, d2, d6 \n" // average rows of V
+ "vst1.8 {d0}, [%2]! \n" // store 8 U.
+ "vst1.8 {d2}, [%3]! \n" // store 8 V.
+ "bgt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(stride_uyvy), // %1
"+r"(dst_u), // %2
@@ -1062,20 +1661,43 @@ void UYVYToUVRow_NEON(const uint8_t* src_uyvy,
);
}
+void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "add %1, %0, %1 \n" // stride + src_yuy2
+ "1: \n"
+ "vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2.
+ "subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
+ "vld2.8 {q2, q3}, [%1]! \n" // load next row YUY2.
+ "vrhadd.u8 q4, q1, q3 \n" // average rows of UV
+ "vst1.8 {q4}, [%2]! \n" // store 8 UV.
+ "bgt 1b \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(stride_yuy2), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8",
+ "d9" // Clobber List
+ );
+}
+
// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
void ARGBShuffleRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
asm volatile(
- "vld1.8 {q2}, [%3] \n" // shuffler
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 4 pixels.
- "subs %2, %2, #4 \n" // 4 processed per loop
- "vtbl.8 d2, {d0, d1}, d4 \n" // look up 2 first pixels
- "vtbl.8 d3, {d0, d1}, d5 \n" // look up 2 next pixels
- "vst1.8 {q1}, [%1]! \n" // store 4.
- "bgt 1b \n"
+ "vld1.8 {q2}, [%3] \n" // shuffler
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 4 pixels.
+ "subs %2, %2, #4 \n" // 4 processed per loop
+ "vtbl.8 d2, {d0, d1}, d4 \n" // look up 2 first pixels
+ "vtbl.8 d3, {d0, d1}, d5 \n" // look up 2 next pixels
+ "vst1.8 {q1}, [%1]! \n" // store 4.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -1090,13 +1712,13 @@ void I422ToYUY2Row_NEON(const uint8_t* src_y,
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
- "vld2.8 {d0, d2}, [%0]! \n" // load 16 Ys
- "vld1.8 {d1}, [%1]! \n" // load 8 Us
- "vld1.8 {d3}, [%2]! \n" // load 8 Vs
- "subs %4, %4, #16 \n" // 16 pixels
- "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 YUY2/16 pixels.
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {d0, d2}, [%0]! \n" // load 16 Ys
+ "vld1.8 {d1}, [%1]! \n" // load 8 Us
+ "vld1.8 {d3}, [%2]! \n" // load 8 Vs
+ "subs %4, %4, #16 \n" // 16 pixels
+ "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 YUY2/16 pixels.
+ "bgt 1b \n"
: "+r"(src_y), // %0
"+r"(src_u), // %1
"+r"(src_v), // %2
@@ -1112,13 +1734,13 @@ void I422ToUYVYRow_NEON(const uint8_t* src_y,
uint8_t* dst_uyvy,
int width) {
asm volatile(
- "1: \n"
- "vld2.8 {d1, d3}, [%0]! \n" // load 16 Ys
- "vld1.8 {d0}, [%1]! \n" // load 8 Us
- "vld1.8 {d2}, [%2]! \n" // load 8 Vs
- "subs %4, %4, #16 \n" // 16 pixels
- "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 UYVY/16 pixels.
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {d1, d3}, [%0]! \n" // load 16 Ys
+ "vld1.8 {d0}, [%1]! \n" // load 8 Us
+ "vld1.8 {d2}, [%2]! \n" // load 8 Vs
+ "subs %4, %4, #16 \n" // 16 pixels
+ "vst4.8 {d0, d1, d2, d3}, [%3]! \n" // Store 8 UYVY/16 pixels.
+ "bgt 1b \n"
: "+r"(src_y), // %0
"+r"(src_u), // %1
"+r"(src_v), // %2
@@ -1132,110 +1754,87 @@ void ARGBToRGB565Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTORGB565
- "vst1.8 {q0}, [%1]! \n" // store 8 pixels RGB565.
- "bgt 1b \n"
+ "vst1.8 {q2}, [%1]! \n" // store 8 pixels RGB565.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_rgb565), // %1
"+r"(width) // %2
:
- : "cc", "memory", "q0", "q8", "q9", "q10", "q11");
+ : "cc", "memory", "q0", "q1", "q2", "d6");
}
void ARGBToRGB565DitherRow_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width) {
asm volatile(
- "vdup.32 d2, %2 \n" // dither4
- "1: \n"
- "vld4.8 {d20, d21, d22, d23}, [%1]! \n" // load 8 pixels of ARGB.
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vqadd.u8 d20, d20, d2 \n"
- "vqadd.u8 d21, d21, d2 \n"
- "vqadd.u8 d22, d22, d2 \n" // add for dither
+ "vdup.32 d7, %3 \n" // dither4
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vqadd.u8 d0, d0, d7 \n"
+ "vqadd.u8 d2, d2, d7 \n"
+ "vqadd.u8 d4, d4, d7 \n" // add for dither
ARGBTORGB565
- "vst1.8 {q0}, [%0]! \n" // store 8 RGB565.
- "bgt 1b \n"
- : "+r"(dst_rgb) // %0
- : "r"(src_argb), // %1
- "r"(dither4), // %2
- "r"(width) // %3
- : "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11");
+ "vst1.8 {q2}, [%1]! \n" // store 8 RGB565.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_rgb), // %1
+ "+r"(width) // %2
+ : "r"(dither4) // %3
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
void ARGBToARGB1555Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb1555,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTOARGB1555
- "vst1.8 {q0}, [%1]! \n" // store 8 ARGB1555.
- "bgt 1b \n"
+ "vst1.8 {q3}, [%1]! \n" // store 8 ARGB1555.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb1555), // %1
"+r"(width) // %2
:
- : "cc", "memory", "q0", "q8", "q9", "q10", "q11");
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
void ARGBToARGB4444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb4444,
int width) {
asm volatile(
- "vmov.u8 d4, #0x0f \n" // bits to clear with
+ "vmov.u8 d7, #0x0f \n" // bits to clear with
// vbic.
- "1: \n"
- "vld4.8 {d20, d21, d22, d23}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTOARGB4444
- "vst1.8 {q0}, [%1]! \n" // store 8 ARGB4444.
- "bgt 1b \n"
+ "vst1.8 {q0}, [%1]! \n" // store 8 ARGB4444.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb4444), // %1
"+r"(width) // %2
:
- : "cc", "memory", "q0", "q8", "q9", "q10", "q11");
-}
-
-void ARGBToYRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width) {
- asm volatile(
- "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d27 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "q0", "q1", "q2", "q12", "q13");
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels
- "subs %2, %2, #16 \n" // 16 processed per loop
- "vst1.8 {q3}, [%1]! \n" // store 16 A's.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "vst1.8 {q3}, [%1]! \n" // store 16 A's.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_a), // %1
"+r"(width) // %2
@@ -1244,66 +1843,133 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
);
}
-void ARGBToYJRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "vmov.u8 d24, #15 \n" // B * 0.11400 coefficient
- "vmov.u8 d25, #75 \n" // G * 0.58700 coefficient
- "vmov.u8 d26, #38 \n" // R * 0.29900 coefficient
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 15 bit to 8 bit Y
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%1] \n" // load 8 ARGB
+ "pld [%0, #448] \n"
+ "vld1.8 {d3}, [%0]! \n" // replace A with 8 Y
+ "pld [%1, #448] \n"
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n"
+ "vld4.8 {d4, d5, d6, d7}, [%1] \n" // load 8 ARGB
+ "vld1.8 {d7}, [%0]! \n" // replace A with 8 Y
+ "vst4.8 {d4, d5, d6, d7}, [%1]! \n"
+ "subs %2, %2, #16 \n"
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
:
- : "cc", "memory", "q0", "q1", "q2", "q12", "q13");
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
// 8x1 pixels.
-void ARGBToUV444Row_NEON(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "vmov.u8 d24, #112 \n" // UB / VR 0.875
- // coefficient
- "vmov.u8 d25, #74 \n" // UG -0.5781 coefficient
- "vmov.u8 d26, #38 \n" // UR -0.2969 coefficient
- "vmov.u8 d27, #18 \n" // VB -0.1406 coefficient
- "vmov.u8 d28, #94 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlsl.u8 q2, d1, d25 \n" // G
- "vmlsl.u8 q2, d2, d26 \n" // R
- "vadd.u16 q2, q2, q15 \n" // +128 -> unsigned
-
- "vmull.u8 q3, d2, d24 \n" // R
- "vmlsl.u8 q3, d1, d28 \n" // G
- "vmlsl.u8 q3, d0, d27 \n" // B
- "vadd.u16 q3, q3, q15 \n" // +128 -> unsigned
-
- "vqshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit U
- "vqshrn.u16 d1, q3, #8 \n" // 16 bit to 8 bit V
-
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%2]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q12", "q13", "q14",
- "q15");
+ "vld1.8 {d24}, [%4] \n" // load kRGBToU
+ "vld1.8 {d25}, [%5] \n" // load kRGBToV
+ "vld1.16 {d26[0]}, [%6] \n" // load kAddUV[0]
+ "vmovl.s8 q10, d24 \n" // U coeffs (8 shorts)
+ "vmovl.s8 q11, d25 \n" // V coeffs (8 shorts)
+ "vdup.16 q6, d26[0] \n" // bias
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+
+ "vmovl.u8 q4, d0 \n" // B
+ "vmovl.u8 q5, d1 \n" // G
+ "vmovl.u8 q7, d2 \n" // R
+ "vmovl.u8 q8, d3 \n" // A
+
+ "vdup.16 q12, d20[0] \n"
+ "vmul.s16 q2, q4, q12 \n" // U = B * U0
+ "vdup.16 q12, d20[1] \n"
+ "vmla.s16 q2, q5, q12 \n" // U += G * U1
+ "vdup.16 q12, d20[2] \n"
+ "vmla.s16 q2, q7, q12 \n" // U += R * U2
+ "vdup.16 q12, d20[3] \n"
+ "vmla.s16 q2, q8, q12 \n" // U += A * U3
+
+ "vdup.16 q12, d22[0] \n"
+ "vmul.s16 q3, q4, q12 \n" // V = B * V0
+ "vdup.16 q12, d22[1] \n"
+ "vmla.s16 q3, q5, q12 \n" // V += G * V1
+ "vdup.16 q12, d22[2] \n"
+ "vmla.s16 q3, q7, q12 \n" // V += R * V2
+ "vdup.16 q12, d22[3] \n"
+ "vmla.s16 q3, q8, q12 \n" // V += A * V3
+
+ "vsubhn.s16 d0, q6, q2 \n" // 128.0 - U
+ "vsubhn.s16 d1, q6, q3 \n" // 128.0 - V
+
+ "vst1.8 {d0}, [%1]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%2]! \n" // store 8 pixels V.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(&c->kRGBToU), // %4
+ "r"(&c->kRGBToV), // %5
+ "r"(&c->kAddUV) // %6
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8",
+ "q10", "q11", "q12", "q13");
+}
+
+// 8x1 pixels.
+void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "vld1.8 {d24}, [%4] \n" // load kRGBToU
+ "vld1.8 {d25}, [%5] \n" // load kRGBToV
+ "vld1.16 {d26[0]}, [%6] \n" // load kAddUV[0]
+ "vmovl.s8 q10, d24 \n" // U coeffs (8 shorts)
+ "vmovl.s8 q11, d25 \n" // V coeffs (8 shorts)
+ "vdup.16 q6, d26[0] \n" // bias
+ "1: \n"
+ "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 RGB pixels.
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+
+ "vmovl.u8 q4, d0 \n" // B
+ "vmovl.u8 q5, d1 \n" // G
+ "vmovl.u8 q7, d2 \n" // R
+
+ "vdup.16 q12, d20[0] \n"
+ "vmul.s16 q2, q4, q12 \n" // U = B * U0
+ "vdup.16 q12, d20[1] \n"
+ "vmla.s16 q2, q5, q12 \n" // U += G * U1
+ "vdup.16 q12, d20[2] \n"
+ "vmla.s16 q2, q7, q12 \n" // U += R * U2
+
+ "vdup.16 q12, d22[0] \n"
+ "vmul.s16 q3, q4, q12 \n" // V = B * V0
+ "vdup.16 q12, d22[1] \n"
+ "vmla.s16 q3, q5, q12 \n" // V += G * V1
+ "vdup.16 q12, d22[2] \n"
+ "vmla.s16 q3, q7, q12 \n" // V += R * V2
+
+ "vsubhn.s16 d0, q6, q2 \n" // 128.0 - U
+ "vsubhn.s16 d1, q6, q3 \n" // 128.0 - V
+
+ "vst1.8 {d0}, [%1]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%2]! \n" // store 8 pixels V.
+ "bgt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(&c->kRGBToU), // %4
+ "r"(&c->kRGBToV), // %5
+ "r"(&c->kAddUV) // %6
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10",
+ "q11", "q12", "q13");
}
// clang-format off
@@ -1312,240 +1978,145 @@ void ARGBToUV444Row_NEON(const uint8_t* src_argb,
"vmul.s16 q8, " #QB ", q10 \n" /* B */ \
"vmls.s16 q8, " #QG ", q11 \n" /* G */ \
"vmls.s16 q8, " #QR ", q12 \n" /* R */ \
- "vadd.u16 q8, q8, q15 \n" /* +128 -> unsigned */ \
"vmul.s16 q9, " #QR ", q10 \n" /* R */ \
"vmls.s16 q9, " #QG ", q14 \n" /* G */ \
"vmls.s16 q9, " #QB ", q13 \n" /* B */ \
- "vadd.u16 q9, q9, q15 \n" /* +128 -> unsigned */ \
- "vqshrn.u16 d0, q8, #8 \n" /* 16 bit to 8 bit U */ \
- "vqshrn.u16 d1, q9, #8 \n" /* 16 bit to 8 bit V */
+ "vaddhn.u16 d0, q8, q15 \n" /* +128 -> unsigned */ \
+ "vaddhn.u16 d1, q9, q15 \n" /* +128 -> unsigned */
// clang-format on
// TODO(fbarchard): Consider vhadd vertical, then vpaddl horizontal, avoid shr.
-void ARGBToUVRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels.
- "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ARGB pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ARGB pixels.
- "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
- RGBTOUV(q0, q1, q2)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_stride_argb), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
- "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
- );
-}
-
-// TODO(fbarchard): Subsample match C code.
-void ARGBToUVJRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient
- "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient
- "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient
- "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient
- "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels.
- "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ARGB pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ARGB pixels.
- "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
- RGBTOUV(q0, q1, q2)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_stride_argb), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
- "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
- );
-}
-
-void BGRAToUVRow_NEON(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_bgra
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 BGRA pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 BGRA pixels.
- "vpaddl.u8 q3, q3 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q2 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // R 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more BGRA pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 BGRA pixels.
- "vpadal.u8 q3, q7 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q6 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q1, q1, #1 \n" // 2x average
- "vrshr.u16 q2, q2, #1 \n"
- "vrshr.u16 q3, q3, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
- RGBTOUV(q3, q2, q1)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(src_stride_bgra), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
- "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
- );
-}
-
-void ABGRToUVRow_NEON(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_abgr
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ABGR pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ABGR pixels.
- "vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more ABGR pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 ABGR pixels.
- "vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
- RGBTOUV(q2, q1, q0)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(src_stride_abgr), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
- "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
- );
+void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_argb_1 = src_argb + src_stride_argb;
+ asm volatile(
+ "vld1.8 {d24}, [%5] \n" // load kRGBToU (8 bytes,
+ // only 4 used)
+ "vld1.8 {d25}, [%6] \n" // load kRGBToV
+ "vmovl.s8 q14, d24 \n" // U coeffs in d28
+ "vmovl.s8 q15, d25 \n" // V coeffs in d30
+ "vmov.u16 q11, #0x8000 \n" // 128.0 bias
+
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
+ // pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
+ "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
+ "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts.
+ "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more pixels.
+ "vld4.8 {d9, d11, d13, d15}, [%1]! \n"
+ "vpadal.u8 q0, q4 \n" // B
+ "vpadal.u8 q1, q5 \n" // G
+ "vpadal.u8 q2, q6 \n" // R
+ "vpadal.u8 q3, q7 \n" // A
+
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
+ "vrshr.u16 q3, q3, #2 \n"
+
+ "vdup.16 q12, d28[0] \n"
+ "vmul.s16 q8, q0, q12 \n" // U = B * U0
+ "vdup.16 q12, d28[1] \n"
+ "vmla.s16 q8, q1, q12 \n" // U += G * U1
+ "vdup.16 q12, d28[2] \n"
+ "vmla.s16 q8, q2, q12 \n" // U += R * U2
+ "vdup.16 q12, d28[3] \n"
+ "vmla.s16 q8, q3, q12 \n" // U += A * U3
+
+ "vdup.16 q12, d30[0] \n"
+ "vmul.s16 q9, q0, q12 \n" // V = B * V0
+ "vdup.16 q12, d30[1] \n"
+ "vmla.s16 q9, q1, q12 \n" // V += G * V1
+ "vdup.16 q12, d30[2] \n"
+ "vmla.s16 q9, q2, q12 \n" // V += R * V2
+ "vdup.16 q12, d30[3] \n"
+ "vmla.s16 q9, q3, q12 \n" // V += A * V3
+
+ "vsubhn.s16 d0, q11, q8 \n" // 128.0 - U
+ "vsubhn.s16 d1, q11, q9 \n" // 128.0 - V
+
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb_1), // %1
+ "+r"(dst_u), // %2
+ "+r"(dst_v), // %3
+ "+r"(width) // %4
+ : "r"(&c->kRGBToU), // %5
+ "r"(&c->kRGBToV) // %6
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8",
+ "q9", "q11", "q12", "q14", "q15");
}
-void RGBAToUVRow_NEON(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_rgba
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 RGBA pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 RGBA pixels.
- "vpaddl.u8 q0, q1 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q2 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q3 \n" // R 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more RGBA pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 RGBA pixels.
- "vpadal.u8 q0, q5 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q6 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q7 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
- RGBTOUV(q0, q1, q2)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
- : "+r"(src_rgba), // %0
- "+r"(src_stride_rgba), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
- "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
- );
+void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
+ asm volatile(
+ "vld1.8 {d24}, [%5] \n" // load kRGBToU
+ "vld1.8 {d25}, [%6] \n" // load kRGBToV
+ "vmovl.s8 q14, d24 \n" // U coeffs in d28
+ "vmovl.s8 q15, d25 \n" // V coeffs in d30
+ "vmov.u16 q11, #0x8000 \n" // 128.0 bias
+
+ "1: \n"
+ "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB pixels.
+ "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
+ "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
+ "vld3.8 {d6, d8, d10}, [%1]! \n" // load 8 more pixels.
+ "vld3.8 {d7, d9, d11}, [%1]! \n"
+ "vpadal.u8 q0, q3 \n" // B
+ "vpadal.u8 q1, q4 \n" // G
+ "vpadal.u8 q2, q5 \n" // R
+
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
+
+ "vdup.16 q12, d28[0] \n"
+ "vmul.s16 q8, q0, q12 \n" // U = B * U0
+ "vdup.16 q12, d28[1] \n"
+ "vmla.s16 q8, q1, q12 \n" // U += G * U1
+ "vdup.16 q12, d28[2] \n"
+ "vmla.s16 q8, q2, q12 \n" // U += R * U2
+
+ "vdup.16 q12, d30[0] \n"
+ "vmul.s16 q9, q0, q12 \n" // V = B * V0
+ "vdup.16 q12, d30[1] \n"
+ "vmla.s16 q9, q1, q12 \n" // V += G * V1
+ "vdup.16 q12, d30[2] \n"
+ "vmla.s16 q9, q2, q12 \n" // V += R * V2
+
+ "vsubhn.s16 d0, q11, q8 \n" // 128.0 - U
+ "vsubhn.s16 d1, q11, q9 \n" // 128.0 - V
+
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(src_rgb_1), // %1
+ "+r"(dst_u), // %2
+ "+r"(dst_v), // %3
+ "+r"(width) // %4
+ : "r"(&c->kRGBToU), // %5
+ "r"(&c->kRGBToV) // %6
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", "q11",
+ "q12", "q14", "q15");
}
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
@@ -1554,34 +2125,34 @@ void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_v,
int width) {
asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_rgb24
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB24 pixels.
- "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB24 pixels.
- "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
- "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RGB24 pixels.
- "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RGB24 pixels.
- "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
+ "add %1, %0, %1 \n" // src_stride + src_rgb24
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
+ "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB24 pixels.
+ "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB24 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
+ "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
+ "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RGB24 pixels.
+ "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RGB24 pixels.
+ "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
+ "vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
+
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
+
RGBTOUV(q0, q1, q2)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
: "+r"(src_rgb24), // %0
"+r"(src_stride_rgb24), // %1
"+r"(dst_u), // %2
@@ -1599,34 +2170,34 @@ void RAWToUVRow_NEON(const uint8_t* src_raw,
uint8_t* dst_v,
int width) {
asm volatile (
- "add %1, %0, %1 \n" // src_stride + src_raw
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RAW pixels.
- "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RAW pixels.
- "vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
- "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RAW pixels.
- "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RAW pixels.
- "vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts.
-
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
-
- "subs %4, %4, #16 \n" // 32 processed per loop.
+ "add %1, %0, %1 \n" // src_stride + src_raw
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
+ "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RAW pixels.
+ "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RAW pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
+ "vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
+ "vld3.8 {d8, d10, d12}, [%1]! \n" // load 8 more RAW pixels.
+ "vld3.8 {d9, d11, d13}, [%1]! \n" // load last 8 RAW pixels.
+ "vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
+ "vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts.
+
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
+
RGBTOUV(q2, q1, q0)
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
: "+r"(src_raw), // %0
"+r"(src_stride_raw), // %1
"+r"(dst_u), // %2
@@ -1645,55 +2216,54 @@ void RGB565ToUVRow_NEON(const uint8_t* src_rgb565,
uint8_t* dst_v,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
+ "add %1, %0, %1 \n" // src_stride + src_argb
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
RGB565TOARGB
- "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%0]! \n" // next 8 RGB565 pixels.
+ "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%0]! \n" // next 8 RGB565 pixels.
RGB565TOARGB
- "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+ "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // load 8 RGB565 pixels.
+ "vld1.8 {q0}, [%1]! \n" // load 8 RGB565 pixels.
RGB565TOARGB
- "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // next 8 RGB565 pixels.
+ "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%1]! \n" // next 8 RGB565 pixels.
RGB565TOARGB
- "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
-
- "vrshr.u16 q4, q4, #1 \n" // 2x average
- "vrshr.u16 q5, q5, #1 \n"
- "vrshr.u16 q6, q6, #1 \n"
-
- "subs %4, %4, #16 \n" // 16 processed per loop.
- "vmul.s16 q8, q4, q10 \n" // B
- "vmls.s16 q8, q5, q11 \n" // G
- "vmls.s16 q8, q6, q12 \n" // R
- "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned
- "vmul.s16 q9, q6, q10 \n" // R
- "vmls.s16 q9, q5, q14 \n" // G
- "vmls.s16 q9, q4, q13 \n" // B
- "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned
+ "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+
+ "vrshr.u16 q4, q4, #2 \n" // average of 4
+ "vrshr.u16 q5, q5, #2 \n"
+ "vrshr.u16 q6, q6, #2 \n"
+
+ "vmul.s16 q8, q4, q10 \n" // B
+ "vmls.s16 q8, q5, q11 \n" // G
+ "vmls.s16 q8, q6, q12 \n" // R
+ "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned
+ "vmul.s16 q9, q6, q10 \n" // R
+ "vmls.s16 q9, q5, q14 \n" // G
+ "vmls.s16 q9, q4, q13 \n" // B
+ "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned
"vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U
"vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(src_stride_rgb565), // %1
"+r"(dst_u), // %2
@@ -1711,55 +2281,54 @@ void ARGB1555ToUVRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_v,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
+ "add %1, %0, %1 \n" // src_stride + src_argb
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
RGB555TOARGB
- "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%0]! \n" // next 8 ARGB1555 pixels.
+ "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%0]! \n" // next 8 ARGB1555 pixels.
RGB555TOARGB
- "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+ "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // load 8 ARGB1555 pixels.
+ "vld1.8 {q0}, [%1]! \n" // load 8 ARGB1555 pixels.
RGB555TOARGB
- "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // next 8 ARGB1555 pixels.
+ "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%1]! \n" // next 8 ARGB1555 pixels.
RGB555TOARGB
- "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
-
- "vrshr.u16 q4, q4, #1 \n" // 2x average
- "vrshr.u16 q5, q5, #1 \n"
- "vrshr.u16 q6, q6, #1 \n"
-
- "subs %4, %4, #16 \n" // 16 processed per loop.
- "vmul.s16 q8, q4, q10 \n" // B
- "vmls.s16 q8, q5, q11 \n" // G
- "vmls.s16 q8, q6, q12 \n" // R
- "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned
- "vmul.s16 q9, q6, q10 \n" // R
- "vmls.s16 q9, q5, q14 \n" // G
- "vmls.s16 q9, q4, q13 \n" // B
- "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned
+ "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+
+ "vrshr.u16 q4, q4, #2 \n" // average of 4
+ "vrshr.u16 q5, q5, #2 \n"
+ "vrshr.u16 q6, q6, #2 \n"
+
+ "vmul.s16 q8, q4, q10 \n" // B
+ "vmls.s16 q8, q5, q11 \n" // G
+ "vmls.s16 q8, q6, q12 \n" // R
+ "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned
+ "vmul.s16 q9, q6, q10 \n" // R
+ "vmls.s16 q9, q5, q14 \n" // G
+ "vmls.s16 q9, q4, q13 \n" // B
+ "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned
"vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U
"vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(src_stride_argb1555), // %1
"+r"(dst_u), // %2
@@ -1776,56 +2345,46 @@ void ARGB4444ToUVRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
- asm volatile(
- "add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
+ asm volatile (
+ "add %1, %0, %1 \n" // src_stride + src_argb
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
ARGB4444TOARGB
- "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%0]! \n" // next 8 ARGB4444 pixels.
+ "vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%0]! \n" // next 8 ARGB4444 pixels.
ARGB4444TOARGB
- "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+ "vpaddl.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpaddl.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpaddl.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // load 8 ARGB4444 pixels.
+ "vld1.8 {q0}, [%1]! \n" // load 8 ARGB4444 pixels.
ARGB4444TOARGB
- "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
- "vld1.8 {q0}, [%1]! \n" // next 8 ARGB4444 pixels.
+ "vpadal.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d12, d2 \n" // R 8 bytes -> 4 shorts.
+ "vld1.8 {q0}, [%1]! \n" // next 8 ARGB4444 pixels.
ARGB4444TOARGB
- "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
- "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
- "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
-
- "vrshr.u16 q4, q4, #1 \n" // 2x average
- "vrshr.u16 q5, q5, #1 \n"
- "vrshr.u16 q6, q6, #1 \n"
-
- "subs %4, %4, #16 \n" // 16 processed per loop.
- "vmul.s16 q8, q4, q10 \n" // B
- "vmls.s16 q8, q5, q11 \n" // G
- "vmls.s16 q8, q6, q12 \n" // R
- "vadd.u16 q8, q8, q15 \n" // +128 -> unsigned
- "vmul.s16 q9, q6, q10 \n" // R
- "vmls.s16 q9, q5, q14 \n" // G
- "vmls.s16 q9, q4, q13 \n" // B
- "vadd.u16 q9, q9, q15 \n" // +128 -> unsigned
- "vqshrn.u16 d0, q8, #8 \n" // 16 bit to 8 bit U
- "vqshrn.u16 d1, q9, #8 \n" // 16 bit to 8 bit V
- "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
- "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
- "bgt 1b \n"
+ "vpadal.u8 d9, d0 \n" // B 8 bytes -> 4 shorts.
+ "vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
+ "vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
+
+ "vrshr.u16 q0, q4, #2 \n" // average of 4
+ "vrshr.u16 q1, q5, #2 \n"
+ "vrshr.u16 q2, q6, #2 \n"
+
+ RGBTOUV(q0, q1, q2)
+ "vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
+ "vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
+ "bgt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(src_stride_argb4444), // %1
"+r"(dst_u), // %2
@@ -1838,21 +2397,21 @@ void ARGB4444ToUVRow_NEON(const uint8_t* src_argb4444,
void RGB565ToYRow_NEON(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
asm volatile(
- "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d24, #25 \n" // B * 0.1016 coefficient
+ "vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
+ "vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
+ "vmov.u8 d27, #16 \n" // Add 16 constant
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
RGB565TOARGB
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d27 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
+ "vmull.u8 q2, d0, d24 \n" // B
+ "vmlal.u8 q2, d1, d25 \n" // G
+ "vmlal.u8 q2, d2, d26 \n" // R
+ "vqrshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit Y
+ "vqadd.u8 d0, d27 \n"
+ "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
+ "bgt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1864,21 +2423,21 @@ void ARGB1555ToYRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_y,
int width) {
asm volatile(
- "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d24, #25 \n" // B * 0.1016 coefficient
+ "vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
+ "vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
+ "vmov.u8 d27, #16 \n" // Add 16 constant
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGB1555TOARGB
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d27 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
+ "vmull.u8 q2, d0, d24 \n" // B
+ "vmlal.u8 q2, d1, d25 \n" // G
+ "vmlal.u8 q2, d2, d26 \n" // R
+ "vqrshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit Y
+ "vqadd.u8 d0, d27 \n"
+ "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
+ "bgt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1890,21 +2449,21 @@ void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_y,
int width) {
asm volatile(
- "vmov.u8 d24, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d25, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d26, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 d24, #25 \n" // B * 0.1016 coefficient
+ "vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
+ "vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
+ "vmov.u8 d27, #16 \n" // Add 16 constant
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
ARGB4444TOARGB
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d27 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
+ "vmull.u8 q2, d0, d24 \n" // B
+ "vmlal.u8 q2, d1, d25 \n" // G
+ "vmlal.u8 q2, d2, d26 \n" // R
+ "vqrshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit Y
+ "vqadd.u8 d0, d27 \n"
+ "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
+ "bgt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1912,119 +2471,178 @@ void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
: "cc", "memory", "q0", "q1", "q2", "q3", "q12", "q13");
}
-void BGRAToYRow_NEON(const uint8_t* src_bgra, uint8_t* dst_y, int width) {
+void ARGBToAR64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
asm volatile(
- "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d7, #16 \n" // Add 16 constant
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of BGRA.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q8, d1, d4 \n" // R
- "vmlal.u8 q8, d2, d5 \n" // G
- "vmlal.u8 q8, d3, d6 \n" // B
- "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d7 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(dst_y), // %1
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n"
+ "vld1.8 {q2}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmov.u8 q1, q0 \n"
+ "vmov.u8 q3, q2 \n"
+ "vst2.8 {q0, q1}, [%1]! \n" // store 4 pixels
+ "vst2.8 {q2, q3}, [%1]! \n" // store 4 pixels
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ar64), // %1
"+r"(width) // %2
:
- : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8");
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
-void ABGRToYRow_NEON(const uint8_t* src_abgr, uint8_t* dst_y, int width) {
+static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+
+void ARGBToAB64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
asm volatile(
- "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d7, #16 \n" // Add 16 constant
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ABGR.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q8, d0, d4 \n" // R
- "vmlal.u8 q8, d1, d5 \n" // G
- "vmlal.u8 q8, d2, d6 \n" // B
- "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d7 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8");
+ "vld1.8 {q4}, [%3] \n" // shuffler
+
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n"
+ "vld1.8 {q2}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vtbl.8 d2, {d0, d1}, d8 \n"
+ "vtbl.8 d3, {d0, d1}, d9 \n"
+ "vtbl.8 d6, {d4, d5}, d8 \n"
+ "vtbl.8 d7, {d4, d5}, d9 \n"
+ "vmov.u8 q0, q1 \n"
+ "vmov.u8 q2, q3 \n"
+ "vst2.8 {q0, q1}, [%1]! \n" // store 4 pixels
+ "vst2.8 {q2, q3}, [%1]! \n" // store 4 pixels
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ab64), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleARGBToABGR) // %3
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4");
}
-void RGBAToYRow_NEON(const uint8_t* src_rgba, uint8_t* dst_y, int width) {
+void AR64ToARGBRow_NEON(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "vmov.u8 d4, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d6, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d7, #16 \n" // Add 16 constant
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of RGBA.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q8, d1, d4 \n" // B
- "vmlal.u8 q8, d2, d5 \n" // G
- "vmlal.u8 q8, d3, d6 \n" // R
- "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d7 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_rgba), // %0
- "+r"(dst_y), // %1
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n"
+ "vld1.16 {q1}, [%0]! \n"
+ "vld1.16 {q2}, [%0]! \n"
+ "vld1.16 {q3}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vshrn.u16 d0, q0, #8 \n"
+ "vshrn.u16 d1, q1, #8 \n"
+ "vshrn.u16 d4, q2, #8 \n"
+ "vshrn.u16 d5, q3, #8 \n"
+ "vst1.8 {q0}, [%1]! \n" // store 4 pixels
+ "vst1.8 {q2}, [%1]! \n" // store 4 pixels
+ "bgt 1b \n"
+ : "+r"(src_ar64), // %0
+ "+r"(dst_argb), // %1
"+r"(width) // %2
:
- : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8");
+ : "cc", "memory", "q0", "q1", "q2", "q3");
}
-void RGB24ToYRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_y, int width) {
+static const uvec8 kShuffleAB64ToARGB = {5, 3, 1, 7, 13, 11, 9, 15};
+
+void AB64ToARGBRow_NEON(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "vmov.u8 d4, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d6, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d7, #16 \n" // Add 16 constant
- "1: \n"
- "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 pixels of RGB24.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q8, d0, d4 \n" // B
- "vmlal.u8 q8, d1, d5 \n" // G
- "vmlal.u8 q8, d2, d6 \n" // R
- "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d7 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_rgb24), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8");
+ "vld1.8 {d8}, [%3] \n" // shuffler
+
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n"
+ "vld1.16 {q1}, [%0]! \n"
+ "vld1.16 {q2}, [%0]! \n"
+ "vld1.16 {q3}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vtbl.8 d0, {d0, d1}, d8 \n"
+ "vtbl.8 d1, {d2, d3}, d8 \n"
+ "vtbl.8 d4, {d4, d5}, d8 \n"
+ "vtbl.8 d5, {d6, d7}, d8 \n"
+ "vst1.8 {q0}, [%1]! \n" // store 4 pixels
+ "vst1.8 {q2}, [%1]! \n" // store 4 pixels
+ "bgt 1b \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleAB64ToARGB) // %3
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4");
}
-void RAWToYRow_NEON(const uint8_t* src_raw, uint8_t* dst_y, int width) {
+// ARGB expects first 3 values to contain RGB and 4th value is ignored.
+void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "vmov.u8 d4, #33 \n" // R * 0.2578 coefficient
- "vmov.u8 d5, #65 \n" // G * 0.5078 coefficient
- "vmov.u8 d6, #13 \n" // B * 0.1016 coefficient
- "vmov.u8 d7, #16 \n" // Add 16 constant
- "1: \n"
- "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 pixels of RAW.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q8, d0, d4 \n" // B
- "vmlal.u8 q8, d1, d5 \n" // G
- "vmlal.u8 q8, d2, d6 \n" // R
- "vqrshrun.s16 d0, q8, #7 \n" // 16 bit to 8 bit Y
- "vqadd.u8 d0, d7 \n"
- "vst1.8 {d0}, [%1]! \n" // store 8 pixels Y.
- "bgt 1b \n"
- : "+r"(src_raw), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "q8");
+ "vld1.8 {d24}, [%3] \n" // load kRGBToY
+ "vld1.16 {d25[0]}, [%4] \n" // load kAddY[0]
+ "vdup.8 d20, d24[0] \n" // B
+ "vdup.8 d21, d24[1] \n" // G
+ "vdup.8 d22, d24[2] \n" // R
+ "vdup.8 d23, d24[3] \n" // A
+ "vdup.16 q12, d25[0] \n" // bias
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 pixels
+ "subs %1, %1, #16 \n" // 16 processed per loop.
+ "vmull.u8 q8, d0, d20 \n" // B
+ "vmull.u8 q9, d1, d20 \n"
+ "vmlal.u8 q8, d2, d21 \n" // G
+ "vmlal.u8 q9, d3, d21 \n"
+ "vmlal.u8 q8, d4, d22 \n" // R
+ "vmlal.u8 q9, d5, d22 \n"
+ "vmlal.u8 q8, d6, d23 \n" // A
+ "vmlal.u8 q9, d7, d23 \n"
+ "vaddhn.u16 d0, q8, q12 \n" // 16 bit to 8 bit Y
+ "vaddhn.u16 d1, q9, q12 \n"
+ "vst1.8 {d0, d1}, [%2]! \n" // store 16 pixels Y.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(width), // %1
+ "+r"(dst_y) // %2
+ : "r"(&c->kRGBToY), // %3
+ "r"(&c->kAddY) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
+ "d24", "d25");
+}
+
+void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "vld1.8 {d24}, [%3] \n" // load kRGBToY
+ "vld1.16 {d25[0]}, [%4] \n" // load kAddY[0]
+ "vdup.8 d20, d24[0] \n" // BY
+ "vdup.8 d21, d24[1] \n" // GY
+ "vdup.8 d22, d24[2] \n" // RY
+ "vdup.16 q12, d25[0] \n" // AY
+ "1: \n"
+ "vld3.8 {d2, d4, d6}, [%0]! \n" // load 16 pixels of
+ // RGB24.
+ "vld3.8 {d3, d5, d7}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 processed per loop.
+ "vmull.u8 q8, d2, d20 \n" // B
+ "vmull.u8 q9, d3, d20 \n"
+ "vmlal.u8 q8, d4, d21 \n" // G
+ "vmlal.u8 q9, d5, d21 \n"
+ "vmlal.u8 q8, d6, d22 \n" // R
+ "vmlal.u8 q9, d7, d22 \n"
+ "vaddhn.u16 d0, q8, q12 \n" // 16 bit to 8 bit Y
+ "vaddhn.u16 d1, q9, q12 \n"
+ "vst1.8 {d0, d1}, [%1]! \n" // store 16 pixels Y.
+ "bgt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(&c->kRGBToY), // %3
+ "r"(&c->kAddY) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
+ "d24", "d25");
}
// Bilinear filter 16x2 -> 16x1
@@ -2035,115 +2653,272 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
int source_y_fraction) {
int y1_fraction = source_y_fraction;
asm volatile(
- "cmp %4, #0 \n"
- "beq 100f \n"
- "add %2, %1 \n"
- "cmp %4, #128 \n"
- "beq 50f \n"
-
- "vdup.8 d5, %4 \n"
- "rsb %4, #256 \n"
- "vdup.8 d4, %4 \n"
+ "cmp %4, #0 \n"
+ "beq 100f \n"
+ "add %2, %1 \n"
+ "cmp %4, #128 \n"
+ "beq 50f \n"
+
+ "vdup.8 d5, %4 \n"
+ "rsb %4, #256 \n"
+ "vdup.8 d4, %4 \n"
// General purpose row blend.
- "1: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vmull.u8 q13, d0, d4 \n"
- "vmull.u8 q14, d1, d4 \n"
- "vmlal.u8 q13, d2, d5 \n"
- "vmlal.u8 q14, d3, d5 \n"
- "vrshrn.u16 d0, q13, #8 \n"
- "vrshrn.u16 d1, q14, #8 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 1b \n"
- "b 99f \n"
+ "1: \n"
+ "vld1.8 {q0}, [%1]! \n"
+ "vld1.8 {q1}, [%2]! \n"
+ "subs %3, %3, #16 \n"
+ "vmull.u8 q13, d0, d4 \n"
+ "vmull.u8 q14, d1, d4 \n"
+ "vmlal.u8 q13, d2, d5 \n"
+ "vmlal.u8 q14, d3, d5 \n"
+ "vrshrn.u16 d0, q13, #8 \n"
+ "vrshrn.u16 d1, q14, #8 \n"
+ "vst1.8 {q0}, [%0]! \n"
+ "bgt 1b \n"
+ "b 99f \n"
// Blend 50 / 50.
- "50: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 50b \n"
- "b 99f \n"
+ "50: \n"
+ "vld1.8 {q0}, [%1]! \n"
+ "vld1.8 {q1}, [%2]! \n"
+ "subs %3, %3, #16 \n"
+ "vrhadd.u8 q0, q1 \n"
+ "vst1.8 {q0}, [%0]! \n"
+ "bgt 50b \n"
+ "b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
- "vld1.8 {q0}, [%1]! \n"
- "subs %3, %3, #16 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 100b \n"
+ "100: \n"
+ "vld1.8 {q0}, [%1]! \n"
+ "subs %3, %3, #16 \n"
+ "vst1.8 {q0}, [%0]! \n"
+ "bgt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_stride), // %2
"+r"(dst_width), // %3
"+r"(y1_fraction) // %4
:
- : "cc", "memory", "q0", "q1", "d4", "d5", "q13", "q14");
+ : "cc", "memory", "q0", "q1", "d4", "d5", "q13", "q14");
+}
+
+// Bilinear filter 8x2 -> 8x1
+void InterpolateRow_16_NEON(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ asm volatile(
+ "cmp %4, #0 \n"
+ "beq 100f \n"
+ "cmp %4, #128 \n"
+ "beq 50f \n"
+
+ "vdup.16 d17, %4 \n"
+ "vdup.16 d16, %5 \n"
+ // General purpose row blend.
+ "1: \n"
+ "vld1.16 {q0}, [%1]! \n"
+ "vld1.16 {q1}, [%2]! \n"
+ "subs %3, %3, #8 \n"
+ "vmull.u16 q2, d0, d16 \n"
+ "vmull.u16 q3, d1, d16 \n"
+ "vmlal.u16 q2, d2, d17 \n"
+ "vmlal.u16 q3, d3, d17 \n"
+ "vrshrn.u32 d0, q2, #8 \n"
+ "vrshrn.u32 d1, q3, #8 \n"
+ "vst1.16 {q0}, [%0]! \n"
+ "bgt 1b \n"
+ "b 99f \n"
+
+ // Blend 50 / 50.
+ "50: \n"
+ "vld1.16 {q0}, [%1]! \n"
+ "vld1.16 {q1}, [%2]! \n"
+ "subs %3, %3, #8 \n"
+ "vrhadd.u16 q0, q1 \n"
+ "vst1.16 {q0}, [%0]! \n"
+ "bgt 50b \n"
+ "b 99f \n"
+
+ // Blend 100 / 0 - Copy row unchanged.
+ "100: \n"
+ "vld1.16 {q0}, [%1]! \n"
+ "subs %3, %3, #8 \n"
+ "vst1.16 {q0}, [%0]! \n"
+ "bgt 100b \n"
+
+ "99: \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(src_ptr1), // %2
+ "+r"(dst_width) // %3
+ : "r"(y1_fraction), // %4
+ "r"(y0_fraction) // %5
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q8");
+}
+
+// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
+void ARGBBlendRow_NEON(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "subs %3, #8 \n"
+ "blt 89f \n"
+ // Blend 8 pixels.
+ "8: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB0.
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 pixels of ARGB1.
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vmull.u8 q10, d4, d3 \n" // db * a
+ "vmull.u8 q11, d5, d3 \n" // dg * a
+ "vmull.u8 q12, d6, d3 \n" // dr * a
+ "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8
+ "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8
+ "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8
+ "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256
+ "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256
+ "vqadd.u8 q0, q0, q2 \n" // + sbg
+ "vqadd.u8 d2, d2, d6 \n" // + sr
+ "vmov.u8 d3, #255 \n" // a = 255
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 pixels of ARGB.
+ "bge 8b \n"
+
+ "89: \n"
+ "adds %3, #8-1 \n"
+ "blt 99f \n"
+
+ // Blend 1 pixels.
+ "1: \n"
+ "vld4.8 {d0[0],d1[0],d2[0],d3[0]}, [%0]! \n" // load 1 pixel ARGB0.
+ "vld4.8 {d4[0],d5[0],d6[0],d7[0]}, [%1]! \n" // load 1 pixel ARGB1.
+ "subs %3, %3, #1 \n" // 1 processed per loop.
+ "vmull.u8 q10, d4, d3 \n" // db * a
+ "vmull.u8 q11, d5, d3 \n" // dg * a
+ "vmull.u8 q12, d6, d3 \n" // dr * a
+ "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8
+ "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8
+ "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8
+ "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256
+ "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256
+ "vqadd.u8 q0, q0, q2 \n" // + sbg
+ "vqadd.u8 d2, d2, d6 \n" // + sr
+ "vmov.u8 d3, #255 \n" // a = 255
+ "vst4.8 {d0[0],d1[0],d2[0],d3[0]}, [%2]! \n" // store 1 pixel.
+ "bge 1b \n"
+
+ "99: \n"
+
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12");
}
-// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
-void ARGBBlendRow_NEON(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
+void ARGBOrRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
asm volatile(
- "subs %3, #8 \n"
- "blt 89f \n"
- // Blend 8 pixels.
- "8: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB0.
- "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 pixels of ARGB1.
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vmull.u8 q10, d4, d3 \n" // db * a
- "vmull.u8 q11, d5, d3 \n" // dg * a
- "vmull.u8 q12, d6, d3 \n" // dr * a
- "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8
- "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8
- "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8
- "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256
- "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256
- "vqadd.u8 q0, q0, q2 \n" // + sbg
- "vqadd.u8 d2, d2, d6 \n" // + sr
- "vmov.u8 d3, #255 \n" // a = 255
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 pixels of ARGB.
- "bge 8b \n"
-
- "89: \n"
- "adds %3, #8-1 \n"
- "blt 99f \n"
+ "vdup.32 q2, %3 \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n"
+ "subs %2, %2, #8 \n"
+ "vorr q0, q0, q2 \n"
+ "vorr q1, q1, q2 \n"
+ "vst1.8 {q0, q1}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(value) // %3
+ : "memory", "cc", "q0", "q1", "q2");
+}
- // Blend 1 pixels.
+void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "vmov.u8 d28, #255 \n"
+ "vmov.i16 q15, #1 \n"
"1: \n"
- "vld4.8 {d0[0],d1[0],d2[0],d3[0]}, [%0]! \n" // load 1 pixel ARGB0.
- "vld4.8 {d4[0],d5[0],d6[0],d7[0]}, [%1]! \n" // load 1 pixel ARGB1.
- "subs %3, %3, #1 \n" // 1 processed per loop.
- "vmull.u8 q10, d4, d3 \n" // db * a
- "vmull.u8 q11, d5, d3 \n" // dg * a
- "vmull.u8 q12, d6, d3 \n" // dr * a
- "vqrshrn.u16 d20, q10, #8 \n" // db >>= 8
- "vqrshrn.u16 d21, q11, #8 \n" // dg >>= 8
- "vqrshrn.u16 d22, q12, #8 \n" // dr >>= 8
- "vqsub.u8 q2, q2, q10 \n" // dbg - dbg * a / 256
- "vqsub.u8 d6, d6, d22 \n" // dr - dr * a / 256
- "vqadd.u8 q0, q0, q2 \n" // + sbg
- "vqadd.u8 d2, d2, d6 \n" // + sr
- "vmov.u8 d3, #255 \n" // a = 255
- "vst4.8 {d0[0],d1[0],d2[0],d3[0]}, [%2]! \n" // store 1 pixel.
- "bge 1b \n"
-
- "99: \n"
-
- : "+r"(src_argb0), // %0
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n"
+ "subs %3, %3, #8 \n"
+ "vmov q8, q15 \n"
+ "vmlal.u8 q8, d7, d3 \n"
+ "vsra.u16 q8, q8, #8 \n"
+ "vshrn.i16 d16, q8, #8 \n"
+ "vsub.i8 d16, d3, d16 \n"
+ "vadd.i8 d7, d7, d16 \n"
+ "vceq.i8 d17, d7, #0 \n"
+ "vorr d18, d3, d17 \n"
+ "vorr d19, d7, d17 \n"
+ "vmull.u8 q8, d18, d28 \n"
+ "vmovl.u8 q9, d19 \n"
+ "vmovl.u16 q10, d16 \n"
+ "vmovl.u16 q11, d17 \n"
+ "vmovl.u16 q12, d18 \n"
+ "vmovl.u16 q13, d19 \n"
+ "vcvt.f32.u32 q10, q10 \n"
+ "vcvt.f32.u32 q11, q11 \n"
+ "vcvt.f32.u32 q12, q12 \n"
+ "vcvt.f32.u32 q13, q13 \n"
+ "vrecpe.f32 q8, q12 \n"
+ "vrecpe.f32 q9, q13 \n"
+ "vrecps.f32 q4, q12, q8 \n"
+ "vrecps.f32 q5, q13, q9 \n"
+ "vmul.f32 q8, q8, q4 \n"
+ "vmul.f32 q9, q9, q5 \n"
+ "vrecps.f32 q4, q12, q8 \n"
+ "vrecps.f32 q5, q13, q9 \n"
+ "vmul.f32 q8, q8, q4 \n"
+ "vmul.f32 q9, q9, q5 \n"
+ "vsra.u32 q8, q8, #29 \n"
+ "vsra.u32 q9, q9, #29 \n"
+ "vmul.f32 q10, q10, q8 \n"
+ "vmul.f32 q11, q11, q9 \n"
+ "vcvt.u32.f32 q10, q10 \n"
+ "vcvt.u32.f32 q11, q11 \n"
+ "vmovn.i32 d16, q10 \n"
+ "vmovn.i32 d17, q11 \n"
+ "vmovn.i16 d16, q8 \n"
+ "vsub.i8 d17, d28, d16 \n"
+ "vmov q9, q15 \n"
+ "vmov q10, q15 \n"
+ "vmov q11, q15 \n"
+ "vmlal.u8 q9, d4, d17 \n"
+ "vmlal.u8 q10, d5, d17 \n"
+ "vmlal.u8 q11, d6, d17 \n"
+ "vmlal.u8 q9, d0, d16 \n"
+ "vmlal.u8 q10, d1, d16 \n"
+ "vmlal.u8 q11, d2, d16 \n"
+ "vsra.u16 q9, q9, #8 \n"
+ "vsra.u16 q10, q10, #8 \n"
+ "vsra.u16 q11, q11, #8 \n"
+ "vshrn.i16 d0, q9, #8 \n"
+ "vshrn.i16 d1, q10, #8 \n"
+ "vshrn.i16 d2, q11, #8 \n"
+ "vmov d3, d7 \n"
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
:
- : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12");
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", "q10",
+ "q11", "q12", "q13", "q14", "q15");
}
// Attenuate 8 pixels at a time.
@@ -2151,23 +2926,25 @@ void ARGBAttenuateRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
asm volatile(
+ "vmov.u16 q15, #0x00ff \n" // 255 for rounding up
+
// Attenuate 8 pixels.
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q10, d0, d3 \n" // b * a
- "vmull.u8 q11, d1, d3 \n" // g * a
- "vmull.u8 q12, d2, d3 \n" // r * a
- "vqrshrn.u16 d0, q10, #8 \n" // b >>= 8
- "vqrshrn.u16 d1, q11, #8 \n" // g >>= 8
- "vqrshrn.u16 d2, q12, #8 \n" // r >>= 8
- "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmull.u8 q10, d0, d3 \n" // b * a
+ "vmull.u8 q11, d1, d3 \n" // g * a
+ "vmull.u8 q12, d2, d3 \n" // r * a
+ "vaddhn.u16 d0, q10, q15 \n" // (b + 255) >> 8
+ "vaddhn.u16 d1, q11, q15 \n" // (g + 255) >> 8
+ "vaddhn.u16 d2, q12, q15 \n" // (r + 255) >> 8
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
:
- : "cc", "memory", "q0", "q1", "q10", "q11", "q12");
+ : "cc", "memory", "q0", "q1", "q10", "q11", "q12", "q15");
}
// Quantize 8 ARGB pixels (32 bytes).
@@ -2178,32 +2955,32 @@ void ARGBQuantizeRow_NEON(uint8_t* dst_argb,
int interval_offset,
int width) {
asm volatile(
- "vdup.u16 q8, %2 \n"
- "vshr.u16 q8, q8, #1 \n" // scale >>= 1
- "vdup.u16 q9, %3 \n" // interval multiply.
- "vdup.u16 q10, %4 \n" // interval add
+ "vdup.u16 q8, %2 \n"
+ "vshr.u16 q8, q8, #1 \n" // scale >>= 1
+ "vdup.u16 q9, %3 \n" // interval multiply.
+ "vdup.u16 q10, %4 \n" // interval add
// 8 pixel loop.
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0] \n" // load 8 pixels of ARGB.
- "subs %1, %1, #8 \n" // 8 processed per loop.
- "vmovl.u8 q0, d0 \n" // b (0 .. 255)
- "vmovl.u8 q1, d2 \n"
- "vmovl.u8 q2, d4 \n"
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0] \n" // load 8 pixels of ARGB.
+ "subs %1, %1, #8 \n" // 8 processed per loop.
+ "vmovl.u8 q0, d0 \n" // b (0 .. 255)
+ "vmovl.u8 q1, d2 \n"
+ "vmovl.u8 q2, d4 \n"
"vqdmulh.s16 q0, q0, q8 \n" // b * scale
"vqdmulh.s16 q1, q1, q8 \n" // g
"vqdmulh.s16 q2, q2, q8 \n" // r
- "vmul.u16 q0, q0, q9 \n" // b * interval_size
- "vmul.u16 q1, q1, q9 \n" // g
- "vmul.u16 q2, q2, q9 \n" // r
- "vadd.u16 q0, q0, q10 \n" // b + interval_offset
- "vadd.u16 q1, q1, q10 \n" // g
- "vadd.u16 q2, q2, q10 \n" // r
- "vqmovn.u16 d0, q0 \n"
- "vqmovn.u16 d2, q1 \n"
- "vqmovn.u16 d4, q2 \n"
- "vst4.8 {d0, d2, d4, d6}, [%0]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vmul.u16 q0, q0, q9 \n" // b * interval_size
+ "vmul.u16 q1, q1, q9 \n" // g
+ "vmul.u16 q2, q2, q9 \n" // r
+ "vadd.u16 q0, q0, q10 \n" // b + interval_offset
+ "vadd.u16 q1, q1, q10 \n" // g
+ "vadd.u16 q2, q2, q10 \n" // r
+ "vqmovn.u16 d0, q0 \n"
+ "vqmovn.u16 d2, q1 \n"
+ "vqmovn.u16 d4, q2 \n"
+ "vst4.8 {d0, d2, d4, d6}, [%0]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
: "r"(scale), // %2
@@ -2220,28 +2997,28 @@ void ARGBShadeRow_NEON(const uint8_t* src_argb,
int width,
uint32_t value) {
asm volatile(
- "vdup.u32 q0, %3 \n" // duplicate scale value.
- "vzip.u8 d0, d1 \n" // d0 aarrggbb.
- "vshr.u16 q0, q0, #1 \n" // scale / 2.
+ "vdup.u32 q0, %3 \n" // duplicate scale value.
+ "vzip.u8 d0, d1 \n" // d0 aarrggbb.
+ "vshr.u16 q0, q0, #1 \n" // scale / 2.
// 8 pixel loop.
- "1: \n"
- "vld4.8 {d20, d22, d24, d26}, [%0]! \n" // load 8 pixels of ARGB.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmovl.u8 q10, d20 \n" // b (0 .. 255)
- "vmovl.u8 q11, d22 \n"
- "vmovl.u8 q12, d24 \n"
- "vmovl.u8 q13, d26 \n"
+ "1: \n"
+ "vld4.8 {d20, d22, d24, d26}, [%0]! \n" // load 8 pixels of ARGB.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmovl.u8 q10, d20 \n" // b (0 .. 255)
+ "vmovl.u8 q11, d22 \n"
+ "vmovl.u8 q12, d24 \n"
+ "vmovl.u8 q13, d26 \n"
"vqrdmulh.s16 q10, q10, d0[0] \n" // b * scale * 2
"vqrdmulh.s16 q11, q11, d0[1] \n" // g
"vqrdmulh.s16 q12, q12, d0[2] \n" // r
"vqrdmulh.s16 q13, q13, d0[3] \n" // a
- "vqmovn.u16 d20, q10 \n"
- "vqmovn.u16 d22, q11 \n"
- "vqmovn.u16 d24, q12 \n"
- "vqmovn.u16 d26, q13 \n"
- "vst4.8 {d20, d22, d24, d26}, [%1]! \n" // store 8 pixels of ARGB.
- "bgt 1b \n"
+ "vqmovn.u16 d20, q10 \n"
+ "vqmovn.u16 d22, q11 \n"
+ "vqmovn.u16 d24, q12 \n"
+ "vqmovn.u16 d26, q13 \n"
+ "vst4.8 {d20, d22, d24, d26}, [%1]! \n" // store 8 pixels of ARGB.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2251,23 +3028,23 @@ void ARGBShadeRow_NEON(const uint8_t* src_argb,
// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels
// Similar to ARGBToYJ but stores ARGB.
-// C code is (15 * b + 75 * g + 38 * r + 64) >> 7;
+// C code is (29 * b + 150 * g + 77 * r + 128) >> 8;
void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
asm volatile(
- "vmov.u8 d24, #15 \n" // B * 0.11400 coefficient
- "vmov.u8 d25, #75 \n" // G * 0.58700 coefficient
- "vmov.u8 d26, #38 \n" // R * 0.29900 coefficient
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmull.u8 q2, d0, d24 \n" // B
- "vmlal.u8 q2, d1, d25 \n" // G
- "vmlal.u8 q2, d2, d26 \n" // R
- "vqrshrun.s16 d0, q2, #7 \n" // 15 bit to 8 bit B
- "vmov d1, d0 \n" // G
- "vmov d2, d0 \n" // R
- "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
+ "vmov.u8 d24, #29 \n" // B * 0.1140 coefficient
+ "vmov.u8 d25, #150 \n" // G * 0.5870 coefficient
+ "vmov.u8 d26, #77 \n" // R * 0.2990 coefficient
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmull.u8 q2, d0, d24 \n" // B
+ "vmlal.u8 q2, d1, d25 \n" // G
+ "vmlal.u8 q2, d2, d26 \n" // R
+ "vqrshrn.u16 d0, q2, #8 \n" // 16 bit to 8 bit B
+ "vmov d1, d0 \n" // G
+ "vmov d2, d0 \n" // R
+ "vst4.8 {d0, d1, d2, d3}, [%1]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2281,37 +3058,37 @@ void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
// r = (r * 50 + g * 98 + b * 24) >> 7
void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width) {
asm volatile(
- "vmov.u8 d20, #17 \n" // BB coefficient
- "vmov.u8 d21, #68 \n" // BG coefficient
- "vmov.u8 d22, #35 \n" // BR coefficient
- "vmov.u8 d24, #22 \n" // GB coefficient
- "vmov.u8 d25, #88 \n" // GG coefficient
- "vmov.u8 d26, #45 \n" // GR coefficient
- "vmov.u8 d28, #24 \n" // BB coefficient
- "vmov.u8 d29, #98 \n" // BG coefficient
- "vmov.u8 d30, #50 \n" // BR coefficient
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0] \n" // load 8 ARGB pixels.
- "subs %1, %1, #8 \n" // 8 processed per loop.
- "vmull.u8 q2, d0, d20 \n" // B to Sepia B
- "vmlal.u8 q2, d1, d21 \n" // G
- "vmlal.u8 q2, d2, d22 \n" // R
- "vmull.u8 q3, d0, d24 \n" // B to Sepia G
- "vmlal.u8 q3, d1, d25 \n" // G
- "vmlal.u8 q3, d2, d26 \n" // R
- "vmull.u8 q8, d0, d28 \n" // B to Sepia R
- "vmlal.u8 q8, d1, d29 \n" // G
- "vmlal.u8 q8, d2, d30 \n" // R
- "vqshrn.u16 d0, q2, #7 \n" // 16 bit to 8 bit B
- "vqshrn.u16 d1, q3, #7 \n" // 16 bit to 8 bit G
- "vqshrn.u16 d2, q8, #7 \n" // 16 bit to 8 bit R
- "vst4.8 {d0, d1, d2, d3}, [%0]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
+ "vmov.u8 d20, #17 \n" // BB coefficient
+ "vmov.u8 d21, #68 \n" // BG coefficient
+ "vmov.u8 d22, #35 \n" // BR coefficient
+ "vmov.u8 d24, #22 \n" // GB coefficient
+ "vmov.u8 d25, #88 \n" // GG coefficient
+ "vmov.u8 d26, #45 \n" // GR coefficient
+ "vmov.u8 d28, #24 \n" // BB coefficient
+ "vmov.u8 d29, #98 \n" // BG coefficient
+ "vmov.u8 d30, #50 \n" // BR coefficient
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0] \n" // load 8 ARGB pixels.
+ "subs %1, %1, #8 \n" // 8 processed per loop.
+ "vmull.u8 q2, d0, d20 \n" // B to Sepia B
+ "vmlal.u8 q2, d1, d21 \n" // G
+ "vmlal.u8 q2, d2, d22 \n" // R
+ "vmull.u8 q3, d0, d24 \n" // B to Sepia G
+ "vmlal.u8 q3, d1, d25 \n" // G
+ "vmlal.u8 q3, d2, d26 \n" // R
+ "vmull.u8 q8, d0, d28 \n" // B to Sepia R
+ "vmlal.u8 q8, d1, d29 \n" // G
+ "vmlal.u8 q8, d2, d30 \n" // R
+ "vqshrn.u16 d0, q2, #7 \n" // 16 bit to 8 bit B
+ "vqshrn.u16 d1, q3, #7 \n" // 16 bit to 8 bit G
+ "vqshrn.u16 d2, q8, #7 \n" // 16 bit to 8 bit R
+ "vst4.8 {d0, d1, d2, d3}, [%0]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
:
- : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12", "q13",
- "q14", "q15");
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q10", "q11", "q12",
+ "q13", "q14", "q15");
}
// Tranform 8 ARGB pixels (32 bytes) with color matrix.
@@ -2322,51 +3099,51 @@ void ARGBColorMatrixRow_NEON(const uint8_t* src_argb,
const int8_t* matrix_argb,
int width) {
asm volatile(
- "vld1.8 {q2}, [%3] \n" // load 3 ARGB vectors.
- "vmovl.s8 q0, d4 \n" // B,G coefficients s16.
- "vmovl.s8 q1, d5 \n" // R,A coefficients s16.
-
- "1: \n"
- "vld4.8 {d16, d18, d20, d22}, [%0]! \n" // load 8 ARGB pixels.
- "subs %2, %2, #8 \n" // 8 processed per loop.
- "vmovl.u8 q8, d16 \n" // b (0 .. 255) 16 bit
- "vmovl.u8 q9, d18 \n" // g
- "vmovl.u8 q10, d20 \n" // r
- "vmovl.u8 q11, d22 \n" // a
- "vmul.s16 q12, q8, d0[0] \n" // B = B * Matrix B
- "vmul.s16 q13, q8, d1[0] \n" // G = B * Matrix G
- "vmul.s16 q14, q8, d2[0] \n" // R = B * Matrix R
- "vmul.s16 q15, q8, d3[0] \n" // A = B * Matrix A
- "vmul.s16 q4, q9, d0[1] \n" // B += G * Matrix B
- "vmul.s16 q5, q9, d1[1] \n" // G += G * Matrix G
- "vmul.s16 q6, q9, d2[1] \n" // R += G * Matrix R
- "vmul.s16 q7, q9, d3[1] \n" // A += G * Matrix A
- "vqadd.s16 q12, q12, q4 \n" // Accumulate B
- "vqadd.s16 q13, q13, q5 \n" // Accumulate G
- "vqadd.s16 q14, q14, q6 \n" // Accumulate R
- "vqadd.s16 q15, q15, q7 \n" // Accumulate A
- "vmul.s16 q4, q10, d0[2] \n" // B += R * Matrix B
- "vmul.s16 q5, q10, d1[2] \n" // G += R * Matrix G
- "vmul.s16 q6, q10, d2[2] \n" // R += R * Matrix R
- "vmul.s16 q7, q10, d3[2] \n" // A += R * Matrix A
- "vqadd.s16 q12, q12, q4 \n" // Accumulate B
- "vqadd.s16 q13, q13, q5 \n" // Accumulate G
- "vqadd.s16 q14, q14, q6 \n" // Accumulate R
- "vqadd.s16 q15, q15, q7 \n" // Accumulate A
- "vmul.s16 q4, q11, d0[3] \n" // B += A * Matrix B
- "vmul.s16 q5, q11, d1[3] \n" // G += A * Matrix G
- "vmul.s16 q6, q11, d2[3] \n" // R += A * Matrix R
- "vmul.s16 q7, q11, d3[3] \n" // A += A * Matrix A
- "vqadd.s16 q12, q12, q4 \n" // Accumulate B
- "vqadd.s16 q13, q13, q5 \n" // Accumulate G
- "vqadd.s16 q14, q14, q6 \n" // Accumulate R
- "vqadd.s16 q15, q15, q7 \n" // Accumulate A
+ "vld1.8 {q2}, [%3] \n" // load 3 ARGB vectors.
+ "vmovl.s8 q0, d4 \n" // B,G coefficients s16.
+ "vmovl.s8 q1, d5 \n" // R,A coefficients s16.
+
+ "1: \n"
+ "vld4.8 {d16, d18, d20, d22}, [%0]! \n" // load 8 ARGB pixels.
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vmovl.u8 q8, d16 \n" // b (0 .. 255) 16 bit
+ "vmovl.u8 q9, d18 \n" // g
+ "vmovl.u8 q10, d20 \n" // r
+ "vmovl.u8 q11, d22 \n" // a
+ "vmul.s16 q12, q8, d0[0] \n" // B = B * Matrix B
+ "vmul.s16 q13, q8, d1[0] \n" // G = B * Matrix G
+ "vmul.s16 q14, q8, d2[0] \n" // R = B * Matrix R
+ "vmul.s16 q15, q8, d3[0] \n" // A = B * Matrix A
+ "vmul.s16 q4, q9, d0[1] \n" // B += G * Matrix B
+ "vmul.s16 q5, q9, d1[1] \n" // G += G * Matrix G
+ "vmul.s16 q6, q9, d2[1] \n" // R += G * Matrix R
+ "vmul.s16 q7, q9, d3[1] \n" // A += G * Matrix A
+ "vqadd.s16 q12, q12, q4 \n" // Accumulate B
+ "vqadd.s16 q13, q13, q5 \n" // Accumulate G
+ "vqadd.s16 q14, q14, q6 \n" // Accumulate R
+ "vqadd.s16 q15, q15, q7 \n" // Accumulate A
+ "vmul.s16 q4, q10, d0[2] \n" // B += R * Matrix B
+ "vmul.s16 q5, q10, d1[2] \n" // G += R * Matrix G
+ "vmul.s16 q6, q10, d2[2] \n" // R += R * Matrix R
+ "vmul.s16 q7, q10, d3[2] \n" // A += R * Matrix A
+ "vqadd.s16 q12, q12, q4 \n" // Accumulate B
+ "vqadd.s16 q13, q13, q5 \n" // Accumulate G
+ "vqadd.s16 q14, q14, q6 \n" // Accumulate R
+ "vqadd.s16 q15, q15, q7 \n" // Accumulate A
+ "vmul.s16 q4, q11, d0[3] \n" // B += A * Matrix B
+ "vmul.s16 q5, q11, d1[3] \n" // G += A * Matrix G
+ "vmul.s16 q6, q11, d2[3] \n" // R += A * Matrix R
+ "vmul.s16 q7, q11, d3[3] \n" // A += A * Matrix A
+ "vqadd.s16 q12, q12, q4 \n" // Accumulate B
+ "vqadd.s16 q13, q13, q5 \n" // Accumulate G
+ "vqadd.s16 q14, q14, q6 \n" // Accumulate R
+ "vqadd.s16 q15, q15, q7 \n" // Accumulate A
"vqshrun.s16 d16, q12, #6 \n" // 16 bit to 8 bit B
"vqshrun.s16 d18, q13, #6 \n" // 16 bit to 8 bit G
"vqshrun.s16 d20, q14, #6 \n" // 16 bit to 8 bit R
"vqshrun.s16 d22, q15, #6 \n" // 16 bit to 8 bit A
- "vst4.8 {d16, d18, d20, d22}, [%1]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
+ "vst4.8 {d16, d18, d20, d22}, [%1]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2376,27 +3153,27 @@ void ARGBColorMatrixRow_NEON(const uint8_t* src_argb,
}
// Multiply 2 rows of ARGB pixels together, 8 pixels at a time.
-void ARGBMultiplyRow_NEON(const uint8_t* src_argb0,
+void ARGBMultiplyRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d1, d3, d5, d7}, [%1]! \n" // load 8 more ARGB
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vmull.u8 q0, d0, d1 \n" // multiply B
- "vmull.u8 q1, d2, d3 \n" // multiply G
- "vmull.u8 q2, d4, d5 \n" // multiply R
- "vmull.u8 q3, d6, d7 \n" // multiply A
- "vrshrn.u16 d0, q0, #8 \n" // 16 bit to 8 bit B
- "vrshrn.u16 d1, q1, #8 \n" // 16 bit to 8 bit G
- "vrshrn.u16 d2, q2, #8 \n" // 16 bit to 8 bit R
- "vrshrn.u16 d3, q3, #8 \n" // 16 bit to 8 bit A
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.8 {d1, d3, d5, d7}, [%1]! \n" // load 8 more ARGB
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vmull.u8 q0, d0, d1 \n" // multiply B
+ "vmull.u8 q1, d2, d3 \n" // multiply G
+ "vmull.u8 q2, d4, d5 \n" // multiply R
+ "vmull.u8 q3, d6, d7 \n" // multiply A
+ "vrshrn.u16 d0, q0, #8 \n" // 16 bit to 8 bit B
+ "vrshrn.u16 d1, q1, #8 \n" // 16 bit to 8 bit G
+ "vrshrn.u16 d2, q2, #8 \n" // 16 bit to 8 bit R
+ "vrshrn.u16 d3, q3, #8 \n" // 16 bit to 8 bit A
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2405,21 +3182,21 @@ void ARGBMultiplyRow_NEON(const uint8_t* src_argb0,
}
// Add 2 rows of ARGB pixels together, 8 pixels at a time.
-void ARGBAddRow_NEON(const uint8_t* src_argb0,
+void ARGBAddRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vqadd.u8 q0, q0, q2 \n" // add B, G
- "vqadd.u8 q1, q1, q3 \n" // add R, A
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vqadd.u8 q0, q0, q2 \n" // add B, G
+ "vqadd.u8 q1, q1, q3 \n" // add R, A
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2428,21 +3205,21 @@ void ARGBAddRow_NEON(const uint8_t* src_argb0,
}
// Subtract 2 rows of ARGB pixels, 8 pixels at a time.
-void ARGBSubtractRow_NEON(const uint8_t* src_argb0,
+void ARGBSubtractRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vqsub.u8 q0, q0, q2 \n" // subtract B, G
- "vqsub.u8 q1, q1, q3 \n" // subtract R, A
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vqsub.u8 q0, q0, q2 \n" // subtract B, G
+ "vqsub.u8 q1, q1, q3 \n" // subtract R, A
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2450,6 +3227,72 @@ void ARGBSubtractRow_NEON(const uint8_t* src_argb0,
: "cc", "memory", "q0", "q1", "q2", "q3");
}
+// Blend 32 pixels at a time.
+// dst = (((a) * src0) + ((255 - a) * src1) + 255) >> 8
+void BlendPlaneRow_NEON(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ asm volatile(
+ "vmov.u16 q15, #255 \n"
+ "subs %4, %4, #32 \n"
+ "blt 19f \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load 32 src0
+ "vld1.8 {q2, q3}, [%1]! \n" // load 32 src1
+ "vld1.8 {q8, q9}, [%2]! \n" // load 32 alpha
+ "subs %4, %4, #32 \n" // 32 processed per loop
+ "vmvn.8 q10, q8 \n" // 255 - alpha0
+ "vmvn.8 q11, q9 \n" // 255 - alpha1
+ "vmull.u8 q4, d0, d16 \n" // low src0 * alpha
+ "pld [%0, #448] \n"
+ "vmull.u8 q5, d1, d17 \n" // high src0 * alpha
+ "pld [%1, #448] \n"
+ "vmull.u8 q6, d2, d18 \n"
+ "pld [%2, #448] \n"
+ "vmull.u8 q7, d3, d19 \n"
+ "vmlal.u8 q4, d4, d20 \n" // low + src1 * (255 -
+ // alpha)
+ "vmlal.u8 q5, d5, d21 \n" // high + src1 * (255 -
+ // alpha)
+ "vmlal.u8 q6, d6, d22 \n"
+ "vmlal.u8 q7, d7, d23 \n"
+ "vaddhn.u16 d0, q4, q15 \n" // (low + 255) >> 8
+ "vaddhn.u16 d1, q5, q15 \n" // (high + 255) >> 8
+ "vaddhn.u16 d2, q6, q15 \n"
+ "vaddhn.u16 d3, q7, q15 \n"
+ "vst1.8 {q0, q1}, [%3]! \n" // store 32 dst
+ "bge 1b \n"
+ "19: \n"
+ "adds %4, %4, #32 \n"
+ "ble 99f \n"
+
+ // 16 pixel tail
+ "vld1.8 {q0}, [%0]! \n" // load 16 src0
+ "vld1.8 {q1}, [%1]! \n" // load 16 src1
+ "vld1.8 {q2}, [%2]! \n" // load 16 alpha
+ "vmvn.8 q12, q2 \n" // 255 - alpha
+ "vmull.u8 q4, d0, d4 \n" // low src0 * alpha
+ "vmull.u8 q5, d1, d5 \n" // high src0 * alpha
+ "vmlal.u8 q4, d2, d24 \n" // low + src1 * (255 -
+ // alpha)
+ "vmlal.u8 q5, d3, d25 \n" // high + src1 * (255 -
+ // alpha)
+ "vaddhn.u16 d0, q4, q15 \n" // + 255 >> 8
+ "vaddhn.u16 d1, q5, q15 \n" // + 255 >> 8
+ "vst1.8 {q0}, [%3]! \n" // store 16 dst
+ "99: \n"
+ : "+r"(src0), // %0
+ "+r"(src1), // %1
+ "+r"(alpha), // %2
+ "+r"(dst), // %3
+ "+r"(width) // %4
+ :
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8",
+ "q9", "q10", "q11", "q12", "q15");
+}
+
// Adds Sobel X and Sobel Y and stores Sobel into ARGB.
// A = 255
// R = Sobel
@@ -2460,17 +3303,17 @@ void SobelRow_NEON(const uint8_t* src_sobelx,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d3, #255 \n" // alpha
+ "vmov.u8 d3, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
- "vld1.8 {d0}, [%0]! \n" // load 8 sobelx.
- "vld1.8 {d1}, [%1]! \n" // load 8 sobely.
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vqadd.u8 d0, d0, d1 \n" // add
- "vmov.u8 d1, d0 \n"
- "vmov.u8 d2, d0 \n"
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d0}, [%0]! \n" // load 8 sobelx.
+ "vld1.8 {d1}, [%1]! \n" // load 8 sobely.
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vqadd.u8 d0, d0, d1 \n" // add
+ "vmov.u8 d1, d0 \n"
+ "vmov.u8 d2, d0 \n"
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_argb), // %2
@@ -2486,13 +3329,13 @@ void SobelToPlaneRow_NEON(const uint8_t* src_sobelx,
int width) {
asm volatile(
// 16 pixel loop.
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load 16 sobelx.
- "vld1.8 {q1}, [%1]! \n" // load 16 sobely.
- "subs %3, %3, #16 \n" // 16 processed per loop.
- "vqadd.u8 q0, q0, q1 \n" // add
- "vst1.8 {q0}, [%2]! \n" // store 16 pixels.
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 16 sobelx.
+ "vld1.8 {q1}, [%1]! \n" // load 16 sobely.
+ "subs %3, %3, #16 \n" // 16 processed per loop.
+ "vqadd.u8 q0, q0, q1 \n" // add
+ "vst1.8 {q0}, [%2]! \n" // store 16 pixels.
+ "bgt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_y), // %2
@@ -2511,15 +3354,15 @@ void SobelXYRow_NEON(const uint8_t* src_sobelx,
uint8_t* dst_argb,
int width) {
asm volatile(
- "vmov.u8 d3, #255 \n" // alpha
+ "vmov.u8 d3, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
- "vld1.8 {d2}, [%0]! \n" // load 8 sobelx.
- "vld1.8 {d0}, [%1]! \n" // load 8 sobely.
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vqadd.u8 d1, d0, d2 \n" // add
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d2}, [%0]! \n" // load 8 sobelx.
+ "vld1.8 {d0}, [%1]! \n" // load 8 sobely.
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vqadd.u8 d1, d0, d2 \n" // add
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 ARGB pixels.
+ "bgt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_argb), // %2
@@ -2538,24 +3381,24 @@ void SobelXRow_NEON(const uint8_t* src_y0,
uint8_t* dst_sobelx,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {d0}, [%0],%5 \n" // top
- "vld1.8 {d1}, [%0],%6 \n"
- "vsubl.u8 q0, d0, d1 \n"
- "vld1.8 {d2}, [%1],%5 \n" // center * 2
- "vld1.8 {d3}, [%1],%6 \n"
- "vsubl.u8 q1, d2, d3 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vld1.8 {d2}, [%2],%5 \n" // bottom
- "vld1.8 {d3}, [%2],%6 \n"
- "subs %4, %4, #8 \n" // 8 pixels
- "vsubl.u8 q1, d2, d3 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vabs.s16 q0, q0 \n"
- "vqmovn.u16 d0, q0 \n"
- "vst1.8 {d0}, [%3]! \n" // store 8 sobelx
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d0}, [%0],%5 \n" // top
+ "vld1.8 {d1}, [%0],%6 \n"
+ "subs %4, %4, #8 \n" // 8 pixels
+ "vsubl.u8 q0, d0, d1 \n"
+ "vld1.8 {d2}, [%1],%5 \n" // center * 2
+ "vld1.8 {d3}, [%1],%6 \n"
+ "vsubl.u8 q1, d2, d3 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vld1.8 {d2}, [%2],%5 \n" // bottom
+ "vld1.8 {d3}, [%2],%6 \n"
+ "vsubl.u8 q1, d2, d3 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vabs.s16 q0, q0 \n"
+ "vqmovn.u16 d0, q0 \n"
+ "vst1.8 {d0}, [%3]! \n" // store 8 sobelx
+ "bgt 1b \n"
: "+r"(src_y0), // %0
"+r"(src_y1), // %1
"+r"(src_y2), // %2
@@ -2576,24 +3419,24 @@ void SobelYRow_NEON(const uint8_t* src_y0,
uint8_t* dst_sobely,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {d0}, [%0],%4 \n" // left
- "vld1.8 {d1}, [%1],%4 \n"
- "vsubl.u8 q0, d0, d1 \n"
- "vld1.8 {d2}, [%0],%4 \n" // center * 2
- "vld1.8 {d3}, [%1],%4 \n"
- "vsubl.u8 q1, d2, d3 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vld1.8 {d2}, [%0],%5 \n" // right
- "vld1.8 {d3}, [%1],%5 \n"
- "subs %3, %3, #8 \n" // 8 pixels
- "vsubl.u8 q1, d2, d3 \n"
- "vadd.s16 q0, q0, q1 \n"
- "vabs.s16 q0, q0 \n"
- "vqmovn.u16 d0, q0 \n"
- "vst1.8 {d0}, [%2]! \n" // store 8 sobely
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d0}, [%0],%4 \n" // left
+ "vld1.8 {d1}, [%1],%4 \n"
+ "subs %3, %3, #8 \n" // 8 pixels
+ "vsubl.u8 q0, d0, d1 \n"
+ "vld1.8 {d2}, [%0],%4 \n" // center * 2
+ "vld1.8 {d3}, [%1],%4 \n"
+ "vsubl.u8 q1, d2, d3 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vld1.8 {d2}, [%0],%5 \n" // right
+ "vld1.8 {d3}, [%1],%5 \n"
+ "vsubl.u8 q1, d2, d3 \n"
+ "vadd.s16 q0, q0, q1 \n"
+ "vabs.s16 q0, q0 \n"
+ "vqmovn.u16 d0, q0 \n"
+ "vst1.8 {d0}, [%2]! \n" // store 8 sobely
+ "bgt 1b \n"
: "+r"(src_y0), // %0
"+r"(src_y1), // %1
"+r"(dst_sobely), // %2
@@ -2605,59 +3448,46 @@ void SobelYRow_NEON(const uint8_t* src_y0,
}
// %y passes a float as a scalar vector for vector * scalar multiply.
-// the regoster must be d0 to d15 and indexed with [0] or [1] to access
+// the register must be d0 to d15 and indexed with [0] or [1] to access
// the float in the first or second float of the d-reg
-void HalfFloat1Row_NEON(const uint16_t* src,
- uint16_t* dst,
- float /*unused*/,
- int width) {
- asm volatile(
-
- "1: \n"
- "vld1.8 {q1}, [%0]! \n" // load 8 shorts
- "subs %2, %2, #8 \n" // 8 pixels per loop
- "vmovl.u16 q2, d2 \n" // 8 int's
- "vmovl.u16 q3, d3 \n"
- "vcvt.f32.u32 q2, q2 \n" // 8 floats
- "vcvt.f32.u32 q3, q3 \n"
- "vmul.f32 q2, q2, %y3 \n" // adjust exponent
- "vmul.f32 q3, q3, %y3 \n"
- "vqshrn.u32 d2, q2, #13 \n" // isolate halffloat
- "vqshrn.u32 d3, q3, #13 \n"
- "vst1.8 {q1}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "w"(1.9259299444e-34f) // %3
- : "cc", "memory", "q1", "q2", "q3");
-}
-
void HalfFloatRow_NEON(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {q1}, [%0]! \n" // load 8 shorts
- "subs %2, %2, #8 \n" // 8 pixels per loop
- "vmovl.u16 q2, d2 \n" // 8 int's
- "vmovl.u16 q3, d3 \n"
- "vcvt.f32.u32 q2, q2 \n" // 8 floats
- "vcvt.f32.u32 q3, q3 \n"
- "vmul.f32 q2, q2, %y3 \n" // adjust exponent
- "vmul.f32 q3, q3, %y3 \n"
- "vqshrn.u32 d2, q2, #13 \n" // isolate halffloat
- "vqshrn.u32 d3, q3, #13 \n"
- "vst1.8 {q1}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "w"(scale * 1.9259299444e-34f) // %3
- : "cc", "memory", "q1", "q2", "q3");
+ "1: \n"
+ "vld1.16 {q0, q1}, [%0]! \n" // load 16 shorts
+ "subs %2, %2, #16 \n" // 16 pixels per loop
+ "vmovl.u16 q8, d0 \n"
+ "vmovl.u16 q9, d1 \n"
+ "vmovl.u16 q10, d2 \n"
+ "vmovl.u16 q11, d3 \n"
+ "vcvt.f32.u32 q8, q8 \n"
+ "vcvt.f32.u32 q9, q9 \n"
+ "vcvt.f32.u32 q10, q10 \n"
+ "vcvt.f32.u32 q11, q11 \n"
+ "vmul.f32 q8, q8, %y3 \n" // * scale
+ "vmul.f32 q9, q9, %y3 \n"
+ "vmul.f32 q10, q10, %y3 \n"
+ "vmul.f32 q11, q11, %y3 \n"
+ "vmul.f32 q8, q8, %y4 \n" // adjust exponent
+ "vmul.f32 q9, q9, %y4 \n"
+ "vmul.f32 q10, q10, %y4 \n"
+ "vmul.f32 q11, q11, %y4 \n"
+ "vqshrn.u32 d0, q8, #13 \n" // isolate halffloat
+ "vqshrn.u32 d1, q9, #13 \n"
+ "vqshrn.u32 d2, q10, #13 \n"
+ "vqshrn.u32 d3, q11, #13 \n"
+ "vst1.16 {q0, q1}, [%1]! \n" // store 16 fp16
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "w"(scale), // %3
+ "w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
+ : "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11");
}
void ByteToFloatRow_NEON(const uint8_t* src,
@@ -2666,18 +3496,18 @@ void ByteToFloatRow_NEON(const uint8_t* src,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {d2}, [%0]! \n" // load 8 bytes
- "subs %2, %2, #8 \n" // 8 pixels per loop
- "vmovl.u8 q1, d2 \n" // 8 shorts
- "vmovl.u16 q2, d2 \n" // 8 ints
- "vmovl.u16 q3, d3 \n"
- "vcvt.f32.u32 q2, q2 \n" // 8 floats
- "vcvt.f32.u32 q3, q3 \n"
- "vmul.f32 q2, q2, %y3 \n" // scale
- "vmul.f32 q3, q3, %y3 \n"
- "vst1.8 {q2, q3}, [%1]! \n" // store 8 floats
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {d2}, [%0]! \n" // load 8 bytes
+ "subs %2, %2, #8 \n" // 8 pixels per loop
+ "vmovl.u8 q1, d2 \n" // 8 shorts
+ "vmovl.u16 q2, d2 \n" // 8 ints
+ "vmovl.u16 q3, d3 \n"
+ "vcvt.f32.u32 q2, q2 \n" // 8 floats
+ "vcvt.f32.u32 q3, q3 \n"
+ "vmul.f32 q2, q2, %y3 \n" // scale
+ "vmul.f32 q3, q3, %y3 \n"
+ "vst1.8 {q2, q3}, [%1]! \n" // store 8 floats
+ "bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -2694,26 +3524,26 @@ void GaussCol_NEON(const uint16_t* src0,
uint32_t* dst,
int width) {
asm volatile(
- "vmov.u16 d6, #4 \n" // constant 4
- "vmov.u16 d7, #6 \n" // constant 6
-
- "1: \n"
- "vld1.16 {q1}, [%0]! \n" // load 8 samples, 5 rows
- "vld1.16 {q2}, [%4]! \n"
- "vaddl.u16 q0, d2, d4 \n" // * 1
- "vaddl.u16 q1, d3, d5 \n" // * 1
- "vld1.16 {q2}, [%1]! \n"
- "vmlal.u16 q0, d4, d6 \n" // * 4
- "vmlal.u16 q1, d5, d6 \n" // * 4
- "vld1.16 {q2}, [%2]! \n"
- "vmlal.u16 q0, d4, d7 \n" // * 6
- "vmlal.u16 q1, d5, d7 \n" // * 6
- "vld1.16 {q2}, [%3]! \n"
- "vmlal.u16 q0, d4, d6 \n" // * 4
- "vmlal.u16 q1, d5, d6 \n" // * 4
- "subs %6, %6, #8 \n" // 8 processed per loop
- "vst1.32 {q0, q1}, [%5]! \n" // store 8 samples
- "bgt 1b \n"
+ "vmov.u16 d6, #4 \n" // constant 4
+ "vmov.u16 d7, #6 \n" // constant 6
+
+ "1: \n"
+ "vld1.16 {q1}, [%0]! \n" // load 8 samples, 5 rows
+ "vld1.16 {q2}, [%4]! \n"
+ "subs %6, %6, #8 \n" // 8 processed per loop
+ "vaddl.u16 q0, d2, d4 \n" // * 1
+ "vaddl.u16 q1, d3, d5 \n" // * 1
+ "vld1.16 {q2}, [%1]! \n"
+ "vmlal.u16 q0, d4, d6 \n" // * 4
+ "vmlal.u16 q1, d5, d6 \n" // * 4
+ "vld1.16 {q2}, [%2]! \n"
+ "vmlal.u16 q0, d4, d7 \n" // * 6
+ "vmlal.u16 q1, d5, d7 \n" // * 6
+ "vld1.16 {q2}, [%3]! \n"
+ "vmlal.u16 q0, d4, d6 \n" // * 4
+ "vmlal.u16 q1, d5, d6 \n" // * 4
+ "vst1.32 {q0, q1}, [%5]! \n" // store 8 samples
+ "bgt 1b \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(src2), // %2
@@ -2731,12 +3561,13 @@ void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width) {
const uint32_t* src2 = src + 2;
const uint32_t* src3 = src + 3;
asm volatile(
- "vmov.u32 q10, #4 \n" // constant 4
- "vmov.u32 q11, #6 \n" // constant 6
+ "vmov.u32 q10, #4 \n" // constant 4
+ "vmov.u32 q11, #6 \n" // constant 6
- "1: \n"
+ "1: \n"
"vld1.32 {q0, q1}, [%0]! \n" // load 12 source samples
"vld1.32 {q2}, [%0] \n"
+ "subs %5, %5, #8 \n" // 8 processed per loop
"vadd.u32 q0, q0, q1 \n" // * 1
"vadd.u32 q1, q1, q2 \n" // * 1
"vld1.32 {q2, q3}, [%2]! \n"
@@ -2748,7 +3579,6 @@ void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width) {
"vadd.u32 q3, q3, q9 \n"
"vmla.u32 q0, q2, q10 \n" // * 4
"vmla.u32 q1, q3, q10 \n" // * 4
- "subs %5, %5, #8 \n" // 8 processed per loop
"vqshrn.u32 d0, q0, #8 \n" // round and pack
"vqshrn.u32 d1, q1, #8 \n"
"vst1.u16 {q0}, [%4]! \n" // store 8 samples
@@ -2769,17 +3599,17 @@ void NV21ToYUV24Row_NEON(const uint8_t* src_y,
uint8_t* dst_yuv24,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {q2}, [%0]! \n" // load 16 Y values
- "vld2.8 {d0, d2}, [%1]! \n" // load 8 VU values
- "vmov d1, d0 \n"
- "vzip.u8 d0, d1 \n" // VV
- "vmov d3, d2 \n"
- "vzip.u8 d2, d3 \n" // UU
- "subs %3, %3, #16 \n" // 16 pixels per loop
- "vst3.8 {d0, d2, d4}, [%2]! \n" // store 16 YUV pixels
- "vst3.8 {d1, d3, d5}, [%2]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {q2}, [%0]! \n" // load 16 Y values
+ "vld2.8 {d0, d2}, [%1]! \n" // load 8 VU values
+ "subs %3, %3, #16 \n" // 16 pixels per loop
+ "vmov d1, d0 \n"
+ "vzip.u8 d0, d1 \n" // VV
+ "vmov d3, d2 \n"
+ "vzip.u8 d2, d3 \n" // UU
+ "vst3.8 {d0, d2, d4}, [%2]! \n" // store 16 YUV pixels
+ "vst3.8 {d1, d3, d5}, [%2]! \n"
+ "bgt 1b \n"
: "+r"(src_y), // %0
"+r"(src_vu), // %1
"+r"(dst_yuv24), // %2
@@ -2793,24 +3623,24 @@ void AYUVToUVRow_NEON(const uint8_t* src_ayuv,
uint8_t* dst_uv,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // src_stride + src_AYUV
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
+ "add %1, %0, %1 \n" // src_stride + src_AYUV
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
// pixels.
- "vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
+ "subs %3, %3, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
+ "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
// pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 AYUV
+ "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 AYUV
// pixels.
- "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
+ "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vqrshrun.s16 d1, q0, #2 \n" // 2x2 average
"vqrshrun.s16 d0, q1, #2 \n"
- "subs %3, %3, #16 \n" // 16 processed per loop.
- "vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels UV.
- "bgt 1b \n"
+ "vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels UV.
+ "bgt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(src_stride_ayuv), // %1
"+r"(dst_uv), // %2
@@ -2824,24 +3654,24 @@ void AYUVToVURow_NEON(const uint8_t* src_ayuv,
uint8_t* dst_vu,
int width) {
asm volatile(
- "add %1, %0, %1 \n" // src_stride + src_AYUV
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
+ "add %1, %0, %1 \n" // src_stride + src_AYUV
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
// pixels.
- "vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
- "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
+ "subs %3, %3, #16 \n" // 16 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
+ "vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
// pixels.
- "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 AYUV
+ "vld4.8 {d9, d11, d13, d15}, [%1]! \n" // load last 8 AYUV
// pixels.
- "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
+ "vpadal.u8 q0, q4 \n" // B 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vqrshrun.s16 d0, q0, #2 \n" // 2x2 average
"vqrshrun.s16 d1, q1, #2 \n"
- "subs %3, %3, #16 \n" // 16 processed per loop.
- "vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels VU.
- "bgt 1b \n"
+ "vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels VU.
+ "bgt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(src_stride_ayuv), // %1
"+r"(dst_vu), // %2
@@ -2854,12 +3684,12 @@ void AYUVToVURow_NEON(const uint8_t* src_ayuv,
// Similar to ARGBExtractAlphaRow_NEON
void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV pixels
- "subs %2, %2, #16 \n" // 16 processed per loop
- "vst1.8 {q2}, [%1]! \n" // store 16 Y's.
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV pixels
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "vst1.8 {q2}, [%1]! \n" // store 16 Y's.
+ "bgt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -2870,13 +3700,13 @@ void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
// Convert UV plane of NV12 to VU of NV21.
void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
asm volatile(
- "1: \n"
- "vld2.8 {d0, d2}, [%0]! \n" // load 16 UV values
- "vld2.8 {d1, d3}, [%0]! \n"
- "vorr.u8 q2, q0, q0 \n" // move U after V
- "subs %2, %2, #16 \n" // 16 pixels per loop
- "vst2.8 {q1, q2}, [%1]! \n" // store 16 VU pixels
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {d0, d2}, [%0]! \n" // load 16 UV values
+ "vld2.8 {d1, d3}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 pixels per loop
+ "vmov.u8 q2, q0 \n" // move U after V
+ "vst2.8 {q1, q2}, [%1]! \n" // store 16 VU pixels
+ "bgt 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_vu), // %1
"+r"(width) // %2
@@ -2884,6 +3714,719 @@ void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
: "cc", "memory", "q0", "q1", "q2");
}
+void HalfMergeUVRow_NEON(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ const uint8_t* src_u_1 = src_u + src_stride_u;
+ const uint8_t* src_v_1 = src_v + src_stride_v;
+ asm volatile(
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 16 U values
+ "vld1.8 {q1}, [%2]! \n" // load 16 V values
+ "vld1.8 {q2}, [%1]! \n"
+ "vld1.8 {q3}, [%3]! \n"
+ "subs %5, %5, #16 \n" // 16 src pixels per loop
+ "vpaddl.u8 q0, q0 \n" // half size
+ "vpaddl.u8 q1, q1 \n"
+ "vpadal.u8 q0, q2 \n"
+ "vpadal.u8 q1, q3 \n"
+ "vqrshrn.u16 d0, q0, #2 \n"
+ "vqrshrn.u16 d1, q1, #2 \n"
+ "vst2.8 {d0, d1}, [%4]! \n" // store 8 UV pixels
+ "bgt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_u_1), // %1
+ "+r"(src_v), // %2
+ "+r"(src_v_1), // %3
+ "+r"(dst_uv), // %4
+ "+r"(width) // %5
+ :
+ : "cc", "memory", "q0", "q1", "q2", "q3");
+}
+
+void SplitUVRow_16_NEON(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ int shift = depth - 16; // Negative for right shift.
+ asm volatile(
+ "vdup.16 q2, %4 \n"
+ "1: \n"
+ "vld2.16 {q0, q1}, [%0]! \n" // load 8 UV
+ "subs %3, %3, #8 \n" // 8 src pixels per loop
+ "vshl.u16 q0, q0, q2 \n"
+ "vshl.u16 q1, q1, q2 \n"
+ "vst1.16 {q0}, [%1]! \n" // store 8 U pixels
+ "vst1.16 {q1}, [%2]! \n" // store 8 V pixels
+ "bgt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "q0", "q1", "q2");
+}
+
+void MergeUVRow_16_NEON(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ asm volatile(
+ "vdup.16 q2, %4 \n"
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n" // load 8 U
+ "vld1.16 {q1}, [%1]! \n" // load 8 V
+ "subs %3, %3, #8 \n" // 8 src pixels per loop
+ "vshl.u16 q0, q0, q2 \n"
+ "vshl.u16 q1, q1, q2 \n"
+ "vst2.16 {q0, q1}, [%2]! \n" // store 8 UV pixels
+ "bgt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "q0", "q1", "q2");
+}
+
+void MultiplyRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ asm volatile(
+ "vdup.16 q2, %3 \n"
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n"
+ "vld1.16 {q1}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
+ "vmul.u16 q0, q0, q2 \n"
+ "vmul.u16 q1, q1, q2 \n"
+ "vst1.16 {q0}, [%1]! \n"
+ "vst1.16 {q1}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "cc", "memory", "q0", "q1", "q2");
+}
+
+void DivideRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ asm volatile(
+ "vdup.16 d8, %3 \n"
+ "1: \n"
+ "vld1.16 {q2, q3}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
+ "vmull.u16 q0, d4, d8 \n"
+ "vmull.u16 q1, d5, d8 \n"
+ "vmull.u16 q2, d6, d8 \n"
+ "vmull.u16 q3, d7, d8 \n"
+ "vshrn.u32 d0, q0, #16 \n"
+ "vshrn.u32 d1, q1, #16 \n"
+ "vshrn.u32 d2, q2, #16 \n"
+ "vshrn.u32 d3, q3, #16 \n"
+ "vst1.16 {q0, q1}, [%1]! \n" // store 16 pixels
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "cc", "memory", "q0", "q1", "q2", "q3", "d8");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits = shr 1
+// 16384 = 10 bits = shr 2
+// 4096 = 12 bits = shr 4
+// 256 = 16 bits = shr 8
+void Convert16To8Row_NEON(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
+ asm volatile(
+ "vdup.16 q2, %3 \n"
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n"
+ "vld1.16 {q1}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
+ "vshl.u16 q0, q0, q2 \n" // shr = q2 is negative
+ "vshl.u16 q1, q1, q2 \n"
+ "vqmovn.u16 d0, q0 \n"
+ "vqmovn.u16 d1, q1 \n"
+ "vst1.8 {q0}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(shift) // %3
+ : "cc", "memory", "q0", "q1", "q2");
+}
+
+void HalfRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
+ asm volatile(
+ "vdup.16 q4, %4 \n"
+ "1: \n"
+ "vld1.16 {q0, q1}, [%0]! \n"
+ "vld1.16 {q2, q3}, [%1]! \n"
+ "subs %3, %3, #16 \n" // 16 src pixels per loop
+ "vrhadd.u16 q0, q0, q2 \n"
+ "vrhadd.u16 q1, q1, q3 \n"
+ "vshl.u16 q0, q0, q4 \n" // shr = q4 is negative
+ "vshl.u16 q1, q1, q4 \n"
+ "vqmovn.u16 d0, q0 \n"
+ "vqmovn.u16 d1, q1 \n"
+ "vst1.8 {q0}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(src_uv1), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4");
+}
+
+void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
+ asm volatile(
+ "vdup.16 q4, %4 \n"
+ "1: \n"
+ "vld1.16 {q0, q1}, [%0]! \n"
+ "vld1.16 {q2, q3}, [%1]! \n"
+ "subs %3, %3, #8 \n" // 8 dst pixels per loop
+ "vpaddl.u16 q0, q0 \n"
+ "vpaddl.u16 q1, q1 \n"
+ "vpadal.u16 q0, q2 \n"
+ "vpadal.u16 q1, q3 \n"
+ "vrshrn.u32 d0, q0, #2 \n"
+ "vrshrn.u32 d1, q1, #2 \n"
+ "vshl.u16 q0, q0, q4 \n" // shr = q4 is negative
+ "vqmovn.u16 d0, q0 \n"
+ "vst1.8 {d0}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(src_uv1), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "q4");
+}
+
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "vdup.8 d8, %3 \n"
+ "vdup.8 q5, %4 \n"
+ "1: \n"
+ "vld1.8 {q2, q3}, [%0]! \n"
+ "subs %2, %2, #32 \n" // 32 src pixels per loop
+ "vmull.u8 q0, d4, d8 \n"
+ "vmull.u8 q1, d5, d8 \n"
+ "vmull.u8 q2, d6, d8 \n"
+ "vmull.u8 q3, d7, d8 \n"
+ "vuzp.8 q0, q1 \n"
+ "vuzp.8 q2, q3 \n"
+ "vadd.u8 q0, q1, q5 \n"
+ "vadd.u8 q1, q3, q5 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // store 32 pixels
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "d8", "q5");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 512 = 9 bits
+// 1024 = 10 bits
+// 4096 = 12 bits
+void Convert8To16Row_NEON(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ // (src * 0x0101) >> (16 - bits).
+ // Use negative shift for right shift with vshl.
+ const int shift = bits - 16;
+ asm volatile(
+ "vdup.16 q2, %3 \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n"
+ "vmov q1, q0 \n"
+ "vzip.8 q0, q1 \n"
+ "subs %2, %2, #16 \n"
+ "vshl.u16 q0, q0, q2 \n"
+ "vshl.u16 q1, q1, q2 \n"
+ "vst1.16 {q0, q1}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(shift) // %3
+ : "cc", "memory", "q0", "q1", "q2");
+}
+
+void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
+ asm volatile(
+ "1: \n"
+ "vld1.16 {d0, d1, d2, d3}, [%0]! \n" // load 16
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "vst1.16 {d0, d1, d2, d3}, [%1]! \n" // store 16
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(count) // %2
+ :
+ : "memory", "cc", "q0", "q1");
+}
+
+void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
+ asm volatile(
+ // Start at end of source row.
+ "add %0, %0, %2, lsl #1 \n"
+ "sub %0, %0, #32 \n" // 32 bytes per loop
+ "1: \n"
+ "vld1.16 {q1, q2}, [%0], %3 \n" // src -= 32
+ "subs %2, %2, #16 \n" // 16 pixels per loop.
+ "vrev64.16 q0, q2 \n"
+ "vrev64.16 q1, q1 \n"
+ "vswp d0, d1 \n"
+ "vswp d2, d3 \n"
+ "vst1.16 {q0, q1}, [%1]! \n" // dst += 32
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(-32) // %3
+ : "memory", "cc", "q0", "q1", "q2");
+}
+
+void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vmov.i32 q8, #0x0000ff00 \n"
+ "vmov.i32 q9, #0x000fffff \n"
+ "vshl.i32 q9, q9, #8 \n" // 0x0fffff00
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
+ "subs %2, %2, #8 \n" // 8 pixels per loop
+ "vsri.32 q0, q0, #2 \n"
+ "vsri.32 q1, q1, #2 \n"
+ "vshr.u32 q2, q0, #2 \n"
+ "vshr.u32 q3, q1, #2 \n"
+ "vshr.u32 q4, q0, #4 \n"
+ "vshr.u32 q5, q1, #4 \n"
+ "vbit q0, q4, q9 \n"
+ "vbit q1, q5, q9 \n"
+ "vbit q0, q2, q8 \n"
+ "vbit q1, q3, q8 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // store 8 ARGB pixels
+ "bgt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
+}
+
+void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
+ asm volatile(
+ "vmov.i64 q8, #0x00ffff0000ffff00 \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
+ "subs %2, %2, #8 \n" // 8 pixels per loop
+ "vsri.32 q0, q0, #2 \n"
+ "vsri.32 q1, q1, #2 \n"
+ "vshr.u32 q2, q0, #2 \n"
+ "vshr.u32 q3, q1, #2 \n"
+ "vsli.32 q2, q0, #16 \n"
+ "vsli.32 q3, q1, #16 \n"
+ "vsri.32 q0, q0, #4 \n"
+ "vsri.32 q1, q1, #4 \n"
+ "vsri.32 q0, q0, #16 \n"
+ "vsri.32 q1, q1, #16 \n"
+ "vbit q0, q2, q8 \n"
+ "vbit q1, q3, q8 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // store 8 ABGR pixels
+ "bgt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_abgr), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8");
+}
+
+void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
+ asm volatile(
+ "vmov.i8 q9, #0xff \n"
+ "vshr.u32 q9, q9, #22 \n" // 0x000003ff
+ "vshl.i32 q8, q9, #20 \n" // 0x3ff00000
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
+ "subs %2, %2, #8 \n" // 8 pixels per loop
+ "vshl.i32 q2, q0, #20 \n"
+ "vshl.i32 q3, q1, #20 \n"
+ "vshr.u32 q4, q0, #20 \n"
+ "vshr.u32 q5, q1, #20 \n"
+ "vbit q0, q2, q8 \n"
+ "vbit q1, q3, q8 \n"
+ "vbit q0, q4, q9 \n"
+ "vbit q1, q5, q9 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // store 8 AB30 pixels
+ "bgt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_ab30), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
+}
+
+void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "vmov.i32 q4, #0xff000000 \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load 8 src pixels
+ "vld1.8 {q2, q3}, [%1] \n" // load 8 dst pixels
+ "subs %2, %2, #8 \n" // 8 pixels per loop
+ "vbit q2, q0, q4 \n"
+ "vbit q3, q1, q4 \n"
+ "vst1.8 {q2, q3}, [%1]! \n" // store 8 dst pixels
+ "bgt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4");
+}
+
+void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ uint32_t tmp, addr;
+ asm volatile(
+ "vmov.i32 q8, #0xff000000 \n"
+ "1: \n"
+ "ldrb %3, [%0, #3] \n"
+ "add %4, %5, %3, lsl #2 \n"
+ "vld1.16 {d6[0]}, [%4] \n"
+ "ldrb %3, [%0, #7] \n"
+ "add %4, %5, %3, lsl #2 \n"
+ "vld1.16 {d6[1]}, [%4] \n"
+ "ldrb %3, [%0, #11] \n"
+ "add %4, %5, %3, lsl #2 \n"
+ "vld1.16 {d6[2]}, [%4] \n"
+ "ldrb %3, [%0, #15] \n"
+ "add %4, %5, %3, lsl #2 \n"
+ "vld1.16 {d6[3]}, [%4] \n"
+ "vld1.8 {q0}, [%0]! \n"
+ "subs %2, %2, #4 \n"
+ "vmov q1, q0 \n"
+ "vmov q2, q0 \n"
+ "vzip.8 q1, q2 \n"
+ "vmull.u16 q4, d2, d6[0] \n"
+ "vmull.u16 q5, d3, d6[1] \n"
+ "vmull.u16 q6, d4, d6[2] \n"
+ "vmull.u16 q7, d5, d6[3] \n"
+ "vshrn.i32 d8, q4, #16 \n"
+ "vshrn.i32 d9, q5, #16 \n"
+ "vshrn.i32 d12, q6, #16 \n"
+ "vshrn.i32 d13, q7, #16 \n"
+ "vqmovn.u16 d2, q4 \n"
+ "vqmovn.u16 d3, q6 \n"
+ "vbit q1, q0, q8 \n"
+ "vst1.8 {q1}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ "=&r"(tmp), // %3
+ "=&r"(addr) // %4
+ : "r"(fixed_invtbl8) // %5
+ : "memory", "cc", "q0", "q1", "q2", "d6", "q4", "q5", "q6", "q7", "q8");
+}
+
+void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width) {
+ asm volatile(
+ "vld1.32 {q8, q9}, [%3]! \n" // c0, c1
+ "vld1.32 {q10, q11}, [%3] \n" // c2, c3
+ "1: \n"
+ "vld1.8 {d0}, [%0]! \n" // load 2 pixels
+ "subs %2, %2, #2 \n" // 2 pixels per loop
+ "vmovl.u8 q0, d0 \n"
+ "vmovl.u16 q1, d0 \n"
+ "vmovl.u16 q2, d1 \n"
+ "vcvt.f32.u32 q1, q1 \n" // x0
+ "vcvt.f32.u32 q2, q2 \n" // x1
+ "vmul.f32 q3, q9, q1 \n" // c1 * x0
+ "vmul.f32 q4, q9, q2 \n" // c1 * x1
+ "vmul.f32 q5, q1, q1 \n" // x0 * x0
+ "vmul.f32 q6, q2, q2 \n" // x1 * x1
+ "vadd.f32 q3, q8, q3 \n" // c0 + c1 * x0
+ "vadd.f32 q4, q8, q4 \n" // c0 + c1 * x1
+ "vmla.f32 q3, q10, q5 \n" // + c2 * x0^2
+ "vmla.f32 q4, q10, q6 \n" // + c2 * x1^2
+ "vmul.f32 q5, q5, q1 \n" // x0^3
+ "vmul.f32 q6, q6, q2 \n" // x1^3
+ "vmla.f32 q3, q11, q5 \n" // + c3 * x0^3
+ "vmla.f32 q4, q11, q6 \n" // + c3 * x1^3
+ "vcvt.s32.f32 q3, q3 \n"
+ "vcvt.s32.f32 q4, q4 \n"
+ "vqmovn.s32 d6, q3 \n"
+ "vqmovn.s32 d7, q4 \n"
+ "vqmovun.s16 d6, q3 \n"
+ "vst1.8 {d6}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ "+r"(poly) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9",
+ "q10", "q11");
+}
+
+void ARGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ const uint8_t* table_g = table_argb + 1;
+ const uint8_t* table_r = table_argb + 2;
+ const uint8_t* table_a = table_argb + 3;
+ uint32_t b, g, r, a;
+ asm volatile(
+ "1: \n"
+ "ldrb %2, [%0] \n"
+ "ldrb %3, [%0, #1] \n"
+ "ldrb %4, [%0, #2] \n"
+ "ldrb %5, [%0, #3] \n"
+ "ldrb %2, [%6, %2, lsl #2] \n"
+ "ldrb %3, [%7, %3, lsl #2] \n"
+ "ldrb %4, [%8, %4, lsl #2] \n"
+ "ldrb %5, [%9, %5, lsl #2] \n"
+ "subs %1, %1, #1 \n"
+ "strb %2, [%0] \n"
+ "strb %3, [%0, #1] \n"
+ "strb %4, [%0, #2] \n"
+ "strb %5, [%0, #3] \n"
+ "add %0, %0, #4 \n"
+ "bgt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "=&r"(b), // %2
+ "=&r"(g), // %3
+ "=&r"(r), // %4
+ "=&r"(a) // %5
+ : "r"(table_argb), // %6
+ "r"(table_g), // %7
+ "r"(table_r), // %8
+ "r"(table_a) // %9
+ : "memory", "cc");
+}
+
+void RGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ const uint8_t* table_g = table_argb + 1;
+ const uint8_t* table_r = table_argb + 2;
+ uint32_t b, g, r;
+ asm volatile(
+ "1: \n"
+ "ldrb %2, [%0] \n"
+ "ldrb %3, [%0, #1] \n"
+ "ldrb %4, [%0, #2] \n"
+ "ldrb %2, [%5, %2, lsl #2] \n"
+ "ldrb %3, [%6, %3, lsl #2] \n"
+ "ldrb %4, [%7, %4, lsl #2] \n"
+ "subs %1, %1, #1 \n"
+ "strb %2, [%0] \n"
+ "strb %3, [%0, #1] \n"
+ "strb %4, [%0, #2] \n"
+ "add %0, %0, #4 \n"
+ "bgt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "=&r"(b), // %2
+ "=&r"(g), // %3
+ "=&r"(r) // %4
+ : "r"(table_argb), // %5
+ "r"(table_g), // %6
+ "r"(table_r) // %7
+ : "memory", "cc");
+}
+
+void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff) {
+ uint32_t idx, addr;
+ asm volatile(
+ "vdup.32 q4, %6 \n" // lumacoeff
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load 4 ARGB pixels
+ "subs %2, %2, #4 \n" // 4 pixels per loop
+ "vmull.u8 q1, d0, d8 \n"
+ "vmull.u8 q2, d1, d9 \n"
+ "vpadd.i16 d2, d2, d3 \n"
+ "vpadd.i16 d3, d4, d5 \n"
+ "vpadd.i16 d2, d2, d3 \n"
+ "vbic.i16 d2, #0x8000 \n" // clear bit 15
+ "vshrn.i16 d2, q1, #8 \n" // L_k & 0x7f
+ "vmov d3, d2 \n"
+ "vzip.8 d2, d3 \n"
+ "vmov d3, d2 \n"
+ "vzip.8 d2, d3 \n"
+ "vmov q2, q0 \n"
+ "vzip.8 q2, q1 \n" // q2: p0,1; q1: p2,3
+ "vmov.u16 %3, d4[0] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[0]}, [%4] \n"
+ "vmov.u16 %3, d4[1] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[1]}, [%4] \n"
+ "vmov.u16 %3, d4[2] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[2]}, [%4] \n"
+ "vmov.u16 %3, d5[0] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[4]}, [%4] \n"
+ "vmov.u16 %3, d5[1] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[5]}, [%4] \n"
+ "vmov.u16 %3, d5[2] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d0[6]}, [%4] \n"
+ "vmov.u16 %3, d2[0] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[0]}, [%4] \n"
+ "vmov.u16 %3, d2[1] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[1]}, [%4] \n"
+ "vmov.u16 %3, d2[2] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[2]}, [%4] \n"
+ "vmov.u16 %3, d3[0] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[4]}, [%4] \n"
+ "vmov.u16 %3, d3[1] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[5]}, [%4] \n"
+ "vmov.u16 %3, d3[2] \n"
+ "add %4, %5, %3 \n"
+ "vld1.8 {d1[6]}, [%4] \n"
+ "vst1.8 {q0}, [%1]! \n"
+ "bgt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ "=&r"(idx), // %3
+ "=&r"(addr) // %4
+ : "r"(luma), // %5
+ "r"(lumacoeff) // %6
+ : "memory", "cc", "q0", "q1", "q2", "q4");
+}
+
+void ARGBAffineRow_NEON(const uint8_t* src_argb,
+ int src_argb_stride,
+ uint8_t* dst_argb,
+ const float* uv_dudv,
+ int width) {
+ uint32_t off0, off1;
+ asm volatile(
+ "vld1.32 {d4, d5}, [%5] \n" // d4=[u,v], d5=[du,dv]
+ "vmov d12, d5 \n" // d12=[du,dv]
+ "vdup.32 d10, %4 \n" // [stride, stride]
+ "vmov.i32 d11, #4 \n" // [4, 4]
+ "vtrn.32 d11, d10 \n" // q5=[4, stride, 4,
+ // stride]
+ "subs %1, %1, #4 \n"
+ "blt 2f \n"
+ "vmov d13, d12 \n" // q6=[du, dv, du, dv]
+ "vadd.f32 q4, q6, q6 \n" // 2 * [du, dv, du, dv]
+ "vadd.f32 d5, d4, d12 \n" // [u+du, v+dv]
+ "vadd.f32 q3, q2, q4 \n" // [u+2du, v+2dv, u+3du,
+ // v+3dv]
+ "vadd.f32 q4, q4, q4 \n" // 4 * [du, dv, du, dv]
+ "1: \n"
+ "vcvt.s32.f32 q0, q2 \n"
+ "vcvt.s32.f32 q1, q3 \n"
+ "vadd.f32 q2, q2, q4 \n"
+ "vadd.f32 q3, q3, q4 \n"
+ "vmul.i32 q0, q0, q5 \n"
+ "vmul.i32 q1, q1, q5 \n"
+ "vpadd.i32 d14, d0, d1 \n"
+ "vpadd.i32 d15, d2, d3 \n"
+ "vmov %2, %3, d14 \n"
+ "ldr %2, [%6, %2] \n"
+ "ldr %3, [%6, %3] \n"
+ "str %2, [%0], #4 \n"
+ "str %3, [%0], #4 \n"
+ "vmov %2, %3, d15 \n"
+ "ldr %2, [%6, %2] \n"
+ "ldr %3, [%6, %3] \n"
+ "str %2, [%0], #4 \n"
+ "str %3, [%0], #4 \n"
+ "subs %1, %1, #4 \n"
+ "bge 1b \n"
+ "2: \n"
+ "adds %1, %1, #4 \n"
+ "beq 4f \n"
+ "3: \n"
+ "vcvt.s32.f32 d0, d4 \n"
+ "vadd.f32 d4, d4, d12 \n"
+ "vmul.i32 d0, d0, d10 \n"
+ "vpadd.i32 d0, d0, d0 \n"
+ "vmov.32 %2, d0[0] \n"
+ "ldr %2, [%6, %2] \n"
+ "str %2, [%0], #4 \n"
+ "subs %1, %1, #1 \n"
+ "bgt 3b \n"
+ "4: \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "=&r"(off0), // %2
+ "=&r"(off1) // %3
+ : "r"(src_argb_stride), // %4
+ "r"(uv_dudv), // %5
+ "r"(src_argb) // %6
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7");
+}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)..
#ifdef __cplusplus
diff --git a/libs/libyuv/source/row_neon64.cc b/libs/libyuv/source/row_neon64.cc
index e70953f299..e39360537b 100644
--- a/libs/libyuv/source/row_neon64.cc
+++ b/libs/libyuv/source/row_neon64.cc
@@ -8,6 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <string.h>
+
+#include "libyuv/convert_from_argb.h"
#include "libyuv/row.h"
#ifdef __cplusplus
@@ -15,102 +18,237 @@ namespace libyuv {
extern "C" {
#endif
+// Enable LIBYUV_USE_ST2, LIBYUV_USE_ST3, LIBYUV_USE_ST4 for CPUs that prefer
+// STn over ZIP1+ST1
+// Exynos M1, M2, M3 are slow with ST2, ST3 and ST4 instructions.
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+// v0.8h: Y
+// v1.16b: 8U, 8V
+
// Read 8 Y, 4 U and 4 V from 422
#define READYUV422 \
- "ld1 {v0.8b}, [%0], #8 \n" \
- "ld1 {v1.s}[0], [%1], #4 \n" \
- "ld1 {v1.s}[1], [%2], #4 \n"
+ "ldr d0, [%[src_y]], #8 \n" \
+ "ldr s1, [%[src_u]], #4 \n" \
+ "ldr s2, [%[src_v]], #4 \n" \
+ "zip1 v0.16b, v0.16b, v0.16b \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v1.8b, v1.8b, v1.8b \n" \
+ "zip1 v2.8b, v2.8b, v2.8b \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 4 U and 4 V from 210
+#define READYUV210 \
+ "ldr q2, [%[src_y]], #16 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d3, [%[src_v]], #8 \n" \
+ "shl v0.8h, v2.8h, #6 \n" \
+ "usra v0.8h, v2.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v2.8h, v3.8h, v3.8h \n" \
+ "zip1 v3.8h, v1.8h, v1.8h \n" \
+ "uqshrn v1.8b, v3.8h, #2 \n" \
+ "uqshrn2 v1.16b, v2.8h, #2 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 4 U and 4 V interleaved from 210
+#define READYUVP210 \
+ "ldr q0, [%[src_y]], #16 \n" \
+ "ldr q1, [%[src_uv]], #16 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "tbl v1.16b, {v1.16b}, v2.16b \n"
+
+// Read 8 Y, 4 U and 4 V from 212
+#define READYUV212 \
+ "ldr q2, [%[src_y]], #16 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d3, [%[src_v]], #8 \n" \
+ "shl v0.8h, v2.8h, #4 \n" \
+ "usra v0.8h, v2.8h, #8 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v2.8h, v3.8h, v3.8h \n" \
+ "zip1 v3.8h, v1.8h, v1.8h \n" \
+ "uqshrn v1.8b, v3.8h, #4 \n" \
+ "uqshrn2 v1.16b, v2.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 8 U and 8 V from 410
+#define READYUV410 \
+ "ldr q1, [%[src_y]], #16 \n" \
+ "ldr q2, [%[src_u]], #16 \n" \
+ "ldr q3, [%[src_v]], #16 \n" \
+ "shl v0.8h, v1.8h, #6 \n" \
+ "usra v0.8h, v1.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "uqshrn v1.8b, v2.8h, #2 \n" \
+ "uqshrn2 v1.16b, v3.8h, #2 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 8 U and 8 V interleaved from 410
+#define READYUVP410 \
+ "ldr q0, [%[src_y]], #16 \n" \
+ "ldp q4, q5, [%[src_uv]], #32 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "tbl v1.16b, {v4.16b, v5.16b}, v2.16b \n"
// Read 8 Y, 8 U and 8 V from 444
#define READYUV444 \
- "ld1 {v0.8b}, [%0], #8 \n" \
- "ld1 {v1.d}[0], [%1], #8 \n" \
- "ld1 {v1.d}[1], [%2], #8 \n" \
- "uaddlp v1.8h, v1.16b \n" \
- "rshrn v1.8b, v1.8h, #1 \n"
-
-// Read 8 Y, and set 4 U and 4 V to 128
+ "ldr d0, [%[src_y]], #8 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d2, [%[src_v]], #8 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 448] \n" \
+ "zip1 v0.16b, v0.16b, v0.16b \n" \
+ "prfm pldl1keep, [%[src_v], 448] \n"
+
+// Read 8 Y
#define READYUV400 \
- "ld1 {v0.8b}, [%0], #8 \n" \
- "movi v1.8b , #128 \n"
-
-// Read 8 Y and 4 UV from NV12
+ "ldr d0, [%[src_y]], #8 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v0.16b, v0.16b, v0.16b \n"
+
+static const uvec8 kNV12Table = {0, 0, 2, 2, 4, 4, 6, 6,
+ 1, 1, 3, 3, 5, 5, 7, 7};
+static const uvec8 kNV12InterleavedTable = {0, 0, 4, 4, 8, 8, 12, 12,
+ 2, 2, 6, 6, 10, 10, 14, 14};
+static const uvec8 kNV21Table = {1, 1, 3, 3, 5, 5, 7, 7,
+ 0, 0, 2, 2, 4, 4, 6, 6};
+static const uvec8 kNV21InterleavedTable = {1, 1, 5, 5, 9, 9, 13, 13,
+ 3, 3, 7, 7, 11, 11, 15, 15};
+
+// Read 8 Y and 4 UV from NV12 or NV21
#define READNV12 \
- "ld1 {v0.8b}, [%0], #8 \n" \
- "ld1 {v2.8b}, [%1], #8 \n" \
- "uzp1 v1.8b, v2.8b, v2.8b \n" \
- "uzp2 v3.8b, v2.8b, v2.8b \n" \
- "ins v1.s[1], v3.s[0] \n"
-
-// Read 8 Y and 4 VU from NV21
-#define READNV21 \
- "ld1 {v0.8b}, [%0], #8 \n" \
- "ld1 {v2.8b}, [%1], #8 \n" \
- "uzp1 v3.8b, v2.8b, v2.8b \n" \
- "uzp2 v1.8b, v2.8b, v2.8b \n" \
- "ins v1.s[1], v3.s[0] \n"
+ "ldr d0, [%[src_y]], #8 \n" \
+ "ldr d1, [%[src_uv]], #8 \n" \
+ "zip1 v0.16b, v0.16b, v0.16b \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "tbl v1.16b, {v1.16b}, v2.16b \n" \
+ "prfm pldl1keep, [%[src_uv], 448] \n"
// Read 8 YUY2
#define READYUY2 \
- "ld2 {v0.8b, v1.8b}, [%0], #16 \n" \
- "uzp2 v3.8b, v1.8b, v1.8b \n" \
- "uzp1 v1.8b, v1.8b, v1.8b \n" \
- "ins v1.s[1], v3.s[0] \n"
+ "ld1 {v3.16b}, [%[src_yuy2]], #16 \n" \
+ "trn1 v0.16b, v3.16b, v3.16b \n" \
+ "prfm pldl1keep, [%[src_yuy2], 448] \n" \
+ "tbl v1.16b, {v3.16b}, v2.16b \n"
// Read 8 UYVY
#define READUYVY \
- "ld2 {v2.8b, v3.8b}, [%0], #16 \n" \
- "orr v0.8b, v3.8b, v3.8b \n" \
- "uzp1 v1.8b, v2.8b, v2.8b \n" \
- "uzp2 v3.8b, v2.8b, v2.8b \n" \
- "ins v1.s[1], v3.s[0] \n"
-
-#define YUVTORGB_SETUP \
- "ld1r {v24.8h}, [%[kUVBiasBGR]], #2 \n" \
- "ld1r {v25.8h}, [%[kUVBiasBGR]], #2 \n" \
- "ld1r {v26.8h}, [%[kUVBiasBGR]] \n" \
- "ld1r {v31.4s}, [%[kYToRgb]] \n" \
- "ld2 {v27.8h, v28.8h}, [%[kUVToRB]] \n" \
- "ld2 {v29.8h, v30.8h}, [%[kUVToG]] \n"
-
-#define YUVTORGB(vR, vG, vB) \
- "uxtl v0.8h, v0.8b \n" /* Extract Y */ \
- "shll v2.8h, v1.8b, #8 \n" /* Replicate UV */ \
- "ushll2 v3.4s, v0.8h, #0 \n" /* Y */ \
- "ushll v0.4s, v0.4h, #0 \n" \
- "mul v3.4s, v3.4s, v31.4s \n" \
- "mul v0.4s, v0.4s, v31.4s \n" \
- "sqshrun v0.4h, v0.4s, #16 \n" \
- "sqshrun2 v0.8h, v3.4s, #16 \n" /* Y */ \
- "uaddw v1.8h, v2.8h, v1.8b \n" /* Replicate UV */ \
- "mov v2.d[0], v1.d[1] \n" /* Extract V */ \
- "uxtl v2.8h, v2.8b \n" \
- "uxtl v1.8h, v1.8b \n" /* Extract U */ \
- "mul v3.8h, v1.8h, v27.8h \n" \
- "mul v5.8h, v1.8h, v29.8h \n" \
- "mul v6.8h, v2.8h, v30.8h \n" \
- "mul v7.8h, v2.8h, v28.8h \n" \
- "sqadd v6.8h, v6.8h, v5.8h \n" \
- "sqadd " #vB \
- ".8h, v24.8h, v0.8h \n" /* B */ \
- "sqadd " #vG \
- ".8h, v25.8h, v0.8h \n" /* G */ \
- "sqadd " #vR \
- ".8h, v26.8h, v0.8h \n" /* R */ \
- "sqadd " #vB ".8h, " #vB \
- ".8h, v3.8h \n" /* B */ \
- "sqsub " #vG ".8h, " #vG \
- ".8h, v6.8h \n" /* G */ \
- "sqadd " #vR ".8h, " #vR \
- ".8h, v7.8h \n" /* R */ \
- "sqshrun " #vB ".8b, " #vB \
- ".8h, #6 \n" /* B */ \
- "sqshrun " #vG ".8b, " #vG \
- ".8h, #6 \n" /* G */ \
- "sqshrun " #vR ".8b, " #vR ".8h, #6 \n" /* R */
+ "ld1 {v3.16b}, [%[src_uyvy]], #16 \n" \
+ "trn2 v0.16b, v3.16b, v3.16b \n" \
+ "prfm pldl1keep, [%[src_uyvy], 448] \n" \
+ "tbl v1.16b, {v3.16b}, v2.16b \n"
+
+// UB VR UG VG
+// YG BB BG BR
+#define YUVTORGB_SETUP \
+ "ld4r {v28.16b, v29.16b, v30.16b, v31.16b}, [%[kUVCoeff]] \n" \
+ "ld4r {v24.8h, v25.8h, v26.8h, v27.8h}, [%[kRGBCoeffBias]] \n"
+
+#define YUVTORGB_SETUP_AR30 \
+ YUVTORGB_SETUP \
+ "movi v2.8h, #24 \n" \
+ "add v25.8h, v25.8h, v2.8h \n" \
+ "sub v26.8h, v26.8h, v2.8h \n" \
+ "add v27.8h, v27.8h, v2.8h \n"
+
+// v16.8h: B
+// v17.8h: G
+// v18.8h: R
+
+// Convert from YUV (NV12 or NV21) to 2.14 fixed point RGB.
+// Similar to I4XXTORGB but U/V components are in the low/high halves of v1.
+#define NVTORGB \
+ "umull2 v3.4s, v0.8h, v24.8h \n" \
+ "umull v6.8h, v1.8b, v30.8b \n" \
+ "umull v0.4s, v0.4h, v24.4h \n" \
+ "umlal2 v6.8h, v1.16b, v31.16b \n" /* DG */ \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
+ "umull v4.8h, v1.8b, v28.8b \n" /* DB */ \
+ "umull2 v5.8h, v1.16b, v29.16b \n" /* DR */ \
+ "add v17.8h, v0.8h, v26.8h \n" /* G */ \
+ "add v16.8h, v0.8h, v4.8h \n" /* B */ \
+ "add v18.8h, v0.8h, v5.8h \n" /* R */ \
+ "uqsub v17.8h, v17.8h, v6.8h \n" /* G */ \
+ "uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
+ "uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+
+// Convert from YUV (I444 or I420) to 2.14 fixed point RGB.
+// Similar to NVTORGB but U/V components are in v1/v2.
+#define I4XXTORGB \
+ "umull2 v3.4s, v0.8h, v24.8h \n" \
+ "umull v6.8h, v1.8b, v30.8b \n" \
+ "umull v0.4s, v0.4h, v24.4h \n" \
+ "umlal v6.8h, v2.8b, v31.8b \n" /* DG */ \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
+ "umull v4.8h, v1.8b, v28.8b \n" /* DB */ \
+ "umull v5.8h, v2.8b, v29.8b \n" /* DR */ \
+ "add v17.8h, v0.8h, v26.8h \n" /* G */ \
+ "add v16.8h, v0.8h, v4.8h \n" /* B */ \
+ "add v18.8h, v0.8h, v5.8h \n" /* R */ \
+ "uqsub v17.8h, v17.8h, v6.8h \n" /* G */ \
+ "uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
+ "uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+
+// Convert from YUV I400 to 2.14 fixed point RGB
+#define I400TORGB \
+ "umull2 v3.4s, v0.8h, v24.8h \n" \
+ "umull v0.4s, v0.4h, v24.4h \n" \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
+ "add v17.8h, v0.8h, v26.8h \n" /* G */ \
+ "add v16.8h, v0.8h, v4.8h \n" /* B */ \
+ "add v18.8h, v0.8h, v5.8h \n" /* R */ \
+ "uqsub v17.8h, v17.8h, v6.8h \n" /* G */ \
+ "uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
+ "uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+
+// Convert from 2.14 fixed point RGB To 8 bit RGB
+#define RGBTORGB8 \
+ "uqshrn v17.8b, v17.8h, #6 \n" \
+ "uqshrn v16.8b, v16.8h, #6 \n" \
+ "uqshrn v18.8b, v18.8h, #6 \n"
+
+// Convert from 2.14 fixed point RGB to 8 bit RGB, placing the results in the
+// top half of each lane.
+#define RGBTORGB8_TOP \
+ "uqshl v17.8h, v17.8h, #2 \n" \
+ "uqshl v16.8h, v16.8h, #2 \n" \
+ "uqshl v18.8h, v18.8h, #2 \n"
+
+// Store 2.14 fixed point RGB as AR30 elements
+#define STOREAR30 \
+ /* Inputs: \
+ * v16.8h: xxbbbbbbbbbbxxxx \
+ * v17.8h: xxggggggggggxxxx \
+ * v18.8h: xxrrrrrrrrrrxxxx \
+ * v22.8h: 0011111111110000 (umin limit) \
+ * v23.8h: 1100000000000000 (alpha) \
+ */ \
+ "uqshl v0.8h, v16.8h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl v1.8h, v17.8h, #2 \n" /* ggggggggggxxxxxx */ \
+ "umin v6.8h, v18.8h, v22.8h \n" /* 00rrrrrrrrrrxxxx */ \
+ "shl v4.8h, v1.8h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "orr v5.16b, v6.16b, v23.16b \n" /* 11rrrrrrrrrrxxxx */ \
+ "sri v4.8h, v0.8h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "sri v5.8h, v1.8h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "st2 {v4.8h, v5.8h}, [%[dst_ar30]], #32 \n"
+
+#define YUVTORGB_REGS \
+ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v24", \
+ "v25", "v26", "v27", "v28", "v29", "v30", "v31"
void I444ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -118,27 +256,204 @@ void I444ToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n" /* A */
- "1: \n"
- READYUV444
- YUVTORGB(v22, v21, v20)
- "subs %w4, %w4, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb), // %3
- "+r"(width) // %4
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n" /* A */
+ "1: \n" //
+ READYUV444
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I444ToRGB24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n" //
+ READYUV444
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
+}
+
+void I210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ uint16_t alpha = 0xc000;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "dup v23.8h, %w[alpha] \n"
+ "1: \n" //
+ READYUV210
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [alpha] "r"(alpha) // %[alpha]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ uint16_t alpha = 0xc000;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "dup v23.8h, %w[alpha] \n"
+ "1: \n" //
+ READYUV410
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [alpha] "r"(alpha) // %[alpha]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I212ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "1: \n" //
+ READYUV212
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV210
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV410
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I212ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV212
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
void I422ToARGBRow_NEON(const uint8_t* src_y,
@@ -147,27 +462,240 @@ void I422ToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n" /* A */
- "1: \n"
- READYUV422
- YUVTORGB(v22, v21, v20)
- "subs %w4, %w4, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb), // %3
- "+r"(width) // %4
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n" /* A */
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+uint8_t kP210LoadShuffleIndices[] = {1, 1, 5, 5, 9, 9, 13, 13,
+ 3, 3, 7, 7, 11, 11, 15, 15};
+
+void P210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP210
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [kIndices] "r"(kP210LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+uint8_t kP410LoadShuffleIndices[] = {1, 5, 9, 13, 17, 21, 25, 29,
+ 3, 7, 11, 15, 19, 23, 27, 31};
+
+void P410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP410
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [kIndices] "r"(kP410LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void P210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP210
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [kIndices] "r"(kP210LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void P410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP410
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [kIndices] "r"(kP410LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I422ToAR30Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(
+ YUVTORGB_SETUP_AR30
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I444AlphaToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.8b}, [%[src_a]], #8 \n" READYUV444
+ "subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src_a], 448] \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I410AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.16b}, [%[src_a]], #16 \n" READYUV410
+ "subs %w[width], %w[width], #8 \n"
+ "uqshrn v19.8b, v19.8h, #2 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I210AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.16b}, [%[src_a]], #16 \n" READYUV210
+ "subs %w[width], %w[width], #8 \n"
+ "uqshrn v19.8b, v19.8h, #2 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
@@ -177,28 +705,23 @@ void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "1: \n"
- READYUV422
- YUVTORGB(v22, v21, v20)
- "ld1 {v23.8b}, [%3], #8 \n"
- "subs %w5, %w5, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%4], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(src_a), // %3
- "+r"(dst_argb), // %4
- "+r"(width) // %5
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.8b}, [%[src_a]], #8 \n" READYUV422
+ "subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src_a], 448] \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
void I422ToRGBARow_NEON(const uint8_t* src_y,
@@ -207,27 +730,22 @@ void I422ToRGBARow_NEON(const uint8_t* src_y,
uint8_t* dst_rgba,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v20.8b, #255 \n" /* A */
- "1: \n"
- READYUV422
- YUVTORGB(v23, v22, v21)
- "subs %w4, %w4, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%3], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgba), // %3
- "+r"(width) // %4
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v15.8b, #255 \n" /* A */
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st4 {v15.8b,v16.8b,v17.8b,v18.8b}, [%[dst_rgba]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v15");
}
void I422ToRGB24Row_NEON(const uint8_t* src_y,
@@ -236,34 +754,41 @@ void I422ToRGB24Row_NEON(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "1: \n"
- READYUV422
- YUVTORGB(v22, v21, v20)
- "subs %w4, %w4, #8 \n"
- "st3 {v20.8b,v21.8b,v22.8b}, [%3], #24 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgb24), // %3
- "+r"(width) // %4
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
-#define ARGBTORGB565 \
- "shll v0.8h, v22.8b, #8 \n" /* R */ \
- "shll v21.8h, v21.8b, #8 \n" /* G */ \
- "shll v20.8h, v20.8b, #8 \n" /* B */ \
- "sri v0.8h, v21.8h, #5 \n" /* RG */ \
- "sri v0.8h, v20.8h, #11 \n" /* RGB */
+#define ARGBTORGB565 \
+ /* Inputs: \
+ * v16: bbbbbxxx \
+ * v17: ggggggxx \
+ * v18: rrrrrxxx */ \
+ "shll v18.8h, v18.8b, #8 \n" /* rrrrrrxx00000000 */ \
+ "shll v17.8h, v17.8b, #8 \n" /* gggggxxx00000000 */ \
+ "shll v16.8h, v16.8b, #8 \n" /* bbbbbbxx00000000 */ \
+ "sri v18.8h, v17.8h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri v18.8h, v16.8h, #11 \n" /* rrrrrggggggbbbbb */
+
+#define ARGBTORGB565_FROM_TOP \
+ /* Inputs: \
+ * v16: bbbbbxxxxxxxxxxx \
+ * v17: ggggggxxxxxxxxxx \
+ * v18: rrrrrxxxxxxxxxxx */ \
+ "sri v18.8h, v17.8h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri v18.8h, v16.8h, #11 \n" /* rrrrrggggggbbbbb */
void I422ToRGB565Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -273,33 +798,37 @@ void I422ToRGB565Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB(
- v22, v21,
- v20) "subs %w4, %w4, #8 \n" ARGBTORGB565
- "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels
- // RGB565.
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_rgb565), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30");
+ "1: \n" //
+ READYUV422 "subs %w[width], %w[width], #8 \n" I4XXTORGB
+ RGBTORGB8_TOP ARGBTORGB565_FROM_TOP
+ "st1 {v18.8h}, [%[dst_rgb565]], #16 \n" // store 8 pixels RGB565.
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb565] "+r"(dst_rgb565), // %[dst_rgb565]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS);
}
-#define ARGBTOARGB1555 \
- "shll v0.8h, v23.8b, #8 \n" /* A */ \
- "shll v22.8h, v22.8b, #8 \n" /* R */ \
- "shll v21.8h, v21.8b, #8 \n" /* G */ \
- "shll v20.8h, v20.8b, #8 \n" /* B */ \
- "sri v0.8h, v22.8h, #1 \n" /* AR */ \
- "sri v0.8h, v21.8h, #6 \n" /* ARG */ \
- "sri v0.8h, v20.8h, #11 \n" /* ARGB */
+#define ARGBTOARGB1555 \
+ /* Inputs: \
+ * v16: gggggxxxbbbbbxxx v17: axxxxxxxrrrrrxxx */ \
+ "shl v1.8h, v16.8h, #8 \n" /* bbbbbxxx00000000 */ \
+ "shl v2.8h, v17.8h, #8 \n" /* rrrrrxxx00000000 */ \
+ "sri v17.8h, v2.8h, #1 \n" /* arrrrrxxxrrrrxxx */ \
+ "sri v17.8h, v16.8h, #6 \n" /* arrrrrgggggxxxbb */ \
+ "sri v17.8h, v1.8h, #11 \n" /* arrrrrgggggbbbbb */
+
+#define ARGBTOARGB1555_FROM_TOP \
+ /* Inputs: \
+ * v16: bbbbbxxxxxxxxxxx v17: gggggxxxxxxxxxxx \
+ * v18: rrrrrxxxxxxxxxxx v19: axxxxxxxxxxxxxxx */ \
+ "sri v19.8h, v18.8h, #1 \n" /* arrrrrxxxxxxxxxx */ \
+ "sri v19.8h, v17.8h, #6 \n" /* arrrrrgggggxxxxx */ \
+ "sri v19.8h, v16.8h, #11 \n" /* arrrrrgggggbbbbb */
void I422ToARGB1555Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -309,35 +838,28 @@ void I422ToARGB1555Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n" READYUV422 YUVTORGB(
- v22, v21,
- v20) "subs %w4, %w4, #8 \n" ARGBTOARGB1555
- "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels
- // RGB565.
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb1555), // %3
- "+r"(width) // %4
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30");
+ "movi v19.8h, #0x80, lsl #8 \n"
+ "1: \n" //
+ READYUV422 "subs %w[width], %w[width], #8 \n" //
+ I4XXTORGB RGBTORGB8_TOP ARGBTOARGB1555_FROM_TOP
+ "st1 {v19.8h}, [%[dst_argb1555]], #16 \n" // store 8 pixels
+ // RGB1555.
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb1555] "+r"(dst_argb1555), // %[dst_argb1555]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
-#define ARGBTOARGB4444 \
- /* Input v20.8b<=B, v21.8b<=G, v22.8b<=R, v23.8b<=A, v4.8b<=0x0f */ \
- "ushr v20.8b, v20.8b, #4 \n" /* B */ \
- "bic v21.8b, v21.8b, v4.8b \n" /* G */ \
- "ushr v22.8b, v22.8b, #4 \n" /* R */ \
- "bic v23.8b, v23.8b, v4.8b \n" /* A */ \
- "orr v0.8b, v20.8b, v21.8b \n" /* BG */ \
- "orr v1.8b, v22.8b, v23.8b \n" /* RA */ \
- "zip1 v0.16b, v0.16b, v1.16b \n" /* BGRA */
+#define ARGBTOARGB4444 \
+ /* Input v16.8b<=B, v17.8b<=G, v18.8b<=R, v19.8b<=A */ \
+ "sri v17.8b, v16.8b, #4 \n" /* BG */ \
+ "sri v19.8b, v18.8b, #4 \n" /* RA */ \
+ "zip1 v0.16b, v17.16b, v19.16b \n" /* BGRA */
void I422ToARGB4444Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -345,95 +867,112 @@ void I422ToARGB4444Row_NEON(const uint8_t* src_y,
uint8_t* dst_argb4444,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v4.16b, #0x0f \n" // bits to clear with vbic.
- "1: \n"
- READYUV422
- YUVTORGB(v22, v21, v20)
- "subs %w4, %w4, #8 \n"
- "movi v23.8b, #255 \n"
- ARGBTOARGB4444
- "st1 {v0.8h}, [%3], #16 \n" // store 8 pixels ARGB4444.
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_argb4444), // %3
- "+r"(width) // %4
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "movi v19.8b, #255 \n" ARGBTOARGB4444
+ "st1 {v0.8h}, [%[dst_argb4444]], #16 \n" // store 8
+ // pixels
+ // ARGB4444.
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb4444] "+r"(dst_argb4444), // %[dst_argb4444]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
-void I400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n"
- READYUV400
- YUVTORGB(v22, v21, v20)
- "subs %w2, %w2, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB]"r"(&kYuvI601Constants.kUVToRB),
- [kUVToG]"r"(&kYuvI601Constants.kUVToG),
- [kUVBiasBGR]"r"(&kYuvI601Constants.kUVBiasBGR),
- [kYToRgb]"r"(&kYuvI601Constants.kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+void I400ToARGBRow_NEON(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v1.16b, #128 \n"
+ "movi v19.8b, #255 \n"
+ "umull v6.8h, v1.8b, v30.8b \n"
+ "umlal2 v6.8h, v1.16b, v31.16b \n" /* DG */
+ "umull v4.8h, v1.8b, v28.8b \n" /* DB */
+ "umull2 v5.8h, v1.16b, v29.16b \n" /* DR */
+ "1: \n" //
+ READYUV400 I400TORGB
+ "subs %w[width], %w[width], #8 \n" RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
+#if defined(LIBYUV_USE_ST4)
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
- "movi v23.8b, #255 \n"
- "1: \n"
- "ld1 {v20.8b}, [%0], #8 \n"
- "orr v21.8b, v20.8b, v20.8b \n"
- "orr v22.8b, v20.8b, v20.8b \n"
- "subs %w2, %w2, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n"
- "b.gt 1b \n"
+ "movi v23.8b, #255 \n"
+ "1: \n"
+ "ld1 {v20.8b}, [%0], #8 \n"
+ "subs %w2, %w2, #8 \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v21.8b, v20.8b \n"
+ "mov v22.8b, v20.8b \n"
+ "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n"
+ "b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
:
: "cc", "memory", "v20", "v21", "v22", "v23");
}
+#else
+void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "movi v20.8b, #255 \n"
+ "1: \n"
+ "ldr d16, [%0], #8 \n"
+ "subs %w2, %w2, #8 \n"
+ "zip1 v18.16b, v16.16b, v16.16b \n" // YY
+ "zip1 v19.16b, v16.16b, v20.16b \n" // YA
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip1 v16.16b, v18.16b, v19.16b \n" // YYYA
+ "zip2 v17.16b, v18.16b, v19.16b \n"
+ "stp q16, q17, [%1], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20");
+}
+#endif // LIBYUV_USE_ST4
void NV12ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n"
- READNV12
- YUVTORGB(v22, v21, v20)
- "subs %w3, %w3, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%2], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kNV12Table]] \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12Table] "r"(&kNV12Table)
+ : "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
void NV21ToARGBRow_NEON(const uint8_t* src_y,
@@ -441,26 +980,22 @@ void NV21ToARGBRow_NEON(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n"
- READNV21
- YUVTORGB(v22, v21, v20)
- "subs %w3, %w3, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%2], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_vu), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kNV12Table]] \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12Table] "r"(&kNV21Table)
+ : "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
void NV12ToRGB24Row_NEON(const uint8_t* src_y,
@@ -468,25 +1003,21 @@ void NV12ToRGB24Row_NEON(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "1: \n"
- READNV12
- YUVTORGB(v22, v21, v20)
- "subs %w3, %w3, #8 \n"
- "st3 {v20.8b,v21.8b,v22.8b}, [%2], #24 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_rgb24), // %2
- "+r"(width) // %3
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "ldr q2, [%[kNV12Table]] \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12Table] "r"(&kNV12Table)
+ : "cc", "memory", YUVTORGB_REGS, "v2");
}
void NV21ToRGB24Row_NEON(const uint8_t* src_y,
@@ -494,25 +1025,21 @@ void NV21ToRGB24Row_NEON(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "1: \n"
- READNV21
- YUVTORGB(v22, v21, v20)
- "subs %w3, %w3, #8 \n"
- "st3 {v20.8b,v21.8b,v22.8b}, [%2], #24 \n"
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_vu), // %1
- "+r"(dst_rgb24), // %2
- "+r"(width) // %3
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "ldr q2, [%[kNV12Table]] \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12Table] "r"(&kNV21Table)
+ : "cc", "memory", YUVTORGB_REGS, "v2");
}
void NV12ToRGB565Row_NEON(const uint8_t* src_y,
@@ -522,72 +1049,64 @@ void NV12ToRGB565Row_NEON(const uint8_t* src_y,
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READNV12 YUVTORGB(
- v22, v21,
- v20) "subs %w3, %w3, #8 \n" ARGBTORGB565
- "st1 {v0.8h}, [%2], 16 \n" // store 8 pixels
- // RGB565.
- "b.gt 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_uv), // %1
- "+r"(dst_rgb565), // %2
- "+r"(width) // %3
- : [kUVToRB] "r"(&yuvconstants->kUVToRB),
- [kUVToG] "r"(&yuvconstants->kUVToG),
- [kUVBiasBGR] "r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb] "r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30");
+ "ldr q2, [%[kNV12Table]] \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB
+ RGBTORGB8_TOP ARGBTORGB565_FROM_TOP
+ "st1 {v18.8h}, [%[dst_rgb565]], #16 \n" // store 8
+ // pixels
+ // RGB565.
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb565] "+r"(dst_rgb565), // %[dst_rgb565]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12Table] "r"(&kNV12Table)
+ : "cc", "memory", YUVTORGB_REGS, "v2");
}
void YUY2ToARGBRow_NEON(const uint8_t* src_yuy2,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n"
- READYUY2
- YUVTORGB(v22, v21, v20)
- "subs %w2, %w2, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_yuy2), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kNV21InterleavedTable]] \n"
+ "1: \n" //
+ READYUY2 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV21InterleavedTable] "r"(&kNV21InterleavedTable)
+ : "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
void UYVYToARGBRow_NEON(const uint8_t* src_uyvy,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile (
- YUVTORGB_SETUP
- "movi v23.8b, #255 \n"
- "1: \n"
- READUYVY
- YUVTORGB(v22, v21, v20)
- "subs %w2, %w2, #8 \n"
- "st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], 32 \n"
- "b.gt 1b \n"
- : "+r"(src_uyvy), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : [kUVToRB]"r"(&yuvconstants->kUVToRB),
- [kUVToG]"r"(&yuvconstants->kUVToG),
- [kUVBiasBGR]"r"(&yuvconstants->kUVBiasBGR),
- [kYToRgb]"r"(&yuvconstants->kYToRgb)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
- "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30"
- );
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kNV12InterleavedTable]] \n"
+ "1: \n" //
+ READUYVY "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [kNV12InterleavedTable] "r"(&kNV12InterleavedTable)
+ : "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
// Reads 16 pairs of UV and write even values to dst_u and odd to dst_v.
@@ -596,12 +1115,13 @@ void SplitUVRow_NEON(const uint8_t* src_uv,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pairs of UV
- "subs %w3, %w3, #16 \n" // 16 processed per loop
- "st1 {v0.16b}, [%1], #16 \n" // store U
- "st1 {v1.16b}, [%2], #16 \n" // store V
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pairs of UV
+ "subs %w3, %w3, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.16b}, [%1], #16 \n" // store U
+ "st1 {v1.16b}, [%2], #16 \n" // store V
+ "b.gt 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -611,18 +1131,181 @@ void SplitUVRow_NEON(const uint8_t* src_uv,
);
}
+// Reads 16 byte Y's from tile and writes out 16 Y's.
+// MM21 Y tiles are 16x32 so src_tile_stride = 512 bytes
+// MM21 UV tiles are 8x16 so src_tile_stride = 256 bytes
+// width measured in bytes so 8 UV = 16.
+void DetileRow_NEON(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%0], %3 \n" // load 16 bytes
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 1792] \n" // 7 tiles of 256b ahead
+ "st1 {v0.16b}, [%1], #16 \n" // store 16 bytes
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride) // %3
+ : "cc", "memory", "v0" // Clobber List
+ );
+}
+
+// Reads 16 byte Y's of 16 bits from tile and writes out 16 Y's.
+void DetileRow_16_NEON(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.8h,v1.8h}, [%0], %3 \n" // load 16 pixels
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 3584] \n" // 7 tiles of 512b ahead
+ "st1 {v0.8h,v1.8h}, [%1], #32 \n" // store 16 pixels
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(src_tile_stride * 2) // %3
+ : "cc", "memory", "v0", "v1" // Clobber List
+ );
+}
+
+// Read 16 bytes of UV, detile, and write 8 bytes of U and 8 bytes of V.
+void DetileSplitUVRow_NEON(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld2 {v0.8b,v1.8b}, [%0], %4 \n"
+ "subs %w3, %w3, #16 \n"
+ "prfm pldl1keep, [%0, 1792] \n"
+ "st1 {v0.8b}, [%1], #8 \n"
+ "st1 {v1.8b}, [%2], #8 \n"
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(src_tile_stride) // %4
+ : "cc", "memory", "v0", "v1" // Clobber List
+ );
+}
+
+#if defined(LIBYUV_USE_ST2)
+// Read 16 Y, 8 UV, and write 8 YUY2
+void DetileToYUY2_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%0], %4 \n" // load 16 Ys
+ "subs %w3, %w3, #16 \n" // store 8 YUY2
+ "prfm pldl1keep, [%0, 1792] \n"
+ "ld1 {v1.16b}, [%1], %5 \n" // load 8 UVs
+ "prfm pldl1keep, [%1, 1792] \n"
+ "st2 {v0.16b,v1.16b}, [%2], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_uv), // %1
+ "+r"(dst_yuy2), // %2
+ "+r"(width) // %3
+ : "r"(src_y_tile_stride), // %4
+ "r"(src_uv_tile_stride) // %5
+ : "cc", "memory", "v0", "v1" // Clobber list
+ );
+}
+#else
+// Read 16 Y, 8 UV, and write 8 YUY2
+void DetileToYUY2_NEON(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%0], %4 \n" // load 16 Ys
+ "ld1 {v1.16b}, [%1], %5 \n" // load 8 UVs
+ "subs %w3, %w3, #16 \n"
+ "prfm pldl1keep, [%0, 1792] \n"
+ "zip1 v2.16b, v0.16b, v1.16b \n"
+ "prfm pldl1keep, [%1, 1792] \n"
+ "zip2 v3.16b, v0.16b, v1.16b \n"
+ "st1 {v2.16b,v3.16b}, [%2], #32 \n" // store 8 YUY2
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_uv), // %1
+ "+r"(dst_yuy2), // %2
+ "+r"(width) // %3
+ : "r"(src_y_tile_stride), // %4
+ "r"(src_uv_tile_stride) // %5
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber list
+ );
+}
+#endif
+
+// Unpack MT2T into tiled P010 64 pixels at a time. See
+// tinyurl.com/mtk-10bit-video-format for format documentation.
+void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v7.16b}, [%0], #16 \n"
+ "ld1 {v0.16b-v3.16b}, [%0], #64 \n"
+ "subs %2, %2, #80 \n"
+ "shl v4.16b, v7.16b, #6 \n"
+ "shl v5.16b, v7.16b, #4 \n"
+ "shl v6.16b, v7.16b, #2 \n"
+ "zip1 v16.16b, v4.16b, v0.16b \n"
+ "zip1 v18.16b, v5.16b, v1.16b \n"
+ "zip1 v20.16b, v6.16b, v2.16b \n"
+ "zip1 v22.16b, v7.16b, v3.16b \n"
+ "zip2 v17.16b, v4.16b, v0.16b \n"
+ "zip2 v19.16b, v5.16b, v1.16b \n"
+ "zip2 v21.16b, v6.16b, v2.16b \n"
+ "zip2 v23.16b, v7.16b, v3.16b \n"
+ "sri v16.8h, v16.8h, #10 \n"
+ "sri v17.8h, v17.8h, #10 \n"
+ "sri v18.8h, v18.8h, #10 \n"
+ "sri v19.8h, v19.8h, #10 \n"
+ "st1 {v16.8h-v19.8h}, [%1], #64 \n"
+ "sri v20.8h, v20.8h, #10 \n"
+ "sri v21.8h, v21.8h, #10 \n"
+ "sri v22.8h, v22.8h, #10 \n"
+ "sri v23.8h, v23.8h, #10 \n"
+ "st1 {v20.8h-v23.8h}, [%1], #64 \n"
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(size) // %2
+ :
+ : "cc", "memory", "w0", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+
+#if defined(LIBYUV_USE_ST2)
// Reads 16 U's and V's and writes out 16 pairs of UV.
void MergeUVRow_NEON(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load U
- "ld1 {v1.16b}, [%1], #16 \n" // load V
- "subs %w3, %w3, #16 \n" // 16 processed per loop
- "st2 {v0.16b,v1.16b}, [%2], #32 \n" // store 16 pairs of UV
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load U
+ "ld1 {v1.16b}, [%1], #16 \n" // load V
+ "subs %w3, %w3, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "st2 {v0.16b,v1.16b}, [%2], #32 \n" // store 16 pairs of UV
+ "b.gt 1b \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -632,6 +1315,86 @@ void MergeUVRow_NEON(const uint8_t* src_u,
);
}
+void MergeUVRow_16_NEON(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ asm volatile(
+ "dup v2.8h, %w4 \n"
+ "1: \n"
+ "ld1 {v0.8h}, [%0], #16 \n" // load 8 U
+ "ld1 {v1.8h}, [%1], #16 \n" // load 8 V
+ "subs %w3, %w3, #8 \n" // 8 src pixels per loop
+ "ushl v0.8h, v0.8h, v2.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "ushl v1.8h, v1.8h, v2.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "st2 {v0.8h, v1.8h}, [%2], #32 \n" // store 8 UV pixels
+ "b.gt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "v0", "v1", "v2");
+}
+#else
+// Reads 16 U's and V's and writes out 16 pairs of UV.
+void MergeUVRow_NEON(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load U
+ "ld1 {v1.16b}, [%1], #16 \n" // load V
+ "subs %w3, %w3, #16 \n" // 16 processed per loop
+ "zip1 v2.16b, v0.16b, v1.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip2 v3.16b, v0.16b, v1.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "st1 {v2.16b,v3.16b}, [%2], #32 \n" // store 16 pairs of UV
+ "b.gt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3 // Output registers
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ );
+}
+
+void MergeUVRow_16_NEON(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ asm volatile(
+ "dup v4.8h, %w4 \n"
+ "1: \n"
+ "ld1 {v0.8h}, [%0], #16 \n" // load 8 U
+ "ld1 {v1.8h}, [%1], #16 \n" // load 8 V
+ "subs %w3, %w3, #8 \n" // 8 src pixels per loop
+ "ushl v0.8h, v0.8h, v4.8h \n"
+ "ushl v1.8h, v1.8h, v4.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip1 v2.8h, v0.8h, v1.8h \n"
+ "zip2 v3.8h, v0.8h, v1.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "st1 {v2.8h, v3.8h}, [%2], #32 \n" // store 8 UV pixels
+ "b.gt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v1", "v2", "v3", "v4");
+}
+#endif // LIBYUV_USE_ST2
+
// Reads 16 packed RGB and write to planar dst_r, dst_g, dst_b.
void SplitRGBRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_r,
@@ -639,13 +1402,14 @@ void SplitRGBRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
- "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 RGB
- "subs %w4, %w4, #16 \n" // 16 processed per loop
- "st1 {v0.16b}, [%1], #16 \n" // store R
- "st1 {v1.16b}, [%2], #16 \n" // store G
- "st1 {v2.16b}, [%3], #16 \n" // store B
- "b.gt 1b \n"
+ "1: \n"
+ "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 RGB
+ "subs %w4, %w4, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.16b}, [%1], #16 \n" // store R
+ "st1 {v1.16b}, [%2], #16 \n" // store G
+ "st1 {v2.16b}, [%3], #16 \n" // store B
+ "b.gt 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
@@ -663,13 +1427,16 @@ void MergeRGBRow_NEON(const uint8_t* src_r,
uint8_t* dst_rgb,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load R
- "ld1 {v1.16b}, [%1], #16 \n" // load G
- "ld1 {v2.16b}, [%2], #16 \n" // load B
- "subs %w4, %w4, #16 \n" // 16 processed per loop
- "st3 {v0.16b,v1.16b,v2.16b}, [%3], #48 \n" // store 16 RGB
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load R
+ "ld1 {v1.16b}, [%1], #16 \n" // load G
+ "ld1 {v2.16b}, [%2], #16 \n" // load B
+ "subs %w4, %w4, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "st3 {v0.16b,v1.16b,v2.16b}, [%3], #48 \n" // store 16 RGB
+ "b.gt 1b \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -680,14 +1447,402 @@ void MergeRGBRow_NEON(const uint8_t* src_r,
);
}
+// Reads 16 packed ARGB and write to planar dst_r, dst_g, dst_b, dst_a.
+void SplitARGBRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
+ "subs %w5, %w5, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.16b}, [%3], #16 \n" // store B
+ "st1 {v1.16b}, [%2], #16 \n" // store G
+ "st1 {v2.16b}, [%1], #16 \n" // store R
+ "st1 {v3.16b}, [%4], #16 \n" // store A
+ "b.gt 1b \n"
+ : "+r"(src_rgba), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(dst_a), // %4
+ "+r"(width) // %5
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ );
+}
+
+#if defined(LIBYUV_USE_ST4)
+// Reads 16 planar R's, G's, B's and A's and writes out 16 packed ARGB at a time
+void MergeARGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%2], #16 \n" // load B
+ "ld1 {v1.16b}, [%1], #16 \n" // load G
+ "ld1 {v2.16b}, [%0], #16 \n" // load R
+ "ld1 {v3.16b}, [%3], #16 \n" // load A
+ "subs %w5, %w5, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%4], #64 \n" // store 16ARGB
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ );
+}
+#else
+// Reads 16 planar R's, G's, B's and A's and writes out 16 packed ARGB at a time
+void MergeARGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%2], #16 \n" // load B
+ "ld1 {v1.16b}, [%1], #16 \n" // load G
+ "ld1 {v2.16b}, [%0], #16 \n" // load R
+ "ld1 {v3.16b}, [%3], #16 \n" // load A
+ "subs %w5, %w5, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%2, 448] \n"
+ "zip1 v4.16b, v0.16b, v1.16b \n" // BG
+ "zip1 v5.16b, v2.16b, v3.16b \n" // RA
+ "prfm pldl1keep, [%1, 448] \n"
+ "zip2 v6.16b, v0.16b, v1.16b \n" // BG
+ "zip2 v7.16b, v2.16b, v3.16b \n" // RA
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip1 v0.8h, v4.8h, v5.8h \n" // BGRA
+ "zip2 v1.8h, v4.8h, v5.8h \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "zip1 v2.8h, v6.8h, v7.8h \n"
+ "zip2 v3.8h, v6.8h, v7.8h \n"
+ "st1 {v0.16b,v1.16b,v2.16b,v3.16b}, [%4], #64 \n" // store 16ARGB
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+ "v7" // Clobber List
+ );
+}
+#endif // LIBYUV_USE_ST4
+
+// Reads 16 packed ARGB and write to planar dst_r, dst_g, dst_b.
+void SplitXRGBRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
+ "subs %w4, %w4, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.16b}, [%3], #16 \n" // store B
+ "st1 {v1.16b}, [%2], #16 \n" // store G
+ "st1 {v2.16b}, [%1], #16 \n" // store R
+ "b.gt 1b \n"
+ : "+r"(src_rgba), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ );
+}
+
+// Reads 16 planar R's, G's and B's and writes out 16 packed ARGB at a time
+void MergeXRGBRow_NEON(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "movi v3.16b, #255 \n" // load A(255)
+ "1: \n"
+ "ld1 {v2.16b}, [%0], #16 \n" // load R
+ "ld1 {v1.16b}, [%1], #16 \n" // load G
+ "ld1 {v0.16b}, [%2], #16 \n" // load B
+ "subs %w4, %w4, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%3], #64 \n" // store 16ARGB
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_argb), // %3
+ "+r"(width) // %4
+ : // Input registers
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ );
+}
+
+void MergeXR30Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ int shift = 10 - depth;
+ asm volatile(
+ "movi v30.16b, #255 \n"
+ "ushr v30.4s, v30.4s, #22 \n" // 1023
+ "dup v31.4s, %w5 \n"
+ "1: \n"
+ "ldr d2, [%2], #8 \n" // B
+ "ldr d1, [%1], #8 \n" // G
+ "ldr d0, [%0], #8 \n" // R
+ "subs %w4, %w4, #4 \n"
+ "ushll v2.4s, v2.4h, #0 \n" // B
+ "ushll v1.4s, v1.4h, #0 \n" // G
+ "ushll v0.4s, v0.4h, #0 \n" // R
+ "ushl v2.4s, v2.4s, v31.4s \n" // 000B
+ "ushl v1.4s, v1.4s, v31.4s \n" // G
+ "ushl v0.4s, v0.4s, v31.4s \n" // R
+ "umin v2.4s, v2.4s, v30.4s \n"
+ "umin v1.4s, v1.4s, v30.4s \n"
+ "umin v0.4s, v0.4s, v30.4s \n"
+ "sli v2.4s, v1.4s, #10 \n" // 00GB
+ "sli v2.4s, v0.4s, #20 \n" // 0RGB
+ "orr v2.4s, #0xc0, lsl #24 \n" // ARGB (AR30)
+ "str q2, [%3], #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar30), // %3
+ "+r"(width) // %4
+ : "r"(shift) // %5
+ : "memory", "cc", "v0", "v1", "v2", "v30", "v31");
+}
+
+void MergeXR30Row_10_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int /* depth */,
+ int width) {
+ // Neon has no "shift left and accumulate/orr", so use a multiply-add to
+ // perform the shift instead.
+ int limit = 1023;
+ asm volatile(
+ "dup v5.8h, %w[limit] \n"
+ "movi v6.8h, #16 \n" // 1 << 4
+ "movi v7.8h, #4, lsl #8 \n" // 1 << 10
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // xxxxxxRrrrrrrrrr
+ "ldr q1, [%1], #16 \n" // xxxxxxGggggggggg
+ "ldr q2, [%2], #16 \n" // xxxxxxBbbbbbbbbb
+ "subs %w4, %w4, #8 \n"
+ "umin v0.8h, v0.8h, v5.8h \n" // 000000Rrrrrrrrrr
+ "umin v1.8h, v1.8h, v5.8h \n" // 000000Gggggggggg
+ "movi v4.8h, #0xc0, lsl #8 \n" // 1100000000000000
+ "umin v3.8h, v2.8h, v5.8h \n" // 000000Bbbbbbbbbb
+ "mla v4.8h, v0.8h, v6.8h \n" // 11Rrrrrrrrrr0000
+ "mla v3.8h, v1.8h, v7.8h \n" // ggggggBbbbbbbbbb
+ "usra v4.8h, v1.8h, #6 \n" // 11RrrrrrrrrrGggg
+ "st2 {v3.8h, v4.8h}, [%3], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar30), // %3
+ "+r"(width) // %4
+ : [limit] "r"(limit)
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+
+void MergeAR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ int mask = (1 << depth) - 1;
+ asm volatile(
+
+ "dup v30.8h, %w7 \n"
+ "dup v31.8h, %w6 \n"
+ "1: \n"
+ "ldr q2, [%0], #16 \n" // R
+ "ldr q1, [%1], #16 \n" // G
+ "ldr q0, [%2], #16 \n" // B
+ "ldr q3, [%3], #16 \n" // A
+ "subs %w5, %w5, #8 \n"
+ "umin v2.8h, v2.8h, v30.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "umin v1.8h, v1.8h, v30.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "umin v0.8h, v0.8h, v30.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "umin v3.8h, v3.8h, v30.8h \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "ushl v2.8h, v2.8h, v31.8h \n"
+ "ushl v1.8h, v1.8h, v31.8h \n"
+ "ushl v0.8h, v0.8h, v31.8h \n"
+ "ushl v3.8h, v3.8h, v31.8h \n"
+ "st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%4], #64 \n"
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_ar64), // %4
+ "+r"(width) // %5
+ : "r"(shift), // %6
+ "r"(mask) // %7
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v30", "v31");
+}
+
+void MergeXR64Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ int mask = (1 << depth) - 1;
+ asm volatile(
+
+ "movi v3.16b, #0xff \n" // A (0xffff)
+ "dup v30.8h, %w6 \n"
+ "dup v31.8h, %w5 \n"
+
+ "1: \n"
+ "ldr q2, [%0], #16 \n" // R
+ "ldr q1, [%1], #16 \n" // G
+ "ldr q0, [%2], #16 \n" // B
+ "subs %w4, %w4, #8 \n"
+ "umin v2.8h, v2.8h, v30.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "umin v1.8h, v1.8h, v30.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "umin v0.8h, v0.8h, v30.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "ushl v2.8h, v2.8h, v31.8h \n"
+ "ushl v1.8h, v1.8h, v31.8h \n"
+ "ushl v0.8h, v0.8h, v31.8h \n"
+ "st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%3], #64 \n"
+ "b.gt 1b \n"
+ : "+r"(src_r), // %0
+ "+r"(src_g), // %1
+ "+r"(src_b), // %2
+ "+r"(dst_ar64), // %3
+ "+r"(width) // %4
+ : "r"(shift), // %5
+ "r"(mask) // %6
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v30", "v31");
+}
+
+void MergeARGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ // Shift is 8 - depth, +8 so the result is in the top half of each lane.
+ int shift = 16 - depth;
+ asm volatile(
+ "dup v31.8h, %w6 \n"
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // B
+ "ldr q1, [%1], #16 \n" // G
+ "ldr q2, [%2], #16 \n" // R
+ "ldr q3, [%3], #16 \n" // A
+ "subs %w5, %w5, #8 \n"
+ "uqshl v0.8h, v0.8h, v31.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqshl v1.8h, v1.8h, v31.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqshl v2.8h, v2.8h, v31.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "uqshl v3.8h, v3.8h, v31.8h \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "trn2 v0.16b, v0.16b, v1.16b \n"
+ "trn2 v1.16b, v2.16b, v3.16b \n"
+ "st2 {v0.8h, v1.8h}, [%4], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_b), // %0
+ "+r"(src_g), // %1
+ "+r"(src_r), // %2
+ "+r"(src_a), // %3
+ "+r"(dst_argb), // %4
+ "+r"(width) // %5
+ : "r"(shift) // %6
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v31");
+}
+
+void MergeXRGB16To8Row_NEON(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ // Shift is 8 - depth, +8 so the result is in the top half of each lane.
+ int shift = 16 - depth;
+ asm volatile(
+ "dup v31.8h, %w5 \n"
+ "movi v3.16b, #0xff \n" // A (0xff)
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // B
+ "ldr q1, [%1], #16 \n" // G
+ "ldr q2, [%2], #16 \n" // R
+ "subs %w4, %w4, #8 \n"
+ "uqshl v0.8h, v0.8h, v31.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqshl v1.8h, v1.8h, v31.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqshl v2.8h, v2.8h, v31.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "trn2 v0.16b, v0.16b, v1.16b \n"
+ "trn2 v1.16b, v2.16b, v3.16b \n"
+ "st2 {v0.8h, v1.8h}, [%3], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_b), // %0
+ "+r"(src_g), // %1
+ "+r"(src_r), // %2
+ "+r"(dst_argb), // %3
+ "+r"(width) // %4
+ : "r"(shift) // %5
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v31");
+}
+
// Copy multiple of 32.
void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "1: \n"
- "ldp q0, q1, [%0], #32 \n"
- "subs %w2, %w2, #32 \n" // 32 processed per loop
- "stp q0, q1, [%1], #32 \n"
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "subs %w2, %w2, #32 \n" // 32 processed per loop
+ "stp q0, q1, [%1], #32 \n"
+ "b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2 // Output registers
@@ -699,11 +1854,11 @@ void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
// SetRow writes 'width' bytes using an 8 bit value repeated.
void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
asm volatile(
- "dup v0.16b, %w2 \n" // duplicate 16 bytes
- "1: \n"
- "subs %w1, %w1, #16 \n" // 16 bytes per loop
- "st1 {v0.16b}, [%0], #16 \n" // store
- "b.gt 1b \n"
+ "dup v0.16b, %w2 \n" // duplicate 16 bytes
+ "1: \n"
+ "subs %w1, %w1, #16 \n" // 16 bytes per loop
+ "st1 {v0.16b}, [%0], #16 \n" // store
+ "b.gt 1b \n"
: "+r"(dst), // %0
"+r"(width) // %1
: "r"(v8) // %2
@@ -712,89 +1867,157 @@ void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width) {
asm volatile(
- "dup v0.4s, %w2 \n" // duplicate 4 ints
- "1: \n"
- "subs %w1, %w1, #4 \n" // 4 ints per loop
- "st1 {v0.16b}, [%0], #16 \n" // store
- "b.gt 1b \n"
+ "dup v0.4s, %w2 \n" // duplicate 4 ints
+ "1: \n"
+ "subs %w1, %w1, #4 \n" // 4 ints per loop
+ "st1 {v0.16b}, [%0], #16 \n" // store
+ "b.gt 1b \n"
: "+r"(dst), // %0
"+r"(width) // %1
: "r"(v32) // %2
: "cc", "memory", "v0");
}
+// Shuffle table for reversing the bytes.
+static const uvec8 kShuffleMirror = {15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u,
+ 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
+
void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
// Start at end of source row.
- "add %0, %0, %w2, sxtw \n"
- "sub %0, %0, #16 \n"
- "1: \n"
- "ld1 {v0.16b}, [%0], %3 \n" // src -= 16
- "subs %w2, %w2, #16 \n" // 16 pixels per loop.
- "rev64 v0.16b, v0.16b \n"
- "st1 {v0.D}[1], [%1], #8 \n" // dst += 16
- "st1 {v0.D}[0], [%1], #8 \n"
- "b.gt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "r"((ptrdiff_t)-16) // %3
- : "cc", "memory", "v0");
+ "ld1 {v3.16b}, [%3] \n" // shuffler
+ "add %0, %0, %w2, sxtw \n"
+ "sub %0, %0, #32 \n"
+ "1: \n"
+ "ldr q2, [%0, 16] \n"
+ "ldr q1, [%0], -32 \n" // src -= 32
+ "subs %w2, %w2, #32 \n" // 32 pixels per loop.
+ "tbl v0.16b, {v2.16b}, v3.16b \n"
+ "tbl v1.16b, {v1.16b}, v3.16b \n"
+ "st1 {v0.16b, v1.16b}, [%1], #32 \n" // store 32 pixels
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleMirror) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3");
}
-void MirrorUVRow_NEON(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+// Shuffle table for reversing the UV.
+static const uvec8 kShuffleMirrorUV = {14u, 15u, 12u, 13u, 10u, 11u, 8u, 9u,
+ 6u, 7u, 4u, 5u, 2u, 3u, 0u, 1u};
+
+void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
asm volatile(
// Start at end of source row.
- "add %0, %0, %w3, sxtw #1 \n"
- "sub %0, %0, #16 \n"
- "1: \n"
- "ld2 {v0.8b, v1.8b}, [%0], %4 \n" // src -= 16
- "subs %w3, %w3, #8 \n" // 8 pixels per loop.
- "rev64 v0.8b, v0.8b \n"
- "rev64 v1.8b, v1.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // dst += 8
- "st1 {v1.8b}, [%2], #8 \n"
- "b.gt 1b \n"
- : "+r"(src_uv), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"((ptrdiff_t)-16) // %4
- : "cc", "memory", "v0", "v1");
+ "ld1 {v4.16b}, [%3] \n" // shuffler
+ "add %0, %0, %w2, sxtw #1 \n"
+ "sub %0, %0, #32 \n"
+ "1: \n"
+ "ldr q1, [%0, 16] \n"
+ "ldr q0, [%0], -32 \n" // src -= 32
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop.
+ "tbl v2.16b, {v1.16b}, v4.16b \n"
+ "tbl v3.16b, {v0.16b}, v4.16b \n"
+ "st1 {v2.16b, v3.16b}, [%1], #32 \n" // dst += 32
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleMirrorUV) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
}
-void ARGBMirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
+void MirrorSplitUVRow_NEON(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
asm volatile(
// Start at end of source row.
- "add %0, %0, %w2, sxtw #2 \n"
- "sub %0, %0, #16 \n"
- "1: \n"
- "ld1 {v0.16b}, [%0], %3 \n" // src -= 16
- "subs %w2, %w2, #4 \n" // 4 pixels per loop.
- "rev64 v0.4s, v0.4s \n"
- "st1 {v0.D}[1], [%1], #8 \n" // dst += 16
- "st1 {v0.D}[0], [%1], #8 \n"
- "b.gt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "r"((ptrdiff_t)-16) // %3
- : "cc", "memory", "v0");
+ "ld1 {v4.16b}, [%4] \n" // shuffler
+ "add %0, %0, %w3, sxtw #1 \n"
+ "sub %0, %0, #32 \n"
+ "1: \n"
+ "ldr q1, [%0, 16] \n"
+ "ldr q0, [%0], -32 \n" // src -= 32
+ "subs %w3, %w3, #16 \n" // 16 pixels per loop.
+ "tbl v2.16b, {v1.16b}, v4.16b \n"
+ "tbl v3.16b, {v0.16b}, v4.16b \n"
+ "uzp1 v0.16b, v2.16b, v3.16b \n" // U
+ "uzp2 v1.16b, v2.16b, v3.16b \n" // V
+ "st1 {v0.16b}, [%1], #16 \n" // dst += 16
+ "st1 {v1.16b}, [%2], #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(&kShuffleMirrorUV) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+// Shuffle table for reversing the ARGB.
+static const uvec8 kShuffleMirrorARGB = {12u, 13u, 14u, 15u, 8u, 9u, 10u, 11u,
+ 4u, 5u, 6u, 7u, 0u, 1u, 2u, 3u};
+
+void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ asm volatile(
+ // Start at end of source row.
+ "ld1 {v4.16b}, [%3] \n" // shuffler
+ "add %0, %0, %w2, sxtw #2 \n"
+ "sub %0, %0, #32 \n"
+ "1: \n"
+ "ldr q1, [%0, 16] \n"
+ "ldr q0, [%0], -32 \n" // src -= 32
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop.
+ "tbl v2.16b, {v1.16b}, v4.16b \n"
+ "tbl v3.16b, {v0.16b}, v4.16b \n"
+ "st1 {v2.16b, v3.16b}, [%1], #32 \n" // dst += 32
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleMirrorARGB) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ asm volatile(
+ "ld1 {v3.16b}, [%4] \n" // shuffler
+ "add %0, %0, %w2, sxtw #1 \n" // Start at end of row.
+ "add %0, %0, %w2, sxtw \n"
+ "sub %0, %0, #48 \n"
+
+ "1: \n"
+ "ld3 {v0.16b, v1.16b, v2.16b}, [%0], %3 \n" // src -= 48
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop.
+ "tbl v0.16b, {v0.16b}, v3.16b \n"
+ "tbl v1.16b, {v1.16b}, v3.16b \n"
+ "tbl v2.16b, {v2.16b}, v3.16b \n"
+ "st3 {v0.16b, v1.16b, v2.16b}, [%1], #48 \n" // dst += 48
+ "b.gt 1b \n"
+ : "+r"(src_rgb24), // %0
+ "+r"(dst_rgb24), // %1
+ "+r"(width) // %2
+ : "r"((ptrdiff_t)-48), // %3
+ "r"(&kShuffleMirror) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3");
}
void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v4.8b, #255 \n" // Alpha
- "1: \n"
- "ld3 {v1.8b,v2.8b,v3.8b}, [%0], #24 \n" // load 8 pixels of RGB24.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "st4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%1], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "movi v4.8b, #255 \n" // Alpha
+ "1: \n"
+ "ld3 {v1.8b,v2.8b,v3.8b}, [%0], #24 \n" // load 8 pixels of
+ // RGB24.
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%1], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_rgb24), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -805,14 +2028,15 @@ void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
- "movi v5.8b, #255 \n" // Alpha
- "1: \n"
- "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v3.8b, v1.8b, v1.8b \n" // move g
- "orr v4.8b, v0.8b, v0.8b \n" // move r
- "st4 {v2.8b,v3.8b,v4.8b,v5.8b}, [%1], #32 \n" // store b g r a
- "b.gt 1b \n"
+ "movi v5.8b, #255 \n" // Alpha
+ "1: \n"
+ "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "mov v3.8b, v1.8b \n" // move g
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v4.8b, v0.8b \n" // move r
+ "st4 {v2.8b,v3.8b,v4.8b,v5.8b}, [%1], #32 \n" // store b g r a
+ "b.gt 1b \n"
: "+r"(src_raw), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -821,15 +2045,35 @@ void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
);
}
+void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ asm volatile(
+ "movi v0.8b, #255 \n" // Alpha
+ "1: \n"
+ "ld3 {v3.8b,v4.8b,v5.8b}, [%0], #24 \n" // read r g b
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "mov v2.8b, v4.8b \n" // move g
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v1.8b, v5.8b \n" // move r
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store a b g r
+ "b.gt 1b \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_rgba), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5" // Clobber List
+ );
+}
+
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
asm volatile(
- "1: \n"
- "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v3.8b, v1.8b, v1.8b \n" // move g
- "orr v4.8b, v0.8b, v0.8b \n" // move r
- "st3 {v2.8b,v3.8b,v4.8b}, [%1], #24 \n" // store b g r
- "b.gt 1b \n"
+ "1: \n"
+ "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // read 16 r g b
+ "subs %w2, %w2, #16 \n" // 16 per loop.
+ "mov v3.16b, v1.16b \n" // move g
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v4.16b, v0.16b \n" // move r
+ "st3 {v2.16b,v3.16b,v4.16b}, [%1], #48 \n" // store 16 b g r
+ "b.gt 1b \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
@@ -838,30 +2082,29 @@ void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
);
}
-#define RGB565TOARGB \
- "shrn v6.8b, v0.8h, #5 \n" /* G xxGGGGGG */ \
- "shl v6.8b, v6.8b, #2 \n" /* G GGGGGG00 upper 6 */ \
- "ushr v4.8b, v6.8b, #6 \n" /* G 000000GG lower 2 */ \
- "orr v1.8b, v4.8b, v6.8b \n" /* G */ \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "ushr v0.8h, v0.8h, #11 \n" /* R 000RRRRR */ \
- "xtn2 v2.16b,v0.8h \n" /* R in upper part */ \
- "shl v2.16b, v2.16b, #3 \n" /* R,B BBBBB000 upper 5 */ \
- "ushr v0.16b, v2.16b, #5 \n" /* R,B 00000BBB lower 3 */ \
- "orr v0.16b, v0.16b, v2.16b \n" /* R,B */ \
- "dup v2.2D, v0.D[1] \n" /* R */
+#define RGB565TOARGB \
+ /* Input: v0/v4.8h: RRRRRGGGGGGBBBBB */ \
+ "shrn v1.8b, v0.8h, #3 \n" /* G GGGGGGxx */ \
+ "shrn2 v1.16b, v4.8h, #3 \n" /* G GGGGGGxx */ \
+ "uzp2 v2.16b, v0.16b, v4.16b \n" /* R RRRRRxxx */ \
+ "uzp1 v0.16b, v0.16b, v4.16b \n" /* B xxxBBBBB */ \
+ "sri v1.16b, v1.16b, #6 \n" /* G GGGGGGGG, fill 2 */ \
+ "shl v0.16b, v0.16b, #3 \n" /* B BBBBB000 */ \
+ "sri v2.16b, v2.16b, #5 \n" /* R RRRRRRRR, fill 3 */ \
+ "sri v0.16b, v0.16b, #5 \n" /* R BBBBBBBB, fill 3 */
void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // Alpha
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- RGB565TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "movi v3.16b, #255 \n" // Alpha
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n" RGB565TOARGB
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%1] \n" // store 16 ARGB
+ "add %1, %1, #64 \n"
+ "b.gt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -870,86 +2113,80 @@ void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565,
);
}
-#define ARGB1555TOARGB \
- "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \
- "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \
- "xtn v3.8b, v2.8h \n" /* RRRRR000 AAAAAAAA */ \
- \
- "sshr v2.8h, v0.8h, #15 \n" /* A AAAAAAAA */ \
- "xtn2 v3.16b, v2.8h \n" \
- \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \
- \
- "ushr v1.16b, v3.16b, #5 \n" /* R,A 00000RRR lower 3 */ \
- "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \
- "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \
- \
- "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \
- "orr v2.16b, v1.16b, v3.16b \n" /* R,A */ \
- "dup v1.2D, v0.D[1] \n" \
- "dup v3.2D, v2.D[1] \n"
+#define ARGB1555TOARGB \
+ /* Input: ARRRRRGGGGGBBBBB */ \
+ "shrn v2.8b, v0.8h, #7 \n" /* RRRRRxxx */ \
+ "uzp1 v29.16b, v0.16b, v4.16b \n" /* xxxBBBBB */ \
+ "shrn v1.8b, v0.8h, #2 \n" /* GGGGGxxx */ \
+ "uzp2 v3.16b, v0.16b, v4.16b \n" /* Axxxxxxx */ \
+ "shrn2 v2.16b, v4.8h, #7 \n" /* RRRRRxxx */ \
+ "shl v0.16b, v29.16b, #3 \n" /* BBBBB000 */ \
+ "shrn2 v1.16b, v4.8h, #2 \n" /* GGGGGxxx */ \
+ "sshr v3.16b, v3.16b, #7 \n" /* AAAAAAAA */ \
+ "sri v2.16b, v2.16b, #5 \n" /* RRRRRRRR */ \
+ "sri v1.16b, v1.16b, #5 \n" /* GGGGGGGG */ \
+ "sri v0.16b, v0.16b, #5 \n" /* BBBBBBBB */
// RGB555TOARGB is same as ARGB1555TOARGB but ignores alpha.
-#define RGB555TOARGB \
- "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \
- "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \
- "xtn v3.8b, v2.8h \n" /* RRRRR000 */ \
- \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \
- \
- "ushr v1.16b, v3.16b, #5 \n" /* R 00000RRR lower 3 */ \
- "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \
- "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \
- \
- "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \
- "orr v2.16b, v1.16b, v3.16b \n" /* R */ \
- "dup v1.2D, v0.D[1] \n" /* G */
+#define RGB555TOARGB \
+ /* Input: xRRRRRGGGGGBBBBB */ \
+ "uzp1 v29.16b, v0.16b, v3.16b \n" /* xxxBBBBB */ \
+ "shrn v2.8b, v0.8h, #7 \n" /* RRRRRxxx */ \
+ "shrn v1.8b, v0.8h, #2 \n" /* GGGGGxxx */ \
+ "shl v0.16b, v29.16b, #3 \n" /* BBBBB000 */ \
+ "shrn2 v2.16b, v3.8h, #7 \n" /* RRRRRxxx */ \
+ "shrn2 v1.16b, v3.8h, #2 \n" /* GGGGGxxx */ \
+ \
+ "sri v0.16b, v0.16b, #5 \n" /* BBBBBBBB */ \
+ "sri v2.16b, v2.16b, #5 \n" /* RRRRRRRR */ \
+ "sri v1.16b, v1.16b, #5 \n" /* GGGGGGGG */
void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // Alpha
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 ARGB1555 pixels
+ "prfm pldl1keep, [%0, 448] \n"
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
ARGB1555TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
- // pixels
- "b.gt 1b \n"
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%1] \n" // store 16 ARGB
+ "add %1, %1, #64 \n"
+ "b.gt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v29" // Clobber List
);
}
-#define ARGB4444TOARGB \
- "shrn v1.8b, v0.8h, #8 \n" /* v1(l) AR */ \
- "xtn2 v1.16b, v0.8h \n" /* v1(h) GB */ \
- "shl v2.16b, v1.16b, #4 \n" /* B,R BBBB0000 */ \
- "ushr v3.16b, v1.16b, #4 \n" /* G,A 0000GGGG */ \
- "ushr v0.16b, v2.16b, #4 \n" /* B,R 0000BBBB */ \
- "shl v1.16b, v3.16b, #4 \n" /* G,A GGGG0000 */ \
- "orr v2.16b, v0.16b, v2.16b \n" /* B,R BBBBBBBB */ \
- "orr v3.16b, v1.16b, v3.16b \n" /* G,A GGGGGGGG */ \
- "dup v0.2D, v2.D[1] \n" \
- "dup v1.2D, v3.D[1] \n"
+#define ARGB4444TOARGB \
+ /* Input: v1.8h = AAAARRRR_GGGGBBBB */ \
+ "shl v0.16b, v1.16b, #4 \n" /* RRRR0000_BBBB0000 */ \
+ "sri v1.16b, v1.16b, #4 \n" /* AAAAAAAA_GGGGGGGG */ \
+ "sri v0.16b, v0.16b, #4 \n" /* RRRRRRRR_BBBBBBBB */
+
+#define ARGB4444TORGB \
+ /* Input: v0.8h = xxxxRRRRGGGGBBBB */ \
+ "uzp1 v1.16b, v0.16b, v3.16b \n" /* GGGGBBBB */ \
+ "shrn v2.8b, v0.8h, #4 \n" /* RRRRxxxx */ \
+ "shl v0.16b, v1.16b, #4 \n" /* BBBB0000 */ \
+ "shrn2 v2.16b, v3.8h, #4 \n" /* RRRRxxxx */ \
+ "sri v1.16b, v1.16b, #4 \n" /* GGGGGGGG */ \
+ "sri v2.16b, v2.16b, #4 \n" /* RRRRRRRR */ \
+ "sri v0.16b, v0.16b, #4 \n" /* BBBBBBBB */
void ARGB4444ToARGBRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGB4444TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
- // pixels
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v1.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n" ARGB4444TOARGB
+ "st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 8 ARGB.
+ "b.gt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -958,33 +2195,87 @@ void ARGB4444ToARGBRow_NEON(const uint8_t* src_argb4444,
);
}
+static const int16_t kAR30Row_BoxShifts[] = {0, -6, 0, -6, 0, -6, 0, -6};
+
+static const uint8_t kABGRToAR30Row_BoxIndices[] = {
+ 2, 2, 1, 1, 6, 6, 5, 5, 10, 10, 9, 9, 14, 14, 13, 13,
+ 0, 0, 3, 3, 4, 4, 7, 7, 8, 8, 11, 11, 12, 12, 15, 15};
+static const uint8_t kARGBToAR30Row_BoxIndices[] = {
+ 0, 0, 1, 1, 4, 4, 5, 5, 8, 8, 9, 9, 12, 12, 13, 13,
+ 2, 2, 3, 3, 6, 6, 7, 7, 10, 10, 11, 11, 14, 14, 15, 15};
+
+// ARGB or ABGR as input, reordering based on TBL indices parameter.
+static void ABCDToAR30Row_NEON(const uint8_t* src_abcd,
+ uint8_t* dst_ar30,
+ int width,
+ const uint8_t* indices) {
+ asm volatile(
+ "movi v2.4s, #0xf, msl 16 \n" // 0xfffff
+ "ldr q3, [%[kAR30Row_BoxShifts]] \n"
+ "ldp q4, q5, [%[indices]] \n"
+ "1: \n"
+ "ldp q0, q20, [%[src]], #32 \n"
+ "subs %w[width], %w[width], #8 \n"
+ "tbl v1.16b, {v0.16b}, v5.16b \n"
+ "tbl v21.16b, {v20.16b}, v5.16b \n"
+ "tbl v0.16b, {v0.16b}, v4.16b \n"
+ "tbl v20.16b, {v20.16b}, v4.16b \n"
+ "ushl v0.8h, v0.8h, v3.8h \n"
+ "ushl v20.8h, v20.8h, v3.8h \n"
+ "ushl v1.8h, v1.8h, v3.8h \n"
+ "ushl v21.8h, v21.8h, v3.8h \n"
+ "ushr v0.4s, v0.4s, #6 \n"
+ "ushr v20.4s, v20.4s, #6 \n"
+ "shl v1.4s, v1.4s, #14 \n"
+ "shl v21.4s, v21.4s, #14 \n"
+ "bif v0.16b, v1.16b, v2.16b \n"
+ "bif v20.16b, v21.16b, v2.16b \n"
+ "stp q0, q20, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_abcd), // %[src]
+ [dst] "+r"(dst_ar30), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [kAR30Row_BoxShifts] "r"(kAR30Row_BoxShifts), // %[kAR30Row_BoxShifts]
+ [indices] "r"(indices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v20", "v21");
+}
+
+void ABGRToAR30Row_NEON(const uint8_t* src_abgr, uint8_t* dst_ar30, int width) {
+ ABCDToAR30Row_NEON(src_abgr, dst_ar30, width, kABGRToAR30Row_BoxIndices);
+}
+
+void ARGBToAR30Row_NEON(const uint8_t* src_argb, uint8_t* dst_ar30, int width) {
+ ABCDToAR30Row_NEON(src_argb, dst_ar30, width, kARGBToAR30Row_BoxIndices);
+}
+
void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "st3 {v1.8b,v2.8b,v3.8b}, [%1], #24 \n" // store 8 pixels of
- // RGB24.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n" // store 8 RGB24
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v1", "v2", "v3", "v4" // Clobber List
+ : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
);
}
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
asm volatile(
- "1: \n"
- "ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load b g r a
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v4.8b, v2.8b, v2.8b \n" // mov g
- "orr v5.8b, v1.8b, v1.8b \n" // mov b
- "st3 {v3.8b,v4.8b,v5.8b}, [%1], #24 \n" // store r g b
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load b g r a
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "mov v4.8b, v2.8b \n" // mov g
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v5.8b, v1.8b \n" // mov b
+ "st3 {v3.8b,v4.8b,v5.8b}, [%1], #24 \n" // store r g b
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_raw), // %1
"+r"(width) // %2
@@ -995,11 +2286,12 @@ void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of YUY2.
- "subs %w2, %w2, #16 \n" // 16 processed per loop.
- "st1 {v0.16b}, [%1], #16 \n" // store 16 pixels of Y.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of YUY2.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.16b}, [%1], #16 \n" // store 16 pixels of Y.
+ "b.gt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1010,11 +2302,12 @@ void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
void UYVYToYRow_NEON(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of UYVY.
- "subs %w2, %w2, #16 \n" // 16 processed per loop.
- "st1 {v1.16b}, [%1], #16 \n" // store 16 pixels of Y.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of UYVY.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v1.16b}, [%1], #16 \n" // store 16 pixels of Y.
+ "b.gt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1028,12 +2321,13 @@ void YUY2ToUV422Row_NEON(const uint8_t* src_yuy2,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 YUY2
- "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
- "st1 {v1.8b}, [%1], #8 \n" // store 8 U.
- "st1 {v3.8b}, [%2], #8 \n" // store 8 V.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 YUY2
+ "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v1.8b}, [%1], #8 \n" // store 8 U.
+ "st1 {v3.8b}, [%2], #8 \n" // store 8 V.
+ "b.gt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -1048,12 +2342,13 @@ void UYVYToUV422Row_NEON(const uint8_t* src_uyvy,
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 UYVY
- "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
- "st1 {v0.8b}, [%1], #8 \n" // store 8 U.
- "st1 {v2.8b}, [%2], #8 \n" // store 8 V.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 UYVY
+ "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v0.8b}, [%1], #8 \n" // store 8 U.
+ "st1 {v2.8b}, [%2], #8 \n" // store 8 V.
+ "b.gt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -1070,15 +2365,16 @@ void YUY2ToUVRow_NEON(const uint8_t* src_yuy2,
int width) {
const uint8_t* src_yuy2b = src_yuy2 + stride_yuy2;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
- "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
- "urhadd v1.8b, v1.8b, v5.8b \n" // average rows of U
- "urhadd v3.8b, v3.8b, v7.8b \n" // average rows of V
- "st1 {v1.8b}, [%2], #8 \n" // store 8 U.
- "st1 {v3.8b}, [%3], #8 \n" // store 8 V.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
+ "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
+ "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
+ "urhadd v1.8b, v1.8b, v5.8b \n" // average rows of U
+ "prfm pldl1keep, [%0, 448] \n"
+ "urhadd v3.8b, v3.8b, v7.8b \n" // average rows of V
+ "st1 {v1.8b}, [%2], #8 \n" // store 8 U.
+ "st1 {v3.8b}, [%3], #8 \n" // store 8 V.
+ "b.gt 1b \n"
: "+r"(src_yuy2), // %0
"+r"(src_yuy2b), // %1
"+r"(dst_u), // %2
@@ -1097,15 +2393,16 @@ void UYVYToUVRow_NEON(const uint8_t* src_uyvy,
int width) {
const uint8_t* src_uyvyb = src_uyvy + stride_uyvy;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
- "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
- "urhadd v0.8b, v0.8b, v4.8b \n" // average rows of U
- "urhadd v2.8b, v2.8b, v6.8b \n" // average rows of V
- "st1 {v0.8b}, [%2], #8 \n" // store 8 U.
- "st1 {v2.8b}, [%3], #8 \n" // store 8 V.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
+ "subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
+ "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
+ "urhadd v0.8b, v0.8b, v4.8b \n" // average rows of U
+ "prfm pldl1keep, [%0, 448] \n"
+ "urhadd v2.8b, v2.8b, v6.8b \n" // average rows of V
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 U.
+ "st1 {v2.8b}, [%3], #8 \n" // store 8 V.
+ "b.gt 1b \n"
: "+r"(src_uyvy), // %0
"+r"(src_uyvyb), // %1
"+r"(dst_u), // %2
@@ -1117,19 +2414,43 @@ void UYVYToUVRow_NEON(const uint8_t* src_uyvy,
);
}
+void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ const uint8_t* src_yuy2b = src_yuy2 + stride_yuy2;
+ asm volatile(
+ "1: \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels
+ "subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
+ "ld2 {v2.16b,v3.16b}, [%1], #32 \n" // load next row
+ "urhadd v4.16b, v1.16b, v3.16b \n" // average rows of UV
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v4.16b}, [%2], #16 \n" // store 8 UV.
+ "b.gt 1b \n"
+ : "+r"(src_yuy2), // %0
+ "+r"(src_yuy2b), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4" // Clobber List
+ );
+}
+
// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
void ARGBShuffleRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
asm volatile(
- "ld1 {v2.16b}, [%3] \n" // shuffler
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 4 pixels.
- "subs %w2, %w2, #4 \n" // 4 processed per loop
- "tbl v1.16b, {v0.16b}, v2.16b \n" // look up 4 pixels
- "st1 {v1.16b}, [%1], #16 \n" // store 4.
- "b.gt 1b \n"
+ "ld1 {v2.16b}, [%3] \n" // shuffler
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load 4 pixels.
+ "subs %w2, %w2, #4 \n" // 4 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "tbl v1.16b, {v0.16b}, v2.16b \n" // look up 4 pixels
+ "st1 {v1.16b}, [%1], #16 \n" // store 4.
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -1144,14 +2465,15 @@ void I422ToYUY2Row_NEON(const uint8_t* src_y,
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
- "ld2 {v0.8b, v1.8b}, [%0], #16 \n" // load 16 Ys
- "orr v2.8b, v1.8b, v1.8b \n"
- "ld1 {v1.8b}, [%1], #8 \n" // load 8 Us
- "ld1 {v3.8b}, [%2], #8 \n" // load 8 Vs
- "subs %w4, %w4, #16 \n" // 16 pixels
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v0.8b, v1.8b}, [%0], #16 \n" // load 16 Ys
+ "subs %w4, %w4, #16 \n" // 16 pixels
+ "mov v2.8b, v1.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "ld1 {v1.8b}, [%1], #8 \n" // load 8 Us
+ "ld1 {v3.8b}, [%2], #8 \n" // load 8 Vs
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels.
+ "b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(src_u), // %1
"+r"(src_v), // %2
@@ -1167,14 +2489,15 @@ void I422ToUYVYRow_NEON(const uint8_t* src_y,
uint8_t* dst_uyvy,
int width) {
asm volatile(
- "1: \n"
- "ld2 {v1.8b,v2.8b}, [%0], #16 \n" // load 16 Ys
- "orr v3.8b, v2.8b, v2.8b \n"
- "ld1 {v0.8b}, [%1], #8 \n" // load 8 Us
- "ld1 {v2.8b}, [%2], #8 \n" // load 8 Vs
- "subs %w4, %w4, #16 \n" // 16 pixels
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v1.8b,v2.8b}, [%0], #16 \n" // load 16 Ys
+ "subs %w4, %w4, #16 \n" // 16 pixels
+ "mov v3.8b, v2.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "ld1 {v0.8b}, [%1], #8 \n" // load 8 Us
+ "ld1 {v2.8b}, [%2], #8 \n" // load 8 Vs
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels.
+ "b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(src_u), // %1
"+r"(src_v), // %2
@@ -1188,111 +2511,235 @@ void ARGBToRGB565Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGBTORGB565
- "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels RGB565.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8
+ // pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n" ARGBTORGB565
+ "st1 {v18.16b}, [%1], #16 \n" // store 8 pixels RGB565.
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_rgb565), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v20", "v21", "v22", "v23");
+ : "cc", "memory", "v16", "v17", "v18", "v19");
}
void ARGBToRGB565DitherRow_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb,
- const uint32_t dither4,
+ uint32_t dither4,
int width) {
asm volatile(
- "dup v1.4s, %w2 \n" // dither4
- "1: \n"
- "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n" // load 8 pixels
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqadd v20.8b, v20.8b, v1.8b \n"
- "uqadd v21.8b, v21.8b, v1.8b \n"
- "uqadd v22.8b, v22.8b, v1.8b \n" ARGBTORGB565
- "st1 {v0.16b}, [%0], #16 \n" // store 8 pixels RGB565.
- "b.gt 1b \n"
- : "+r"(dst_rgb) // %0
- : "r"(src_argb), // %1
- "r"(dither4), // %2
- "r"(width) // %3
- : "cc", "memory", "v0", "v1", "v20", "v21", "v22", "v23");
+ "dup v1.4s, %w3 \n" // dither4
+ "1: \n"
+ "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "uqadd v16.8b, v16.8b, v1.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqadd v17.8b, v17.8b, v1.8b \n"
+ "uqadd v18.8b, v18.8b, v1.8b \n" ARGBTORGB565
+ "st1 {v18.16b}, [%1], #16 \n" // store 8 pixels RGB565.
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_rgb), // %1
+ "+r"(width) // %2
+ : "r"(dither4) // %3
+ : "cc", "memory", "v1", "v16", "v17", "v18", "v19");
}
void ARGBToARGB1555Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb1555,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGBTOARGB1555
- "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels
- // ARGB1555.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v16.8h,v17.8h}, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n" ARGBTOARGB1555
+ "st1 {v17.16b}, [%1], #16 \n" // store 8 pixels
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb1555), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v20", "v21", "v22", "v23");
+ : "cc", "memory", "v1", "v2", "v16", "v17");
}
void ARGBToARGB4444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb4444,
int width) {
asm volatile(
- "movi v4.16b, #0x0f \n" // bits to clear with
- // vbic.
- "1: \n"
- "ld4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%0], #32 \n" // load 8 pixels
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGBTOARGB4444
- "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels
- // ARGB4444.
- "b.gt 1b \n"
+ "1: \n"
+ "ld2 {v16.16b, v17.16b}, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "sri v17.16b, v16.16b, #4 \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v17.16b}, [%1], #16 \n" // store 8 pixels
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb4444), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v4", "v20", "v21", "v22", "v23");
+ : "cc", "memory", "v16", "v17");
}
-void ARGBToYRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+#if defined(LIBYUV_USE_ST2)
+void ARGBToAR64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
asm volatile(
- "movi v4.8b, #13 \n" // B * 0.1016 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #33 \n" // R * 0.2578 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v3.8h, v0.8b, v4.8b \n" // B
- "umlal v3.8h, v1.8b, v5.8b \n" // G
- "umlal v3.8h, v2.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q0, q2, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "mov v1.16b, v0.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v3.16b, v2.16b \n"
+ "st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 4 pixels
+ "st2 {v2.16b, v3.16b}, [%1], #32 \n" // store 4 pixels
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
- "+r"(dst_y), // %1
+ "+r"(dst_ar64), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3");
+}
+
+static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+
+void ARGBToAB64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
+ asm volatile(
+ "ldr q4, [%3] \n" // shuffler
+ "1: \n"
+ "ldp q0, q2, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "tbl v0.16b, {v0.16b}, v4.16b \n"
+ "tbl v2.16b, {v2.16b}, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v1.16b, v0.16b \n"
+ "mov v3.16b, v2.16b \n"
+ "st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 4 pixels
+ "st2 {v2.16b, v3.16b}, [%1], #32 \n" // store 4 pixels
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ab64), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleARGBToABGR) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+#else
+void ARGBToAR64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 ARGB pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "zip1 v2.16b, v0.16b, v0.16b \n"
+ "zip2 v3.16b, v0.16b, v0.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip1 v4.16b, v1.16b, v1.16b \n"
+ "zip2 v5.16b, v1.16b, v1.16b \n"
+ "st1 {v2.8h, v3.8h, v4.8h, v5.8h}, [%1], #64 \n" // 8 AR64
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ar64), // %1
"+r"(width) // %2
:
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
+}
+
+static const uvec8 kShuffleARGBToAB64[2] = {
+ {2, 2, 1, 1, 0, 0, 3, 3, 6, 6, 5, 5, 4, 4, 7, 7},
+ {10, 10, 9, 9, 8, 8, 11, 11, 14, 14, 13, 13, 12, 12, 15, 15}};
+
+void ARGBToAB64Row_NEON(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
+ asm volatile(
+ "ldp q6, q7, [%3] \n" // 2 shufflers
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "tbl v2.16b, {v0.16b}, v6.16b \n" // ARGB to AB64
+ "tbl v3.16b, {v0.16b}, v7.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "tbl v4.16b, {v1.16b}, v6.16b \n"
+ "tbl v5.16b, {v1.16b}, v7.16b \n"
+ "st1 {v2.8h, v3.8h, v4.8h, v5.8h}, [%1], #64 \n" // 8 AR64
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_ab64), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleARGBToAB64[0]) // %3
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
+#endif // LIBYUV_USE_ST2
+
+static const uvec8 kShuffleAR64ToARGB = {1, 3, 5, 7, 9, 11, 13, 15,
+ 17, 19, 21, 23, 25, 27, 29, 31};
+
+void AR64ToARGBRow_NEON(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "ldr q4, [%3] \n" // shuffler
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 4 pixels
+ "ldp q2, q3, [%0], #32 \n" // load 4 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "tbl v0.16b, {v0.16b, v1.16b}, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "tbl v2.16b, {v2.16b, v3.16b}, v4.16b \n"
+ "stp q0, q2, [%1], #32 \n" // store 8 pixels
+ "b.gt 1b \n"
+ : "+r"(src_ar64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleAR64ToARGB) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+static const uvec8 kShuffleAB64ToARGB = {5, 3, 1, 7, 13, 11, 9, 15,
+ 21, 19, 17, 23, 29, 27, 25, 31};
+
+void AB64ToARGBRow_NEON(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "ldr q4, [%3] \n" // shuffler
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 4 pixels
+ "ldp q2, q3, [%0], #32 \n" // load 4 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "tbl v0.16b, {v0.16b, v1.16b}, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "tbl v2.16b, {v2.16b, v3.16b}, v4.16b \n"
+ "stp q0, q2, [%1], #32 \n" // store 8 pixels
+ "b.gt 1b \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleAB64ToARGB) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load row 16
- // pixels
- "subs %w2, %w2, #16 \n" // 16 processed per loop
- "st1 {v3.16b}, [%1], #16 \n" // store 16 A's.
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q0, q1, [%0] \n" // load 8 ARGB pixels
+ "ldp q2, q3, [%0, #32] \n" // load next 8 ARGB pixels
+ "add %0, %0, #64 \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v2.8h, v2.8h, v3.8h \n"
+ "uzp2 v0.16b, v0.16b, v2.16b \n"
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "str q0, [%1], #16 \n" // store 16 A's.
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_a), // %1
"+r"(width) // %2
@@ -1301,76 +2748,208 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
);
}
-void ARGBToYJRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width) {
+void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "movi v4.8b, #15 \n" // B * 0.11400 coefficient
- "movi v5.8b, #75 \n" // G * 0.58700 coefficient
- "movi v6.8b, #38 \n" // R * 0.29900 coefficient
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v3.8h, v0.8b, v4.8b \n" // B
- "umlal v3.8h, v1.8b, v5.8b \n" // G
- "umlal v3.8h, v2.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v3.8h, #7 \n" // 15 bit to 8 bit Y
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
+ "1: \n"
+ "ldr q4, [%0], #16 \n"
+ "ldp q0, q1, [%1] \n"
+ "ldp q2, q3, [%1, #32] \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "zip1 v6.16b, v4.16b, v4.16b \n"
+ "zip2 v7.16b, v4.16b, v4.16b \n"
+ "zip1 v4.8h, v6.8h, v6.8h \n"
+ "zip2 v5.8h, v6.8h, v6.8h \n"
+ "zip1 v6.8h, v7.8h, v7.8h \n"
+ "zip2 v7.8h, v7.8h, v7.8h \n"
+ "sli v0.4s, v4.4s, #24 \n"
+ "sli v1.4s, v5.4s, #24 \n"
+ "sli v2.4s, v6.4s, #24 \n"
+ "sli v3.4s, v7.4s, #24 \n"
+ "stp q0, q1, [%1] \n"
+ "stp q2, q3, [%1, #32] \n"
+ "add %1, %1, #64 \n"
+ "subs %w2, %w2, #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
+// Coefficients expressed as negatives to allow 128
+struct RgbUVConstants {
+ int8_t kRGBToU[4];
+ int8_t kRGBToV[4];
+};
+
// 8x1 pixels.
-void ARGBToUV444Row_NEON(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "movi v24.8b, #112 \n" // UB / VR 0.875
- // coefficient
- "movi v25.8b, #74 \n" // UG -0.5781 coefficient
- "movi v26.8b, #38 \n" // UR -0.2969 coefficient
- "movi v27.8b, #18 \n" // VB -0.1406 coefficient
- "movi v28.8b, #94 \n" // VG -0.7344 coefficient
- "movi v29.16b,#0x80 \n" // 128.5
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- // pixels.
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "umull v4.8h, v0.8b, v24.8b \n" // B
- "umlsl v4.8h, v1.8b, v25.8b \n" // G
- "umlsl v4.8h, v2.8b, v26.8b \n" // R
- "add v4.8h, v4.8h, v29.8h \n" // +128 -> unsigned
-
- "umull v3.8h, v2.8b, v24.8b \n" // R
- "umlsl v3.8h, v1.8b, v28.8b \n" // G
- "umlsl v3.8h, v0.8b, v27.8b \n" // B
- "add v3.8h, v3.8h, v29.8h \n" // +128 -> unsigned
-
- "uqshrn v0.8b, v4.8h, #8 \n" // 16 bit to 8 bit U
- "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V
-
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%2], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ "ldr q16, [%[c], #16] \n" // kRGBToU
+ "ldr q17, [%[c], #32] \n" // kRGBToV
+ "ldr s0, [%[c], #64] \n" // kAddUV
+ "sxtl v16.8h, v16.8b \n" // sign extend U coeffs
+ // to 16-bit
+ "sxtl v17.8h, v17.8b \n" // sign extend V coeffs
+ // to 16-bit
+ "dup v20.8h, v16.h[0] \n" // U0
+ "dup v21.8h, v16.h[1] \n" // U1
+ "dup v22.8h, v16.h[2] \n" // U2
+ "dup v23.8h, v16.h[3] \n" // U3
+ "dup v24.8h, v17.h[0] \n" // V0
+ "dup v26.8h, v17.h[1] \n" // V1
+ "dup v27.8h, v17.h[2] \n" // V2
+ "dup v28.8h, v17.h[3] \n" // V3
+ "dup v25.8h, v0.h[0] \n" // kAddUV
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+
+ "uxtl v4.8h, v0.8b \n"
+ "uxtl v5.8h, v1.8b \n"
+ "uxtl v6.8h, v2.8b \n"
+ "uxtl v7.8h, v3.8b \n"
+
+ // U = B*U0 + G*U1 + R*U2 + A*U3
+ "mul v18.8h, v4.8h, v20.8h \n"
+ "mla v18.8h, v5.8h, v21.8h \n"
+ "mla v18.8h, v6.8h, v22.8h \n"
+ "mla v18.8h, v7.8h, v23.8h \n"
+
+ // V = B*V0 + G*V1 + R*V2 + A*V3
+ "mul v19.8h, v4.8h, v24.8h \n"
+ "mla v19.8h, v5.8h, v26.8h \n"
+ "mla v19.8h, v6.8h, v27.8h \n"
+ "mla v19.8h, v7.8h, v28.8h \n"
+
+ "subhn v0.8b, v25.8h, v18.8h \n"
+ "subhn v1.8b, v25.8h, v19.8h \n"
+
+ "st1 {v0.8b}, [%1], #8 \n"
+ "st1 {v1.8b}, [%2], #8 \n"
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width) // %3
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v24", "v25", "v26",
- "v27", "v28", "v29");
+ : [c] "r"(c) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27", "v28");
+}
+
+// 8x1 pixels.
+void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "ldr q16, [%[c], #16] \n" // kRGBToU
+ "ldr q17, [%[c], #32] \n" // kRGBToV
+ "ldr s0, [%[c], #64] \n" // kAddUV
+ "sxtl v16.8h, v16.8b \n" // sign extend U coeffs
+ // to 16-bit
+ "sxtl v17.8h, v17.8b \n" // sign extend V coeffs
+ // to 16-bit
+ "dup v20.8h, v16.h[0] \n" // U0
+ "dup v21.8h, v16.h[1] \n" // U1
+ "dup v22.8h, v16.h[2] \n" // U2
+ "dup v24.8h, v17.h[0] \n" // V0
+ "dup v26.8h, v17.h[1] \n" // V1
+ "dup v27.8h, v17.h[2] \n" // V2
+ "dup v25.8h, v0.h[0] \n" // kAddUV
+ "1: \n"
+ "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 RGB
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+
+ "uxtl v4.8h, v0.8b \n" // B
+ "uxtl v5.8h, v1.8b \n" // G
+ "uxtl v6.8h, v2.8b \n" // R
+
+ // U = B*U0 + G*U1 + R*U2
+ "mul v18.8h, v4.8h, v20.8h \n"
+ "mla v18.8h, v5.8h, v21.8h \n"
+ "mla v18.8h, v6.8h, v22.8h \n"
+
+ // V = B*V0 + G*V1 + R*V2
+ "mul v19.8h, v4.8h, v24.8h \n"
+ "mla v19.8h, v5.8h, v26.8h \n"
+ "mla v19.8h, v6.8h, v27.8h \n"
+
+ "subhn v0.8b, v25.8h, v18.8h \n"
+ "subhn v1.8b, v25.8h, v19.8h \n"
+
+ "st1 {v0.8b}, [%1], #8 \n"
+ "st1 {v1.8b}, [%2], #8 \n"
+ "b.gt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : [c] "r"(c) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v4", "v5", "v6", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v24", "v25", "v26", "v27");
+}
+
+void ARGBToUV444MatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ asm volatile(
+ "ldr q16, [%[c], #16] \n" // kRGBToU
+ "ldr q17, [%[c], #32] \n" // kRGBToV
+ "ldr s0, [%[c], #64] \n" // kAddUV
+ "dup v29.8h, v0.h[0] \n" // 128.0
+ "1: \n"
+ "ldp q0, q1, [%[src]], #32 \n"
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop.
+ "movi v2.4s, #0 \n"
+ "movi v3.4s, #0 \n"
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "usdot v2.4s, v0.16b, v16.16b \n"
+ "usdot v3.4s, v1.16b, v16.16b \n"
+ "usdot v4.4s, v0.16b, v17.16b \n"
+ "usdot v5.4s, v1.16b, v17.16b \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uzp1 v0.8h, v2.8h, v3.8h \n"
+ "uzp1 v1.8h, v4.8h, v5.8h \n"
+ "subhn v0.8b, v29.8h, v0.8h \n" // -signed -> unsigned
+ "subhn v1.8b, v29.8h, v1.8h \n"
+ "str d0, [%[dst_u]], #8 \n" // store 8 pixels U.
+ "str d1, [%[dst_v]], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width) // %[width]
+ : [c] "r"(c) // %[c]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17",
+ "v29");
}
+// RGB to BT601 coefficients
+// UB 0.875 coefficient = 112
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
#define RGBTOUV_SETUP_REG \
- "movi v20.8h, #56, lsl #0 \n" /* UB/VR coefficient (0.875) / 2 */ \
- "movi v21.8h, #37, lsl #0 \n" /* UG coefficient (-0.5781) / 2 */ \
- "movi v22.8h, #19, lsl #0 \n" /* UR coefficient (-0.2969) / 2 */ \
- "movi v23.8h, #9, lsl #0 \n" /* VB coefficient (-0.1406) / 2 */ \
- "movi v24.8h, #47, lsl #0 \n" /* VG coefficient (-0.7344) / 2 */ \
- "movi v25.16b, #0x80 \n" /* 128.5 (0x8080 in 16-bit) */
+ "movi v20.8h, #112 \n" /* UB/VR coefficient (0.875) */ \
+ "movi v21.8h, #74 \n" /* UG coefficient (-0.5781) */ \
+ "movi v22.8h, #38 \n" /* UR coefficient (-0.2969) */ \
+ "movi v23.8h, #18 \n" /* VB coefficient (-0.1406) */ \
+ "movi v24.8h, #94 \n" /* VG coefficient (-0.7344) */ \
+ "movi v25.8h, #0x80, lsl #8 \n" /* 128.0 (0x8000 in 16-bit) */
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
// clang-format off
@@ -1381,210 +2960,155 @@ void ARGBToUV444Row_NEON(const uint8_t* src_argb,
"mls v4.8h, " #QG ",v24.8h \n" /* G */ \
"mls v3.8h, " #QR ",v22.8h \n" /* R */ \
"mls v4.8h, " #QB ",v23.8h \n" /* B */ \
- "add v3.8h, v3.8h, v25.8h \n" /* +128 -> unsigned */ \
- "add v4.8h, v4.8h, v25.8h \n" /* +128 -> unsigned */ \
- "uqshrn v0.8b, v3.8h, #8 \n" /* 16 bit to 8 bit U */ \
- "uqshrn v1.8b, v4.8h, #8 \n" /* 16 bit to 8 bit V */
+ "addhn v0.8b, v3.8h, v25.8h \n" /* +128 -> unsigned */ \
+ "addhn v1.8b, v4.8h, v25.8h \n" /* +128 -> unsigned */
// clang-format on
// TODO(fbarchard): Consider vhadd vertical, then vpaddl horizontal, avoid shr.
// TODO(fbarchard): consider ptrdiff_t for all strides.
-void ARGBToUVRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_argb_1 = src_argb + src_stride_argb;
- asm volatile (
- RGBTOUV_SETUP_REG
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
- "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
-
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
- "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
- RGBTOUV(v0.8h, v1.8h, v2.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_argb_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
- "v20", "v21", "v22", "v23", "v24", "v25"
- );
-}
-
-// TODO(fbarchard): Subsample match C code.
-void ARGBToUVJRow_NEON(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
const uint8_t* src_argb_1 = src_argb + src_stride_argb;
- asm volatile (
- "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2
- "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2
- "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2
- "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2
- "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2
- "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit)
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
- "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
- "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
- RGBTOUV(v0.8h, v1.8h, v2.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_argb_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
- "v20", "v21", "v22", "v23", "v24", "v25"
- );
-}
-
-void BGRAToUVRow_NEON(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_bgra_1 = src_bgra + src_stride_bgra;
- asm volatile (
- RGBTOUV_SETUP_REG
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
- "uaddlp v0.8h, v3.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v3.8h, v2.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v1.16b \n" // R 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more
- "uadalp v0.8h, v7.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v3.8h, v6.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v5.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v3.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
- RGBTOUV(v0.8h, v1.8h, v2.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(src_bgra_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
- "v20", "v21", "v22", "v23", "v24", "v25"
- );
-}
-
-void ABGRToUVRow_NEON(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_abgr_1 = src_abgr + src_stride_abgr;
- asm volatile (
- RGBTOUV_SETUP_REG
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
- "uaddlp v3.8h, v2.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v0.16b \n" // R 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more.
- "uadalp v3.8h, v6.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v2.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v1.8h, v4.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v3.8h, #1 \n" // 2x average
- "urshr v2.8h, v2.8h, #1 \n"
- "urshr v1.8h, v1.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
- RGBTOUV(v0.8h, v2.8h, v1.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(src_abgr_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
- "v20", "v21", "v22", "v23", "v24", "v25"
- );
+ asm volatile(
+ "ldr q16, [%[c], #16] \n" // kRGBToU
+ "ldr q17, [%[c], #32] \n" // kRGBToV
+ "sxtl v16.8h, v16.8b \n" // sign extend U coeffs
+ // to 16-bit
+ "sxtl v17.8h, v17.8b \n" // sign extend V coeffs
+ // to 16-bit
+ "dup v20.8h, v16.h[0] \n" // U0
+ "dup v21.8h, v16.h[1] \n" // U1
+ "dup v22.8h, v16.h[2] \n" // U2
+ "dup v23.8h, v16.h[3] \n" // U3
+ "dup v24.8h, v17.h[0] \n" // V0
+ "dup v26.8h, v17.h[1] \n" // V1
+ "dup v27.8h, v17.h[2] \n" // V2
+ "dup v28.8h, v17.h[3] \n" // V3
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit
+ // (0x8000)
+
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
+ // pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v3.16b \n" // A 16 bytes -> 8 shorts.
+
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16
+ // more.
+ "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v7.16b \n" // A 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
+ "urshr v18.8h, v18.8h, #2 \n"
+
+ // U = B*U0 + G*U1 + R*U2 + A*U3
+ "mul v3.8h, v0.8h, v20.8h \n"
+ "mla v3.8h, v1.8h, v21.8h \n"
+ "mla v3.8h, v2.8h, v22.8h \n"
+ "mla v3.8h, v18.8h, v23.8h \n"
+
+ // V = B*V0 + G*V1 + R*V2 + A*V3
+ "mul v4.8h, v0.8h, v24.8h \n"
+ "mla v4.8h, v1.8h, v26.8h \n"
+ "mla v4.8h, v2.8h, v27.8h \n"
+ "mla v4.8h, v18.8h, v28.8h \n"
+
+ // U = (128.0 - U) >> 8, V = (128.0 - V) >> 8
+ "subhn v0.8b, v25.8h, v3.8h \n"
+ "subhn v1.8b, v25.8h, v4.8h \n"
+
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb_1), // %1
+ "+r"(dst_u), // %2
+ "+r"(dst_v), // %3
+ "+r"(width) // %4
+ : [c] "r"(c) // %5
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28");
}
-void RGBAToUVRow_NEON(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- const uint8_t* src_rgba_1 = src_rgba + src_stride_rgba;
- asm volatile (
- RGBTOUV_SETUP_REG
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
- "uaddlp v0.8h, v1.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v2.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v3.16b \n" // R 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load 16 more.
- "uadalp v0.8h, v5.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v6.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v7.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
- RGBTOUV(v0.8h, v1.8h, v2.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
- : "+r"(src_rgba), // %0
- "+r"(src_rgba_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
- "v20", "v21", "v22", "v23", "v24", "v25"
- );
+void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
+ asm volatile(
+ "ldr q16, [%[c], #16] \n" // kRGBToU
+ "ldr q17, [%[c], #32] \n" // kRGBToV
+ "sxtl v16.8h, v16.8b \n" // sign extend U coeffs
+ "sxtl v17.8h, v17.8b \n" // sign extend V coeffs
+ "dup v20.8h, v16.h[0] \n" // U0
+ "dup v21.8h, v16.h[1] \n" // U1
+ "dup v22.8h, v16.h[2] \n" // U2
+ "dup v24.8h, v17.h[0] \n" // V0
+ "dup v26.8h, v17.h[1] \n" // V1
+ "dup v27.8h, v17.h[2] \n" // V2
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit
+ // (0x8000)
+
+ "1: \n"
+ "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more.
+ "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
+
+ // U = B*U0 + G*U1 + R*U2
+ "mul v3.8h, v0.8h, v20.8h \n"
+ "mla v3.8h, v1.8h, v21.8h \n"
+ "mla v3.8h, v2.8h, v22.8h \n"
+
+ // V = B*V0 + G*V1 + R*V2
+ "mul v4.8h, v0.8h, v24.8h \n"
+ "mla v4.8h, v1.8h, v26.8h \n"
+ "mla v4.8h, v2.8h, v27.8h \n"
+
+ // U = (128.0 - U) >> 8, V = (128.0 - V) >> 8
+ "subhn v0.8b, v25.8h, v3.8h \n"
+ "subhn v1.8b, v25.8h, v4.8h \n"
+
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(src_rgb_1), // %1
+ "+r"(dst_u), // %2
+ "+r"(dst_v), // %3
+ "+r"(width) // %4
+ : [c] "r"(c) // %5
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v20", "v21", "v22", "v24", "v25", "v26", "v27");
}
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
@@ -1595,25 +3119,27 @@ void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
const uint8_t* src_rgb24_1 = src_rgb24 + src_stride_rgb24;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
- "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
- "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more.
- "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
+ "1: \n"
+ "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+ "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more.
+ "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
+
RGBTOUV(v0.8h, v1.8h, v2.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
: "+r"(src_rgb24), // %0
"+r"(src_rgb24_1), // %1
"+r"(dst_u), // %2
@@ -1633,25 +3159,27 @@ void RAWToUVRow_NEON(const uint8_t* src_raw,
const uint8_t* src_raw_1 = src_raw + src_stride_raw;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
- "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 8 RAW pixels.
- "uaddlp v2.8h, v2.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v0.8h, v0.16b \n" // R 16 bytes -> 8 shorts.
- "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 8 more RAW pixels
- "uadalp v2.8h, v6.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v0.8h, v4.16b \n" // R 16 bytes -> 8 shorts.
-
- "urshr v2.8h, v2.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v0.8h, v0.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 32 processed per loop.
+ "1: \n"
+ "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 RAW pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
+ "uaddlp v2.8h, v2.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v0.8h, v0.16b \n" // R 16 bytes -> 8 shorts.
+ "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 8 more RAW pixels
+ "uadalp v2.8h, v6.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v0.8h, v4.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v2.8h, v2.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v0.8h, v0.8h, #2 \n"
+
RGBTOUV(v2.8h, v1.8h, v0.8h)
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
: "+r"(src_raw), // %0
"+r"(src_raw_1), // %1
"+r"(dst_u), // %2
@@ -1663,7 +3191,7 @@ void RAWToUVRow_NEON(const uint8_t* src_raw,
);
}
-// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
+// 16x2 pixels -> 8x1. width is number of rgb pixels. e.g. 16.
void RGB565ToUVRow_NEON(const uint8_t* src_rgb565,
int src_stride_rgb565,
uint8_t* dst_u,
@@ -1671,67 +3199,40 @@ void RGB565ToUVRow_NEON(const uint8_t* src_rgb565,
int width) {
const uint8_t* src_rgb565_1 = src_rgb565 + src_stride_rgb565;
asm volatile(
- "movi v22.8h, #56, lsl #0 \n" // UB / VR coeff (0.875) /
- // 2
- "movi v23.8h, #37, lsl #0 \n" // UG coeff (-0.5781) / 2
- "movi v24.8h, #19, lsl #0 \n" // UR coeff (-0.2969) / 2
- "movi v25.8h, #9 , lsl #0 \n" // VB coeff (-0.1406) / 2
- "movi v26.8h, #47, lsl #0 \n" // VG coeff (-0.7344) / 2
- "movi v27.16b, #0x80 \n" // 128.5 0x8080 in 16bit
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- RGB565TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v20.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 RGB565 pixels.
+ RGBTOUV_SETUP_REG
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGB565TOARGB
- "uaddlp v17.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v19.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v21.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 RGB565 pixels.
+ "ldp q0, q4, [%1], #32 \n" // load 16 RGB565 pixels.
RGB565TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v18.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v20.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 RGB565 pixels.
- RGB565TOARGB
- "uadalp v17.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v19.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v21.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v17.D[0] \n"
- "ins v18.D[1], v19.D[0] \n"
- "ins v20.D[1], v21.D[0] \n"
-
- "urshr v4.8h, v16.8h, #1 \n" // 2x average
- "urshr v5.8h, v18.8h, #1 \n"
- "urshr v6.8h, v20.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
- "mul v16.8h, v4.8h, v22.8h \n" // B
- "mls v16.8h, v5.8h, v23.8h \n" // G
- "mls v16.8h, v6.8h, v24.8h \n" // R
- "add v16.8h, v16.8h, v27.8h \n" // +128 -> unsigned
- "mul v17.8h, v6.8h, v22.8h \n" // R
- "mls v17.8h, v5.8h, v26.8h \n" // G
- "mls v17.8h, v4.8h, v25.8h \n" // B
- "add v17.8h, v17.8h, v27.8h \n" // +128 -> unsigned
- "uqshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit U
- "uqshrn v1.8b, v17.8h, #8 \n" // 16 bit to 8 bit V
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
+ RGBTOUV(v0.8h, v1.8h, v2.8h)
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(src_rgb565_1), // %1
- "+r"(dst_u), // %2
- "+r"(dst_v), // %3
- "+r"(width) // %4
+ "+r"(dst_u), // %2
+ "+r"(dst_v), // %3
+ "+r"(width) // %4
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
- "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
- "v27");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28");
}
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
@@ -1743,51 +3244,30 @@ void ARGB1555ToUVRow_NEON(const uint8_t* src_argb1555,
const uint8_t* src_argb1555_1 = src_argb1555 + src_stride_argb1555;
asm volatile(
RGBTOUV_SETUP_REG
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
- RGB555TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB1555 pixels.
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB1555 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGB555TOARGB
- "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB1555 pixels.
+ "ldp q0, q3, [%1], #32 \n" // load 16 ARGB1555 pixels.
RGB555TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB1555 pixels.
- RGB555TOARGB
- "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v26.D[0] \n"
- "ins v17.D[1], v27.D[0] \n"
- "ins v18.D[1], v28.D[0] \n"
-
- "urshr v4.8h, v16.8h, #1 \n" // 2x average
- "urshr v5.8h, v17.8h, #1 \n"
- "urshr v6.8h, v18.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
- "mul v2.8h, v4.8h, v20.8h \n" // B
- "mls v2.8h, v5.8h, v21.8h \n" // G
- "mls v2.8h, v6.8h, v22.8h \n" // R
- "add v2.8h, v2.8h, v25.8h \n" // +128 -> unsigned
- "mul v3.8h, v6.8h, v20.8h \n" // R
- "mls v3.8h, v5.8h, v24.8h \n" // G
- "mls v3.8h, v4.8h, v23.8h \n" // B
- "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned
- "uqshrn v0.8b, v2.8h, #8 \n" // 16 bit to 8 bit U
- "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
+ RGBTOUV(v0.8h, v1.8h, v2.8h)
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(src_argb1555_1), // %1
"+r"(dst_u), // %2
@@ -1796,7 +3276,7 @@ void ARGB1555ToUVRow_NEON(const uint8_t* src_argb1555,
:
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
"v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
- "v28");
+ "v28", "v29");
}
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
@@ -1807,52 +3287,31 @@ void ARGB4444ToUVRow_NEON(const uint8_t* src_argb4444,
int width) {
const uint8_t* src_argb4444_1 = src_argb4444 + src_stride_argb4444;
asm volatile(
- RGBTOUV_SETUP_REG
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v26.D[0] \n"
- "ins v17.D[1], v27.D[0] \n"
- "ins v18.D[1], v28.D[0] \n"
-
- "urshr v4.8h, v16.8h, #1 \n" // 2x average
- "urshr v5.8h, v17.8h, #1 \n"
- "urshr v6.8h, v18.8h, #1 \n"
-
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
- "mul v2.8h, v4.8h, v20.8h \n" // B
- "mls v2.8h, v5.8h, v21.8h \n" // G
- "mls v2.8h, v6.8h, v22.8h \n" // R
- "add v2.8h, v2.8h, v25.8h \n" // +128 -> unsigned
- "mul v3.8h, v6.8h, v20.8h \n" // R
- "mls v3.8h, v5.8h, v24.8h \n" // G
- "mls v3.8h, v4.8h, v23.8h \n" // B
- "add v3.8h, v3.8h, v25.8h \n" // +128 -> unsigned
- "uqshrn v0.8b, v2.8h, #8 \n" // 16 bit to 8 bit U
- "uqshrn v1.8b, v3.8h, #8 \n" // 16 bit to 8 bit V
- "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
- "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
- "b.gt 1b \n"
+ RGBTOUV_SETUP_REG // sets v20-v25
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB4444 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
+ ARGB4444TORGB
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "ldp q0, q3, [%1], #32 \n" // load 16 ARGB4444 pixels.
+ ARGB4444TORGB
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
+ RGBTOUV(v0.8h, v1.8h, v2.8h)
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
+ "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(src_argb4444_1), // %1
"+r"(dst_u), // %2
@@ -1866,23 +3325,109 @@ void ARGB4444ToUVRow_NEON(const uint8_t* src_argb4444,
);
}
+// Process any of ARGB, ABGR, BGRA, RGBA, by adjusting the ArgbConstants layout.
+static void ARGBToUVMatrixRow_NEON_I8MM_Impl(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src1 = src + src_stride;
+ asm volatile(
+ "movi v23.8h, #0x80, lsl #8 \n" // 128.0 (0x8000 in
+ // 16-bit)
+ "ldr q24, [%[c], #16] \n" // kRGBToU
+ "ldr q25, [%[c], #32] \n" // kRGBToV
+
+ "1: \n"
+ "ld2 {v0.4s, v1.4s}, [%[src]], #32 \n" // load 8 pixels
+ "ld2 {v2.4s, v3.4s}, [%[src]], #32 \n" // load 8 pixels
+ "subs %w[width], %w[width], #16 \n" // 16 processed per loop
+ "uaddl v4.8h, v0.8b, v1.8b \n" // ABCDABCD
+ "uaddl2 v5.8h, v0.16b, v1.16b \n" // ABCDABCD
+ "uaddl v6.8h, v2.8b, v3.8b \n" // ABCDABCD
+ "uaddl2 v7.8h, v2.16b, v3.16b \n" // ABCDABCD
+
+ "ld2 {v0.4s, v1.4s}, [%[src1]], #32 \n" // load 8 pixels
+ "ld2 {v2.4s, v3.4s}, [%[src1]], #32 \n" // load 8 pixels
+ "uaddw v4.8h, v4.8h, v0.8b \n" // ABCDABCD
+ "uaddw2 v5.8h, v5.8h, v0.16b \n" // ABCDABCD
+ "uaddw v6.8h, v6.8h, v2.8b \n" // ABCDABCD
+ "uaddw2 v7.8h, v7.8h, v2.16b \n" // ABCDABCD
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uaddw v4.8h, v4.8h, v1.8b \n" // ABCDABCD
+ "uaddw2 v5.8h, v5.8h, v1.16b \n" // ABCDABCD
+ "uaddw v6.8h, v6.8h, v3.8b \n" // ABCDABCD
+ "uaddw2 v7.8h, v7.8h, v3.16b \n" // ABCDABCD
+ "prfm pldl1keep, [%[src1], 448] \n"
+
+ "rshrn v4.8b, v4.8h, #2 \n" // average of 4 pixels
+ "rshrn v6.8b, v6.8h, #2 \n" // average of 4 pixels
+ "rshrn2 v4.16b, v5.8h, #2 \n" // average of 4 pixels
+ "rshrn2 v6.16b, v7.8h, #2 \n" // average of 4 pixels
+
+ "movi v0.4s, #0 \n" // U
+ "movi v1.4s, #0 \n" // U
+ "usdot v0.4s, v4.16b, v24.16b \n"
+ "usdot v1.4s, v6.16b, v24.16b \n"
+
+ "movi v2.4s, #0 \n" // V
+ "movi v3.4s, #0 \n" // V
+ "usdot v2.4s, v4.16b, v25.16b \n"
+ "usdot v3.4s, v6.16b, v25.16b \n"
+
+ "uzp1 v0.8h, v0.8h, v1.8h \n" // U
+ "uzp1 v1.8h, v2.8h, v3.8h \n" // V
+
+ "subhn v0.8b, v23.8h, v0.8h \n" // +128 -> unsigned
+ "subhn v1.8b, v23.8h, v1.8h \n" // +128 -> unsigned
+
+ "str d0, [%[dst_u]], #8 \n" // store 8 pixels U
+ "str d1, [%[dst_v]], #8 \n" // store 8 pixels V
+ "b.gt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [src1] "+r"(src1), // %[src1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width) // %[width]
+ : [c] "r"(c) // %[c]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v23",
+ "v24", "v25");
+}
+
+void ARGBToUVMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ ARGBToUVMatrixRow_NEON_I8MM_Impl(src_argb, src_stride_argb, dst_u, dst_v,
+ width, c);
+}
+
void RGB565ToYRow_NEON(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
asm volatile(
- "movi v24.8b, #13 \n" // B * 0.1016 coefficient
- "movi v25.8b, #65 \n" // G * 0.5078 coefficient
- "movi v26.8b, #33 \n" // R * 0.2578 coefficient
- "movi v27.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "movi v24.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v25.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v26.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v27.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
RGB565TOARGB
- "umull v3.8h, v0.8b, v24.8b \n" // B
- "umlal v3.8h, v1.8b, v25.8b \n" // G
- "umlal v3.8h, v2.8b, v26.8b \n" // R
- "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v27.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
+ "umull v3.8h, v0.8b, v24.8b \n" // B
+ "umull2 v4.8h, v0.16b, v24.16b \n" // B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v3.8h, v1.8b, v25.8b \n" // G
+ "umlal2 v4.8h, v1.16b, v25.16b \n" // G
+ "umlal v3.8h, v2.8b, v26.8b \n" // R
+ "umlal2 v4.8h, v2.16b, v26.16b \n" // R
+ "uqrshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn v1.8b, v4.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqadd v0.8b, v0.8b, v27.8b \n"
+ "uqadd v1.8b, v1.8b, v27.8b \n"
+ "stp d0, d1, [%1], #16 \n" // store 8 pixels Y.
+ "b.gt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1895,167 +3440,215 @@ void ARGB1555ToYRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_y,
int width) {
asm volatile(
- "movi v4.8b, #13 \n" // B * 0.1016 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #33 \n" // R * 0.2578 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGB1555TOARGB
- "umull v3.8h, v0.8b, v4.8b \n" // B
- "umlal v3.8h, v1.8b, v5.8b \n" // G
- "umlal v3.8h, v2.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
+ "movi v4.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v5.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v6.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v7.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB1555
+ // pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ RGB555TOARGB
+ "umull v16.8h, v0.8b, v4.8b \n" // B
+ "umull2 v17.8h, v0.16b, v4.16b \n" // B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v16.8h, v1.8b, v5.8b \n" // G
+ "umlal2 v17.8h, v1.16b, v5.16b \n" // G
+ "umlal v16.8h, v2.8b, v6.8b \n" // R
+ "umlal2 v17.8h, v2.16b, v6.16b \n" // R
+ "uqrshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn2 v0.16b, v17.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqadd v0.16b, v0.16b, v7.16b \n"
+ "str q0, [%1], #16 \n" // store pixels Y.
+ "b.gt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v29");
}
void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_y,
int width) {
asm volatile(
- "movi v24.8b, #13 \n" // B * 0.1016 coefficient
- "movi v25.8b, #65 \n" // G * 0.5078 coefficient
- "movi v26.8b, #33 \n" // R * 0.2578 coefficient
- "movi v27.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGB4444TOARGB
- "umull v3.8h, v0.8b, v24.8b \n" // B
- "umlal v3.8h, v1.8b, v25.8b \n" // G
- "umlal v3.8h, v2.8b, v26.8b \n" // R
- "sqrshrun v0.8b, v3.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v27.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
+ "movi v24.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v25.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v26.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v27.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB4444
+ // pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ ARGB4444TORGB
+ "umull v16.8h, v0.8b, v24.8b \n" // B
+ "umull2 v17.8h, v0.16b, v24.16b \n" // B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v16.8h, v1.8b, v25.8b \n" // G
+ "umlal2 v17.8h, v1.16b, v25.16b \n" // G
+ "umlal v16.8h, v2.8b, v26.8b \n" // R
+ "umlal2 v17.8h, v2.16b, v26.16b \n" // R
+ "uqrshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn2 v0.16b, v17.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqadd v0.16b, v0.16b, v27.16b \n"
+ "str q0, [%1], #16 \n" // store 8 pixels Y.
+ "b.gt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v24", "v25", "v26", "v27");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v24", "v25",
+ "v26", "v27");
}
-void BGRAToYRow_NEON(const uint8_t* src_bgra, uint8_t* dst_y, int width) {
+// ARGB expects first 3 values to contain RGB and 4th value is ignored.
+void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "movi v4.8b, #33 \n" // R * 0.2578 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #13 \n" // B * 0.1016 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v16.8h, v1.8b, v4.8b \n" // R
- "umlal v16.8h, v2.8b, v5.8b \n" // G
- "umlal v16.8h, v3.8b, v6.8b \n" // B
- "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
-}
-
-void ABGRToYRow_NEON(const uint8_t* src_abgr, uint8_t* dst_y, int width) {
- asm volatile(
- "movi v4.8b, #33 \n" // R * 0.2578 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #13 \n" // B * 0.1016 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v16.8h, v0.8b, v4.8b \n" // R
- "umlal v16.8h, v1.8b, v5.8b \n" // G
- "umlal v16.8h, v2.8b, v6.8b \n" // B
- "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_abgr), // %0
+ "ldr s16, [%3] \n" // load 4 coeffs
+ "ldr s17, [%3, #48] \n" // load kAddY[0]
+ "dup v18.16b, v16.b[0] \n" // B
+ "dup v19.16b, v16.b[1] \n" // G
+ "dup v20.16b, v16.b[2] \n" // R
+ "dup v21.16b, v16.b[3] \n" // A
+ "dup v22.8h, v17.h[0] \n" // bias
+ "1: \n"
+ "ld4 {v2.16b,v3.16b,v4.16b,v5.16b}, [%0], #64 \n" // load 16
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "umull v0.8h, v2.8b, v18.8b \n" // B
+ "umull2 v1.8h, v2.16b, v18.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v0.8h, v3.8b, v19.8b \n" // G
+ "umlal2 v1.8h, v3.16b, v19.16b \n"
+ "umlal v0.8h, v4.8b, v20.8b \n" // R
+ "umlal2 v1.8h, v4.16b, v20.16b \n"
+ "umlal v0.8h, v5.8b, v21.8b \n" // A
+ "umlal2 v1.8h, v5.16b, v21.16b \n"
+ "addhn v0.8b, v0.8h, v22.8h \n" // 16 bit to 8 bit Y
+ "addhn v1.8b, v1.8h, v22.8h \n"
+ "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ : "r"(c) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22");
}
-void RGBAToYRow_NEON(const uint8_t* src_rgba, uint8_t* dst_y, int width) {
+void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "movi v4.8b, #13 \n" // B * 0.1016 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #33 \n" // R * 0.2578 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v16.8h, v1.8b, v4.8b \n" // B
- "umlal v16.8h, v2.8b, v5.8b \n" // G
- "umlal v16.8h, v3.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_rgba), // %0
+ "ldr s16, [%3] \n" // load 4 coeffs
+ "ldr s17, [%3, #48] \n" // load kAddY[0]
+ "dup v18.4s, v16.s[0] \n"
+ "dup v19.8h, v17.h[0] \n"
+ "1: \n"
+ "ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [%0], #64 \n" // load 16
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "movi v0.16b, #0 \n"
+ "movi v1.16b, #0 \n"
+ "movi v2.16b, #0 \n"
+ "movi v3.16b, #0 \n"
+ "udot v0.4s, v4.16b, v18.16b \n"
+ "udot v1.4s, v5.16b, v18.16b \n"
+ "udot v2.4s, v6.16b, v18.16b \n"
+ "udot v3.4s, v7.16b, v18.16b \n"
+ "uzp1 v0.8h, v0.8h, v1.8h \n"
+ "uzp1 v1.8h, v2.8h, v3.8h \n"
+ "addhn v0.8b, v0.8h, v19.8h \n"
+ "addhn v1.8b, v1.8h, v19.8h \n"
+ "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ : "r"(c) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
}
-void RGB24ToYRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_y, int width) {
+void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "movi v4.8b, #13 \n" // B * 0.1016 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #33 \n" // R * 0.2578 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v16.8h, v0.8b, v4.8b \n" // B
- "umlal v16.8h, v1.8b, v5.8b \n" // G
- "umlal v16.8h, v2.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_rgb24), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ "ldr s16, [%3] \n" // load 4 coeffs
+ "ldr s17, [%3, #48] \n" // load kAddY[0]
+ "dup v18.4s, v16.s[0] \n"
+ "dup v19.8h, v17.h[0] \n"
+ "movi v7.16b, #0 \n" // zero alpha
+ "1: \n"
+ "ld3 {v4.16b, v5.16b, v6.16b}, [%0], #48 \n" // load 16 pixels
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "zip1 v24.16b, v4.16b, v5.16b \n" // B0 G0 B1 G1 ...
+ "zip1 v25.16b, v6.16b, v7.16b \n" // R0 0 R1 0 ...
+ "zip2 v26.16b, v4.16b, v5.16b \n" // B8 G8 B9 G9 ...
+ "zip2 v27.16b, v6.16b, v7.16b \n" // R8 0 R9 0 ...
+ "zip1 v20.8h, v24.8h, v25.8h \n" // pixels 0..3
+ "zip2 v21.8h, v24.8h, v25.8h \n" // pixels 4..7
+ "zip1 v22.8h, v26.8h, v27.8h \n" // pixels 8..11
+ "zip2 v23.8h, v26.8h, v27.8h \n" // pixels 12..15
+ "movi v0.16b, #0 \n"
+ "movi v1.16b, #0 \n"
+ "movi v2.16b, #0 \n"
+ "movi v3.16b, #0 \n"
+ "udot v0.4s, v20.16b, v18.16b \n"
+ "udot v1.4s, v21.16b, v18.16b \n"
+ "udot v2.4s, v22.16b, v18.16b \n"
+ "udot v3.4s, v23.16b, v18.16b \n"
+ "uzp1 v0.8h, v0.8h, v1.8h \n"
+ "uzp1 v1.8h, v2.8h, v3.8h \n"
+ "addhn v0.8b, v0.8h, v19.8h \n"
+ "addhn v1.8b, v1.8h, v19.8h \n"
+ "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
+ "b.gt 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(c) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27");
}
-void RAWToYRow_NEON(const uint8_t* src_raw, uint8_t* dst_y, int width) {
+// RGBA expects first value to be A and ignored, then 3 values to contain RGB.
+void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
asm volatile(
- "movi v4.8b, #33 \n" // R * 0.2578 coefficient
- "movi v5.8b, #65 \n" // G * 0.5078 coefficient
- "movi v6.8b, #13 \n" // B * 0.1016 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v16.8h, v0.8b, v4.8b \n" // B
- "umlal v16.8h, v1.8b, v5.8b \n" // G
- "umlal v16.8h, v2.8b, v6.8b \n" // R
- "sqrshrun v0.8b, v16.8h, #7 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
- "b.gt 1b \n"
- : "+r"(src_raw), // %0
+ "ldr s16, [%3] \n" // load 4 coeffs
+ "ldr s17, [%3, #48] \n" // load kAddY[0]
+ "dup v18.16b, v16.b[0] \n" // B
+ "dup v19.16b, v16.b[1] \n" // G
+ "dup v20.16b, v16.b[2] \n" // R
+ "dup v21.8h, v17.h[0] \n" // bias
+ "1: \n"
+ "ld3 {v2.16b,v3.16b,v4.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "umull v0.8h, v2.8b, v18.8b \n" // B
+ "umull2 v1.8h, v2.16b, v18.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v0.8h, v3.8b, v19.8b \n" // G
+ "umlal2 v1.8h, v3.16b, v19.16b \n"
+ "umlal v0.8h, v4.8b, v20.8b \n" // R
+ "umlal2 v1.8h, v4.16b, v20.16b \n"
+ "addhn v0.8b, v0.8h, v21.8h \n" // 16 bit to 8 bit Y
+ "addhn v1.8b, v1.8h, v21.8h \n"
+ "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
+ "b.gt 1b \n"
+ : "+r"(src_rgb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
- :
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ : "r"(c) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v16", "v17", "v18",
+ "v19", "v20", "v21");
}
// Bilinear filter 16x2 -> 16x1
@@ -2064,126 +3657,300 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction) {
- int y1_fraction = source_y_fraction;
- int y0_fraction = 256 - y1_fraction;
+ const int y1_fraction = source_y_fraction;
+ const int y0_fraction = 256 - y1_fraction;
const uint8_t* src_ptr1 = src_ptr + src_stride;
asm volatile(
- "cmp %w4, #0 \n"
- "b.eq 100f \n"
- "cmp %w4, #128 \n"
- "b.eq 50f \n"
+ "cmp %w4, #0 \n"
+ "b.eq 100f \n"
+ "cmp %w4, #128 \n"
+ "b.eq 50f \n"
- "dup v5.16b, %w4 \n"
- "dup v4.16b, %w5 \n"
+ "dup v5.16b, %w4 \n"
+ "dup v4.16b, %w5 \n"
// General purpose row blend.
- "1: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "ld1 {v1.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "umull v2.8h, v0.8b, v4.8b \n"
- "umull2 v3.8h, v0.16b, v4.16b \n"
- "umlal v2.8h, v1.8b, v5.8b \n"
- "umlal2 v3.8h, v1.16b, v5.16b \n"
- "rshrn v0.8b, v2.8h, #8 \n"
- "rshrn2 v0.16b, v3.8h, #8 \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 1b \n"
- "b 99f \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%1], #16 \n"
+ "ld1 {v1.16b}, [%2], #16 \n"
+ "subs %w3, %w3, #16 \n"
+ "umull v2.8h, v0.8b, v4.8b \n"
+ "umull2 v3.8h, v0.16b, v4.16b \n"
+ "umlal v2.8h, v1.8b, v5.8b \n"
+ "umlal2 v3.8h, v1.16b, v5.16b \n"
+ "rshrn v0.8b, v2.8h, #8 \n"
+ "rshrn v1.8b, v3.8h, #8 \n"
+ "stp d0, d1, [%0], #16 \n"
+ "b.gt 1b \n"
+ "b 99f \n"
+
+ // Blend 50 / 50.
+ "50: \n"
+ "ld1 {v0.16b}, [%1], #16 \n"
+ "ld1 {v1.16b}, [%2], #16 \n"
+ "subs %w3, %w3, #16 \n"
+ "urhadd v0.16b, v0.16b, v1.16b \n"
+ "st1 {v0.16b}, [%0], #16 \n"
+ "b.gt 50b \n"
+ "b 99f \n"
+
+ // Blend 100 / 0 - Copy row unchanged.
+ "100: \n"
+ "ld1 {v0.16b}, [%1], #16 \n"
+ "subs %w3, %w3, #16 \n"
+ "st1 {v0.16b}, [%0], #16 \n"
+ "b.gt 100b \n"
+
+ "99: \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(src_ptr1), // %2
+ "+r"(dst_width) // %3
+ : "r"(y1_fraction), // %4
+ "r"(y0_fraction) // %5
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
+}
+
+// Bilinear filter 8x2 -> 8x1
+void InterpolateRow_16_NEON(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ asm volatile(
+ "cmp %w4, #0 \n"
+ "b.eq 100f \n"
+ "cmp %w4, #128 \n"
+ "b.eq 50f \n"
+
+ "dup v5.8h, %w4 \n"
+ "dup v4.8h, %w5 \n"
+ "subs %w3, %w3, #16 \n"
+ "b.lt 2f \n"
+
+ // 16-element unrolled loop (32 bytes).
+ "1: \n"
+ "ldp q0, q1, [%1], #32 \n"
+ "ldp q6, q7, [%2], #32 \n"
+ "subs %w3, %w3, #16 \n"
+ "umull v16.4s, v0.4h, v4.4h \n"
+ "umull2 v17.4s, v0.8h, v4.8h \n"
+ "umull v18.4s, v1.4h, v4.4h \n"
+ "umull2 v19.4s, v1.8h, v4.8h \n"
+ "umlal v16.4s, v6.4h, v5.4h \n"
+ "umlal2 v17.4s, v6.8h, v5.8h \n"
+ "umlal v18.4s, v7.4h, v5.4h \n"
+ "umlal2 v19.4s, v7.8h, v5.8h \n"
+ "rshrn v0.4h, v16.4s, #8 \n"
+ "rshrn v1.4h, v17.4s, #8 \n"
+ "rshrn v2.4h, v18.4s, #8 \n"
+ "rshrn v3.4h, v19.4s, #8 \n"
+ "stp d0, d1, [%0], #16 \n"
+ "stp d2, d3, [%0], #16 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w3, %w3, #16 \n"
+ "b.eq 99f \n"
+
+ // 8-element tail (16 bytes).
+ "ld1 {v0.8h}, [%1] \n"
+ "ld1 {v1.8h}, [%2] \n"
+ "umull v16.4s, v0.4h, v4.4h \n"
+ "umull2 v17.4s, v0.8h, v4.8h \n"
+ "umlal v16.4s, v1.4h, v5.4h \n"
+ "umlal2 v17.4s, v1.8h, v5.8h \n"
+ "rshrn v0.4h, v16.4s, #8 \n"
+ "rshrn v1.4h, v17.4s, #8 \n"
+ "stp d0, d1, [%0] \n"
+ "b 99f \n"
// Blend 50 / 50.
- "50: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "ld1 {v1.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 50b \n"
- "b 99f \n"
+ "50: \n"
+ "ld1 {v0.8h}, [%1], #16 \n"
+ "ld1 {v1.8h}, [%2], #16 \n"
+ "subs %w3, %w3, #8 \n"
+ "urhadd v0.8h, v0.8h, v1.8h \n"
+ "st1 {v0.8h}, [%0], #16 \n"
+ "b.gt 50b \n"
+ "b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "subs %w3, %w3, #16 \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 100b \n"
-
- "99: \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(src_ptr1), // %2
- "+r"(dst_width), // %3
- "+r"(y1_fraction), // %4
- "+r"(y0_fraction) // %5
+ "100: \n"
+ "ld1 {v0.8h}, [%1], #16 \n"
+ "subs %w3, %w3, #8 \n"
+ "st1 {v0.8h}, [%0], #16 \n"
+ "b.gt 100b \n"
+
+ "99: \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(src_ptr1), // %2
+ "+r"(dst_width) // %3
+ : "r"(y1_fraction), // %4
+ "r"(y0_fraction) // %5
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
+}
+
+// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
+void ARGBBlendRow_NEON(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "subs %w3, %w3, #8 \n"
+ "b.lt 89f \n"
+ // Blend 8 pixels.
+ "8: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB0
+ "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 ARGB1
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "umull v16.8h, v4.8b, v3.8b \n" // db * a
+ "prfm pldl1keep, [%0, 448] \n"
+ "umull v17.8h, v5.8b, v3.8b \n" // dg * a
+ "prfm pldl1keep, [%1, 448] \n"
+ "umull v18.8h, v6.8b, v3.8b \n" // dr * a
+ "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8
+ "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8
+ "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8
+ "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256)
+ "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256)
+ "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256)
+ "uqadd v0.8b, v0.8b, v4.8b \n" // + sb
+ "uqadd v1.8b, v1.8b, v5.8b \n" // + sg
+ "uqadd v2.8b, v2.8b, v6.8b \n" // + sr
+ "movi v3.8b, #255 \n" // a = 255
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ // pixels
+ "b.ge 8b \n"
+
+ "89: \n"
+ "adds %w3, %w3, #8-1 \n"
+ "b.lt 99f \n"
+
+ // Blend 1 pixels.
+ "1: \n"
+ "ld4 {v0.b,v1.b,v2.b,v3.b}[0], [%0], #4 \n" // load 1 pixel
+ // ARGB0.
+ "ld4 {v4.b,v5.b,v6.b,v7.b}[0], [%1], #4 \n" // load 1 pixel
+ // ARGB1.
+ "subs %w3, %w3, #1 \n" // 1 processed per loop.
+ "umull v16.8h, v4.8b, v3.8b \n" // db * a
+ "prfm pldl1keep, [%0, 448] \n"
+ "umull v17.8h, v5.8b, v3.8b \n" // dg * a
+ "prfm pldl1keep, [%1, 448] \n"
+ "umull v18.8h, v6.8b, v3.8b \n" // dr * a
+ "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8
+ "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8
+ "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8
+ "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256)
+ "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256)
+ "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256)
+ "uqadd v0.8b, v0.8b, v4.8b \n" // + sb
+ "uqadd v1.8b, v1.8b, v5.8b \n" // + sg
+ "uqadd v2.8b, v2.8b, v6.8b \n" // + sr
+ "movi v3.8b, #255 \n" // a = 255
+ "st4 {v0.b,v1.b,v2.b,v3.b}[0], [%2], #4 \n" // store 1 pixel.
+ "b.ge 1b \n"
+
+ "99: \n"
+
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
:
- : "cc", "memory", "v0", "v1", "v3", "v4", "v5");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18");
+}
+
+void ARGBOrRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ asm volatile(
+ "dup v2.4s, %w3 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "subs %w2, %w2, #8 \n"
+ "orr v0.16b, v0.16b, v2.16b \n"
+ "orr v1.16b, v1.16b, v2.16b \n"
+ "stp q0, q1, [%1], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(value) // %3
+ : "memory", "cc", "v0", "v1", "v2");
}
-// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
-void ARGBBlendRow_NEON(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
+void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
asm volatile(
- "subs %w3, %w3, #8 \n"
- "b.lt 89f \n"
- // Blend 8 pixels.
- "8: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB0
- // pixels
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 ARGB1
- // pixels
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "umull v16.8h, v4.8b, v3.8b \n" // db * a
- "umull v17.8h, v5.8b, v3.8b \n" // dg * a
- "umull v18.8h, v6.8b, v3.8b \n" // dr * a
- "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8
- "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8
- "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8
- "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256)
- "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256)
- "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256)
- "uqadd v0.8b, v0.8b, v4.8b \n" // + sb
- "uqadd v1.8b, v1.8b, v5.8b \n" // + sg
- "uqadd v2.8b, v2.8b, v6.8b \n" // + sr
- "movi v3.8b, #255 \n" // a = 255
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- // pixels
- "b.ge 8b \n"
-
- "89: \n"
- "adds %w3, %w3, #8-1 \n"
- "b.lt 99f \n"
-
- // Blend 1 pixels.
+ "movi v28.8b, #255 \n"
+ "movi v29.8h, #1 \n"
"1: \n"
- "ld4 {v0.b,v1.b,v2.b,v3.b}[0], [%0], #4 \n" // load 1 pixel ARGB0.
- "ld4 {v4.b,v5.b,v6.b,v7.b}[0], [%1], #4 \n" // load 1 pixel ARGB1.
- "subs %w3, %w3, #1 \n" // 1 processed per loop.
- "umull v16.8h, v4.8b, v3.8b \n" // db * a
- "umull v17.8h, v5.8b, v3.8b \n" // dg * a
- "umull v18.8h, v6.8b, v3.8b \n" // dr * a
- "uqrshrn v16.8b, v16.8h, #8 \n" // db >>= 8
- "uqrshrn v17.8b, v17.8h, #8 \n" // dg >>= 8
- "uqrshrn v18.8b, v18.8h, #8 \n" // dr >>= 8
- "uqsub v4.8b, v4.8b, v16.8b \n" // db - (db * a / 256)
- "uqsub v5.8b, v5.8b, v17.8b \n" // dg - (dg * a / 256)
- "uqsub v6.8b, v6.8b, v18.8b \n" // dr - (dr * a / 256)
- "uqadd v0.8b, v0.8b, v4.8b \n" // + sb
- "uqadd v1.8b, v1.8b, v5.8b \n" // + sg
- "uqadd v2.8b, v2.8b, v6.8b \n" // + sr
- "movi v3.8b, #255 \n" // a = 255
- "st4 {v0.b,v1.b,v2.b,v3.b}[0], [%2], #4 \n" // store 1 pixel.
- "b.ge 1b \n"
-
- "99: \n"
-
- : "+r"(src_argb0), // %0
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n"
+ "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n"
+ "subs %w3, %w3, #8 \n"
+ "mov v16.16b, v29.16b \n"
+ "umlal v16.8h, v7.8b, v3.8b \n"
+ "usra v16.8h, v16.8h, #8 \n"
+ "shrn v16.8b, v16.8h, #8 \n"
+ "cmeq v17.8b, v7.8b, #0 \n"
+ "sub v16.8b, v3.8b, v16.8b \n"
+ "add v7.8b, v7.8b, v16.8b \n"
+ "orr v18.8b, v3.8b, v17.8b \n"
+ "orr v19.8b, v7.8b, v17.8b \n"
+ "umull v16.8h, v18.8b, v28.8b \n"
+ "uxtl v17.8h, v19.8b \n"
+ "uxtl v20.4s, v16.4h \n"
+ "uxtl2 v21.4s, v16.8h \n"
+ "uxtl v22.4s, v17.4h \n"
+ "uxtl2 v23.4s, v17.8h \n"
+ "ucvtf v20.4s, v20.4s \n"
+ "ucvtf v21.4s, v21.4s \n"
+ "ucvtf v22.4s, v22.4s \n"
+ "ucvtf v23.4s, v23.4s \n"
+ "fdiv v20.4s, v20.4s, v22.4s \n"
+ "fdiv v21.4s, v21.4s, v23.4s \n"
+ "fcvtzu v20.4s, v20.4s \n"
+ "fcvtzu v21.4s, v21.4s \n"
+ "xtn v16.4h, v20.4s \n"
+ "xtn2 v16.8h, v21.4s \n"
+ "xtn v16.8b, v16.8h \n"
+ "sub v17.8b, v28.8b, v16.8b \n"
+ "mov v20.16b, v29.16b \n"
+ "mov v21.16b, v29.16b \n"
+ "mov v22.16b, v29.16b \n"
+ "umlal v20.8h, v4.8b, v17.8b \n"
+ "umlal v21.8h, v5.8b, v17.8b \n"
+ "umlal v22.8h, v6.8b, v17.8b \n"
+ "umlal v20.8h, v0.8b, v16.8b \n"
+ "umlal v21.8h, v1.8b, v16.8b \n"
+ "umlal v22.8h, v2.8b, v16.8b \n"
+ "usra v20.8h, v20.8h, #8 \n"
+ "usra v21.8h, v21.8h, #8 \n"
+ "usra v22.8h, v22.8h, #8 \n"
+ "shrn v0.8b, v20.8h, #8 \n"
+ "shrn v1.8b, v21.8h, #8 \n"
+ "shrn v2.8b, v22.8h, #8 \n"
+ "mov v3.8b, v7.8b \n"
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
- "v17", "v18");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v28", "v29");
}
// Attenuate 8 pixels at a time.
@@ -2191,24 +3958,26 @@ void ARGBAttenuateRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
asm volatile(
+ "movi v7.8h, #0x00ff \n" // 255 for rounding up
+
// Attenuate 8 pixels.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v4.8h, v0.8b, v3.8b \n" // b * a
- "umull v5.8h, v1.8b, v3.8b \n" // g * a
- "umull v6.8h, v2.8b, v3.8b \n" // r * a
- "uqrshrn v0.8b, v4.8h, #8 \n" // b >>= 8
- "uqrshrn v1.8b, v5.8h, #8 \n" // g >>= 8
- "uqrshrn v2.8b, v6.8h, #8 \n" // r >>= 8
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
- // pixels
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "umull v4.8h, v0.8b, v3.8b \n" // b * a
+ "prfm pldl1keep, [%0, 448] \n"
+ "umull v5.8h, v1.8b, v3.8b \n" // g * a
+ "umull v6.8h, v2.8b, v3.8b \n" // r * a
+ "addhn v0.8b, v4.8h, v7.8h \n" // (b + 255) >> 8
+ "addhn v1.8b, v5.8h, v7.8h \n" // (g + 255) >> 8
+ "addhn v2.8b, v6.8h, v7.8h \n" // (r + 255) >> 8
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
// Quantize 8 ARGB pixels (32 bytes).
@@ -2219,32 +3988,33 @@ void ARGBQuantizeRow_NEON(uint8_t* dst_argb,
int interval_offset,
int width) {
asm volatile(
- "dup v4.8h, %w2 \n"
- "ushr v4.8h, v4.8h, #1 \n" // scale >>= 1
- "dup v5.8h, %w3 \n" // interval multiply.
- "dup v6.8h, %w4 \n" // interval add
+ "dup v4.8h, %w2 \n"
+ "ushr v4.8h, v4.8h, #1 \n" // scale >>= 1
+ "dup v5.8h, %w3 \n" // interval multiply.
+ "dup v6.8h, %w4 \n" // interval add
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB.
- "subs %w1, %w1, #8 \n" // 8 processed per loop.
- "uxtl v0.8h, v0.8b \n" // b (0 .. 255)
- "uxtl v1.8h, v1.8b \n"
- "uxtl v2.8h, v2.8b \n"
- "sqdmulh v0.8h, v0.8h, v4.8h \n" // b * scale
- "sqdmulh v1.8h, v1.8h, v4.8h \n" // g
- "sqdmulh v2.8h, v2.8h, v4.8h \n" // r
- "mul v0.8h, v0.8h, v5.8h \n" // b * interval_size
- "mul v1.8h, v1.8h, v5.8h \n" // g
- "mul v2.8h, v2.8h, v5.8h \n" // r
- "add v0.8h, v0.8h, v6.8h \n" // b + interval_offset
- "add v1.8h, v1.8h, v6.8h \n" // g
- "add v2.8h, v2.8h, v6.8h \n" // r
- "uqxtn v0.8b, v0.8h \n"
- "uqxtn v1.8b, v1.8h \n"
- "uqxtn v2.8b, v2.8h \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB.
+ "subs %w1, %w1, #8 \n" // 8 processed per loop.
+ "uxtl v0.8h, v0.8b \n" // b (0 .. 255)
+ "prfm pldl1keep, [%0, 448] \n"
+ "uxtl v1.8h, v1.8b \n"
+ "uxtl v2.8h, v2.8b \n"
+ "sqdmulh v0.8h, v0.8h, v4.8h \n" // b * scale
+ "sqdmulh v1.8h, v1.8h, v4.8h \n" // g
+ "sqdmulh v2.8h, v2.8h, v4.8h \n" // r
+ "mul v0.8h, v0.8h, v5.8h \n" // b * interval_size
+ "mul v1.8h, v1.8h, v5.8h \n" // g
+ "mul v2.8h, v2.8h, v5.8h \n" // r
+ "add v0.8h, v0.8h, v6.8h \n" // b + interval_offset
+ "add v1.8h, v1.8h, v6.8h \n" // g
+ "add v2.8h, v2.8h, v6.8h \n" // r
+ "uqxtn v0.8b, v0.8h \n"
+ "uqxtn v1.8b, v1.8h \n"
+ "uqxtn v2.8b, v2.8h \n"
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
: "r"(scale), // %2
@@ -2254,35 +4024,37 @@ void ARGBQuantizeRow_NEON(uint8_t* dst_argb,
}
// Shade 8 pixels at a time by specified value.
-// NOTE vqrdmulh.s16 q10, q10, d0[0] must use a scaler register from 0 to 8.
-// Rounding in vqrdmulh does +1 to high if high bit of low s16 is set.
+// sqrdmulh is a rounding instruction, so +1 if high bit of low half of
+// multiply result is set.
void ARGBShadeRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value) {
asm volatile(
- "dup v0.4s, %w3 \n" // duplicate scale value.
- "zip1 v0.8b, v0.8b, v0.8b \n" // v0.8b aarrggbb.
- "ushr v0.8h, v0.8h, #1 \n" // scale / 2.
+ "dup v0.4s, %w3 \n" // duplicate scale value.
+ "zip1 v0.16b, v0.16b, v0.16b \n" // v0.16b
+ // aarrggbbaarrggbb.
+ "ushr v0.8h, v0.8h, #1 \n" // scale / 2.
// 8 pixel loop.
- "1: \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "uxtl v4.8h, v4.8b \n" // b (0 .. 255)
- "uxtl v5.8h, v5.8b \n"
- "uxtl v6.8h, v6.8b \n"
- "uxtl v7.8h, v7.8b \n"
- "sqrdmulh v4.8h, v4.8h, v0.h[0] \n" // b * scale * 2
- "sqrdmulh v5.8h, v5.8h, v0.h[1] \n" // g
- "sqrdmulh v6.8h, v6.8h, v0.h[2] \n" // r
- "sqrdmulh v7.8h, v7.8h, v0.h[3] \n" // a
- "uqxtn v4.8b, v4.8h \n"
- "uqxtn v5.8b, v5.8h \n"
- "uqxtn v6.8b, v6.8h \n"
- "uqxtn v7.8b, v7.8h \n"
- "st4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "uxtl v4.8h, v4.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uxtl v5.8h, v5.8b \n"
+ "uxtl v6.8h, v6.8b \n"
+ "uxtl v7.8h, v7.8b \n"
+ "sqrdmulh v4.8h, v4.8h, v0.8h \n" // argb * scale * 2
+ "sqrdmulh v5.8h, v5.8h, v0.8h \n"
+ "sqrdmulh v6.8h, v6.8h, v0.8h \n"
+ "sqrdmulh v7.8h, v7.8h, v0.8h \n"
+ "uqxtn v4.8b, v4.8h \n"
+ "uqxtn v5.8b, v5.8h \n"
+ "uqxtn v6.8b, v6.8h \n"
+ "uqxtn v7.8b, v7.8h \n"
+ "st1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2292,23 +4064,24 @@ void ARGBShadeRow_NEON(const uint8_t* src_argb,
// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels
// Similar to ARGBToYJ but stores ARGB.
-// C code is (15 * b + 75 * g + 38 * r + 64) >> 7;
+// C code is (29 * b + 150 * g + 77 * r + 128) >> 8;
void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
asm volatile(
- "movi v24.8b, #15 \n" // B * 0.11400 coefficient
- "movi v25.8b, #75 \n" // G * 0.58700 coefficient
- "movi v26.8b, #38 \n" // R * 0.29900 coefficient
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "umull v4.8h, v0.8b, v24.8b \n" // B
- "umlal v4.8h, v1.8b, v25.8b \n" // G
- "umlal v4.8h, v2.8b, v26.8b \n" // R
- "sqrshrun v0.8b, v4.8h, #7 \n" // 15 bit to 8 bit B
- "orr v1.8b, v0.8b, v0.8b \n" // G
- "orr v2.8b, v0.8b, v0.8b \n" // R
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 pixels.
- "b.gt 1b \n"
+ "movi v24.8b, #29 \n" // B * 0.1140 coefficient
+ "movi v25.8b, #150 \n" // G * 0.5870 coefficient
+ "movi v26.8b, #77 \n" // R * 0.2990 coefficient
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "umull v4.8h, v0.8b, v24.8b \n" // B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v4.8h, v1.8b, v25.8b \n" // G
+ "umlal v4.8h, v2.8b, v26.8b \n" // R
+ "uqrshrn v0.8b, v4.8h, #8 \n" // 16 bit to 8 bit B
+ "mov v1.8b, v0.8b \n" // G
+ "mov v2.8b, v0.8b \n" // R
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 pixels.
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2316,6 +4089,38 @@ void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v24", "v25", "v26");
}
+static const uvec8 kARGBGrayRowCoeffs = {29, 150, 77, 0};
+static const uvec8 kARGBGrayRowIndices = {0, 0, 0, 19, 2, 2, 2, 23,
+ 4, 4, 4, 27, 6, 6, 6, 31};
+
+void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "ld1r {v24.4s}, [%[coeffs]] \n"
+ "ldr q25, [%[indices]] \n"
+ "1: \n"
+ "ldp q1, q3, [%[src]], #32 \n" // load 8 ARGB
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop
+ "movi v0.4s, #0 \n"
+ "movi v2.4s, #0 \n"
+ "udot v0.4s, v1.16b, v24.16b \n"
+ "udot v2.4s, v3.16b, v24.16b \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uqrshrn v0.8b, v0.8h, #8 \n"
+ "uqrshrn v2.8b, v2.8h, #8 \n"
+ "tbl v0.16b, {v0.16b, v1.16b}, v25.16b \n" // merge in alpha
+ "tbl v1.16b, {v2.16b, v3.16b}, v25.16b \n"
+ "stp q0, q1, [%[dst]], #32 \n" // store 8 pixels
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [coeffs] "r"(&kARGBGrayRowCoeffs), // %[coeffs]
+ [indices] "r"(&kARGBGrayRowIndices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v24", "v25");
+}
+
// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels.
// b = (r * 35 + g * 68 + b * 17) >> 7
// g = (r * 45 + g * 88 + b * 22) >> 7
@@ -2323,32 +4128,33 @@ void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width) {
asm volatile(
- "movi v20.8b, #17 \n" // BB coefficient
- "movi v21.8b, #68 \n" // BG coefficient
- "movi v22.8b, #35 \n" // BR coefficient
- "movi v24.8b, #22 \n" // GB coefficient
- "movi v25.8b, #88 \n" // GG coefficient
- "movi v26.8b, #45 \n" // GR coefficient
- "movi v28.8b, #24 \n" // BB coefficient
- "movi v29.8b, #98 \n" // BG coefficient
- "movi v30.8b, #50 \n" // BR coefficient
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB pixels.
- "subs %w1, %w1, #8 \n" // 8 processed per loop.
- "umull v4.8h, v0.8b, v20.8b \n" // B to Sepia B
- "umlal v4.8h, v1.8b, v21.8b \n" // G
- "umlal v4.8h, v2.8b, v22.8b \n" // R
- "umull v5.8h, v0.8b, v24.8b \n" // B to Sepia G
- "umlal v5.8h, v1.8b, v25.8b \n" // G
- "umlal v5.8h, v2.8b, v26.8b \n" // R
- "umull v6.8h, v0.8b, v28.8b \n" // B to Sepia R
- "umlal v6.8h, v1.8b, v29.8b \n" // G
- "umlal v6.8h, v2.8b, v30.8b \n" // R
- "uqshrn v0.8b, v4.8h, #7 \n" // 16 bit to 8 bit B
- "uqshrn v1.8b, v5.8h, #7 \n" // 16 bit to 8 bit G
- "uqshrn v2.8b, v6.8h, #7 \n" // 16 bit to 8 bit R
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 pixels.
- "b.gt 1b \n"
+ "movi v20.8b, #17 \n" // BB coefficient
+ "movi v21.8b, #68 \n" // BG coefficient
+ "movi v22.8b, #35 \n" // BR coefficient
+ "movi v24.8b, #22 \n" // GB coefficient
+ "movi v25.8b, #88 \n" // GG coefficient
+ "movi v26.8b, #45 \n" // GR coefficient
+ "movi v28.8b, #24 \n" // BB coefficient
+ "movi v29.8b, #98 \n" // BG coefficient
+ "movi v30.8b, #50 \n" // BR coefficient
+ "1: \n"
+ "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB pixels.
+ "subs %w1, %w1, #8 \n" // 8 processed per loop.
+ "umull v4.8h, v0.8b, v20.8b \n" // B to Sepia B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umlal v4.8h, v1.8b, v21.8b \n" // G
+ "umlal v4.8h, v2.8b, v22.8b \n" // R
+ "umull v5.8h, v0.8b, v24.8b \n" // B to Sepia G
+ "umlal v5.8h, v1.8b, v25.8b \n" // G
+ "umlal v5.8h, v2.8b, v26.8b \n" // R
+ "umull v6.8h, v0.8b, v28.8b \n" // B to Sepia R
+ "umlal v6.8h, v1.8b, v29.8b \n" // G
+ "umlal v6.8h, v2.8b, v30.8b \n" // R
+ "uqshrn v0.8b, v4.8h, #7 \n" // 16 bit to 8 bit B
+ "uqshrn v1.8b, v5.8h, #7 \n" // 16 bit to 8 bit G
+ "uqshrn v2.8b, v6.8h, #7 \n" // 16 bit to 8 bit R
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // store 8 pixels.
+ "b.gt 1b \n"
: "+r"(dst_argb), // %0
"+r"(width) // %1
:
@@ -2356,6 +4162,47 @@ void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width) {
"v21", "v22", "v24", "v25", "v26", "v28", "v29", "v30");
}
+static const uvec8 kARGBSepiaRowCoeffs = {17, 68, 35, 0, 22, 88,
+ 45, 0, 24, 98, 50, 0};
+static const uvec8 kARGBSepiaRowAlphaIndices = {3, 7, 11, 15, 19, 23, 27, 31};
+
+void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width) {
+ asm volatile(
+ "ld3r {v20.4s, v21.4s, v22.4s}, [%[coeffs]] \n"
+ "ldr d23, [%[indices]] \n"
+ "1: \n"
+ "ldp q0, q1, [%[dst]] \n"
+ "subs %w1, %w1, #8 \n"
+ "movi v2.4s, #0 \n"
+ "movi v3.4s, #0 \n"
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "movi v6.4s, #0 \n"
+ "movi v7.4s, #0 \n"
+ "udot v2.4s, v0.16b, v20.16b \n"
+ "udot v3.4s, v1.16b, v20.16b \n"
+ "udot v4.4s, v0.16b, v21.16b \n"
+ "udot v5.4s, v1.16b, v21.16b \n"
+ "udot v6.4s, v0.16b, v22.16b \n"
+ "udot v7.4s, v1.16b, v22.16b \n"
+ "prfm pldl1keep, [%[dst], 448] \n"
+ "uzp1 v6.8h, v6.8h, v7.8h \n"
+ "uzp1 v5.8h, v4.8h, v5.8h \n"
+ "uzp1 v4.8h, v2.8h, v3.8h \n"
+ "tbl v3.16b, {v0.16b, v1.16b}, v23.16b \n"
+ "uqshrn v0.8b, v4.8h, #7 \n"
+ "uqshrn v1.8b, v5.8h, #7 \n"
+ "uqshrn v2.8b, v6.8h, #7 \n"
+ "st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [coeffs] "r"(&kARGBSepiaRowCoeffs), // %[coeffs]
+ [indices] "r"(&kARGBSepiaRowAlphaIndices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
+ "v21", "v22", "v23", "v24", "v25", "v26", "v28", "v29", "v30");
+}
+
// Tranform 8 ARGB pixels (32 bytes) with color matrix.
// TODO(fbarchard): Was same as Sepia except matrix is provided. This function
// needs to saturate. Consider doing a non-saturating version.
@@ -2364,51 +4211,52 @@ void ARGBColorMatrixRow_NEON(const uint8_t* src_argb,
const int8_t* matrix_argb,
int width) {
asm volatile(
- "ld1 {v2.16b}, [%3] \n" // load 3 ARGB vectors.
- "sxtl v0.8h, v2.8b \n" // B,G coefficients s16.
- "sxtl2 v1.8h, v2.16b \n" // R,A coefficients s16.
-
- "1: \n"
- "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 ARGB
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- "uxtl v16.8h, v16.8b \n" // b (0 .. 255) 16 bit
- "uxtl v17.8h, v17.8b \n" // g
- "uxtl v18.8h, v18.8b \n" // r
- "uxtl v19.8h, v19.8b \n" // a
- "mul v22.8h, v16.8h, v0.h[0] \n" // B = B * Matrix B
- "mul v23.8h, v16.8h, v0.h[4] \n" // G = B * Matrix G
- "mul v24.8h, v16.8h, v1.h[0] \n" // R = B * Matrix R
- "mul v25.8h, v16.8h, v1.h[4] \n" // A = B * Matrix A
- "mul v4.8h, v17.8h, v0.h[1] \n" // B += G * Matrix B
- "mul v5.8h, v17.8h, v0.h[5] \n" // G += G * Matrix G
- "mul v6.8h, v17.8h, v1.h[1] \n" // R += G * Matrix R
- "mul v7.8h, v17.8h, v1.h[5] \n" // A += G * Matrix A
- "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
- "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
- "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
- "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
- "mul v4.8h, v18.8h, v0.h[2] \n" // B += R * Matrix B
- "mul v5.8h, v18.8h, v0.h[6] \n" // G += R * Matrix G
- "mul v6.8h, v18.8h, v1.h[2] \n" // R += R * Matrix R
- "mul v7.8h, v18.8h, v1.h[6] \n" // A += R * Matrix A
- "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
- "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
- "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
- "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
- "mul v4.8h, v19.8h, v0.h[3] \n" // B += A * Matrix B
- "mul v5.8h, v19.8h, v0.h[7] \n" // G += A * Matrix G
- "mul v6.8h, v19.8h, v1.h[3] \n" // R += A * Matrix R
- "mul v7.8h, v19.8h, v1.h[7] \n" // A += A * Matrix A
- "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
- "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
- "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
- "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
- "sqshrun v16.8b, v22.8h, #6 \n" // 16 bit to 8 bit B
- "sqshrun v17.8b, v23.8h, #6 \n" // 16 bit to 8 bit G
- "sqshrun v18.8b, v24.8h, #6 \n" // 16 bit to 8 bit R
- "sqshrun v19.8b, v25.8h, #6 \n" // 16 bit to 8 bit A
- "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%1], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "ld1 {v2.16b}, [%3] \n" // load 3 ARGB vectors.
+ "sxtl v0.8h, v2.8b \n" // B,G coefficients s16.
+ "sxtl2 v1.8h, v2.16b \n" // R,A coefficients s16.
+
+ "1: \n"
+ "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 ARGB
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "uxtl v16.8h, v16.8b \n" // b (0 .. 255) 16 bit
+ "prfm pldl1keep, [%0, 448] \n"
+ "uxtl v17.8h, v17.8b \n" // g
+ "uxtl v18.8h, v18.8b \n" // r
+ "uxtl v19.8h, v19.8b \n" // a
+ "mul v22.8h, v16.8h, v0.h[0] \n" // B = B * Matrix B
+ "mul v23.8h, v16.8h, v0.h[4] \n" // G = B * Matrix G
+ "mul v24.8h, v16.8h, v1.h[0] \n" // R = B * Matrix R
+ "mul v25.8h, v16.8h, v1.h[4] \n" // A = B * Matrix A
+ "mul v4.8h, v17.8h, v0.h[1] \n" // B += G * Matrix B
+ "mul v5.8h, v17.8h, v0.h[5] \n" // G += G * Matrix G
+ "mul v6.8h, v17.8h, v1.h[1] \n" // R += G * Matrix R
+ "mul v7.8h, v17.8h, v1.h[5] \n" // A += G * Matrix A
+ "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
+ "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
+ "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
+ "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
+ "mul v4.8h, v18.8h, v0.h[2] \n" // B += R * Matrix B
+ "mul v5.8h, v18.8h, v0.h[6] \n" // G += R * Matrix G
+ "mul v6.8h, v18.8h, v1.h[2] \n" // R += R * Matrix R
+ "mul v7.8h, v18.8h, v1.h[6] \n" // A += R * Matrix A
+ "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
+ "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
+ "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
+ "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
+ "mul v4.8h, v19.8h, v0.h[3] \n" // B += A * Matrix B
+ "mul v5.8h, v19.8h, v0.h[7] \n" // G += A * Matrix G
+ "mul v6.8h, v19.8h, v1.h[3] \n" // R += A * Matrix R
+ "mul v7.8h, v19.8h, v1.h[7] \n" // A += A * Matrix A
+ "sqadd v22.8h, v22.8h, v4.8h \n" // Accumulate B
+ "sqadd v23.8h, v23.8h, v5.8h \n" // Accumulate G
+ "sqadd v24.8h, v24.8h, v6.8h \n" // Accumulate R
+ "sqadd v25.8h, v25.8h, v7.8h \n" // Accumulate A
+ "sqshrun v16.8b, v22.8h, #6 \n" // 16 bit to 8 bit B
+ "sqshrun v17.8b, v23.8h, #6 \n" // 16 bit to 8 bit G
+ "sqshrun v18.8b, v24.8h, #6 \n" // 16 bit to 8 bit R
+ "sqshrun v19.8b, v25.8h, #6 \n" // 16 bit to 8 bit A
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%1], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -2417,29 +4265,86 @@ void ARGBColorMatrixRow_NEON(const uint8_t* src_argb,
"v17", "v18", "v19", "v22", "v23", "v24", "v25");
}
-// TODO(fbarchard): fix vqshrun in ARGBMultiplyRow_NEON and reenable.
+void ARGBColorMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ asm volatile(
+ "ld1 {v31.16b}, [%[matrix_argb]] \n"
+
+ "1: \n"
+ "ld1 {v0.16b, v1.16b}, [%[src_argb]], #32 \n"
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+
+ "movi v16.4s, #0 \n"
+ "movi v17.4s, #0 \n"
+ "movi v18.4s, #0 \n"
+ "movi v19.4s, #0 \n"
+ "movi v20.4s, #0 \n"
+ "movi v21.4s, #0 \n"
+ "movi v22.4s, #0 \n"
+ "movi v23.4s, #0 \n"
+
+ "prfm pldl1keep, [%[src_argb], 448] \n"
+
+ "sudot v16.4s, v31.16b, v0.4b[0] \n"
+ "sudot v17.4s, v31.16b, v0.4b[1] \n"
+ "sudot v18.4s, v31.16b, v0.4b[2] \n"
+ "sudot v19.4s, v31.16b, v0.4b[3] \n"
+ "sudot v20.4s, v31.16b, v1.4b[0] \n"
+ "sudot v21.4s, v31.16b, v1.4b[1] \n"
+ "sudot v22.4s, v31.16b, v1.4b[2] \n"
+ "sudot v23.4s, v31.16b, v1.4b[3] \n"
+
+ "shrn v16.4h, v16.4s, #6 \n"
+ "shrn v18.4h, v18.4s, #6 \n"
+ "shrn v20.4h, v20.4s, #6 \n"
+ "shrn v22.4h, v22.4s, #6 \n"
+ "shrn2 v16.8h, v17.4s, #6 \n"
+ "shrn2 v18.8h, v19.4s, #6 \n"
+ "shrn2 v20.8h, v21.4s, #6 \n"
+ "shrn2 v22.8h, v23.4s, #6 \n"
+
+ "uqxtn v16.8b, v16.8h \n"
+ "uqxtn v18.8b, v18.8h \n"
+ "uqxtn v20.8b, v20.8h \n"
+ "uqxtn v22.8b, v22.8h \n"
+
+ "stp d16, d18, [%[dst_argb]], #16 \n"
+ "stp d20, d22, [%[dst_argb]], #16 \n"
+ "b.gt 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [matrix_argb] "r"(matrix_argb) // %[matrix_argb]
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
+ "v22", "v23", "v31");
+}
+
// Multiply 2 rows of ARGB pixels together, 8 pixels at a time.
-void ARGBMultiplyRow_NEON(const uint8_t* src_argb0,
+void ARGBMultiplyRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "umull v0.8h, v0.8b, v4.8b \n" // multiply B
- "umull v1.8h, v1.8b, v5.8b \n" // multiply G
- "umull v2.8h, v2.8b, v6.8b \n" // multiply R
- "umull v3.8h, v3.8b, v7.8b \n" // multiply A
- "rshrn v0.8b, v0.8h, #8 \n" // 16 bit to 8 bit B
- "rshrn v1.8b, v1.8h, #8 \n" // 16 bit to 8 bit G
- "rshrn v2.8b, v2.8h, #8 \n" // 16 bit to 8 bit R
- "rshrn v3.8b, v3.8h, #8 \n" // 16 bit to 8 bit A
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "ld1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
+ "ld1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "umull v0.8h, v0.8b, v4.8b \n" // multiply B
+ "prfm pldl1keep, [%0, 448] \n"
+ "umull v1.8h, v1.8b, v5.8b \n" // multiply G
+ "prfm pldl1keep, [%1, 448] \n"
+ "umull v2.8h, v2.8b, v6.8b \n" // multiply R
+ "umull v3.8h, v3.8b, v7.8b \n" // multiply A
+ "rshrn v0.8b, v0.8h, #8 \n" // 16 bit to 8 bit B
+ "rshrn v1.8b, v1.8h, #8 \n" // 16 bit to 8 bit G
+ "rshrn v2.8b, v2.8h, #8 \n" // 16 bit to 8 bit R
+ "rshrn v3.8b, v3.8h, #8 \n" // 16 bit to 8 bit A
+ "st1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2448,23 +4353,23 @@ void ARGBMultiplyRow_NEON(const uint8_t* src_argb0,
}
// Add 2 rows of ARGB pixels together, 8 pixels at a time.
-void ARGBAddRow_NEON(const uint8_t* src_argb0,
+void ARGBAddRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqadd v0.8b, v0.8b, v4.8b \n"
- "uqadd v1.8b, v1.8b, v5.8b \n"
- "uqadd v2.8b, v2.8b, v6.8b \n"
- "uqadd v3.8b, v3.8b, v7.8b \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 ARGB
+ "ldp q4, q5, [%1], #32 \n" // load 8 more
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqadd v0.16b, v0.16b, v4.16b \n"
+ "uqadd v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%2], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2473,23 +4378,23 @@ void ARGBAddRow_NEON(const uint8_t* src_argb0,
}
// Subtract 2 rows of ARGB pixels, 8 pixels at a time.
-void ARGBSubtractRow_NEON(const uint8_t* src_argb0,
+void ARGBSubtractRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqsub v0.8b, v0.8b, v4.8b \n"
- "uqsub v1.8b, v1.8b, v5.8b \n"
- "uqsub v2.8b, v2.8b, v6.8b \n"
- "uqsub v3.8b, v3.8b, v7.8b \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
- : "+r"(src_argb0), // %0
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 ARGB
+ "ldp q4, q5, [%1], #32 \n" // load 8 more
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqsub v0.16b, v0.16b, v4.16b \n"
+ "uqsub v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%2], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
"+r"(width) // %3
@@ -2497,6 +4402,68 @@ void ARGBSubtractRow_NEON(const uint8_t* src_argb0,
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
+// Blend 32 pixels at a time.
+// dst = (((a) * src0) + ((255 - a) * src1) + 255) >> 8
+void BlendPlaneRow_NEON(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ asm volatile(
+ "movi v15.8h, #255 \n"
+ "subs %w4, %w4, #32 \n"
+ "blt 19f \n"
+ "1: \n"
+ "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load 32 src0
+ "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load 32 src1
+ "ld1 {v4.16b, v5.16b}, [%2], #32 \n" // load 32 alpha
+ "subs %w4, %w4, #32 \n" // 32 processed per loop
+ "mvn v6.16b, v4.16b \n" // 255 - alpha0
+ "mvn v7.16b, v5.16b \n" // 255 - alpha1
+ "umull v8.8h, v0.8b, v4.8b \n" // low src0 * alpha
+ "prfm pldl1keep, [%0, 448] \n"
+ "umull2 v9.8h, v0.16b, v4.16b \n" // high src0 * alpha
+ "prfm pldl1keep, [%1, 448] \n"
+ "umull v10.8h, v1.8b, v5.8b \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "umull2 v11.8h, v1.16b, v5.16b \n"
+ "umlal v8.8h, v2.8b, v6.8b \n" // low + src1 * (255 -
+ // alpha)
+ "umlal2 v9.8h, v2.16b, v6.16b \n" // high + src1 * (255 -
+ // alpha)
+ "umlal v10.8h, v3.8b, v7.8b \n"
+ "umlal2 v11.8h, v3.16b, v7.16b \n"
+ "addhn v0.8b, v8.8h, v15.8h \n" // (low + 255) >> 8
+ "addhn v1.8b, v9.8h, v15.8h \n" // (high + 255) >> 8
+ "addhn v2.8b, v10.8h, v15.8h \n"
+ "addhn v3.8b, v11.8h, v15.8h \n"
+ "st1 {v0.8b, v1.8b, v2.8b, v3.8b}, [%3], #32 \n" // store 32 dst
+ "b.ge 1b \n"
+ "19: \n"
+ "adds %w4, %w4, #32 \n"
+ "b.le 99f \n"
+
+ // 16 pixel tail
+ "ld1 {v0.16b}, [%0], #16 \n" // load 16 src0
+ "ld1 {v1.16b}, [%1], #16 \n" // load 16 src1
+ "ld1 {v2.16b}, [%2], #16 \n" // load 16 alpha
+ "mvn v3.16b, v2.16b \n" // 255 - alpha
+ "umull v4.8h, v0.8b, v2.8b \n" // low src0 * alpha
+ "umull2 v5.8h, v0.16b, v2.16b \n" // high src0 * alpha
+ "umlal v4.8h, v1.8b, v3.8b \n" // low + src1 * (255 -
+ // alpha)
+ "umlal2 v5.8h, v1.16b, v3.16b \n" // high + src1 * (255 -
+ // alpha)
+ "addhn v0.8b, v4.8h, v15.8h \n" // (low + 255) >> 8
+ "addhn v1.8b, v5.8h, v15.8h \n" // (high + 255) >> 8
+ "st1 {v0.8b, v1.8b}, [%3], #16 \n" // store 16 dst
+ "99: \n"
+ : "+r"(src0), "+r"(src1), "+r"(alpha), "+r"(dst), "+r"(width)
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
+ "v9", "v10", "v11", "v15");
+}
+
// Adds Sobel X and Sobel Y and stores Sobel into ARGB.
// A = 255
// R = Sobel
@@ -2507,17 +4474,19 @@ void SobelRow_NEON(const uint8_t* src_sobelx,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // alpha
+ "movi v3.8b, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
- "ld1 {v0.8b}, [%0], #8 \n" // load 8 sobelx.
- "ld1 {v1.8b}, [%1], #8 \n" // load 8 sobely.
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqadd v0.8b, v0.8b, v1.8b \n" // add
- "orr v1.8b, v0.8b, v0.8b \n"
- "orr v2.8b, v0.8b, v0.8b \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.8b}, [%0], #8 \n" // load 8 sobelx.
+ "ld1 {v1.8b}, [%1], #8 \n" // load 8 sobely.
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "uqadd v0.8b, v0.8b, v1.8b \n" // add
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v1.8b, v0.8b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "mov v2.8b, v0.8b \n"
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_argb), // %2
@@ -2533,13 +4502,15 @@ void SobelToPlaneRow_NEON(const uint8_t* src_sobelx,
int width) {
asm volatile(
// 16 pixel loop.
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 16 sobelx.
- "ld1 {v1.16b}, [%1], #16 \n" // load 16 sobely.
- "subs %w3, %w3, #16 \n" // 16 processed per loop.
- "uqadd v0.16b, v0.16b, v1.16b \n" // add
- "st1 {v0.16b}, [%2], #16 \n" // store 16 pixels.
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load 16 sobelx.
+ "ld1 {v1.16b}, [%1], #16 \n" // load 16 sobely.
+ "subs %w3, %w3, #16 \n" // 16 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqadd v0.16b, v0.16b, v1.16b \n" // add
+ "prfm pldl1keep, [%1, 448] \n"
+ "st1 {v0.16b}, [%2], #16 \n" // store 16 pixels.
+ "b.gt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_y), // %2
@@ -2558,15 +4529,17 @@ void SobelXYRow_NEON(const uint8_t* src_sobelx,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // alpha
+ "movi v3.8b, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
- "ld1 {v2.8b}, [%0], #8 \n" // load 8 sobelx.
- "ld1 {v0.8b}, [%1], #8 \n" // load 8 sobely.
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqadd v1.8b, v0.8b, v2.8b \n" // add
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v2.8b}, [%0], #8 \n" // load 8 sobelx.
+ "ld1 {v0.8b}, [%1], #8 \n" // load 8 sobely.
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqadd v1.8b, v0.8b, v2.8b \n" // add
+ "prfm pldl1keep, [%1, 448] \n"
+ "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "b.gt 1b \n"
: "+r"(src_sobelx), // %0
"+r"(src_sobely), // %1
"+r"(dst_argb), // %2
@@ -2575,6 +4548,214 @@ void SobelXYRow_NEON(const uint8_t* src_sobelx,
: "cc", "memory", "v0", "v1", "v2", "v3");
}
+#ifdef HAS_COMPUTECUMULATIVESUMROW_NEON
+void ComputeCumulativeSumRow_NEON(const uint8_t* row,
+ int32_t* cumsum,
+ const int32_t* previous_cumsum,
+ int width) {
+ asm volatile(
+ "movi v0.4s, #0 \n"
+ "movi v1.16b, #0 \n"
+ "subs %w3, %w3, #4 \n"
+ "b.lt 49f \n"
+
+ // 4 pixel loop.
+ "40: \n"
+ "ldr q2, [%0], #16 \n"
+ "ldp q16, q17, [%2] \n"
+ "ldp q18, q19, [%2, #32] \n"
+ "add %2, %2, #64 \n"
+ "ext v3.16b, v1.16b, v2.16b, #12 \n"
+ "uaddl v4.8h, v2.8b, v3.8b \n"
+ "uaddl2 v5.8h, v2.16b, v3.16b \n"
+ "add v5.8h, v5.8h, v4.8h \n"
+ "uaddw v16.4s, v16.4s, v4.4h \n"
+ "uaddw2 v17.4s, v17.4s, v4.8h \n"
+ "uaddw v18.4s, v18.4s, v5.4h \n"
+ "uaddw2 v19.4s, v19.4s, v5.8h \n"
+ "add v16.4s, v16.4s, v0.4s \n"
+ "add v17.4s, v17.4s, v0.4s \n"
+ "add v18.4s, v18.4s, v0.4s \n"
+ "add v19.4s, v19.4s, v0.4s \n"
+ "uaddw2 v0.4s, v0.4s, v5.8h \n"
+ "stp q16, q17, [%1] \n"
+ "stp q18, q19, [%1, #32] \n"
+ "add %1, %1, #64 \n"
+ "subs %w3, %w3, #4 \n"
+ "b.ge 40b \n"
+
+ "49: \n"
+ "adds %w3, %w3, #3 \n"
+ "b.lt 19f \n"
+
+ // 1 pixel loop.
+ "10: \n"
+ "ldr s2, [%0], #4 \n"
+ "ldr q16, [%2], #16 \n"
+ "ushll v2.8h, v2.8b, #0 \n"
+ "uaddw v0.4s, v0.4s, v2.4h \n"
+ "add v16.4s, v16.4s, v0.4s \n"
+ "str q16, [%1], #16 \n"
+ "subs %w3, %w3, #1 \n"
+ "b.ge 10b \n"
+
+ "19: \n"
+ : "+r"(row), // %0
+ "+r"(cumsum), // %1
+ "+r"(previous_cumsum), // %2
+ "+r"(width) // %3
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18",
+ "v19");
+}
+#endif // HAS_COMPUTECUMULATIVESUMROW_NEON
+
+#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_NEON
+void CumulativeSumToAverageRow_NEON(const int32_t* topleft,
+ const int32_t* botleft,
+ int width,
+ int area,
+ uint8_t* dst,
+ int count) {
+ const int32_t* topright;
+ const int32_t* botright;
+ int scale;
+ asm volatile(
+ "ucvtf s4, %w7 \n"
+ "fmov s5, #1.0 \n"
+ "fdiv s4, s5, s4 \n"
+ "subs %w3, %w3, #4 \n"
+ "b.lt 49f \n"
+ "add %4, %0, %8 \n"
+ "add %5, %1, %8 \n"
+ "sub %w6, %w7, #2 \n"
+ "cmp %w6, #126 \n"
+ "b.hi 40f \n"
+ "mov %w6, #32767 \n"
+ "add %w6, %w6, %w7 \n"
+ "udiv %w6, %w6, %w7 \n"
+ "dup v6.8h, %w6 \n"
+
+ // 4 pixel small loop.
+ "4: \n"
+ "ldp q0, q1, [%0] \n"
+ "ldp q2, q3, [%0, #32] \n"
+ "add %0, %0, #64 \n"
+ "ldp q16, q17, [%4] \n"
+ "ldp q18, q19, [%4, #32] \n"
+ "add %4, %4, #64 \n"
+ "ldp q20, q21, [%1] \n"
+ "ldp q22, q23, [%1, #32] \n"
+ "add %1, %1, #64 \n"
+ "ldp q24, q25, [%5] \n"
+ "ldp q26, q27, [%5, #32] \n"
+ "add %5, %5, #64 \n"
+ "sub v0.4s, v0.4s, v16.4s \n"
+ "sub v1.4s, v1.4s, v17.4s \n"
+ "sub v2.4s, v2.4s, v18.4s \n"
+ "sub v3.4s, v3.4s, v19.4s \n"
+ "sub v24.4s, v24.4s, v20.4s \n"
+ "sub v25.4s, v25.4s, v21.4s \n"
+ "sub v26.4s, v26.4s, v22.4s \n"
+ "sub v27.4s, v27.4s, v23.4s \n"
+ "add v0.4s, v0.4s, v24.4s \n"
+ "add v1.4s, v1.4s, v25.4s \n"
+ "add v2.4s, v2.4s, v26.4s \n"
+ "add v3.4s, v3.4s, v27.4s \n"
+ "uzp1 v0.8h, v0.8h, v1.8h \n"
+ "uzp1 v2.8h, v2.8h, v3.8h \n"
+ "sqdmulh v0.8h, v0.8h, v6.8h \n"
+ "sqdmulh v2.8h, v2.8h, v6.8h \n"
+ "uzp1 v0.16b, v0.16b, v2.16b \n"
+ "str q0, [%2], #16 \n"
+ "subs %w3, %w3, #4 \n"
+ "b.ge 4b \n"
+ "b 49f \n"
+
+ // 4 pixel loop.
+ "40: \n"
+ "ldp q0, q1, [%0] \n"
+ "ldp q2, q3, [%0, #32] \n"
+ "add %0, %0, #64 \n"
+ "ldp q16, q17, [%4] \n"
+ "ldp q18, q19, [%4, #32] \n"
+ "add %4, %4, #64 \n"
+ "ldp q20, q21, [%1] \n"
+ "ldp q22, q23, [%1, #32] \n"
+ "add %1, %1, #64 \n"
+ "ldp q24, q25, [%5] \n"
+ "ldp q26, q27, [%5, #32] \n"
+ "add %5, %5, #64 \n"
+ "sub v0.4s, v0.4s, v16.4s \n"
+ "sub v1.4s, v1.4s, v17.4s \n"
+ "sub v2.4s, v2.4s, v18.4s \n"
+ "sub v3.4s, v3.4s, v19.4s \n"
+ "sub v24.4s, v24.4s, v20.4s \n"
+ "sub v25.4s, v25.4s, v21.4s \n"
+ "sub v26.4s, v26.4s, v22.4s \n"
+ "sub v27.4s, v27.4s, v23.4s \n"
+ "add v0.4s, v0.4s, v24.4s \n"
+ "add v1.4s, v1.4s, v25.4s \n"
+ "add v2.4s, v2.4s, v26.4s \n"
+ "add v3.4s, v3.4s, v27.4s \n"
+ "scvtf v0.4s, v0.4s \n"
+ "scvtf v1.4s, v1.4s \n"
+ "scvtf v2.4s, v2.4s \n"
+ "scvtf v3.4s, v3.4s \n"
+ "fmul v0.4s, v0.4s, v4.s[0] \n"
+ "fmul v1.4s, v1.4s, v4.s[0] \n"
+ "fmul v2.4s, v2.4s, v4.s[0] \n"
+ "fmul v3.4s, v3.4s, v4.s[0] \n"
+ "fcvtzs v0.4s, v0.4s \n"
+ "fcvtzs v1.4s, v1.4s \n"
+ "fcvtzs v2.4s, v2.4s \n"
+ "fcvtzs v3.4s, v3.4s \n"
+ "uqxtn v0.4h, v0.4s \n"
+ "uqxtn2 v0.8h, v1.4s \n"
+ "uqxtn v2.4h, v2.4s \n"
+ "uqxtn2 v2.8h, v3.4s \n"
+ "uqxtn v0.8b, v0.8h \n"
+ "uqxtn2 v0.16b, v2.8h \n"
+ "str q0, [%2], #16 \n"
+ "subs %w3, %w3, #4 \n"
+ "b.ge 40b \n"
+
+ "49: \n"
+ "adds %w3, %w3, #3 \n"
+ "b.lt 19f \n"
+
+ // 1 pixel loop.
+ "10: \n"
+ "ldr q1, [%0, %8] \n"
+ "ldr q0, [%0], #16 \n"
+ "ldr q3, [%1, %8] \n"
+ "ldr q2, [%1], #16 \n"
+ "sub v0.4s, v0.4s, v1.4s \n"
+ "sub v3.4s, v3.4s, v2.4s \n"
+ "add v0.4s, v0.4s, v3.4s \n"
+ "scvtf v0.4s, v0.4s \n"
+ "fmul v0.4s, v0.4s, v4.s[0] \n"
+ "fcvtzs v0.4s, v0.4s \n"
+ "uqxtn v0.4h, v0.4s \n"
+ "uqxtn v0.8b, v0.8h \n"
+ "str s0, [%2], #4 \n"
+ "subs %w3, %w3, #1 \n"
+ "b.ge 10b \n"
+ "19: \n"
+ : "+r"(topleft), // %0
+ "+r"(botleft), // %1
+ "+r"(dst), // %2
+ "+r"(count), // %3
+ "=&r"(topright), // %4
+ "=&r"(botright), // %5
+ "=&r"(scale) // %6
+ : "r"(area), // %7
+ "r"((ptrdiff_t)width * 4) // %8
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27");
+}
+#endif // HAS_CUMULATIVESUMTOAVERAGEROW_NEON
+
// SobelX as a matrix is
// -1 0 1
// -2 0 2
@@ -2585,24 +4766,27 @@ void SobelXRow_NEON(const uint8_t* src_y0,
uint8_t* dst_sobelx,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.8b}, [%0],%5 \n" // top
- "ld1 {v1.8b}, [%0],%6 \n"
- "usubl v0.8h, v0.8b, v1.8b \n"
- "ld1 {v2.8b}, [%1],%5 \n" // center * 2
- "ld1 {v3.8b}, [%1],%6 \n"
- "usubl v1.8h, v2.8b, v3.8b \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "ld1 {v2.8b}, [%2],%5 \n" // bottom
- "ld1 {v3.8b}, [%2],%6 \n"
- "subs %w4, %w4, #8 \n" // 8 pixels
- "usubl v1.8h, v2.8b, v3.8b \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "abs v0.8h, v0.8h \n"
- "uqxtn v0.8b, v0.8h \n"
- "st1 {v0.8b}, [%3], #8 \n" // store 8 sobelx
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.8b}, [%0],%5 \n" // top
+ "ld1 {v1.8b}, [%0],%6 \n"
+ "subs %w4, %w4, #8 \n" // 8 pixels
+ "usubl v0.8h, v0.8b, v1.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "ld1 {v2.8b}, [%1],%5 \n" // center * 2
+ "ld1 {v3.8b}, [%1],%6 \n"
+ "usubl v1.8h, v2.8b, v3.8b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "ld1 {v2.8b}, [%2],%5 \n" // bottom
+ "ld1 {v3.8b}, [%2],%6 \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "usubl v1.8h, v2.8b, v3.8b \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "abs v0.8h, v0.8h \n"
+ "uqxtn v0.8b, v0.8h \n"
+ "st1 {v0.8b}, [%3], #8 \n" // store 8 sobelx
+ "b.gt 1b \n"
: "+r"(src_y0), // %0
"+r"(src_y1), // %1
"+r"(src_y2), // %2
@@ -2623,24 +4807,26 @@ void SobelYRow_NEON(const uint8_t* src_y0,
uint8_t* dst_sobely,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.8b}, [%0],%4 \n" // left
- "ld1 {v1.8b}, [%1],%4 \n"
- "usubl v0.8h, v0.8b, v1.8b \n"
- "ld1 {v2.8b}, [%0],%4 \n" // center * 2
- "ld1 {v3.8b}, [%1],%4 \n"
- "usubl v1.8h, v2.8b, v3.8b \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "ld1 {v2.8b}, [%0],%5 \n" // right
- "ld1 {v3.8b}, [%1],%5 \n"
- "subs %w3, %w3, #8 \n" // 8 pixels
- "usubl v1.8h, v2.8b, v3.8b \n"
- "add v0.8h, v0.8h, v1.8h \n"
- "abs v0.8h, v0.8h \n"
- "uqxtn v0.8b, v0.8h \n"
- "st1 {v0.8b}, [%2], #8 \n" // store 8 sobely
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.8b}, [%0],%4 \n" // left
+ "ld1 {v1.8b}, [%1],%4 \n"
+ "subs %w3, %w3, #8 \n" // 8 pixels
+ "usubl v0.8h, v0.8b, v1.8b \n"
+ "ld1 {v2.8b}, [%0],%4 \n" // center * 2
+ "ld1 {v3.8b}, [%1],%4 \n"
+ "usubl v1.8h, v2.8b, v3.8b \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "ld1 {v2.8b}, [%0],%5 \n" // right
+ "ld1 {v3.8b}, [%1],%5 \n"
+ "usubl v1.8h, v2.8b, v3.8b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "abs v0.8h, v0.8h \n"
+ "uqxtn v0.8b, v0.8h \n"
+ "st1 {v0.8b}, [%2], #8 \n" // store 8 sobely
+ "b.gt 1b \n"
: "+r"(src_y0), // %0
"+r"(src_y1), // %1
"+r"(dst_sobely), // %2
@@ -2651,23 +4837,83 @@ void SobelYRow_NEON(const uint8_t* src_y0,
);
}
-// Caveat - rounds float to half float whereas scaling version truncates.
-void HalfFloat1Row_NEON(const uint16_t* src,
- uint16_t* dst,
- float /*unused*/,
- int width) {
+void HalfFloatRow_NEON(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
asm volatile(
- "1: \n"
- "ld1 {v1.16b}, [%0], #16 \n" // load 8 shorts
- "subs %w2, %w2, #8 \n" // 8 pixels per loop
- "uxtl v2.4s, v1.4h \n" // 8 int's
- "uxtl2 v3.4s, v1.8h \n"
- "scvtf v2.4s, v2.4s \n" // 8 floats
- "scvtf v3.4s, v3.4s \n"
- "fcvtn v1.4h, v2.4s \n" // 8 half floats
- "fcvtn2 v1.8h, v3.4s \n"
- "st1 {v1.16b}, [%1], #16 \n" // store 8 shorts
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 16 shorts
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop
+ "uxtl v2.4s, v0.4h \n"
+ "uxtl v4.4s, v1.4h \n"
+ "uxtl2 v3.4s, v0.8h \n"
+ "uxtl2 v5.4s, v1.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "scvtf v2.4s, v2.4s \n"
+ "scvtf v4.4s, v4.4s \n"
+ "scvtf v3.4s, v3.4s \n"
+ "scvtf v5.4s, v5.4s \n"
+ "fmul v2.4s, v2.4s, %3.s[0] \n" // * scale
+ "fmul v4.4s, v4.4s, %3.s[0] \n"
+ "fmul v3.4s, v3.4s, %3.s[0] \n"
+ "fmul v5.4s, v5.4s, %3.s[0] \n"
+ "fmul v2.4s, v2.4s, %4.s[0] \n" // adjust exponent
+ "fmul v4.4s, v4.4s, %4.s[0] \n"
+ "fmul v3.4s, v3.4s, %4.s[0] \n"
+ "fmul v5.4s, v5.4s, %4.s[0] \n"
+ "uqshrn v0.4h, v2.4s, #13 \n" // isolate halffloat
+ "uqshrn v1.4h, v4.4s, #13 \n"
+ "uqshrn2 v0.8h, v3.4s, #13 \n"
+ "uqshrn2 v1.8h, v5.4s, #13 \n"
+ "stp q0, q1, [%1], #32 \n" // store 16 fp16
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "w"(scale), // %3
+ "w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
+}
+
+void ByteToFloatRow_NEON(const uint8_t* src,
+ float* dst,
+ float scale,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v1.8b}, [%0], #8 \n" // load 8 bytes
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop
+ "uxtl v1.8h, v1.8b \n" // 8 shorts
+ "prfm pldl1keep, [%0, 448] \n"
+ "uxtl v2.4s, v1.4h \n" // 8 ints
+ "uxtl2 v3.4s, v1.8h \n"
+ "scvtf v2.4s, v2.4s \n" // 8 floats
+ "scvtf v3.4s, v3.4s \n"
+ "fmul v2.4s, v2.4s, %3.s[0] \n" // scale
+ "fmul v3.4s, v3.4s, %3.s[0] \n"
+ "st1 {v2.16b, v3.16b}, [%1], #32 \n" // store 8 floats
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "w"(scale) // %3
+ : "cc", "memory", "v1", "v2", "v3");
+}
+
+// Convert FP16 Half Floats to FP32 Floats
+void ConvertFP16ToFP32Row_NEON(const uint16_t* src, // fp16
+ float* dst,
+ int width) {
+ asm volatile(
+ "1: \n"
+ "ld1 {v1.8h}, [%0], #16 \n" // load 8 halffloats
+ "subs %w2, %w2, #8 \n" // 8 floats per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "fcvtl v2.4s, v1.4h \n" // 8 floats
+ "fcvtl2 v3.4s, v1.8h \n"
+ "stp q2, q3, [%1], #32 \n" // store 8 floats
+ "b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -2675,52 +4921,64 @@ void HalfFloat1Row_NEON(const uint16_t* src,
: "cc", "memory", "v1", "v2", "v3");
}
-void HalfFloatRow_NEON(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
+// Convert FP16 Half Floats to FP32 Floats
+// Read a column and write a row
+void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
+ int src_stride, // stride in elements
+ float* dst,
+ int width) {
asm volatile(
- "1: \n"
- "ld1 {v1.16b}, [%0], #16 \n" // load 8 shorts
- "subs %w2, %w2, #8 \n" // 8 pixels per loop
- "uxtl v2.4s, v1.4h \n" // 8 int's
- "uxtl2 v3.4s, v1.8h \n"
- "scvtf v2.4s, v2.4s \n" // 8 floats
- "scvtf v3.4s, v3.4s \n"
- "fmul v2.4s, v2.4s, %3.s[0] \n" // adjust exponent
- "fmul v3.4s, v3.4s, %3.s[0] \n"
- "uqshrn v1.4h, v2.4s, #13 \n" // isolate halffloat
- "uqshrn2 v1.8h, v3.4s, #13 \n"
- "st1 {v1.16b}, [%1], #16 \n" // store 8 shorts
- "b.gt 1b \n"
+ "subs %w2, %w2, #8 \n" // Is there 8 rows?
+ "b.lt 2f \n"
+ "1: \n"
+ "ld1 {v0.h}[0], [%0], %3 \n" // load 8 halffloats
+ "ld1 {v0.h}[1], [%0], %3 \n"
+ "ld1 {v0.h}[2], [%0], %3 \n"
+ "ld1 {v0.h}[3], [%0], %3 \n"
+ "ld1 {v1.h}[0], [%0], %3 \n"
+ "ld1 {v1.h}[1], [%0], %3 \n"
+ "ld1 {v1.h}[2], [%0], %3 \n"
+ "ld1 {v1.h}[3], [%0], %3 \n"
+ "subs %w2, %w2, #8 \n" // 8 rows per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "fcvtl v2.4s, v0.4h \n" // 4 floats
+ "fcvtl v3.4s, v1.4h \n" // 4 more floats
+ "stp q2, q3, [%1], #32 \n" // store 8 floats
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w2, %w2, #8 \n" // Restore remainder.
+ "b.eq 3f \n"
+ "4: \n"
+ "ld1 {v1.h}[0], [%0], %3 \n" // load 1 halffloats
+ "subs %w2, %w2, #1 \n" // 1 floats per loop
+ "fcvtl v2.4s, v1.4h \n" // 1 floats
+ "str s2, [%1], #4 \n" // store 1 floats
+ "b.gt 4b \n"
+ "3: \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
- : "w"(scale * 1.9259299444e-34f) // %3
- : "cc", "memory", "v1", "v2", "v3");
+ : "r"((ptrdiff_t)src_stride * 2) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3");
}
-void ByteToFloatRow_NEON(const uint8_t* src,
- float* dst,
- float scale,
- int width) {
+// Convert FP32 Floats to FP16 Half Floats
+void ConvertFP32ToFP16Row_NEON(const float* src,
+ uint16_t* dst, // fp16
+ int width) {
asm volatile(
- "1: \n"
- "ld1 {v1.8b}, [%0], #8 \n" // load 8 bytes
- "subs %w2, %w2, #8 \n" // 8 pixels per loop
- "uxtl v1.8h, v1.8b \n" // 8 shorts
- "uxtl v2.4s, v1.4h \n" // 8 ints
- "uxtl2 v3.4s, v1.8h \n"
- "scvtf v2.4s, v2.4s \n" // 8 floats
- "scvtf v3.4s, v3.4s \n"
- "fmul v2.4s, v2.4s, %3.s[0] \n" // scale
- "fmul v3.4s, v3.4s, %3.s[0] \n"
- "st1 {v2.16b, v3.16b}, [%1], #32 \n" // store 8 floats
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q2, q3, [%0], #32 \n" // load 8 floats
+ "subs %w2, %w2, #8 \n" // 8 floats per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "fcvtn v1.4h, v2.4s \n" // 8 fp16 halffloats
+ "fcvtn2 v1.8h, v3.4s \n"
+ "str q1, [%1], #16 \n" // store 8 fp16 halffloats
+ "b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
- : "w"(scale) // %3
+ :
: "cc", "memory", "v1", "v2", "v3");
}
@@ -2730,20 +4988,21 @@ float ScaleMaxSamples_NEON(const float* src,
int width) {
float fmax;
asm volatile(
- "movi v5.4s, #0 \n" // max
- "movi v6.4s, #0 \n"
-
- "1: \n"
- "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "fmul v3.4s, v1.4s, %4.s[0] \n" // scale
- "fmul v4.4s, v2.4s, %4.s[0] \n" // scale
- "fmax v5.4s, v5.4s, v1.4s \n" // max
- "fmax v6.4s, v6.4s, v2.4s \n"
- "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples
- "b.gt 1b \n"
- "fmax v5.4s, v5.4s, v6.4s \n" // max
- "fmaxv %s3, v5.4s \n" // signed max acculator
+ "movi v5.4s, #0 \n" // max
+ "movi v6.4s, #0 \n"
+
+ "1: \n"
+ "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "fmul v3.4s, v1.4s, %4.s[0] \n" // scale
+ "prfm pldl1keep, [%0, 448] \n"
+ "fmul v4.4s, v2.4s, %4.s[0] \n" // scale
+ "fmax v5.4s, v5.4s, v1.4s \n" // max
+ "fmax v6.4s, v6.4s, v2.4s \n"
+ "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples
+ "b.gt 1b \n"
+ "fmax v5.4s, v5.4s, v6.4s \n" // max
+ "fmaxv %s3, v5.4s \n" // signed max acculator
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width), // %2
@@ -2759,21 +5018,22 @@ float ScaleSumSamples_NEON(const float* src,
int width) {
float fsum;
asm volatile(
- "movi v5.4s, #0 \n" // max
- "movi v6.4s, #0 \n" // max
-
- "1: \n"
- "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "fmul v3.4s, v1.4s, %4.s[0] \n" // scale
- "fmul v4.4s, v2.4s, %4.s[0] \n"
- "fmla v5.4s, v1.4s, v1.4s \n" // sum of squares
- "fmla v6.4s, v2.4s, v2.4s \n"
- "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples
- "b.gt 1b \n"
- "faddp v5.4s, v5.4s, v6.4s \n"
- "faddp v5.4s, v5.4s, v5.4s \n"
- "faddp %3.4s, v5.4s, v5.4s \n" // sum
+ "movi v5.4s, #0 \n" // max
+ "movi v6.4s, #0 \n" // max
+
+ "1: \n"
+ "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "fmul v3.4s, v1.4s, %4.s[0] \n" // scale
+ "prfm pldl1keep, [%0, 448] \n"
+ "fmul v4.4s, v2.4s, %4.s[0] \n"
+ "fmla v5.4s, v1.4s, v1.4s \n" // sum of squares
+ "fmla v6.4s, v2.4s, v2.4s \n"
+ "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples
+ "b.gt 1b \n"
+ "faddp v5.4s, v5.4s, v6.4s \n"
+ "faddp v5.4s, v5.4s, v5.4s \n"
+ "faddp %3.4s, v5.4s, v5.4s \n" // sum
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width), // %2
@@ -2785,13 +5045,14 @@ float ScaleSumSamples_NEON(const float* src,
void ScaleSamples_NEON(const float* src, float* dst, float scale, int width) {
asm volatile(
- "1: \n"
- "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "fmul v1.4s, v1.4s, %3.s[0] \n" // scale
- "fmul v2.4s, v2.4s, %3.s[0] \n" // scale
- "st1 {v1.4s, v2.4s}, [%1], #32 \n" // store 8 samples
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "fmul v1.4s, v1.4s, %3.s[0] \n" // scale
+ "fmul v2.4s, v2.4s, %3.s[0] \n" // scale
+ "st1 {v1.4s, v2.4s}, [%1], #32 \n" // store 8 samples
+ "b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -2808,26 +5069,31 @@ void GaussCol_NEON(const uint16_t* src0,
uint32_t* dst,
int width) {
asm volatile(
- "movi v6.8h, #4 \n" // constant 4
- "movi v7.8h, #6 \n" // constant 6
-
- "1: \n"
- "ld1 {v1.8h}, [%0], #16 \n" // load 8 samples, 5 rows
- "ld1 {v2.8h}, [%4], #16 \n"
- "uaddl v0.4s, v1.4h, v2.4h \n" // * 1
- "uaddl2 v1.4s, v1.8h, v2.8h \n" // * 1
- "ld1 {v2.8h}, [%1], #16 \n"
- "umlal v0.4s, v2.4h, v6.4h \n" // * 4
- "umlal2 v1.4s, v2.8h, v6.8h \n" // * 4
- "ld1 {v2.8h}, [%2], #16 \n"
- "umlal v0.4s, v2.4h, v7.4h \n" // * 6
- "umlal2 v1.4s, v2.8h, v7.8h \n" // * 6
- "ld1 {v2.8h}, [%3], #16 \n"
- "umlal v0.4s, v2.4h, v6.4h \n" // * 4
- "umlal2 v1.4s, v2.8h, v6.8h \n" // * 4
- "subs %w6, %w6, #8 \n" // 8 processed per loop
- "st1 {v0.4s,v1.4s}, [%5], #32 \n" // store 8 samples
- "b.gt 1b \n"
+ "movi v6.8h, #4 \n" // constant 4
+ "movi v7.8h, #6 \n" // constant 6
+
+ "1: \n"
+ "ld1 {v1.8h}, [%0], #16 \n" // load 8 samples, 5 rows
+ "ld1 {v2.8h}, [%4], #16 \n"
+ "subs %w6, %w6, #8 \n" // 8 processed per loop
+ "uaddl v0.4s, v1.4h, v2.4h \n" // * 1
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddl2 v1.4s, v1.8h, v2.8h \n" // * 1
+ "ld1 {v2.8h}, [%1], #16 \n"
+ "umlal v0.4s, v2.4h, v6.4h \n" // * 4
+ "prfm pldl1keep, [%1, 448] \n"
+ "umlal2 v1.4s, v2.8h, v6.8h \n" // * 4
+ "ld1 {v2.8h}, [%2], #16 \n"
+ "umlal v0.4s, v2.4h, v7.4h \n" // * 6
+ "prfm pldl1keep, [%2, 448] \n"
+ "umlal2 v1.4s, v2.8h, v7.8h \n" // * 6
+ "ld1 {v2.8h}, [%3], #16 \n"
+ "umlal v0.4s, v2.4h, v6.4h \n" // * 4
+ "prfm pldl1keep, [%3, 448] \n"
+ "umlal2 v1.4s, v2.8h, v6.8h \n" // * 4
+ "st1 {v0.4s,v1.4s}, [%5], #32 \n" // store 8 samples
+ "prfm pldl1keep, [%4, 448] \n"
+ "b.gt 1b \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(src2), // %2
@@ -2845,27 +5111,28 @@ void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width) {
const uint32_t* src2 = src + 2;
const uint32_t* src3 = src + 3;
asm volatile(
- "movi v6.4s, #4 \n" // constant 4
- "movi v7.4s, #6 \n" // constant 6
-
- "1: \n"
- "ld1 {v0.4s,v1.4s,v2.4s}, [%0], %6 \n" // load 12 source samples
- "add v0.4s, v0.4s, v1.4s \n" // * 1
- "add v1.4s, v1.4s, v2.4s \n" // * 1
- "ld1 {v2.4s,v3.4s}, [%2], #32 \n"
- "mla v0.4s, v2.4s, v7.4s \n" // * 6
- "mla v1.4s, v3.4s, v7.4s \n" // * 6
- "ld1 {v2.4s,v3.4s}, [%1], #32 \n"
- "ld1 {v4.4s,v5.4s}, [%3], #32 \n"
- "add v2.4s, v2.4s, v4.4s \n" // add rows for * 4
- "add v3.4s, v3.4s, v5.4s \n"
- "mla v0.4s, v2.4s, v6.4s \n" // * 4
- "mla v1.4s, v3.4s, v6.4s \n" // * 4
- "subs %w5, %w5, #8 \n" // 8 processed per loop
- "uqrshrn v0.4h, v0.4s, #8 \n" // round and pack
- "uqrshrn2 v0.8h, v1.4s, #8 \n"
- "st1 {v0.8h}, [%4], #16 \n" // store 8 samples
- "b.gt 1b \n"
+ "movi v6.4s, #4 \n" // constant 4
+ "movi v7.4s, #6 \n" // constant 6
+
+ "1: \n"
+ "ld1 {v0.4s,v1.4s,v2.4s}, [%0], %6 \n" // load 12 source samples
+ "subs %w5, %w5, #8 \n" // 8 processed per loop
+ "add v0.4s, v0.4s, v1.4s \n" // * 1
+ "add v1.4s, v1.4s, v2.4s \n" // * 1
+ "ld1 {v2.4s,v3.4s}, [%2], #32 \n"
+ "mla v0.4s, v2.4s, v7.4s \n" // * 6
+ "mla v1.4s, v3.4s, v7.4s \n" // * 6
+ "ld1 {v2.4s,v3.4s}, [%1], #32 \n"
+ "ld1 {v4.4s,v5.4s}, [%3], #32 \n"
+ "add v2.4s, v2.4s, v4.4s \n" // add rows for * 4
+ "add v3.4s, v3.4s, v5.4s \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mla v0.4s, v2.4s, v6.4s \n" // * 4
+ "mla v1.4s, v3.4s, v6.4s \n" // * 4
+ "uqrshrn v0.4h, v0.4s, #8 \n" // round and pack
+ "uqrshrn2 v0.8h, v1.4s, #8 \n"
+ "st1 {v0.8h}, [%4], #16 \n" // store 8 samples
+ "b.gt 1b \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
@@ -2876,20 +5143,104 @@ void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width) {
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
+static const vecf32 kGaussCoefficients = {4.0f, 6.0f, 1.0f / 256.0f, 0.0f};
+
+// filter 5 rows with 1, 4, 6, 4, 1 coefficients to produce 1 row.
+void GaussCol_F32_NEON(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width) {
+ asm volatile(
+ "ld2r {v6.4s, v7.4s}, [%7] \n" // constants 4 and 6
+
+ "1: \n"
+ "ld1 {v0.4s, v1.4s}, [%0], #32 \n" // load 8 samples, 5 rows
+ "ld1 {v2.4s, v3.4s}, [%1], #32 \n"
+ "subs %w6, %w6, #8 \n" // 8 processed per loop
+ "fmla v0.4s, v2.4s, v6.4s \n" // * 4
+ "ld1 {v4.4s, v5.4s}, [%2], #32 \n"
+ "fmla v1.4s, v3.4s, v6.4s \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "fmla v0.4s, v4.4s, v7.4s \n" // * 6
+ "ld1 {v2.4s, v3.4s}, [%3], #32 \n"
+ "fmla v1.4s, v5.4s, v7.4s \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "fmla v0.4s, v2.4s, v6.4s \n" // * 4
+ "ld1 {v4.4s, v5.4s}, [%4], #32 \n"
+ "fmla v1.4s, v3.4s, v6.4s \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "fadd v0.4s, v0.4s, v4.4s \n" // * 1
+ "prfm pldl1keep, [%3, 448] \n"
+ "fadd v1.4s, v1.4s, v5.4s \n"
+ "prfm pldl1keep, [%4, 448] \n"
+ "st1 {v0.4s, v1.4s}, [%5], #32 \n" // store 8 samples
+ "b.gt 1b \n"
+ : "+r"(src0), // %0
+ "+r"(src1), // %1
+ "+r"(src2), // %2
+ "+r"(src3), // %3
+ "+r"(src4), // %4
+ "+r"(dst), // %5
+ "+r"(width) // %6
+ : "r"(&kGaussCoefficients) // %7
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+
+// filter 5 rows with 1, 4, 6, 4, 1 coefficients to produce 1 row.
+void GaussRow_F32_NEON(const float* src, float* dst, int width) {
+ asm volatile(
+ "ld3r {v6.4s, v7.4s, v8.4s}, [%3] \n" // constants 4, 6, 1/256
+
+ "1: \n"
+ "ld1 {v0.4s, v1.4s, v2.4s}, [%0], %4 \n" // load 12 samples, 5
+ // rows
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "fadd v0.4s, v0.4s, v1.4s \n" // * 1
+ "ld1 {v4.4s, v5.4s}, [%0], %5 \n"
+ "fadd v1.4s, v1.4s, v2.4s \n"
+ "fmla v0.4s, v4.4s, v7.4s \n" // * 6
+ "ld1 {v2.4s, v3.4s}, [%0], %4 \n"
+ "fmla v1.4s, v5.4s, v7.4s \n"
+ "ld1 {v4.4s, v5.4s}, [%0], %6 \n"
+ "fadd v2.4s, v2.4s, v4.4s \n"
+ "fadd v3.4s, v3.4s, v5.4s \n"
+ "fmla v0.4s, v2.4s, v6.4s \n" // * 4
+ "fmla v1.4s, v3.4s, v6.4s \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "fmul v0.4s, v0.4s, v8.4s \n" // / 256
+ "fmul v1.4s, v1.4s, v8.4s \n"
+ "st1 {v0.4s, v1.4s}, [%1], #32 \n" // store 8 samples
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(&kGaussCoefficients), // %3
+ "r"(8LL), // %4
+ "r"(-4LL), // %5
+ "r"(20LL) // %6
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8");
+}
+
+#if defined(LIBYUV_USE_ST3)
// Convert biplanar NV21 to packed YUV24
void NV21ToYUV24Row_NEON(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_yuv24,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v2.16b}, [%0], #16 \n" // load 16 Y values
- "ld2 {v0.8b, v1.8b}, [%1], #16 \n" // load 8 VU values
- "zip1 v0.16b, v0.16b, v0.16b \n" // replicate V values
- "zip1 v1.16b, v1.16b, v1.16b \n" // replicate U values
- "subs %w3, %w3, #16 \n" // 16 pixels per loop
- "st3 {v0.16b,v1.16b,v2.16b}, [%2], #48 \n" // store 16 YUV pixels
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v2.16b}, [%0], #16 \n" // load 16 Y values
+ "ld2 {v0.8b, v1.8b}, [%1], #16 \n" // load 8 VU values
+ "subs %w3, %w3, #16 \n" // 16 pixels per loop
+ "zip1 v0.16b, v0.16b, v0.16b \n" // replicate V values
+ "prfm pldl1keep, [%0, 448] \n"
+ "zip1 v1.16b, v1.16b, v1.16b \n" // replicate U values
+ "prfm pldl1keep, [%1, 448] \n"
+ "st3 {v0.16b,v1.16b,v2.16b}, [%2], #48 \n" // store 16 YUV pixels
+ "b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(src_vu), // %1
"+r"(dst_yuv24), // %2
@@ -2897,7 +5248,44 @@ void NV21ToYUV24Row_NEON(const uint8_t* src_y,
:
: "cc", "memory", "v0", "v1", "v2");
}
+#else
+static const uvec8 kYUV24Shuffle[3] = {
+ {16, 17, 0, 16, 17, 1, 18, 19, 2, 18, 19, 3, 20, 21, 4, 20},
+ {21, 5, 22, 23, 6, 22, 23, 7, 24, 25, 8, 24, 25, 9, 26, 27},
+ {10, 26, 27, 11, 28, 29, 12, 28, 29, 13, 30, 31, 14, 30, 31, 15}};
+
+// Convert biplanar NV21 to packed YUV24
+// NV21 has VU in memory for chroma.
+// YUV24 is VUY in memory
+void NV21ToYUV24Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ asm volatile(
+ "ld1 {v5.16b,v6.16b,v7.16b}, [%4] \n" // 3 shuffler constants
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load 16 Y values
+ "ld1 {v1.16b}, [%1], #16 \n" // load 8 VU values
+ "subs %w3, %w3, #16 \n" // 16 pixels per loop
+ "tbl v2.16b, {v0.16b,v1.16b}, v5.16b \n" // weave into YUV24
+ "prfm pldl1keep, [%0, 448] \n"
+ "tbl v3.16b, {v0.16b,v1.16b}, v6.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "tbl v4.16b, {v0.16b,v1.16b}, v7.16b \n"
+ "st1 {v2.16b,v3.16b,v4.16b}, [%2], #48 \n" // store 16 YUV pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_vu), // %1
+ "+r"(dst_yuv24), // %2
+ "+r"(width) // %3
+ : "r"(&kYUV24Shuffle[0]) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+#endif // LIBYUV_USE_ST3
+
+// Note ST2 8b version is faster than zip+ST1
+// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
void AYUVToUVRow_NEON(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_uv,
@@ -2905,18 +5293,20 @@ void AYUVToUVRow_NEON(const uint8_t* src_ayuv,
const uint8_t* src_ayuv_1 = src_ayuv + src_stride_ayuv;
asm volatile(
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ayuv
- "uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
- "uadalp v0.8h, v4.16b \n" // V 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // U 16 bytes -> 8 shorts.
- "uqrshrn v3.8b, v0.8h, #2 \n" // 2x2 average
- "uqrshrn v2.8b, v1.8h, #2 \n"
- "subs %w3, %w3, #16 \n" // 16 processed per loop.
- "st2 {v2.8b,v3.8b}, [%2], #16 \n" // store 8 pixels UV.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ayuv
+ "subs %w3, %w3, #16 \n" // 16 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
+ "uadalp v0.8h, v4.16b \n" // V 16 bytes -> 8 shorts.
+ "uadalp v1.8h, v5.16b \n" // U 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqrshrn v3.8b, v0.8h, #2 \n" // 2x2 average
+ "uqrshrn v2.8b, v1.8h, #2 \n"
+ "st2 {v2.8b,v3.8b}, [%2], #16 \n" // store 8 pixels UV.
+ "b.gt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(src_ayuv_1), // %1
"+r"(dst_uv), // %2
@@ -2932,19 +5322,20 @@ void AYUVToVURow_NEON(const uint8_t* src_ayuv,
const uint8_t* src_ayuv_1 = src_ayuv + src_stride_ayuv;
asm volatile(
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
- // pixels.
- "uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
- "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
- "uadalp v0.8h, v4.16b \n" // V 16 bytes -> 8 shorts.
- "uadalp v1.8h, v5.16b \n" // U 16 bytes -> 8 shorts.
- "uqrshrn v0.8b, v0.8h, #2 \n" // 2x2 average
- "uqrshrn v1.8b, v1.8h, #2 \n"
- "subs %w3, %w3, #16 \n" // 16 processed per loop.
- "st2 {v0.8b,v1.8b}, [%2], #16 \n" // store 8 pixels VU.
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ayuv
+ "subs %w3, %w3, #16 \n" // 16 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%1], #64 \n" // load next 16
+ "uadalp v0.8h, v4.16b \n" // V 16 bytes -> 8 shorts.
+ "uadalp v1.8h, v5.16b \n" // U 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqrshrn v0.8b, v0.8h, #2 \n" // 2x2 average
+ "uqrshrn v1.8b, v1.8h, #2 \n"
+ "st2 {v0.8b,v1.8b}, [%2], #16 \n" // store 8 pixels VU.
+ "b.gt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(src_ayuv_1), // %1
"+r"(dst_vu), // %2
@@ -2956,12 +5347,12 @@ void AYUVToVURow_NEON(const uint8_t* src_ayuv,
// Copy row of AYUV Y's into Y
void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
- // pixels
- "subs %w2, %w2, #16 \n" // 16 pixels per loop
- "st1 {v2.16b}, [%1], #16 \n" // store 16 Y pixels
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop
+ "prfm pldl1keep, [%0, 448] \n"
+ "st1 {v2.16b}, [%1], #16 \n" // store 16 Y pixels
+ "b.gt 1b \n"
: "+r"(src_ayuv), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -2972,19 +5363,758 @@ void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
// Convert UV plane of NV12 to VU of NV21.
void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
asm volatile(
- "1: \n"
- "ld2 {v0.16b, v1.16b}, [%0], #32 \n" // load 16 UV values
- "orr v2.16b, v0.16b, v0.16b \n" // move U after V
- "subs %w2, %w2, #16 \n" // 16 pixels per loop
- "st2 {v1.16b, v2.16b}, [%1], #32 \n" // store 16 VU pixels
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v0.16b}, [%0], 16 \n" // load 16 UV values
+ "ld1 {v1.16b}, [%0], 16 \n"
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop
+ "rev16 v0.16b, v0.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "rev16 v1.16b, v1.16b \n"
+ "stp q0, q1, [%1], 32 \n" // store 16 VU pixels
+ "b.gt 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_vu), // %1
"+r"(width) // %2
:
+ : "cc", "memory", "v0", "v1");
+}
+
+void HalfMergeUVRow_NEON(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ const uint8_t* src_u_1 = src_u + src_stride_u;
+ const uint8_t* src_v_1 = src_v + src_stride_v;
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.16b}, [%0], #16 \n" // load 16 U values
+ "ld1 {v1.16b}, [%2], #16 \n" // load 16 V values
+ "ld1 {v2.16b}, [%1], #16 \n"
+ "ld1 {v3.16b}, [%3], #16 \n"
+ "subs %w5, %w5, #16 \n" // 16 src pixels per loop
+ "uaddlp v0.8h, v0.16b \n" // half size
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v1.8h, v1.16b \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "uadalp v0.8h, v2.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v1.8h, v3.16b \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "uqrshrn v0.8b, v0.8h, #2 \n"
+ "uqrshrn v1.8b, v1.8h, #2 \n"
+ "st2 {v0.8b, v1.8b}, [%4], #16 \n" // store 8 UV pixels
+ "b.gt 1b \n"
+ : "+r"(src_u), // %0
+ "+r"(src_u_1), // %1
+ "+r"(src_v), // %2
+ "+r"(src_v_1), // %3
+ "+r"(dst_uv), // %4
+ "+r"(width) // %5
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3");
+}
+
+void SplitUVRow_16_NEON(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ int shift = depth - 16; // Negative for right shift.
+ asm volatile(
+ "dup v2.8h, %w4 \n"
+ "1: \n"
+ "ld2 {v0.8h, v1.8h}, [%0], #32 \n" // load 8 UV
+ "subs %w3, %w3, #8 \n" // 8 src pixels per loop
+ "ushl v0.8h, v0.8h, v2.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "ushl v1.8h, v1.8h, v2.8h \n"
+ "st1 {v0.8h}, [%1], #16 \n" // store 8 U pixels
+ "st1 {v1.8h}, [%2], #16 \n" // store 8 V pixels
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "v0", "v1", "v2");
+}
+
+void MultiplyRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ asm volatile(
+ "dup v2.8h, %w3 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 src pixels per loop
+ "mul v0.8h, v0.8h, v2.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mul v1.8h, v1.8h, v2.8h \n"
+ "stp q0, q1, [%1], #32 \n" // store 16 pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "cc", "memory", "v0", "v1", "v2");
+}
+
+void DivideRow_16_NEON(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ asm volatile(
+ "dup v4.8h, %w3 \n"
+ "1: \n"
+ "ldp q2, q3, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 src pixels per loop
+ "umull v0.4s, v2.4h, v4.4h \n"
+ "umull2 v1.4s, v2.8h, v4.8h \n"
+ "umull v2.4s, v3.4h, v4.4h \n"
+ "umull2 v3.4s, v3.8h, v4.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
+ "stp q0, q1, [%1], #32 \n" // store 16 pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits = shr 1
+// 16384 = 10 bits = shr 2
+// 4096 = 12 bits = shr 4
+// 256 = 16 bits = shr 8
+void Convert16To8Row_NEON(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ asm volatile(
+ "dup v2.8h, %w3 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 src pixels per loop
+ "uqshl v0.8h, v0.8h, v2.8h \n"
+ "uqshl v1.8h, v1.8h, v2.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.16b, v0.16b, v1.16b \n"
+ "str q0, [%1], #16 \n" // store 16 pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(shift) // %3
+ : "cc", "memory", "v0", "v1", "v2");
+}
+
+void HalfRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ asm volatile(
+ "dup v4.8h, %w4 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "ldp q2, q3, [%1], #32 \n"
+ "subs %w3, %w3, #16 \n"
+ "urhadd v0.8h, v0.8h, v2.8h \n"
+ "urhadd v1.8h, v1.8h, v3.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqshl v0.8h, v0.8h, v4.8h \n"
+ "uqshl v1.8h, v1.8h, v4.8h \n"
+ "uzp2 v0.16b, v0.16b, v1.16b \n"
+ "str q0, [%2], #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(src_uv1), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ asm volatile(
+ "dup v4.8h, %w4 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "ldp q2, q3, [%1], #32 \n"
+ "subs %w3, %w3, #8 \n"
+ "uaddlp v0.4s, v0.8h \n"
+ "uaddlp v1.4s, v1.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uadalp v0.4s, v2.8h \n"
+ "uadalp v1.4s, v3.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "rshrn v0.4h, v0.4s, #2 \n"
+ "rshrn2 v0.8h, v1.4s, #2 \n"
+ "uqshl v0.8h, v0.8h, v4.8h \n"
+ "uzp2 v0.16b, v0.16b, v0.16b \n"
+ "str d0, [%2], #8 \n"
+ "b.gt 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(src_uv1), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(shift) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4");
+}
+
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "dup v4.16b, %w3 \n" // scale
+ "dup v5.16b, %w4 \n" // bias
+ "1: \n"
+ "ldp q2, q3, [%0], #32 \n"
+ "subs %w2, %w2, #32 \n" // 32 pixels per loop
+ "umull v0.8h, v2.8b, v4.8b \n"
+ "umull2 v1.8h, v2.16b, v4.16b \n"
+ "umull v2.8h, v3.8b, v4.8b \n"
+ "umull2 v3.8h, v3.16b, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.16b, v0.16b, v1.16b \n"
+ "uzp2 v1.16b, v2.16b, v3.16b \n"
+ "add v0.16b, v0.16b, v5.16b \n" // add bias (16)
+ "add v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%1], #32 \n" // store 32 pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 1024 = 10 bits
+// 4096 = 12 bits
+// 65536 = 16 bits
+void Convert8To16Row_NEON(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ // (src * 0x0101) >> (16 - bits).
+ // Use negative shift for right shift with ushl.
+ const int shift = bits - 16;
+ asm volatile(
+ "dup v2.8h, %w[shift] \n"
+ "1: \n"
+ "ldr q0, [%[src]], #16 \n"
+ "zip2 v1.16b, v0.16b, v0.16b \n"
+ "zip1 v0.16b, v0.16b, v0.16b \n"
+ "subs %w[width], %w[width], #16 \n"
+ "ushl v1.8h, v1.8h, v2.8h \n"
+ "ushl v0.8h, v0.8h, v2.8h \n"
+ "stp q0, q1, [%[dst]], #32 \n"
+ "b.ne 1b \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [shift] "r"(shift) // %[shift]
: "cc", "memory", "v0", "v1", "v2");
}
+void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
+ asm volatile(
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "stp q0, q1, [%1], #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(count) // %2
+ :
+ : "memory", "cc", "v0", "v1");
+}
+
+void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
+ asm volatile(
+ // Start at end of source row.
+ "ld1 {v4.16b}, [%3] \n" // shuffler
+ "add %0, %0, %w2, sxtw #1 \n"
+ "sub %0, %0, #32 \n"
+ "1: \n"
+ "ldr q1, [%0, 16] \n"
+ "ldr q0, [%0], -32 \n" // src -= 32
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop.
+ "tbl v2.16b, {v1.16b}, v4.16b \n"
+ "tbl v3.16b, {v0.16b}, v4.16b \n"
+ "stp q2, q3, [%1], #32 \n" // dst += 32
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "r"(&kShuffleMirrorUV) // %3
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4");
+}
+
+void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "movi v16.4s, #0xff, lsl #8 \n" // 0x0000ff00
+ "movi v17.4s, #0x0f, msl #16 \n" // 0x000fffff
+ "shl v17.4s, v17.4s, #8 \n" // 0x0fffff00
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop
+ "sri v0.4s, v0.4s, #2 \n"
+ "sri v1.4s, v1.4s, #2 \n"
+ "ushr v2.4s, v0.4s, #2 \n"
+ "ushr v3.4s, v1.4s, #2 \n"
+ "ushr v4.4s, v0.4s, #4 \n"
+ "ushr v5.4s, v1.4s, #4 \n"
+ "bit v0.16b, v4.16b, v17.16b \n"
+ "bit v1.16b, v5.16b, v17.16b \n"
+ "bit v0.16b, v2.16b, v16.16b \n"
+ "bit v1.16b, v3.16b, v16.16b \n"
+ "stp q0, q1, [%1], #32 \n" // store 8 ARGB pixels
+ "b.gt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
+}
+
+void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
+ asm volatile(
+ "movi v16.2d, #0x00ffff0000ffff00 \n"
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop
+ "sri v0.4s, v0.4s, #2 \n"
+ "sri v1.4s, v1.4s, #2 \n"
+ "ushr v2.4s, v0.4s, #2 \n"
+ "ushr v3.4s, v1.4s, #2 \n"
+ "sli v2.4s, v0.4s, #16 \n"
+ "sli v3.4s, v1.4s, #16 \n"
+ "sri v0.4s, v0.4s, #4 \n"
+ "sri v1.4s, v1.4s, #4 \n"
+ "sri v0.4s, v0.4s, #16 \n"
+ "sri v1.4s, v1.4s, #16 \n"
+ "bit v0.16b, v2.16b, v16.16b \n"
+ "bit v1.16b, v3.16b, v16.16b \n"
+ "stp q0, q1, [%1], #32 \n" // store 8 ABGR pixels
+ "b.gt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_abgr), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v16");
+}
+
+void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
+ asm volatile(
+ "movi v17.16b, #0xff \n"
+ "ushr v17.4s, v17.4s, #22 \n" // 0x000003ff
+ "shl v16.4s, v17.4s, #20 \n" // 0x3ff00000
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop
+ "shl v2.4s, v0.4s, #20 \n"
+ "shl v3.4s, v1.4s, #20 \n"
+ "ushr v4.4s, v0.4s, #20 \n"
+ "ushr v5.4s, v1.4s, #20 \n"
+ "bit v0.16b, v2.16b, v16.16b \n"
+ "bit v1.16b, v3.16b, v16.16b \n"
+ "bit v0.16b, v4.16b, v17.16b \n"
+ "bit v1.16b, v5.16b, v17.16b \n"
+ "stp q0, q1, [%1], #32 \n" // store 8 AB30 pixels
+ "b.gt 1b \n"
+ : "+r"(src_ar30), // %0
+ "+r"(dst_ab30), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
+}
+
+void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "movi v4.4s, #0xff, lsl #24 \n" // 0xff000000
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 src pixels
+ "ldp q2, q3, [%1] \n" // load 8 dst pixels
+ "subs %w2, %w2, #8 \n" // 8 pixels per loop
+ "bit v2.16b, v0.16b, v4.16b \n"
+ "bit v3.16b, v1.16b, v4.16b \n"
+ "stp q2, q3, [%1], #32 \n" // store 8 dst pixels
+ "b.gt 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4");
+}
+
+void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ uint64_t tmp;
+ asm volatile(
+ "movi v16.4s, #0xff, lsl #24 \n" // 0xff000000
+ "1: \n"
+ "ldrb %w3, [%0, #3] \n"
+ "ldr s4, [%4, %w3, uxtw #2] \n"
+ "ldrb %w3, [%0, #7] \n"
+ "ldr s5, [%4, %w3, uxtw #2] \n"
+ "ldrb %w3, [%0, #11] \n"
+ "ldr s6, [%4, %w3, uxtw #2] \n"
+ "ldrb %w3, [%0, #15] \n"
+ "ldr s7, [%4, %w3, uxtw #2] \n"
+ "ldr q0, [%0], #16 \n"
+ "subs %w2, %w2, #4 \n"
+ "zip1 v1.16b, v0.16b, v0.16b \n"
+ "zip2 v2.16b, v0.16b, v0.16b \n"
+ "umull v3.4s, v1.4h, v4.h[0] \n"
+ "umull2 v5.4s, v1.8h, v5.h[0] \n"
+ "umull v6.4s, v2.4h, v6.h[0] \n"
+ "umull2 v7.4s, v2.8h, v7.h[0] \n"
+ "uzp2 v3.8h, v3.8h, v5.8h \n"
+ "uzp2 v6.8h, v6.8h, v7.8h \n"
+ "uqxtn v1.8b, v3.8h \n"
+ "uqxtn2 v1.16b, v6.8h \n"
+ "bit v1.16b, v0.16b, v16.16b \n"
+ "str q1, [%1], #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ "=&r"(tmp) // %3
+ : "r"(fixed_invtbl8) // %4
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+}
+
+void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width) {
+ asm volatile(
+ "ldp q16, q17, [%3] \n" // c0, c1
+ "ldp q18, q19, [%3, #32] \n" // c2, c3
+ "subs %w2, %w2, #4 \n"
+ "b.lt 2f \n"
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // load 4 pixels
+ "subs %w2, %w2, #4 \n"
+ "uxtl v1.8h, v0.8b \n"
+ "uxtl2 v3.8h, v0.16b \n"
+ "uxtl v0.4s, v1.4h \n"
+ "uxtl2 v1.4s, v1.8h \n"
+ "uxtl v2.4s, v3.4h \n"
+ "uxtl2 v3.4s, v3.8h \n"
+ "ucvtf v0.4s, v0.4s \n" // x0
+ "ucvtf v1.4s, v1.4s \n" // x1
+ "ucvtf v2.4s, v2.4s \n" // x2
+ "ucvtf v3.4s, v3.4s \n" // x3
+ "fmul v20.4s, v0.4s, v0.4s \n" // x0^2
+ "fmul v21.4s, v1.4s, v1.4s \n" // x1^2
+ "fmul v22.4s, v2.4s, v2.4s \n" // x2^2
+ "fmul v23.4s, v3.4s, v3.4s \n" // x3^2
+ "fmul v4.4s, v17.4s, v0.4s \n" // c1 * x0
+ "fmul v5.4s, v17.4s, v1.4s \n" // c1 * x1
+ "fmul v6.4s, v17.4s, v2.4s \n" // c1 * x2
+ "fmul v7.4s, v17.4s, v3.4s \n" // c1 * x3
+ "fmul v0.4s, v20.4s, v0.4s \n" // x0^3
+ "fmul v1.4s, v21.4s, v1.4s \n" // x1^3
+ "fmul v2.4s, v22.4s, v2.4s \n" // x2^3
+ "fmul v3.4s, v23.4s, v3.4s \n" // x3^3
+ "fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x0
+ "fadd v5.4s, v16.4s, v5.4s \n" // c0 + c1 * x1
+ "fadd v6.4s, v16.4s, v6.4s \n" // c0 + c1 * x2
+ "fadd v7.4s, v16.4s, v7.4s \n" // c0 + c1 * x3
+ "fmla v4.4s, v18.4s, v20.4s \n" // + c2 * x0^2
+ "fmla v5.4s, v18.4s, v21.4s \n" // + c2 * x1^2
+ "fmla v6.4s, v18.4s, v22.4s \n" // + c2 * x2^2
+ "fmla v7.4s, v18.4s, v23.4s \n" // + c2 * x3^2
+ "fmla v4.4s, v19.4s, v0.4s \n" // + c3 * x0^3
+ "fmla v5.4s, v19.4s, v1.4s \n" // + c3 * x1^3
+ "fmla v6.4s, v19.4s, v2.4s \n" // + c3 * x2^3
+ "fmla v7.4s, v19.4s, v3.4s \n" // + c3 * x3^3
+ "fcvtzs v4.4s, v4.4s \n"
+ "fcvtzs v5.4s, v5.4s \n"
+ "fcvtzs v6.4s, v6.4s \n"
+ "fcvtzs v7.4s, v7.4s \n"
+ "sqxtn v4.4h, v4.4s \n"
+ "sqxtn2 v4.8h, v5.4s \n"
+ "sqxtn v6.4h, v6.4s \n"
+ "sqxtn2 v6.8h, v7.4s \n"
+ "sqxtun v4.8b, v4.8h \n"
+ "sqxtun2 v4.16b, v6.8h \n"
+ "str q4, [%1], #16 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w2, %w2, #4 \n"
+ "b.eq 3f \n"
+ "ldr d0, [%0], #8 \n" // load 2 pixels
+ "uxtl v0.8h, v0.8b \n"
+ "uxtl v1.4s, v0.4h \n"
+ "uxtl2 v2.4s, v0.8h \n"
+ "ucvtf v1.4s, v1.4s \n" // x0
+ "ucvtf v2.4s, v2.4s \n" // x1
+ "fmul v5.4s, v1.4s, v1.4s \n" // x0^2
+ "fmul v6.4s, v2.4s, v2.4s \n" // x1^2
+ "fmul v3.4s, v17.4s, v1.4s \n" // c1 * x0
+ "fmul v4.4s, v17.4s, v2.4s \n" // c1 * x1
+ "fmul v1.4s, v5.4s, v1.4s \n" // x0^3
+ "fmul v2.4s, v6.4s, v2.4s \n" // x1^3
+ "fadd v3.4s, v16.4s, v3.4s \n" // c0 + c1 * x0
+ "fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x1
+ "fmla v3.4s, v18.4s, v5.4s \n" // + c2 * x0^2
+ "fmla v4.4s, v18.4s, v6.4s \n" // + c2 * x1^2
+ "fmla v3.4s, v19.4s, v1.4s \n" // + c3 * x0^3
+ "fmla v4.4s, v19.4s, v2.4s \n" // + c3 * x1^3
+ "fcvtzs v3.4s, v3.4s \n"
+ "fcvtzs v4.4s, v4.4s \n"
+ "sqxtn v3.4h, v3.4s \n"
+ "sqxtn2 v3.8h, v4.4s \n"
+ "sqxtun v3.8b, v3.8h \n"
+ "str d3, [%1], #8 \n"
+ "3: \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(poly) // %3
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+
+void ARGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ uint64_t b, g, r, a;
+ asm volatile(
+ "1: \n"
+ "ldrb %w2, [%0] \n"
+ "ldrb %w3, [%0, #1] \n"
+ "ldrb %w4, [%0, #2] \n"
+ "ldrb %w5, [%0, #3] \n"
+ "subs %w1, %w1, #1 \n"
+ "ldr %w2, [%6, %w2, uxtw #2] \n"
+ "ldr %w3, [%6, %w3, uxtw #2] \n"
+ "ldr %w4, [%6, %w4, uxtw #2] \n"
+ "ldr %w5, [%6, %w5, uxtw #2] \n"
+ "bfi %w3, %w2, #0, #8 \n"
+ "lsr %w4, %w4, #16 \n"
+ "lsr %w5, %w5, #24 \n"
+ "strh %w3, [%0] \n"
+ "strb %w4, [%0, #2] \n"
+ "strb %w5, [%0, #3] \n"
+ "add %0, %0, #4 \n"
+ "b.gt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "=&r"(b), // %2
+ "=&r"(g), // %3
+ "=&r"(r), // %4
+ "=&r"(a) // %5
+ : "r"(table_argb) // %6
+ : "memory", "cc");
+}
+
+void RGBColorTableRow_NEON(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ uint64_t b, g, r;
+ asm volatile(
+ "1: \n"
+ "ldrb %w2, [%0] \n"
+ "ldrb %w3, [%0, #1] \n"
+ "ldrb %w4, [%0, #2] \n"
+ "subs %w1, %w1, #1 \n"
+ "ldr %w2, [%5, %w2, uxtw #2] \n"
+ "ldr %w3, [%5, %w3, uxtw #2] \n"
+ "ldr %w4, [%5, %w4, uxtw #2] \n"
+ "bfi %w3, %w2, #0, #8 \n"
+ "lsr %w4, %w4, #16 \n"
+ "strh %w3, [%0] \n"
+ "strb %w4, [%0, #2] \n"
+ "add %0, %0, #4 \n"
+ "b.gt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "=&r"(b), // %2
+ "=&r"(g), // %3
+ "=&r"(r) // %4
+ : "r"(table_argb) // %5
+ : "memory", "cc");
+}
+
+static const uvec8 kLumaTblIdx = {1, 1, 1, 1, 3, 3, 3, 3,
+ 5, 5, 5, 5, 7, 7, 7, 7};
+
+void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff) {
+ uint64_t p0, p1, t0, t1;
+ asm volatile(
+ "dup v4.4s, %w8 \n" // broadcast 4 byte coeffs
+ "ldr q5, [%9] \n"
+ "movi v6.4s, #0xff, lsl #24 \n"
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // load 4 ARGB pixels
+ "subs %w2, %w2, #4 \n" // 4 pixels per loop
+ "umull v1.8h, v0.8b, v4.8b \n"
+ "umull2 v2.8h, v0.16b, v4.16b \n"
+ "addp v1.8h, v1.8h, v2.8h \n"
+ "addp v1.8h, v1.8h, v1.8h \n"
+ "bic v1.8h, #0x80, lsl #8 \n" // clear bit 15
+ "tbl v1.16b, {v1.16b}, v5.16b \n"
+ "zip2 v2.16b, v0.16b, v1.16b \n" // offsets pixels 2, 3
+ "zip1 v1.16b, v0.16b, v1.16b \n" // offsets pixels 0, 1
+ "umov %3, v1.d[0] \n"
+ "umov %4, v1.d[1] \n"
+ "add %5, %7, %w3, uxth \n"
+ "ubfx %6, %3, #16, #16 \n"
+ "ldr b3, [%5] \n"
+ "add %6, %7, %6 \n"
+ "ubfx %5, %3, #32, #16 \n"
+ "ld1 {v3.b}[1], [%6] \n"
+ "add %5, %7, %5 \n"
+ "add %6, %7, %w4, uxth \n"
+ "ld1 {v3.b}[2], [%5] \n"
+ "ld1 {v3.b}[4], [%6] \n"
+ "ubfx %5, %4, #16, #16 \n"
+ "ubfx %6, %4, #32, #16 \n"
+ "add %5, %7, %5 \n"
+ "add %6, %7, %6 \n"
+ "ld1 {v3.b}[5], [%5] \n"
+ "ld1 {v3.b}[6], [%6] \n"
+ "umov %3, v2.d[0] \n"
+ "umov %4, v2.d[1] \n"
+ "add %5, %7, %w3, uxth \n"
+ "ubfx %6, %3, #16, #16 \n"
+ "ld1 {v3.b}[8], [%5] \n"
+ "add %6, %7, %6 \n"
+ "ubfx %5, %3, #32, #16 \n"
+ "ld1 {v3.b}[9], [%6] \n"
+ "add %5, %7, %5 \n"
+ "add %6, %7, %w4, uxth \n"
+ "ld1 {v3.b}[10], [%5] \n"
+ "ld1 {v3.b}[12], [%6] \n"
+ "ubfx %5, %4, #16, #16 \n"
+ "ubfx %6, %4, #32, #16 \n"
+ "add %5, %7, %5 \n"
+ "add %6, %7, %6 \n"
+ "ld1 {v3.b}[13], [%5] \n"
+ "ld1 {v3.b}[14], [%6] \n"
+ "bit v3.16b, v0.16b, v6.16b \n"
+ "str q3, [%1], #16 \n"
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width), // %2
+ "=&r"(p0), // %3
+ "=&r"(p1), // %4
+ "=&r"(t0), // %5
+ "=&r"(t1) // %6
+ : "r"(luma), // %7
+ "r"(lumacoeff), // %8
+ "r"(&kLumaTblIdx) // %9
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
+}
+
+void ARGBAffineRow_NEON(const uint8_t* src_argb,
+ int src_argb_stride,
+ uint8_t* dst_argb,
+ const float* uv_dudv,
+ int width) {
+ int64_t off0, off1;
+ asm volatile(
+ "ld1r {v2.2d}, [%2], #8 \n" // [u, v, u, v]
+ "ld1r {v6.2d}, [%2] \n" // [du, dv, du, dv]
+ "dup v5.2s, %w5 \n" // [stride, stride]
+ "movi v4.2s, #4 \n" // [4, 4]
+ "zip1 v5.4s, v4.4s, v5.4s \n" // [4, stride, 4, stride]
+ "subs %w1, %w1, #4 \n"
+ "b.lt 2f \n"
+ "fadd v4.4s, v6.4s, v6.4s \n" // 2 * [du, dv, du, dv]
+ "fadd v3.2s, v2.2s, v6.2s \n" // [u+du, v+dv]
+ "ins v2.d[1], v3.d[0] \n" // [u, v, u+du, v+dv]
+ "fadd v3.4s, v2.4s, v4.4s \n" // [u+2du, v+2dv, u+3du,
+ // v+3dv]
+ "fadd v4.4s, v4.4s, v4.4s \n" // 4 * [du, dv, du, dv]
+ "1: \n"
+ "fcvtzs v0.4s, v2.4s \n"
+ "fcvtzs v1.4s, v3.4s \n"
+ "fadd v2.4s, v2.4s, v4.4s \n"
+ "fadd v3.4s, v3.4s, v4.4s \n"
+ "mul v0.4s, v0.4s, v5.4s \n"
+ "mul v1.4s, v1.4s, v5.4s \n"
+ "addp v7.4s, v0.4s, v1.4s \n"
+ "smov %3, v7.s[0] \n"
+ "smov %4, v7.s[1] \n"
+ "ldr %w3, [%6, %3] \n"
+ "ldr %w4, [%6, %4] \n"
+ "stp %w3, %w4, [%0], #8 \n"
+ "smov %3, v7.s[2] \n"
+ "smov %4, v7.s[3] \n"
+ "ldr %w3, [%6, %3] \n"
+ "ldr %w4, [%6, %4] \n"
+ "stp %w3, %w4, [%0], #8 \n"
+ "subs %w1, %w1, #4 \n"
+ "b.ge 1b \n"
+ "2: \n"
+ "adds %w1, %w1, #4 \n"
+ "b.eq 4f \n"
+ "3: \n"
+ "fcvtzs v0.2s, v2.2s \n"
+ "fadd v2.2s, v2.2s, v6.2s \n"
+ "mul v0.2s, v0.2s, v5.2s \n"
+ "addp v0.2s, v0.2s, v0.2s \n"
+ "smov %3, v0.s[0] \n"
+ "ldr %w3, [%6, %3] \n"
+ "str %w3, [%0], #4 \n"
+ "subs %w1, %w1, #1 \n"
+ "b.gt 3b \n"
+ "4: \n"
+ : "+r"(dst_argb), // %0
+ "+r"(width), // %1
+ "+r"(uv_dudv), // %2
+ "=&r"(off0), // %3
+ "=&r"(off1) // %4
+ : "r"(src_argb_stride), // %5
+ "r"(src_argb) // %6
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/row_rvv.cc b/libs/libyuv/source/row_rvv.cc
new file mode 100644
index 0000000000..e137d58d0e
--- /dev/null
+++ b/libs/libyuv/source/row_rvv.cc
@@ -0,0 +1,3199 @@
+/*
+ * Copyright 2023 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+/*
+ * Copyright (c) 2023 SiFive, Inc. All rights reserved.
+ *
+ * Contributed by Darren Hsieh <darren.hsieh@sifive.com>
+ * Contributed by Bruce Lai <bruce.lai@sifive.com>
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+#include "libyuv/video_common.h"
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+// This module is for RVV (RISC-V Vector extension)
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
+#include <assert.h>
+
+// vsetvli/vsetivli write the vl and vtype CSRs. Only "vl" is declared, and
+// only for GCC; clang historically rejected it. The "vtype" register is
+// intentionally not declared, and whether it should be is unresolved.
+// TODO(fbarchard): consider adding "vtype" (and "vl" for clang) to the
+// clobber list once confirmed with compiler maintainers.
+#ifndef RVV_VL_CLOBBER
+#ifdef __clang__
+#define RVV_VL_CLOBBER
+#else
+#define RVV_VL_CLOBBER "vl",
+#endif
+#endif
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Fill YUV -> RGB conversion constants into registers
+// NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+// register) is set to round-to-nearest-up mode(0).
+#define YUVTORGB_SETUP \
+ "csrwi vxrm, 0 \n" \
+ "lbu %[ub], 0(%[yuvconst]) \n" \
+ "lbu %[vr], 1(%[yuvconst]) \n" \
+ "lbu %[ug], 2(%[yuvconst]) \n" \
+ "lbu %[vg], 3(%[yuvconst]) \n" \
+ "lhu %[yg], 16(%[yuvconst])\n" \
+ "lh %[bb], 18(%[yuvconst])\n" \
+ "lh %[bg], 20(%[yuvconst])\n" \
+ "lh %[br], 22(%[yuvconst])\n" \
+ "addi %[bb], %[bb], 32 \n" \
+ "addi %[bg], %[bg], -32 \n" \
+ "addi %[br], %[br], 32 \n" \
+ "li %[k0101], 0x0101 \n"
+
+// Fill YUV -> AR30 conversion constants into registers
+#define YUVTORGB_SETUP_AR30 \
+ "csrwi vxrm, 0 \n" \
+ "lbu %[ub], 0(%[yuvconst]) \n" \
+ "lbu %[vr], 1(%[yuvconst]) \n" \
+ "lbu %[ug], 2(%[yuvconst]) \n" \
+ "lbu %[vg], 3(%[yuvconst]) \n" \
+ "lhu %[yg], 16(%[yuvconst])\n" \
+ "lh %[bb], 18(%[yuvconst])\n" \
+ "lh %[bg], 20(%[yuvconst])\n" \
+ "lh %[br], 22(%[yuvconst])\n" \
+ "addi %[bb], %[bb], 24 \n" \
+ "addi %[bg], %[bg], -24 \n" \
+ "addi %[br], %[br], 24 \n" \
+ "li %[k0101], 0x0101 \n"
+
+// Read [2*VLEN/8] Y, [VLEN/8] U and [VLEN/8] V from 422
+#define READYUV422 \
+ "addi %[vl], %[w], 1 \n" \
+ "srli %[vl], %[vl], 1 \n" \
+ "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \
+ "vle8.v v0, (%[src_u]) \n" \
+ "vle8.v v1, (%[src_v]) \n" \
+ "vwcvtu.x.x.v v2, v0 \n" \
+ "vwcvtu.x.x.v v4, v1 \n" \
+ "vsetvli zero, zero, e16, m2, ta, ma \n" \
+ "vmul.vx v2, v2, %[k0101] \n" \
+ "vmul.vx v4, v4, %[k0101] \n" \
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \
+ "vle8.v v0, (%[src_y]) \n" \
+ "vwcvtu.x.x.v v8, v0 \n"
+
+// Read [2*VLEN/8] Y, [2*VLEN/8] U, and [2*VLEN/8] V from 444
+#define READYUV444 \
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \
+ "vle8.v v0, (%[src_y]) \n" \
+ "vle8.v v2, (%[src_u]) \n" \
+ "vle8.v v4, (%[src_v]) \n" \
+ "vwcvtu.x.x.v v8, v0 \n"
+
+// Read [2*VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_uv
+#define READNV12 \
+ "addi %[vl], %[w], 1 \n" \
+ "srli %[vl], %[vl], 1 \n" \
+ "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \
+ "vlseg2e8.v v0, (%[src_uv]) \n" \
+ "vwcvtu.x.x.v v2, v0 \n" \
+ "vwcvtu.x.x.v v4, v1 \n" \
+ "vsetvli zero, zero, e16, m2, ta, ma \n" \
+ "vmul.vx v2, v2, %[k0101] \n" \
+ "vmul.vx v4, v4, %[k0101] \n" \
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \
+ "vle8.v v0, (%[src_y]) \n" \
+ "vwcvtu.x.x.v v8, v0 \n"
+
+// Read 2*[VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_vu
+#define READNV21 \
+ "addi %[vl], %[w], 1 \n" \
+ "srli %[vl], %[vl], 1 \n" \
+ "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \
+ "vlseg2e8.v v0, (%[src_vu]) \n" \
+ "vwcvtu.x.x.v v4, v0 \n" \
+ "vwcvtu.x.x.v v2, v1 \n" \
+ "vsetvli zero, zero, e16, m2, ta, ma \n" \
+ "vmul.vx v2, v2, %[k0101] \n" \
+ "vmul.vx v4, v4, %[k0101] \n" \
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \
+ "vle8.v v0, (%[src_y]) \n" \
+ "vwcvtu.x.x.v v8, v0 \n"
+
+// Convert from YUV to fixed point RGB
+#define YUVTORGB \
+ "vwmulu.vx v12, v2, %[ug] \n" \
+ "vwmaccu.vx v12, %[vg], v4 \n" \
+ "vwmulu.vx v0, v2, %[ub] \n" \
+ "vsetvli zero, zero, e16, m4, ta, ma \n" \
+ "vmul.vx v8, v8, %[k0101] \n" \
+ "vmulhu.vx v8, v8, %[yg] \n" \
+ "vadd.vx v16, v8, %[bg] \n" \
+ "vadd.vv v0, v8, v0 \n" \
+ "vsetvli zero, zero, e8, m2, ta, ma \n" \
+ "vwmaccu.vx v8, %[vr], v4 \n" \
+ "vsetvli zero, zero, e16, m4, ta, ma \n" \
+ "vssubu.vv v12, v16, v12 \n" \
+ "vssubu.vx v0, v0, %[bb] \n" \
+ "vssubu.vx v8, v8, %[br] \n"
+
+// Convert from fixed point RGB To 8 bit RGB
+#define RGBTORGB8 \
+ "vsetvli zero, zero, e8, m2, ta, ma \n" \
+ "vnclipu.wi v26, v12, 6 \n" \
+ "vnclipu.wi v24, v0, 6 \n" \
+ "vnclipu.wi v28, v8, 6 \n"
+
+#define YUVTORGB_REGS \
+ "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9", "v10", "v11", "v12", "v13", \
+ "v14", "v15", "v16", "v17", "v18", "v19", "v24", "v25", "v26", "v27", \
+ "v28", "v29"
+
+#ifdef HAS_ARGBTOAR64ROW_RVV
+void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width) {
+ int vl;
+ asm volatile(
+ "slli %[w], %[w], 2 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v16, (%[src_argb]) \n"
+ "vwcvtu.x.x.v v8, v16 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmul.vx v8, v8, %[k0101] \n"
+ "vse16.v v8, (%[dst_ar64]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ar64], %[dst_ar64], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_ar64] "+r"(dst_ar64), // %[dst_ar64]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k0101] "r"(0x0101)
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_ARGBTOAB64ROW_RVV
+void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ "vlseg4e8.v v4, (%[src_argb]) \n"
+ "vwcvtu.x.x.v v12, v4 \n"
+ "vwcvtu.x.x.v v10, v5 \n"
+ "vwcvtu.x.x.v v8, v6 \n"
+ "vwcvtu.x.x.v v14, v7 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vmul.vx v8, v8, %[k0101] \n"
+ "vmul.vx v10, v10, %[k0101] \n"
+ "vmul.vx v12, v12, %[k0101] \n"
+ "vmul.vx v14, v14, %[k0101] \n"
+ "vsseg4e16.v v8, (%[dst_ab64]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ab64], %[dst_ab64], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_ab64] "+r"(dst_ab64), // %[dst_ab64]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k0101] "r"(0x0101)
+ : RVV_VL_CLOBBER "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
+ "v11", "v12", "v13", "v14", "v15");
+}
+#endif
+
+#ifdef HAS_AR64TOARGBROW_RVV
+void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width) {
+ int vl;
+ asm volatile(
+ "slli %[w], %[w], 2 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle16.v v8, (%[src_ar64]) \n"
+ "vnsrl.wi v16, v8, 8 \n"
+ "vse8.v v16, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ar64], %[src_ar64], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_ar64] "+r"(src_ar64), // %[src_ar64]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_AR64TOAB64ROW_RVV
+void AR64ToAB64Row_RVV(const uint16_t* src_ar64,
+ uint16_t* dst_ab64,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m2, ta, ma \n"
+ "vlseg4e16.v v8, (%[src_ar64]) \n"
+ "vmv2r.v v16, v12 \n"
+ "vmv2r.v v18, v10 \n"
+ "vmv2r.v v20, v8 \n"
+ "vmv2r.v v22, v14 \n"
+ "vsseg4e16.v v16, (%[dst_ab64]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 3 \n"
+ "add %[src_ar64], %[src_ar64], %[vl]\n"
+ "add %[dst_ab64], %[dst_ab64], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_ar64] "+r"(src_ar64), // %[src_ar64]
+ [dst_ab64] "+r"(dst_ab64), // %[dst_ab64]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_AB64TOARGBROW_RVV
+void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ "vlseg4e16.v v8, (%[src_ab64]) \n"
+ "vnsrl.wi v4, v12, 8 \n"
+ "vnsrl.wi v5, v10, 8 \n"
+ "vnsrl.wi v6, v8, 8 \n"
+ "vnsrl.wi v7, v14, 8 \n"
+ "vsseg4e8.v v4, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ab64], %[src_ab64], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_ab64] "+r"(src_ab64), // %[src_ab64]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
+ "v11", "v12", "v13", "v14", "v15");
+}
+#endif
+
+#ifdef HAS_RAWTOARGBROW_RVV
+void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vlseg3e8.v v16, (%[src_raw]) \n"
+ "vmv2r.v v8, v20 \n"
+ "vmv2r.v v10, v18 \n"
+ "vmv2r.v v12, v16 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_raw], %[src_raw], %[vl] \n"
+ "add %[src_raw], %[src_raw], %[tmp]\n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [src_raw] "+r"(src_raw), // %[src_raw]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21");
+}
+#endif
+
+#ifdef HAS_RAWTORGBAROW_RVV
+void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v8, -1 \n"
+
+ "1: \n"
+ "vlseg3e8.v v16, (%[src_raw]) \n"
+ "vmv2r.v v10, v20 \n"
+ "vmv2r.v v12, v18 \n"
+ "vmv2r.v v14, v16 \n"
+ "vsseg4e8.v v8, (%[dst_rgba]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_raw], %[src_raw], %[vl] \n"
+ "add %[src_raw], %[src_raw], %[tmp]\n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_rgba], %[dst_rgba], %[vl]\n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [src_raw] "+r"(src_raw), // %[src_raw]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21");
+}
+#endif
+
+#ifdef HAS_RAWTORGB24ROW_RVV
+void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg3e8.v v16, (%[src_raw]) \n"
+ "vmv2r.v v8, v20 \n"
+ "vmv2r.v v10, v18 \n"
+ "vmv2r.v v12, v16 \n"
+ "vsseg3e8.v v8, (%[dst_rgb24]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[tmp], %[tmp], %[vl] \n"
+ "add %[src_raw], %[src_raw], %[tmp]\n"
+ "add %[dst_rgb24], %[dst_rgb24], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_raw] "+r"(src_raw), // %[src_raw]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v16",
+ "v17", "v18", "v19", "v20", "v21");
+}
+#endif
+
+#ifdef HAS_ARGBTORAWROW_RVV
+void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vmv2r.v v16, v12 \n"
+ "vmv2r.v v18, v10 \n"
+ "vmv2r.v v20, v8 \n"
+ "vsseg3e8.v v16, (%[dst_raw]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[tmp]\n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[dst_raw], %[dst_raw], %[vl] \n"
+ "add %[dst_raw], %[dst_raw], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_raw] "+r"(dst_raw), // %[dst_raw]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21");
+}
+#endif
+
+#ifdef HAS_ARGBTORGB24ROW_RVV
+void ARGBToRGB24Row_RVV(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vsseg3e8.v v8, (%[dst_rgb24]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[tmp]\n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "add %[dst_rgb24], %[dst_rgb24], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_ARGBTOABGRROW_RVV
+void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vmv2r.v v16, v12 \n"
+ "vmv2r.v v18, v10 \n"
+ "vmv2r.v v20, v8 \n"
+ "vmv2r.v v22, v14 \n"
+ "vsseg4e8.v v16, (%[dst_abgr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "add %[dst_abgr], %[dst_abgr], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_abgr] "+r"(dst_abgr), // %[dst_abgr]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_ARGBTOBGRAROW_RVV
+void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vle32.v v8, (%[src_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "vsrl.vi v12, v8, 16 \n"
+ "vsll.vi v8, v8, 16 \n"
+ "vor.vv v8, v8, v12 \n"
+ "slli %[vl], %[vl], 1 \n"
+ "vsetvli zero, %[vl], e16, m4, ta, ma \n"
+ "vsrl.vi v12, v8, 8 \n"
+ "vsll.vi v8, v8, 8 \n"
+ "vor.vv v8, v8, v12 \n"
+ "vse16.v v8, (%[dst_bgra]) \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "add %[dst_bgra], %[dst_bgra], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_bgra] "+r"(dst_bgra), // %[dst_bgra]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_ARGBTORGBAROW_RVV
+void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vle32.v v8, (%[src_argb]) \n"
+ "vsrl.vi v12, v8, 24 \n"
+ "vsll.vi v8, v8, 8 \n"
+ "vor.vv v8, v8, v12 \n"
+ "vse32.v v8, (%[dst_rgba]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "add %[dst_rgba], %[dst_rgba], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_RGBATOARGBROW_RVV
+void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vle32.v v8, (%[src_rgba]) \n"
+ "vsrl.vi v12, v8, 8 \n"
+ "vsll.vi v8, v8, 24 \n"
+ "vor.vv v8, v8, v12 \n"
+ "vse32.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_rgba], %[src_rgba], %[vl]\n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_rgba] "+r"(src_rgba), // %[src_rgba]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_RGB24TOARGBROW_RVV
+void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vlseg3e8.v v8, (%[src_rgb24]) \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_rgb24], %[src_rgb24], %[vl]\n"
+ "add %[src_rgb24], %[src_rgb24], %[tmp]\n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [src_rgb24] "+r"(src_rgb24), // %[src_rgb24]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_I444TOARGBROW_RVV
+void I444ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v30, -1 \n"
+
+ "1: \n" //
+ READYUV444 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_u], %[src_u], %[vl] \n"
+ "add %[src_v], %[src_v], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_I444ALPHATOARGBROW_RVV
+void I444AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READYUV444 //
+ "vle8.v v30, (%[src_a]) \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_u], %[src_u], %[vl] \n"
+ "add %[src_v], %[src_v], %[vl] \n"
+ "add %[src_a], %[src_a], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_I444TORGB24ROW_RVV
+void I444ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READYUV444 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg3e8.v v24, (%[dst_rgb24]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_u], %[src_u], %[vl] \n"
+ "add %[src_v], %[src_v], %[vl] \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
+}
+#endif
+
+#ifdef HAS_I422TOARGBROW_RVV
+void I422ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v30, -1 \n"
+
+ "1: \n" //
+ READYUV422 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "srli %[k0101], %[vl], 1 \n"
+ "add %[src_u], %[src_u], %[k0101] \n"
+ "add %[src_v], %[src_v], %[k0101] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "li %[k0101], 0x0101 \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_I422ALPHATOARGBROW_RVV
+void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READYUV422 //
+ "vle8.v v30, (%[src_a]) \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_a], %[src_a], %[vl] \n"
+ "srli %[k0101], %[vl], 1 \n"
+ "add %[src_u], %[src_u], %[k0101] \n"
+ "add %[src_v], %[src_v], %[k0101] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "li %[k0101], 0x0101 \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_I422TORGBAROW_RVV
+void I422ToRGBARow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READYUV422 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vmv.v.i v22, -1 \n"
+ "vsseg4e8.v v22, (%[dst_rgba]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "srli %[k0101], %[vl], 1 \n"
+ "add %[src_u], %[src_u], %[k0101] \n"
+ "add %[src_v], %[src_v], %[k0101] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_rgba], %[dst_rgba], %[vl]\n"
+ "li %[k0101], 0x0101 \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v22", "v23");
+}
+#endif
+
+#ifdef HAS_I422TORGB24ROW_RVV
+void I422ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READYUV422 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg3e8.v v24, (%[dst_rgb24]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "srli %[k0101], %[vl], 1 \n"
+ "add %[src_u], %[src_u], %[k0101] \n"
+ "add %[src_v], %[src_v], %[k0101] \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "li %[k0101], 0x0101 \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
+}
+#endif
+
+#ifdef HAS_I422TOAR30ROW_RVV
+void I422ToAR30Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP_AR30 //
+
+ "1: \n"
+ "addi %[vl], %[w], 1 \n"
+ "srli %[vl], %[vl], 1 \n"
+ "vsetvli %[vl], %[vl], e8, mf2, ta, ma \n"
+ "vle8.v v0, (%[src_u]) \n"
+ "vle8.v v1, (%[src_v]) \n"
+ "vwcvtu.x.x.v v2, v0 \n"
+ "vwcvtu.x.x.v v4, v1 \n"
+ "vsetvli zero, zero, e16, m1, ta, ma \n"
+ "vmul.vx v2, v2, %[k0101] \n"
+ "vmul.vx v4, v4, %[k0101] \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ "vle8.v v0, (%[src_y]) \n"
+ "vwcvtu.x.x.v v8, v0 \n"
+ "vwmulu.vx v12, v2, %[ug] \n"
+ "vwmaccu.vx v12, %[vg], v4 \n"
+ "vwmulu.vx v0, v2, %[ub] \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vmul.vx v8, v8, %[k0101] \n"
+ "vmulhu.vx v8, v8, %[yg] \n"
+ "vadd.vx v16, v8, %[bg] \n"
+ "vadd.vv v0, v8, v0 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwmaccu.vx v8, %[vr], v4 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vssubu.vv v12, v16, v12 \n"
+ "vssubu.vx v0, v0, %[bb] \n"
+ "vssubu.vx v8, v8, %[br] \n"
+ "vsrl.vi v0, v0, 4 \n"
+ "vsrl.vi v12, v12, 4 \n"
+ "vsrl.vi v8, v8, 4 \n"
+ "li %[k0101], 1023 \n"
+ "vminu.vx v0, v0, %[k0101] \n"
+ "vminu.vx v12, v12, %[k0101] \n"
+ "vminu.vx v8, v8, %[k0101] \n"
+ "vwcvtu.x.x.v v16, v0 \n"
+ "li %[k0101], 1024 \n"
+ "vwmaccu.vx v16, %[k0101], v12 \n"
+ "li %[k0101], 0x0c00 \n"
+ "vor.vx v8, v8, %[k0101] \n"
+ "vwcvtu.x.x.v v20, v8 \n"
+ "vsetvli zero, zero, e32, m4, ta, ma \n"
+ "vsll.vi v20, v20, 20 \n"
+ "vor.vv v16, v16, v20 \n"
+ "vse32.v v16, (%[dst_ar30]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "srli %[k0101], %[vl], 1 \n"
+ "add %[src_u], %[src_u], %[k0101] \n"
+ "add %[src_v], %[src_v], %[k0101] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_ar30], %[dst_ar30], %[vl]\n"
+ "li %[k0101], 0x0101 \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v4", "v8", "v9", "v12",
+ "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_I400TOARGBROW_RVV
+void I400ToARGBRow_RVV(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ const bool is_yb_positive = (yuvconstants->kRGBCoeffBias[4] >= 0);
+ uint16_t yg = yuvconstants->kRGBCoeffBias[0];
+ uint16_t yb = is_yb_positive
+ ? (uint16_t)(yuvconstants->kRGBCoeffBias[4] - 32)
+ : (uint16_t)(-yuvconstants->kRGBCoeffBias[4] + 32);
+ size_t k0101;
+ if (is_yb_positive) {
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+ "li %[k0101], 0x0101 \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v20, %[yg] \n"
+ "vmv.v.x v24, %[yb] \n"
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v0, (%[src_y]) \n"
+ "vwcvtu.x.x.v v16, v0 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmul.vx v16, v16, %[k0101] \n"
+ "vmulhu.vv v16, v16, v20 \n"
+ "vsaddu.vv v16, v16, v24 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnclipu.wi v8, v16, 6 \n"
+ "vmv2r.v v10, v8 \n"
+ "vmv2r.v v12, v8 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yg] "r"(yg), // %[yg]
+ [yb] "r"(yb) // %[yb]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
+ "v23", "v24", "v25", "v26", "v27");
+ } else {
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+ "li %[k0101], 0x0101 \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v20, %[yg] \n"
+ "vmv.v.x v24, %[yb] \n"
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v0, (%[src_y]) \n"
+ "vwcvtu.x.x.v v16, v0 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmul.vx v16, v16, %[k0101] \n"
+ "vmulhu.vv v16, v16, v20 \n"
+ "vssubu.vv v16, v16, v24 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnclipu.wi v8, v16, 6 \n"
+ "vmv2r.v v10, v8 \n"
+ "vmv2r.v v12, v8 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yg] "r"(yg), // %[yg]
+ [yb] "r"(yb) // %[yb]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
+ "v23", "v24", "v25", "v26", "v27");
+ }
+}
+#endif
+
+#ifdef HAS_J400TOARGBROW_RVV
+void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ size_t vl;
+ asm volatile(
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vle8.v v8, (%[src_y]) \n"
+ "vmv2r.v v10, v8 \n"
+ "vmv2r.v v12, v8 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_COPYROW_RVV
+void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m4, ta, ma \n"
+ "vle8.v v8, (%[src]) \n"
+ "vse8.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src], %[src], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
+}
+#endif
+
+#ifdef HAS_NV12TOARGBROW_RVV
+void NV12ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v30, -1 \n"
+
+ "1: \n" //
+ READNV12 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_NV12TORGB24ROW_RVV
+void NV12ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READNV12 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg3e8.v v24, (%[dst_rgb24]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
+}
+#endif
+
+#ifdef HAS_NV21TOARGBROW_RVV
+void NV21ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+ "vsetvli zero, %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v30, -1 \n"
+
+ "1: \n" //
+ READNV21 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_vu], %[src_vu], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_vu] "+r"(src_vu), // %[src_vu]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
+}
+#endif
+
+#ifdef HAS_NV21TORGB24ROW_RVV
+void NV21ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t ub, vr, ug, vg;
+ size_t yg, bb, bg, br;
+ size_t k0101;
+ asm volatile( //
+ YUVTORGB_SETUP //
+
+ "1: \n" //
+ READNV21 //
+ "sub %[w], %[w], %[vl] \n" //
+ YUVTORGB //
+ RGBTORGB8 //
+ "vsseg3e8.v v24, (%[dst_rgb24]) \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[src_vu], %[src_vu], %[vl] \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_vu] "+r"(src_vu), // %[src_vu]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [ub] "=&r"(ub), // %[ub]
+ [vr] "=&r"(vr), // %[vr]
+ [ug] "=&r"(ug), // %[ug]
+ [vg] "=&r"(vg), // %[vg]
+ [yg] "=&r"(yg), // %[yg]
+ [bb] "=&r"(bb), // %[bb]
+ [bg] "=&r"(bg), // %[bg]
+ [br] "=&r"(br), // %[br]
+ [k0101] "=&r"(k0101) // %[k0101]
+ : [yuvconst] "r"(yuvconstants) // %[yuvconst]
+ : RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
+}
+#endif
+
+// Bilinear filter [VLEN/8]x2 -> [VLEN/8]x1
+#ifdef HAS_INTERPOLATEROW_RVV
+void InterpolateRow_RVV(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ size_t dst_w = (size_t)dst_width;
+ size_t vl;
+ assert(source_y_fraction >= 0);
+ assert(source_y_fraction < 256);
+ // Blend 100 / 0 - Copy row unchanged.
+ if (y1_fraction == 0) {
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[dst_w], e8, m4, ta, ma\n"
+ "vle8.v v8, (%[src_ptr]) \n"
+ "vse8.v v8, (%[dst_ptr]) \n"
+ "sub %[dst_w], %[dst_w], %[vl] \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "bgtz %[dst_w], 1b \n"
+ : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_w] "+r"(dst_w), // %[dst_w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
+ return;
+ }
+ // Blend 50 / 50.
+ if (y1_fraction == 128) {
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[dst_w], e8, m4, ta, ma\n"
+ "vle8.v v8, (%[src_ptr]) \n"
+ "vle8.v v12, (%[src_ptr1]) \n"
+ "vaaddu.vv v8, v8, v12 \n"
+ "vse8.v v8, (%[dst_ptr]) \n"
+ "sub %[dst_w], %[dst_w], %[vl] \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[src_ptr1], %[src_ptr1], %[vl]\n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "bgtz %[dst_w], 1b \n"
+ : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_w] "+r"(dst_w), // %[dst_w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13",
+ "v14", "v15");
+ return;
+ }
+ // General purpose row blend.
+ asm volatile(
+ "vsetvli zero, %[dst_w], e16, m4, ta, ma\n"
+ "vmv.v.x v12, %[c128] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[dst_w], e8, m2, ta, ma\n"
+ "vle8.v v16, (%[src_ptr]) \n"
+ "vmv4r.v v8, v12 \n"
+ "vwmaccu.vx v8, %[y0_fraction], v16 \n"
+ "vle8.v v16, (%[src_ptr1]) \n"
+ "vwmaccu.vx v8, %[y1_fraction], v16 \n"
+ "vnsrl.wi v16, v8, 8 \n"
+ "vse8.v v16, (%[dst_ptr]) \n"
+ "sub %[dst_w], %[dst_w], %[vl] \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[src_ptr1], %[src_ptr1], %[vl]\n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "bgtz %[dst_w], 1b \n"
+ : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_w] "+r"(dst_w), // %[dst_w]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction), // %[y1_fraction]
+ [c128] "r"(128) // %[c128]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_SPLITRGBROW_RVV
+void SplitRGBRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg3e8.v v8, (%[src_rgb]) \n"
+ "vse8.v v8, (%[dst_r]) \n"
+ "vse8.v v10, (%[dst_g]) \n"
+ "vse8.v v12, (%[dst_b]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_r], %[dst_r], %[vl] \n"
+ "add %[dst_g], %[dst_g], %[vl] \n"
+ "add %[dst_b], %[dst_b], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_rgb], %[src_rgb], %[vl] \n"
+ "add %[src_rgb], %[src_rgb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_rgb] "+r"(src_rgb), // %[src_rgb]
+ [dst_r] "+r"(dst_r), // %[dst_r]
+ [dst_g] "+r"(dst_g), // %[dst_g]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13");
+}
+#endif
+
+#ifdef HAS_MERGERGBROW_RVV
+void MergeRGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ size_t vl, tmp;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v8, (%[src_r]) \n"
+ "vle8.v v10, (%[src_g]) \n"
+ "vle8.v v12, (%[src_b]) \n"
+ "vsseg3e8.v v8, (%[dst_rgb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_r], %[src_r], %[vl] \n"
+ "add %[src_g], %[src_g], %[vl] \n"
+ "add %[src_b], %[src_b], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[dst_rgb], %[dst_rgb], %[vl] \n"
+ "add %[dst_rgb], %[dst_rgb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_r] "+r"(src_r), // %[src_r]
+ [src_g] "+r"(src_g), // %[src_g]
+ [src_b] "+r"(src_b), // %[src_b]
+ [dst_rgb] "+r"(dst_rgb), // %[dst_rgb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13");
+}
+#endif
+
+#ifdef HAS_SPLITARGBROW_RVV
+void SplitARGBRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vse8.v v14, (%[dst_a]) \n"
+ "vse8.v v12, (%[dst_r]) \n"
+ "vse8.v v10, (%[dst_g]) \n"
+ "vse8.v v8, (%[dst_b]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_a], %[dst_a], %[vl] \n"
+ "add %[dst_r], %[dst_r], %[vl] \n"
+ "add %[dst_g], %[dst_g], %[vl] \n"
+ "add %[dst_b], %[dst_b], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_r] "+r"(dst_r), // %[dst_r]
+ [dst_g] "+r"(dst_g), // %[dst_g]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_MERGEARGBROW_RVV
+void MergeARGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v12, (%[src_r]) \n"
+ "vle8.v v10, (%[src_g]) \n"
+ "vle8.v v8, (%[src_b]) \n"
+ "vle8.v v14, (%[src_a]) \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_r], %[src_r], %[vl] \n"
+ "add %[src_g], %[src_g], %[vl] \n"
+ "add %[src_b], %[src_b], %[vl] \n"
+ "add %[src_a], %[src_a], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_r] "+r"(src_r), // %[src_r]
+ [src_g] "+r"(src_g), // %[src_g]
+ [src_b] "+r"(src_b), // %[src_b]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SPLITXRGBROW_RVV
+void SplitXRGBRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vse8.v v12, (%[dst_r]) \n"
+ "vse8.v v10, (%[dst_g]) \n"
+ "vse8.v v8, (%[dst_b]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_r], %[dst_r], %[vl] \n"
+ "add %[dst_g], %[dst_g], %[vl] \n"
+ "add %[dst_b], %[dst_b], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_r] "+r"(dst_r), // %[dst_r]
+ [dst_g] "+r"(dst_g), // %[dst_g]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_MERGEXRGBROW_RVV
+void MergeXRGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vle8.v v12, (%[src_r]) \n"
+ "vle8.v v10, (%[src_g]) \n"
+ "vle8.v v8, (%[src_b]) \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_r], %[src_r], %[vl] \n"
+ "add %[src_g], %[src_g], %[vl] \n"
+ "add %[src_b], %[src_b], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [src_r] "+r"(src_r), // %[src_r]
+ [src_g] "+r"(src_g), // %[src_g]
+ [src_b] "+r"(src_b), // %[src_b]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SPLITUVROW_RVV
+void SplitUVRow_RVV(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m4, ta, ma \n"
+ "vlseg2e8.v v8, (%[src_uv]) \n"
+ "vse8.v v8, (%[dst_u]) \n"
+ "vse8.v v12, (%[dst_v]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_u], %[dst_u], %[vl] \n"
+ "add %[dst_v], %[dst_v], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_MERGEUVROW_RVV
+void MergeUVRow_RVV(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m4, ta, ma \n"
+ "vle8.v v8, (%[src_u]) \n"
+ "vle8.v v12, (%[src_v]) \n"
+ "vsseg2e8.v v8, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_u], %[src_u], %[vl] \n"
+ "add %[src_v], %[src_v], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SWAPUVROW_RVV
+void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ assert(width != 0);
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m4, ta, ma \n"
+ "vle16.v v8, (%[src_uv]) \n"
+ "vsrl.vi v12, v8, 8 \n"
+ "vsll.vi v8, v8, 8 \n"
+ "vor.vv v8, v8, v12 \n"
+ "vse16.v v8, (%[dst_vu]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "add %[dst_vu], %[dst_vu], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_vu] "+r"(dst_vu), // %[dst_vu]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+// ARGB expects first 3 values to contain RGB and 4th value is ignored
+#ifdef HAS_ARGBTOYMATRIXROW_RVV
+void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ size_t vl, tmp;
+ asm volatile(
+ "lhu %[tmp], 48(%[c]) \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v8, %[tmp] \n"
+ "lbu %[vl], 0(%[c]) \n"
+ "lbu %[tmp], 1(%[c]) \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vmv.v.x v12, %[vl] \n"
+ "vmv.v.x v14, %[tmp] \n"
+ "lbu %[vl], 2(%[c]) \n"
+ "lbu %[tmp], 3(%[c]) \n"
+ "vmv.v.x v16, %[vl] \n"
+ "vmv.v.x v18, %[tmp] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v24, (%[src_argb]) \n"
+ "vwmulu.vv v20, v24, v12 \n"
+ "vwmaccu.vv v20, v14, v26 \n"
+ "vwmaccu.vv v20, v16, v28 \n"
+ "vwmaccu.vv v20, v18, v30 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vv v20, v20, v8 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v24, v20, 8 \n"
+ "vse8.v v24, (%[dst_y]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_y], %[dst_y], %[vl] \n"
+ "slli %[tmp], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_RGBTOYMATRIXROW_RVV
+void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ size_t vl, tmp;
+ asm volatile(
+ "lhu %[tmp], 48(%[c]) \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v8, %[tmp] \n"
+ "lbu %[vl], 0(%[c]) \n"
+ "lbu %[tmp], 1(%[c]) \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vmv.v.x v12, %[vl] \n"
+ "vmv.v.x v14, %[tmp] \n"
+ "lbu %[vl], 2(%[c]) \n"
+ "vmv.v.x v16, %[vl] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg3e8.v v24, (%[src_rgb]) \n"
+ "vwmulu.vv v20, v28, v16 \n"
+ "vwmaccu.vv v20, v14, v26 \n"
+ "vwmaccu.vv v20, v12, v24 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vv v20, v20, v8 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v18, v20, 8 \n"
+ "vse8.v v18, (%[dst_y]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_y], %[dst_y], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_rgb], %[src_rgb], %[vl] \n"
+ "add %[src_rgb], %[src_rgb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_rgb] "+r"(src_rgb), // %[src_rgb]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29");
+}
+#endif
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_RVV
+void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ size_t vl, tmp;
+ asm volatile(
+ "lh %[tmp], 64(%[c]) \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v16, %[tmp] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "lb %[vl], 16(%[c]) \n"
+ "lb %[tmp], 17(%[c]) \n"
+ "vmv.v.x v0, %[vl] \n"
+ "vmv.v.x v2, %[tmp] \n"
+ "lb %[vl], 18(%[c]) \n"
+ "lb %[tmp], 19(%[c]) \n"
+ "vmv.v.x v4, %[vl] \n"
+ "vmv.v.x v6, %[tmp] \n"
+ "lb %[vl], 32(%[c]) \n"
+ "lb %[tmp], 33(%[c]) \n"
+ "vmv.v.x v8, %[vl] \n"
+ "vmv.v.x v10, %[tmp] \n"
+ "lb %[vl], 34(%[c]) \n"
+ "lb %[tmp], 35(%[c]) \n"
+ "vmv.v.x v12, %[vl] \n"
+ "vmv.v.x v14, %[tmp] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v24, (%[src_argb]) \n"
+ "vwmulsu.vv v20, v0, v24 \n"
+ "vwmaccsu.vv v20, v2, v26 \n"
+ "vwmaccsu.vv v20, v4, v28 \n"
+ "vwmaccsu.vv v20, v6, v30 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vsub.vv v20, v16, v20 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v20, v20, 8 \n"
+ "vse8.v v20, (%[dst_u]) \n"
+ "vwmulsu.vv v20, v8, v24 \n"
+ "vwmaccsu.vv v20, v10, v26 \n"
+ "vwmaccsu.vv v20, v12, v28 \n"
+ "vwmaccsu.vv v20, v14, v30 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vsub.vv v20, v16, v20 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v20, v20, 8 \n"
+ "vse8.v v20, (%[dst_v]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_u], %[dst_u], %[vl] \n"
+ "add %[dst_v], %[dst_v], %[vl] \n"
+ "slli %[tmp], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28", "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_ARGBTOUVMATRIXROW_RVV
+void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ const uint8_t* src_argb_0 = src_argb;
+ const uint8_t* src_argb_1 = src_argb + src_stride_argb;
+ int w = width >> 1;
+ if (w > 0) {
+ size_t w_len = (size_t)w;
+ size_t vl, tmp;
+ asm volatile(
+ "lh %[tmp], 64(%[c]) \n"
+ "vsetvli zero, %[w_len], e16, m2, ta, ma\n"
+ "vmv.v.x v8, %[tmp] \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "lb %[vl], 16(%[c]) \n"
+ "lb %[tmp], 17(%[c]) \n"
+ "vmv.v.x v10, %[vl] \n"
+ "vmv.v.x v11, %[tmp] \n"
+ "lb %[vl], 18(%[c]) \n"
+ "lb %[tmp], 19(%[c]) \n"
+ "vmv.v.x v12, %[vl] \n"
+ "vmv.v.x v13, %[tmp] \n"
+ "lb %[vl], 32(%[c]) \n"
+ "lb %[tmp], 33(%[c]) \n"
+ "vmv.v.x v14, %[vl] \n"
+ "vmv.v.x v15, %[tmp] \n"
+ "lb %[vl], 34(%[c]) \n"
+ "lb %[tmp], 35(%[c]) \n"
+ "vmv.v.x v16, %[vl] \n"
+ "vmv.v.x v17, %[tmp] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w_len], e8, m1, ta, ma\n"
+ "vlseg8e8.v v20, (%[src_argb_0]) \n"
+ "vlseg8e8.v v0, (%[src_argb_1]) \n"
+ "vwaddu.vv v18, v20, v24 \n"
+ "vwaddu.wv v18, v18, v0 \n"
+ "vwaddu.wv v18, v18, v4 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v20, v18, 2 \n"
+ "vwaddu.vv v18, v21, v25 \n"
+ "vwaddu.wv v18, v18, v1 \n"
+ "vwaddu.wv v18, v18, v5 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v21, v18, 2 \n"
+ "vwaddu.vv v18, v22, v26 \n"
+ "vwaddu.wv v18, v18, v2 \n"
+ "vwaddu.wv v18, v18, v6 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v22, v18, 2 \n"
+ "vwaddu.vv v18, v23, v27 \n"
+ "vwaddu.wv v18, v18, v3 \n"
+ "vwaddu.wv v18, v18, v7 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v23, v18, 2 \n"
+ "vwmulsu.vv v18, v10, v20 \n"
+ "vwmaccsu.vv v18, v11, v21 \n"
+ "vwmaccsu.vv v18, v12, v22 \n"
+ "vwmaccsu.vv v18, v13, v23 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vsub.vv v18, v8, v18 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v24, v18, 8 \n"
+ "vse8.v v24, (%[dst_u]) \n"
+ "vwmulsu.vv v18, v14, v20 \n"
+ "vwmaccsu.vv v18, v15, v21 \n"
+ "vwmaccsu.vv v18, v16, v22 \n"
+ "vwmaccsu.vv v18, v17, v23 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vsub.vv v18, v8, v18 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v20, v18, 8 \n"
+ "vse8.v v20, (%[dst_v]) \n"
+ "sub %[w_len], %[w_len], %[vl] \n"
+ "add %[dst_u], %[dst_u], %[vl] \n"
+ "add %[dst_v], %[dst_v], %[vl] \n"
+ "slli %[tmp], %[vl], 3 \n"
+ "add %[src_argb_0], %[src_argb_0], %[tmp]\n"
+ "add %[src_argb_1], %[src_argb_1], %[tmp]\n"
+ "bgtz %[w_len], 1b \n"
+ : [src_argb_0] "+r"(src_argb_0), // %[src_argb_0]
+ [src_argb_1] "+r"(src_argb_1), // %[src_argb_1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [w_len] "+r"(w_len), // %[w_len]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+ "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27");
+ }
+ if (width & 1) {
+ uint8_t b = (src_argb_0[0] + src_argb_1[0] + 1) >> 1;
+ uint8_t g = (src_argb_0[1] + src_argb_1[1] + 1) >> 1;
+ uint8_t r = (src_argb_0[2] + src_argb_1[2] + 1) >> 1;
+ uint8_t a = (src_argb_0[3] + src_argb_1[3] + 1) >> 1;
+ dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g +
+ c->kRGBToU[2] * r + c->kRGBToU[3] * a)) >>
+ 8;
+ dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g +
+ c->kRGBToV[2] * r + c->kRGBToV[3] * a)) >>
+ 8;
+ }
+}
+#endif
+
+#ifdef HAS_RGBTOUV444MATRIXROW_RVV
+void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ size_t vl, tmp;
+ asm volatile(
+ "lh %[tmp], 64(%[c]) \n"
+ "vsetvli zero, %[w], e16, m4, ta, ma \n"
+ "vmv.v.x v16, %[tmp] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "lb %[vl], 16(%[c]) \n"
+ "lb %[tmp], 17(%[c]) \n"
+ "vmv.v.x v0, %[vl] \n"
+ "vmv.v.x v2, %[tmp] \n"
+ "lb %[vl], 18(%[c]) \n"
+ "vmv.v.x v4, %[vl] \n"
+ "lb %[vl], 32(%[c]) \n"
+ "lb %[tmp], 33(%[c]) \n"
+ "vmv.v.x v8, %[vl] \n"
+ "vmv.v.x v10, %[tmp] \n"
+ "lb %[vl], 34(%[c]) \n"
+ "vmv.v.x v12, %[vl] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg3e8.v v24, (%[src_rgb]) \n"
+ "vwmulsu.vv v20, v4, v28 \n"
+ "vwmaccsu.vv v20, v2, v26 \n"
+ "vwmaccsu.vv v20, v0, v24 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vsub.vv v20, v16, v20 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v20, v20, 8 \n"
+ "vse8.v v20, (%[dst_u]) \n"
+ "vwmulsu.vv v20, v12, v28 \n"
+ "vwmaccsu.vv v20, v10, v26 \n"
+ "vwmaccsu.vv v20, v8, v24 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vsub.vv v20, v16, v20 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v20, v20, 8 \n"
+ "vse8.v v20, (%[dst_v]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_u], %[dst_u], %[vl] \n"
+ "add %[dst_v], %[dst_v], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[src_rgb], %[src_rgb], %[vl] \n"
+ "add %[src_rgb], %[src_rgb], %[tmp]\n"
+ "bgtz %[w], 1b \n"
+ : [src_rgb] "+r"(src_rgb), // %[src_rgb]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9",
+ "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21",
+ "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29");
+}
+#endif
+
+#ifdef HAS_RGBTOUVMATRIXROW_RVV
+void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ assert(width != 0);
+ const uint8_t* src_rgb_0 = src_rgb;
+ const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
+ int w = width >> 1;
+ if (w > 0) {
+ size_t w_len = (size_t)w;
+ size_t vl, tmp;
+ asm volatile(
+ "lh %[tmp], 64(%[c]) \n"
+ "vsetvli zero, %[w_len], e16, m2, ta, ma\n"
+ "vmv.v.x v8, %[tmp] \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "lb %[vl], 16(%[c]) \n"
+ "lb %[tmp], 17(%[c]) \n"
+ "vmv.v.x v10, %[vl] \n"
+ "vmv.v.x v11, %[tmp] \n"
+ "lb %[vl], 18(%[c]) \n"
+ "vmv.v.x v12, %[vl] \n"
+ "lb %[vl], 32(%[c]) \n"
+ "lb %[tmp], 33(%[c]) \n"
+ "vmv.v.x v14, %[vl] \n"
+ "vmv.v.x v15, %[tmp] \n"
+ "lb %[vl], 34(%[c]) \n"
+ "vmv.v.x v16, %[vl] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w_len], e8, m1, ta, ma\n"
+ "vlseg6e8.v v20, (%[src_rgb_0]) \n"
+ "vlseg6e8.v v0, (%[src_rgb_1]) \n"
+ "vwaddu.vv v18, v20, v23 \n"
+ "vwaddu.wv v18, v18, v0 \n"
+ "vwaddu.wv v18, v18, v3 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v20, v18, 2 \n"
+ "vwaddu.vv v18, v21, v24 \n"
+ "vwaddu.wv v18, v18, v1 \n"
+ "vwaddu.wv v18, v18, v4 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v21, v18, 2 \n"
+ "vwaddu.vv v18, v22, v25 \n"
+ "vwaddu.wv v18, v18, v2 \n"
+ "vwaddu.wv v18, v18, v5 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vi v18, v18, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v22, v18, 2 \n"
+ "vwmulsu.vv v18, v12, v22 \n"
+ "vwmaccsu.vv v18, v11, v21 \n"
+ "vwmaccsu.vv v18, v10, v20 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vsub.vv v18, v8, v18 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v24, v18, 8 \n"
+ "vse8.v v24, (%[dst_u]) \n"
+ "vwmulsu.vv v18, v16, v22 \n"
+ "vwmaccsu.vv v18, v15, v21 \n"
+ "vwmaccsu.vv v18, v14, v20 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vsub.vv v18, v8, v18 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v20, v18, 8 \n"
+ "vse8.v v20, (%[dst_v]) \n"
+ "sub %[w_len], %[w_len], %[vl] \n"
+ "add %[dst_u], %[dst_u], %[vl] \n"
+ "add %[dst_v], %[dst_v], %[vl] \n"
+ "slli %[tmp], %[vl], 1 \n"
+ "add %[tmp], %[tmp], %[vl] \n"
+ "slli %[tmp], %[tmp], 1 \n"
+ "add %[src_rgb_0], %[src_rgb_0], %[tmp]\n"
+ "add %[src_rgb_1], %[src_rgb_1], %[tmp]\n"
+ "bgtz %[w_len], 1b \n"
+ : [src_rgb_0] "+r"(src_rgb_0), // %[src_rgb_0]
+ [src_rgb_1] "+r"(src_rgb_1), // %[src_rgb_1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [w_len] "+r"(w_len), // %[w_len]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp] "=&r"(tmp) // %[tmp]
+ : [c] "r"(c) // %[c]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8",
+ "v9", "v10", "v11", "v12", "v14", "v15", "v16", "v18", "v19", "v20",
+ "v21", "v22", "v23", "v24", "v25");
+ }
+ if (width & 1) {
+ uint8_t b = (src_rgb_0[0] + src_rgb_1[0] + 1) >> 1;
+ uint8_t g = (src_rgb_0[1] + src_rgb_1[1] + 1) >> 1;
+ uint8_t r = (src_rgb_0[2] + src_rgb_1[2] + 1) >> 1;
+ dst_u[0] = (c->kAddUV[0] -
+ (c->kRGBToU[0] * b + c->kRGBToU[1] * g + c->kRGBToU[2] * r)) >>
+ 8;
+ dst_v[0] = (c->kAddUV[0] -
+ (c->kRGBToV[0] * b + c->kRGBToV[1] * g + c->kRGBToV[2] * r)) >>
+ 8;
+ }
+}
+#endif
+
+// Blend src_argb over src_argb1 and store to dst_argb.
+// dst_argb may be src_argb or src_argb1.
+// src_argb: RGB values have already been pre-multiplied by the a.
+#ifdef HAS_ARGBBLENDROW_RVV
+void ARGBBlendRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "vsetvli %[vl], zero, e8, m2, ta, ma \n"
+ "vmv.v.i v14, -1 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v16, (%[src_argb]) \n"
+ "vlseg4e8.v v24, (%[src_argb1]) \n"
+ "vmulhu.vv v8, v24, v22 \n"
+ "vmulhu.vv v10, v26, v22 \n"
+ "vmulhu.vv v12, v28, v22 \n"
+ "vsub.vv v8, v24, v8 \n"
+ "vsub.vv v10, v26, v10 \n"
+ "vsub.vv v12, v28, v12 \n"
+ "vsaddu.vv v8, v8, v16 \n"
+ "vsaddu.vv v10, v10, v18 \n"
+ "vsaddu.vv v12, v12, v20 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "add %[src_argb1], %[src_argb1], %[vl]\n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src_argb1] "+r"(src_argb1), // %[src_argb1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_ARGBORROW_RVV
+void ARGBOrRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ size_t vl;
+ int w = width * 4;
+ asm volatile(
+ "vsetvli %[vl], zero, e32, m8, ta, ma \n"
+ "vmv.v.x v16, %[val] \n"
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m8, ta, ma \n"
+ "vle8.v v8, (%[src]) \n"
+ "vor.vv v8, v8, v16 \n"
+ "vse8.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src], %[src], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(w), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [val] "r"(value) // %[val]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_ARGBALPHABLENDROW_RVV
+void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v16, (%[src_argb]) \n"
+ "vlseg4e8.v v24, (%[src_argb1]) \n"
+ "vwmulu.vv v4, v30, v22 \n"
+ "vwmulu.vx v8, v22, %[k255] \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vi v4, v4, 1 \n"
+ "vmulhu.vx v4, v4, %[k0101] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v0, v4, 0 \n"
+ "vsub.vv v0, v22, v0 \n"
+ "vadd.vv v30, v30, v0 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vzext.vf2 v12, v30 \n"
+ "vdivu.vv v8, v8, v12 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v0, v8, 0 \n"
+ "vrsub.vx v2, v0, %[k255] \n"
+ "vwmulu.vv v4, v24, v2 \n"
+ "vwmulu.vv v8, v26, v2 \n"
+ "vwmulu.vv v12, v28, v2 \n"
+ "vwmaccu.vv v4, v0, v16 \n"
+ "vwmaccu.vv v8, v0, v18 \n"
+ "vwmaccu.vv v12, v0, v20 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vi v4, v4, 1 \n"
+ "vadd.vi v8, v8, 1 \n"
+ "vadd.vi v12, v12, 1 \n"
+ "vmulhu.vx v4, v4, %[k0101] \n"
+ "vmulhu.vx v8, v8, %[k0101] \n"
+ "vmulhu.vx v12, v12, %[k0101] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v24, v4, 0 \n"
+ "vnsrl.wi v26, v8, 0 \n"
+ "vnsrl.wi v28, v12, 0 \n"
+ "vsseg4e8.v v24, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl] \n"
+ "add %[src_argb1], %[src_argb1], %[vl]\n"
+ "add %[dst_argb], %[dst_argb], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src_argb1] "+r"(src_argb1), // %[src_argb1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k0101] "r"(0x0101), // %[k0101]
+ [k255] "r"(255) // %[k255]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28", "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_BLENDPLANEROW_RVV
+void BlendPlaneRow_RVV(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v16, (%[src0]) \n"
+ "vle8.v v18, (%[src1]) \n"
+ "vle8.v v20, (%[alpha]) \n"
+ "vrsub.vx v22, v20, %[k255] \n"
+ "vwmulu.vv v8, v20, v16 \n"
+ "vwmaccu.vv v8, v22, v18 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vx v8, v8, %[k255] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v16, v8, 8 \n"
+ "vse8.v v16, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src0], %[src0], %[vl] \n"
+ "add %[src1], %[src1], %[vl] \n"
+ "add %[alpha], %[alpha], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src0] "+r"(src0), // %[src0]
+ [src1] "+r"(src1), // %[src1]
+ [alpha] "+r"(alpha), // %[alpha]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k255] "r"(255)
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+// Attenuate: (f * a + 255) >> 8
+#ifdef HAS_ARGBATTENUATEROW_RVV
+void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_argb]) \n"
+ "vwmulu.vv v16, v8, v14 \n"
+ "vwmulu.vv v20, v10, v14 \n"
+ "vwmulu.vv v24, v12, v14 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vadd.vx v16, v16, %[k255] \n"
+ "vadd.vx v20, v20, %[k255] \n"
+ "vadd.vx v24, v24, %[k255] \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v8, v16, 8 \n"
+ "vnsrl.wi v10, v20, 8 \n"
+ "vnsrl.wi v12, v24, 8 \n"
+ "vsseg4e8.v v8, (%[dst_argb]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k255] "r"(255)
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27");
+}
+#endif
+
+#ifdef HAS_ARGBMULTIPLYROW_RVV
+void ARGBMultiplyRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ assert(width != 0);
+ size_t w = (size_t)width * 4;
+ size_t vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v8, (%[src0]) \n"
+ "vle8.v v10, (%[src1]) \n"
+ "vwmulu.vv v12, v8, v10 \n"
+ "vwaddu.wx v12, v12, %[k128] \n"
+ "vnsrl.wi v8, v12, 8 \n"
+ "vse8.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src0], %[src0], %[vl] \n"
+ "add %[src1], %[src1], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src0] "+r"(src_argb), // %[src0]
+ [src1] "+r"(src_argb1), // %[src1]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(w), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [k128] "r"(128) // %[k128]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_ARGBEXTRACTALPHAROW_RVV
+void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m4, ta, ma \n"
+ "vle32.v v8, (%[src_argb]) \n"
+ "vnsrl.wi v16, v8, 16 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v8, v16, 8 \n"
+ "vse8.v v8, (%[dst_a]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_a], %[dst_a], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_argb], %[src_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19");
+}
+#endif
+
+#ifdef HAS_ARGBCOPYYTOALPHAROW_RVV
+void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "vle8.v v16, (%[src]) \n"
+ "vle32.v v8, (%[dst]) \n"
+ "vzext.vf4 v24, v16 \n"
+ "vand.vx v8, v8, %[mask] \n"
+ "vsll.vi v24, v24, 24 \n"
+ "vor.vv v8, v8, v24 \n"
+ "vse32.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src], %[src], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [mask] "r"(0x00ffffffu) // %[mask]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
+ "v31");
+}
+#endif
+
+#ifdef HAS_CONVERT16TO8ROW_RVV
+void Convert16To8Row_RVV(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle16.v v8, (%[src_y]) \n"
+ "vnsrl.wx v16, v8, %[shift] \n"
+ "vse8.v v16, (%[dst_y]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_y], %[dst_y], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_CONVERT8TO16ROW_RVV
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 512 = 9 bits
+// 1024 = 10 bits
+// 4096 = 12 bits
+// 65536 = 16 bits
+void Convert8To16Row_RVV(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle8.v v16, (%[src_y]) \n"
+ "vwcvtu.x.x.v v8, v16 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmul.vx v8, v8, %[k0101] \n"
+ "vsrl.vx v8, v8, %[shift] \n"
+ "vse16.v v8, (%[dst_y]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_y], %[dst_y], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(16 - bits), [k0101] "r"(0x0101)
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_MULTIPLYROW_16_RVV
+void MultiplyRow_16_RVV(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m4, ta, ma \n"
+ "vle16.v v8, (%[src_y]) \n"
+ "vmul.vx v8, v8, %[scale] \n"
+ "vse16.v v8, (%[dst_y]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_y], %[src_y], %[vl] \n"
+ "add %[dst_y], %[dst_y], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale) // %[scale]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
+}
+#endif
+
+#ifdef HAS_HALFROW_16TO8_RVV
+void HalfRow_16To8_RVV(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ int vl;
+ asm volatile(
+ "slli %[src_uv1], %[src_uv1], 1 \n"
+ "add %[src_uv1], %[src_uv], %[src_uv1]\n"
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m4, ta, ma \n"
+ "vle16.v v8, (%[src_uv]) \n"
+ "vle16.v v12, (%[src_uv1]) \n"
+ "vaaddu.vv v8, v8, v12 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wx v16, v8, %[shift] \n"
+ "vse8.v v16, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "add %[src_uv1], %[src_uv1], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [src_uv1] "+r"(src_uv_stride), // %[src_uv1]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_HALFWIDTHROW_16TO8_RVV
+void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ int vl;
+ asm volatile(
+ "slli %[t], %[t], 1 \n"
+ "add %[t], %[s], %[t] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m2, ta, ma \n"
+ "vlseg2e16.v v16, (%[s]) \n"
+ "vlseg2e16.v v20, (%[t]) \n"
+ "vwaddu.vx v8, v16, %[c2] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vnsrl.wi v16, v8, 2 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wx v8, v16, %[shift] \n"
+ "vse8.v v8, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[s], %[s], %[vl] \n"
+ "add %[t], %[t], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [s] "+r"(src_uv), // %[s]
+ [t] "+r"(src_uv_stride), // %[t]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(__builtin_clz((int32_t)scale) - 15), // %[shift]
+ [c2] "r"(2) // %[c2]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_ARGBMIRRORROW_RVV
+void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
+ int vl;
+ ptrdiff_t byte_vl;
+ src += (ptrdiff_t)width * 4;
+ asm volatile(
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "slli %[byte_vl], %[vl], 2 \n"
+ "vid.v v16 \n"
+ "addi %[vl], %[vl], -1 \n"
+ "vrsub.vx v16, v16, %[vl] \n"
+ "addi %[vl], %[vl], 1 \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bltz %[w], 2f \n"
+ "1: \n"
+ "sub %[src], %[src], %[byte_vl] \n"
+ "vle32.v v8, (%[src]) \n"
+ "vrgather.vv v0, v8, v16 \n"
+ "vse32.v v0, (%[dst]) \n"
+ "add %[dst], %[dst], %[byte_vl] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bgez %[w], 1b \n"
+ "2: \n"
+ "add %[w], %[w], %[vl] \n"
+ "beqz %[w], 3f \n"
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "slli %[byte_vl], %[vl], 2 \n"
+ "vid.v v16 \n"
+ "addi %[vl], %[vl], -1 \n"
+ "vrsub.vx v16, v16, %[vl] \n"
+ "sub %[src], %[src], %[byte_vl] \n"
+ "vle32.v v8, (%[src]) \n"
+ "vrgather.vv v0, v8, v16 \n"
+ "vse32.v v0, (%[dst]) \n"
+ "3: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [byte_vl] "=&r"(byte_vl) // %[byte_vl]
+ :
+ : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif // HAS_ARGBMIRRORROW_RVV
+
+#ifdef HAS_ARGBSETROW_RVV
+void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width) {
+ size_t vl;
+ asm volatile(
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "vmv.v.x v8, %[v32] \n"
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "vse32.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bnez %[w], 1b \n"
+ : [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [v32] "r"(v32) // %[v32]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif // HAS_ARGBSETROW_RVV
+
+#ifdef HAS_ARGBSHUFFLEROW_RVV
+// Shuffle the 4 bytes of each ARGB pixel with a 16 byte shuffler table (only
+// the first 4 entries are used, the pattern repeats every pixel), e.g.
+// {0, 1, 2, 3} is a copy and {2, 1, 0, 3} swaps R and B.
+// Byte index vector: index[i] = (i & ~3) + shuffler[i & 3], gathered with a
+// 16 bit index so any VLEN works (e8 indices would be limited to 256 bytes).
+void ARGBShuffleRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ size_t vl;
+ size_t w = (size_t)width * 4; // bytes
+ // The 4 shuffler bytes packed little endian so a 32 bit broadcast lays them
+ // out as s0 s1 s2 s3 s0 s1 s2 s3 ... in memory order.
+ uint32_t pattern = (uint32_t)shuffler[0] | ((uint32_t)shuffler[1] << 8) |
+ ((uint32_t)shuffler[2] << 16) |
+ ((uint32_t)shuffler[3] << 24);
+ asm volatile(
+ // v16..v23 (e16, m8): index = (i & ~3) + shuffler[i & 3].
+ "vsetvli %[vl], %[w], e16, m8, ta, ma \n"
+ "vid.v v16 \n"
+ "vand.vi v16, v16, -4 \n"
+ "srli %[vl], %[vl], 2 \n"
+ "vsetvli zero, %[vl], e32, m4, ta, ma \n"
+ "vmv.v.x v8, %[pattern] \n" // s0 s1 s2 s3 ...
+ "vsetvli %[vl], %[w], e16, m8, ta, ma \n"
+ "vzext.vf2 v24, v8 \n" // bytes -> e16
+ "vadd.vv v16, v16, v24 \n"
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m4, ta, ma \n"
+ "vle8.v v8, (%[src]) \n"
+ "vrgatherei16.vv v0, v8, v16 \n"
+ "vse8.v v0, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src], %[src], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bnez %[w], 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(w), // %[w]
+ [vl] "=&r"(vl) // %[vl]
+ : [pattern] "r"(pattern) // %[pattern]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10",
+ "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29", "v30", "v31");
+}
+#endif // HAS_ARGBSHUFFLEROW_RVV
+
+#ifdef HAS_MIRRORUVROW_RVV
+void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ int vl;
+ ptrdiff_t byte_vl;
+ src_uv += (ptrdiff_t)width * 2;
+ asm volatile(
+ "vsetvli %[vl], %[w], e16, m8, ta, ma \n"
+ "slli %[byte_vl], %[vl], 1 \n"
+ "vid.v v16 \n"
+ "addi %[vl], %[vl], -1 \n"
+ "vrsub.vx v16, v16, %[vl] \n"
+ "addi %[vl], %[vl], 1 \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bltz %[w], 2f \n"
+ "1: \n"
+ "sub %[src], %[src], %[byte_vl] \n"
+ "vle16.v v8, (%[src]) \n"
+ "vrgather.vv v0, v8, v16 \n"
+ "vse16.v v0, (%[dst]) \n"
+ "add %[dst], %[dst], %[byte_vl] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "bgez %[w], 1b \n"
+ "2: \n"
+ "add %[w], %[w], %[vl] \n"
+ "beqz %[w], 3f \n"
+ "vsetvli %[vl], %[w], e16, m8, ta, ma \n"
+ "slli %[byte_vl], %[vl], 1 \n"
+ "vid.v v16 \n"
+ "addi %[vl], %[vl], -1 \n"
+ "vrsub.vx v16, v16, %[vl] \n"
+ "sub %[src], %[src], %[byte_vl] \n"
+ "vle16.v v8, (%[src]) \n"
+ "vrgather.vv v0, v8, v16 \n"
+ "vse16.v v0, (%[dst]) \n"
+ "3: \n"
+ : [src] "+r"(src_uv), // %[src]
+ [dst] "+r"(dst_uv), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [byte_vl] "=&r"(byte_vl) // %[byte_vl]
+ :
+ : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+}
+#endif // HAS_MIRRORUVROW_RVV
+
+#ifdef HAS_SSIM4X4ROW_RVV
+#define SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \
+ "vlseg4e8.v v0, (" #ptr_a \
+ ") \n" \
+ "vlseg4e8.v v4, (" #ptr_b \
+ ") \n" \
+ "vwaddu.vv v10, v0, v1 \n" \
+ "vwaddu.vv v11, v2, v3 \n" \
+ "vwaddu.vv v12, v4, v5 \n" \
+ "vwaddu.vv v13, v6, v7 \n" \
+ "vwmulu.vv v14, v0, v4 \n" \
+ "vwmulu.vv v15, v1, v5 \n" \
+ "vwmulu.vv v24, v2, v6 \n" \
+ "vwmulu.vv v25, v3, v7 \n" \
+ "vwmulu.vv v26, v0, v0 \n" \
+ "vwmulu.vv v27, v1, v1 \n" \
+ "vwmulu.vv v28, v2, v2 \n" \
+ "vwmulu.vv v29, v3, v3 \n" \
+ "vwmulu.vv v0, v4, v4 \n" \
+ "vwmulu.vv v1, v5, v5 \n" \
+ "vwmulu.vv v2, v6, v6 \n" \
+ "vwmulu.vv v3, v7, v7 \n"
+
+#define SSIM4X4_ACCUM_RVV(ptr_a, ptr_b) \
+ "vsetvli zero, zero, e8, mf2, ta, ma \n" \
+ SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \
+ "vsetvli zero, zero, e16, m1, ta, ma \n" \
+ "vadd.vv v10, v10, v11 \n" \
+ "vadd.vv v12, v12, v13 \n" \
+ "vadd.vv v8, v8, v10 \n" \
+ "vadd.vv v9, v9, v12 \n" \
+ "vwaddu.wv v22, v22, v14 \n" \
+ "vwaddu.wv v22, v22, v15 \n" \
+ "vwaddu.wv v22, v22, v24 \n" \
+ "vwaddu.wv v22, v22, v25 \n" \
+ "vwaddu.wv v20, v20, v26 \n" \
+ "vwaddu.wv v20, v20, v27 \n" \
+ "vwaddu.wv v20, v20, v28 \n" \
+ "vwaddu.wv v20, v20, v29 \n" \
+ "vwaddu.wv v20, v20, v0 \n" \
+ "vwaddu.wv v20, v20, v1 \n" \
+ "vwaddu.wv v20, v20, v2 \n" \
+ "vwaddu.wv v20, v20, v3 \n"
+
+void Ssim4x4Row_RVV(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ size_t vl;
+ size_t byte_vl;
+ const uint8_t* tmp_a;
+ const uint8_t* tmp_b;
+ width >>= 2;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, mf2, ta, ma \n"
+ "add %[tmp_a], %[src_a], %[stride_a]\n"
+ "add %[tmp_b], %[src_b], %[stride_b]\n"
+ SSIM4X4_LOAD_RVV(%[src_a], %[src_b])
+ "vsetvli zero, zero, e16, m1, ta, ma \n"
+ "vadd.vv v8, v10, v11 \n"
+ "vadd.vv v9, v12, v13 \n"
+ "vwaddu.vv v22, v14, v15 \n"
+ "vwaddu.wv v22, v22, v24 \n"
+ "vwaddu.wv v22, v22, v25 \n"
+ "vwaddu.vv v20, v26, v27 \n"
+ "vwaddu.wv v20, v20, v28 \n"
+ "vwaddu.wv v20, v20, v29 \n"
+ "vwaddu.wv v20, v20, v0 \n"
+ "vwaddu.wv v20, v20, v1 \n"
+ "vwaddu.wv v20, v20, v2 \n"
+ "vwaddu.wv v20, v20, v3 \n"
+ SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
+ "add %[tmp_a], %[tmp_a], %[stride_a]\n"
+ "add %[tmp_b], %[tmp_b], %[stride_b]\n"
+ SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
+ "add %[tmp_a], %[tmp_a], %[stride_a]\n"
+ "add %[tmp_b], %[tmp_b], %[stride_b]\n"
+ SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
+ "vwcvtu.x.x.v v16, v8 \n"
+ "vwcvtu.x.x.v v18, v9 \n"
+ "vsetvli zero, zero, e32, m2, ta, ma \n"
+ "vsseg4e32.v v16, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[byte_vl], %[vl], 2 \n"
+ "add %[src_a], %[src_a], %[byte_vl]\n"
+ "add %[src_b], %[src_b], %[byte_vl]\n"
+ "slli %[byte_vl], %[vl], 4 \n"
+ "add %[dst], %[dst], %[byte_vl] \n"
+ "bgtz %[w], 1b \n"
+ : [src_a] "+r"(src_a), // %[src_a]
+ [src_b] "+r"(src_b), // %[src_b]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(width), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [byte_vl] "=&r"(byte_vl), // %[byte_vl]
+ [tmp_a] "=&r"(tmp_a), // %[tmp_a]
+ [tmp_b] "=&r"(tmp_b) // %[tmp_b]
+ : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a]
+ [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b]
+ : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27", "v28", "v29");
+}
+#undef SSIM4X4_LOAD_RVV
+#undef SSIM4X4_ACCUM_RVV
+#endif // HAS_SSIM4X4ROW_RVV
+
+#ifdef HAS_SSIMWINDOWROW_RVV
+float SsimWindowRow_RVV(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ size_t vl;
+ size_t byte_vl;
+ const uint32_t* row0_next = row0 + 4;
+ const uint32_t* row1_next = row1 + 4;
+ float total;
+ asm volatile(
+ "vsetvli %[vl], zero, e32, m2, ta, ma \n"
+ "vmv.v.i v0, 0 \n"
+ "vmv.s.x v4, zero \n"
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m2, tu, ma \n"
+ "vlseg4e32.v v8, (%[row0]) \n"
+ "vlseg4e32.v v16, (%[row0_next]) \n"
+ "vadd.vv v8, v8, v16 \n"
+ "vadd.vv v10, v10, v18 \n"
+ "vadd.vv v12, v12, v20 \n"
+ "vadd.vv v14, v14, v22 \n"
+ "vlseg4e32.v v16, (%[row1]) \n"
+ "vlseg4e32.v v24, (%[row1_next]) \n"
+ "vadd.vv v16, v16, v24 \n"
+ "vadd.vv v18, v18, v26 \n"
+ "vadd.vv v20, v20, v28 \n"
+ "vadd.vv v22, v22, v30 \n"
+ "vadd.vv v8, v8, v16 \n"
+ "vadd.vv v10, v10, v18 \n"
+ "vadd.vv v12, v12, v20 \n"
+ "vadd.vv v14, v14, v22 \n"
+ "vmul.vv v16, v8, v10 \n"
+ "vsll.vi v16, v16, 1 \n"
+ "vmul.vv v18, v8, v8 \n"
+ "vmacc.vv v18, v10, v10 \n"
+ "vsll.vi v14, v14, 7 \n"
+ "vsub.vv v14, v14, v16 \n"
+ "vadd.vx v16, v16, %[c1] \n"
+ "vadd.vx v14, v14, %[c2] \n"
+ "vsll.vi v12, v12, 6 \n"
+ "vsub.vv v12, v12, v18 \n"
+ "vadd.vx v18, v18, %[c1] \n"
+ "vadd.vx v12, v12, %[c2] \n"
+ "vfcvt.f.x.v v16, v16 \n"
+ "vfcvt.f.x.v v14, v14 \n"
+ "vfcvt.f.x.v v18, v18 \n"
+ "vfcvt.f.x.v v12, v12 \n"
+ "vfmul.vv v16, v16, v14 \n"
+ "vfmul.vv v18, v18, v12 \n"
+ "vfdiv.vv v8, v16, v18 \n"
+ "vfadd.vv v0, v0, v8 \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[byte_vl], %[vl], 4 \n"
+ "add %[row0], %[row0], %[byte_vl] \n"
+ "add %[row0_next], %[row0_next], %[byte_vl]\n"
+ "add %[row1], %[row1], %[byte_vl] \n"
+ "add %[row1_next], %[row1_next], %[byte_vl]\n"
+ "bgtz %[w], 1b \n"
+ "vsetvli %[vl], zero, e32, m2, ta, ma \n"
+ "vfredusum.vs v4, v0, v4 \n"
+ "vfmv.f.s %[total], v4 \n"
+ : [row0] "+r"(row0), // %[row0]
+ [row0_next] "+r"(row0_next), // %[row0_next]
+ [row1] "+r"(row1), // %[row1]
+ [row1_next] "+r"(row1_next), // %[row1_next]
+ [w] "+r"(num_windows), // %[w]
+ [vl] "=&r"(vl), // %[vl]
+ [byte_vl] "=&r"(byte_vl), // %[byte_vl]
+ [total] "=f"(total) // %[total]
+ : [c1] "r"(26634), // %[c1]
+ [c2] "r"(239708) // %[c2]
+ : "memory", RVV_VL_CLOBBER "vtype", "v0", "v1", "v4", "v8", "v9", "v10",
+ "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",
+ "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
+ "v31");
+ return total;
+}
+#endif // HAS_SSIMWINDOWROW_RVV
+
+#ifdef HAS_ARGBDETECTROW_RVV
+uint32_t ARGBDetectRow_RVV(const uint8_t* argb, int width) {
+ size_t vl;
+ size_t byte_vl;
+ long idx = -1;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m8, ta, ma \n"
+ "slli %[byte_vl], %[vl], 2 \n"
+ "vsetvli zero, %[byte_vl], e8, m8, ta, ma \n"
+ "vle8.v v8, (%[argb]) \n"
+ "vsetvli zero, %[vl], e32, m8, ta, ma \n"
+ "vor.vx v8, v8, %[mask_rgb] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[argb], %[argb], %[byte_vl] \n"
+ "vsetvli zero, %[byte_vl], e8, m8, ta, ma \n"
+ "vmsne.vi v0, v8, -1 \n"
+ "vfirst.m %[idx], v0 \n"
+ "bgez %[idx], 2f \n"
+ "bgtz %[w], 1b \n"
+ "2: \n"
+ : [argb] "+r"(argb), [w] "+r"(width), [vl] "=&r"(vl),
+ [byte_vl] "=&r"(byte_vl), [idx] "+r"(idx)
+ : [mask_rgb] "r"(0x00ffff00u)
+ : "memory", RVV_VL_CLOBBER "vtype", "v0", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15");
+ if (idx >= 0) {
+ return (idx & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
+ }
+ return 0;
+}
+#endif // HAS_ARGBDETECTROW_RVV
+
+#ifdef HAS_SUMSQUAREERROR_RVV
+uint32_t SumSquareError_RVV(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ size_t vl;
+ uint32_t sse;
+ asm volatile(
+ "vsetvli %[vl], zero, e32, m8, ta, ma \n"
+ "vmv.v.i v24, 0 \n"
+ "vmv.s.x v0, zero \n"
+ "1: \n"
+ "vsetvli %[vl], %[count], e8, m2, tu, ma \n"
+ "vle8.v v8, (%[src_a]) \n"
+ "vle8.v v10, (%[src_b]) \n"
+ "vwsubu.vv v12, v8, v10 \n"
+ "sub %[count], %[count], %[vl] \n"
+ "add %[src_a], %[src_a], %[vl] \n"
+ "add %[src_b], %[src_b], %[vl] \n"
+ "vsetvli zero, %[vl], e16, m4, tu, ma \n"
+ "vwmacc.vv v24, v12, v12 \n"
+ "bgtz %[count], 1b \n"
+ "vsetvli %[vl], zero, e32, m8, ta, ma \n"
+ "vredsum.vs v0, v24, v0 \n"
+ "vmv.x.s %[sse], v0 \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [vl] "=&r"(vl), [sse] "=r"(sse)
+ :
+ : "memory", RVV_VL_CLOBBER "vtype", "v0", "v8", "v9", "v10", "v11", "v12",
+ "v13", "v14", "v15", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
+ "v31");
+ return sse;
+}
+#endif // HAS_SUMSQUAREERROR_RVV
+
+#ifdef HAS_HAMMINGDISTANCE_RVV
+static const uint8_t kNibblePopcnt_RVV[16] = {0, 1, 1, 2, 1, 2, 2, 3,
+ 1, 2, 2, 3, 2, 3, 3, 4};
+
+uint32_t HammingDistance_RVV(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ size_t vl;
+ uint32_t diff;
+ asm volatile(
+ "vsetivli zero, 16, e8, m1, ta, ma \n"
+ "vle8.v v4, (%[lut]) \n"
+ "vsetvli %[vl], zero, e32, m4, ta, ma \n"
+ "vmv.v.i v24, 0 \n"
+ "vsetvli %[vl], zero, e32, m1, ta, ma \n"
+ "vmv.s.x v0, zero \n"
+ "1: \n"
+ "vsetvli %[vl], %[count], e8, m1, tu, ma \n"
+ "vle8.v v8, (%[src_a]) \n"
+ "vle8.v v9, (%[src_b]) \n"
+ "vxor.vv v8, v8, v9 \n"
+ "vand.vi v10, v8, 15 \n"
+ "vsrl.vi v8, v8, 4 \n"
+ "vrgather.vv v11, v4, v10 \n"
+ "vrgather.vv v12, v4, v8 \n"
+ "vwaddu.vv v16, v11, v12 \n"
+ "sub %[count], %[count], %[vl] \n"
+ "add %[src_a], %[src_a], %[vl] \n"
+ "add %[src_b], %[src_b], %[vl] \n"
+ "vsetvli zero, %[vl], e16, m2, tu, ma \n"
+ "vwaddu.wv v24, v24, v16 \n"
+ "bgtz %[count], 1b \n"
+ "vsetvli %[vl], zero, e32, m4, ta, ma \n"
+ "vredsum.vs v0, v24, v0 \n"
+ "vsetvli %[vl], zero, e32, m1, ta, ma \n"
+ "vmv.x.s %[diff], v0 \n"
+ : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
+ [vl] "=&r"(vl), [diff] "=r"(diff)
+ : [lut] "r"(kNibblePopcnt_RVV)
+ : "memory", RVV_VL_CLOBBER "vtype", "v0", "v4", "v8", "v9", "v10", "v11",
+ "v12", "v16", "v17", "v24", "v25", "v26", "v27");
+ return diff;
+}
+#endif // HAS_HAMMINGDISTANCE_RVV
+
+#ifdef HAS_HASHDJB2_RVV
+static const uint32_t kDjb2Muls_RVV[17] = {
+ 0x92d9e201, // 33 ^ 16
+ 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
+ 0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
+ 0x00008c61, 0x00000441, 0x00000021, 0x00000001,
+};
+
+uint32_t HashDjb2_RVV(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ uint32_t tmp;
+ const uint32_t c16 = 0x92d9e201u;
+ const int rem = count & 15;
+ const uint32_t* tail_muls = &kDjb2Muls_RVV[17 - rem];
+ const uint32_t pow_n = kDjb2Muls_RVV[16 - rem];
+ asm volatile(
+ "vsetivli zero, 16, e32, m4, ta, ma \n"
+ "vmv.s.x v4, zero \n"
+ "addi %[count], %[count], -16 \n"
+ "bltz %[count], 2f \n"
+ "vle32.v v16, (%[muls16]) \n"
+ "1: \n"
+ "vle8.v v8, (%[src]) \n"
+ "vzext.vf4 v12, v8 \n"
+ "vmul.vv v12, v12, v16 \n"
+ "vredsum.vs v0, v12, v4 \n"
+ "vmv.x.s %[tmp], v0 \n"
+ "mul %[hash], %[hash], %[c16] \n"
+ "add %[hash], %[hash], %[tmp] \n"
+ "addi %[src], %[src], 16 \n"
+ "addi %[count], %[count], -16 \n"
+ "bgez %[count], 1b \n"
+ "2: \n"
+ "addi %[count], %[count], 16 \n"
+ "beqz %[count], 3f \n"
+ "vsetvli zero, %[count], e32, m4, ta, ma \n"
+ "vle32.v v16, (%[tail_muls]) \n"
+ "vle8.v v8, (%[src]) \n"
+ "vzext.vf4 v12, v8 \n"
+ "vmul.vv v12, v12, v16 \n"
+ "vredsum.vs v0, v12, v4 \n"
+ "vmv.x.s %[tmp], v0 \n"
+ "mul %[hash], %[hash], %[pow_n] \n"
+ "add %[hash], %[hash], %[tmp] \n"
+ "3: \n"
+ :
+ [src] "+r"(src), [count] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
+ : [muls16] "r"(kDjb2Muls_RVV + 1), [c16] "r"(c16),
+ [tail_muls] "r"(tail_muls), [pow_n] "r"(pow_n)
+ : "memory", RVV_VL_CLOBBER "vtype", "v0", "v4", "v8", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19");
+ return hash;
+}
+#endif // HAS_HASHDJB2_RVV
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
diff --git a/libs/libyuv/source/row_sme.cc b/libs/libyuv/source/row_sme.cc
new file mode 100644
index 0000000000..21abac9220
--- /dev/null
+++ b/libs/libyuv/source/row_sme.cc
@@ -0,0 +1,1169 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+#include "libyuv/row_sve.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+__arm_locally_streaming void I444ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // A
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+__arm_locally_streaming void I444ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
+}
+
+__arm_locally_streaming void I400ToARGBRow_SME(
+ const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToRGB565Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToARGB1555Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I422ToARGB4444Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I422ToRGBARow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToAR30Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422AlphaToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I444AlphaToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void NV12ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV21ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV12ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV21ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
+}
+
+__arm_locally_streaming void YUY2ToARGBRow_SME(
+ const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void UYVYToARGBRow_SME(
+ const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I210ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I210AlphaToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I210ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void P210ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void P210ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I410ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I410AlphaToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I410ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void P410ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void P410ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I212ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I212ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void MultiplyRow_16_SME(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "mov z0.h, %w[scale] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "incb %[src_y] \n"
+ "mul z1.h, z0.h, z1.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "mul z1.h, z0.h, z1.h \n"
+ "st1h {z1.h}, p0, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale) // %[scale]
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ width *= 4;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src_argb]] \n"
+ "ld1b {z1.b}, p0/z, [%[src_argb1]] \n"
+ "incb %[src_argb] \n"
+ "incb %[src_argb1] \n"
+ "umullb z2.h, z0.b, z1.b \n"
+ "umullt z1.h, z0.b, z1.b \n"
+ "rshrnb z0.b, z2.h, #8 \n"
+ "rshrnt z0.b, z1.h, #8 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src_argb]] \n"
+ "ld1b {z1.b}, p0/z, [%[src_argb1]] \n"
+ "umullb z2.h, z0.b, z1.b \n"
+ "umullt z1.h, z0.b, z1.b \n"
+ "rshrnb z0.b, z2.h, #8 \n"
+ "rshrnt z0.b, z1.h, #8 \n"
+ "st1b {z0.b}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src_argb1] "+r"(src_argb1), // %[src_argb1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "p0", "p1");
+}
+
+__arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_v] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "p0");
+}
+
+__arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "mov z0.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_u]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_v]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_v] \n"
+ "lsl z1.h, p0/m, z1.h, z0.h \n"
+ "lsl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_u]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_v]] \n"
+ "lsl z1.h, p0/m, z1.h, z0.h \n"
+ "lsl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "memory", "cc", "z0", "z1", "z2", "p0");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits = shr 1
+// 16384 = 10 bits = shr 2
+// 4096 = 12 bits = shr 4
+// 256 = 16 bits = shr 8
+__arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "dup z0.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_y], #1, mul vl] \n"
+ "incb %[src_y], all, mul #2 \n"
+ "uqshl z1.h, p0/m, z1.h, z0.h \n"
+ "uqshl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uzp2 z1.b, z1.b, z2.b \n"
+ "st1b {z1.b}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ // We need separate predicates for the load and store instructions since
+ // they are operating on different element sizes (.b vs .h).
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "whilelt p2.b, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "ld1h {z2.h}, p1/z, [%[src_y], #1, mul vl] \n"
+ "uqshl z1.h, p0/m, z1.h, z0.h \n"
+ "uqshl z2.h, p1/m, z2.h, z0.h \n"
+ "uzp2 z1.b, z1.b, z2.b \n"
+ "st1b {z1.b}, p2, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "p0", "p1", "p2");
+}
+
+__arm_locally_streaming void CopyRow_SME(const uint8_t* src,
+ uint8_t* dst,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "incb %[src] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "p0");
+}
+
+__arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0");
+}
+
+__arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+
+ if (y0_fraction == 0) {
+ CopyRow_SME(src_ptr1, dst_ptr, width);
+ return;
+ }
+ if (y0_fraction == 128) {
+ HalfRow_SME(dst_ptr, src_ptr, src_stride, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ CopyRow_SME(src_ptr, dst_ptr, width);
+ return;
+ }
+
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "dup z0.b, %w[y0_fraction] \n"
+ "dup z1.b, %w[y1_fraction] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction) // %[y1_fraction]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+__arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1h {z2.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "st1h {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0");
+}
+
+__arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ if (y0_fraction == 0) {
+ CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr,
+ width * sizeof(uint16_t));
+ return;
+ }
+ if (y0_fraction == 128) {
+ HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr,
+ width * sizeof(uint16_t));
+ return;
+ }
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "dup z0.h, %w[y0_fraction] \n"
+ "dup z1.h, %w[y1_fraction] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "st1h {z3.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "st1h {z3.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction) // %[y1_fraction]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+__arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0");
+}
+
+__arm_locally_streaming void HalfWidthRow_16To8_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int scale,
+ int width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "incb %[src_ptr1], all, mul #2 \n"
+ "uaddlb z4.s, z0.h, z1.h \n"
+ "uaddlt z5.s, z0.h, z1.h \n"
+ "uaddlb z6.s, z2.h, z3.h \n"
+ "uaddlt z7.s, z2.h, z3.h \n"
+ "add z4.s, z4.s, z6.s \n"
+ "add z5.s, z5.s, z7.s \n"
+ "rshrnb z0.h, z4.s, #2 \n"
+ "rshrnt z0.h, z5.s, #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z0.h, p0/m, z0.h, z31.h \n"
+ "shrnb z0.b, z0.h, #8 \n"
+ "st1b {z0.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n"
+ "uaddlb z4.s, z0.h, z1.h \n"
+ "uaddlt z5.s, z0.h, z1.h \n"
+ "uaddlb z6.s, z2.h, z3.h \n"
+ "uaddlt z7.s, z2.h, z3.h \n"
+ "add z4.s, z4.s, z6.s \n"
+ "add z5.s, z5.s, z7.s \n"
+ "rshrnb z0.h, z4.s, #2 \n"
+ "rshrnt z0.h, z5.s, #2 \n"
+ "uqshl z0.h, p0/m, z0.h, z31.h \n"
+ "shrnb z0.b, z0.h, #8 \n"
+ "st1b {z0.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
+ "p0");
+}
+
+__arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
+}
+
+#define CONVERT8TO16_SVE \
+ "ld1b {z0.h}, p0/z, [%[src]] \n" \
+ "ld1b {z1.h}, p1/z, [%[src], #1, mul vl] \n" \
+ "incb %[src] \n" \
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n" \
+ "trn1 z0.b, z0.b, z0.b \n" \
+ "trn1 z1.b, z1.b, z1.b \n" \
+ "lsr z0.h, p0/m, z0.h, z2.h \n" \
+ "lsr z1.h, p1/m, z1.h, z2.h \n" \
+ "prfm pldl1keep, [%[src], 448] \n" \
+ "st1h {z0.h}, p0, [%[dst]] \n" \
+ "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n" \
+ "incb %[dst], all, mul #2 \n"
+
+__arm_locally_streaming void Convert8To16Row_SME(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ // (src * 0x0101) >> (16 - bits).
+ const int shift = 16 - bits;
+
+ uint64_t vl;
+ asm volatile(
+ "dup z2.h, %w[shift] \n"
+ "cnth %[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "ptrue p1.h \n"
+ "1: \n" //
+ CONVERT8TO16_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n" //
+ CONVERT8TO16_SVE
+
+ "99: \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "p0", "p1");
+}
+
+__arm_locally_streaming void ARGBToUVMatrixRow_SME(
+ const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ int8_t uvconstants[8] = {(int8_t)c->kRGBToU[0], (int8_t)c->kRGBToU[1],
+ (int8_t)c->kRGBToU[2], (int8_t)c->kRGBToU[3],
+ (int8_t)c->kRGBToV[0], (int8_t)c->kRGBToV[1],
+ (int8_t)c->kRGBToV[2], (int8_t)c->kRGBToV[3]};
+ ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width,
+ uvconstants);
+}
+
+__arm_locally_streaming void ARGBToYMatrixRow_SME(
+ const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
+}
+
+#ifdef HAS_SSIM4X4ROW_SME
+__arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
+}
+#endif // HAS_SSIM4X4ROW_SME
+
+#ifdef HAS_SSIMWINDOWROW_SME
+__arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ return SsimWindowRow_SVE_SC(row0, row1, num_windows);
+}
+#endif // HAS_SSIMWINDOWROW_SME
+
+#ifdef HAS_ARGBORROW_SME
+__arm_locally_streaming void ARGBOrRow_SME(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width);
+}
+#endif // HAS_ARGBORROW_SME
+
+#ifdef HAS_ARGBALPHABLENDROW_SME
+__arm_locally_streaming void ARGBAlphaBlendRow_SME(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width);
+}
+#endif // HAS_ARGBALPHABLENDROW_SME
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/row_sve.cc b/libs/libyuv/source/row_sve.cc
new file mode 100644
index 0000000000..cb5f739934
--- /dev/null
+++ b/libs/libyuv/source/row_sve.cc
@@ -0,0 +1,1785 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <string.h>
+
+#include "libyuv/row_sve.h"
+#include "libyuv/compare_row.h"
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+#define RGBTOARGB8_SVE_TOP_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
+ "uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
+ "uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
+ "uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
+ "uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
+
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+void I444ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
+}
+
+void I400ToARGBRow_SVE2(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
+}
+
+void I422ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422ToRGB565Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+}
+
+void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
+ width);
+}
+
+void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
+ width);
+}
+
+void I422ToRGBARow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
+}
+
+void I422ToAR30Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void NV12ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void NV21ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
+}
+
+void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
+}
+
+void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
+}
+
+void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ int8_t uvconstants[8] = {(int8_t)c->kRGBToU[0], (int8_t)c->kRGBToU[1],
+ (int8_t)c->kRGBToU[2], (int8_t)c->kRGBToU[3],
+ (int8_t)c->kRGBToV[0], (int8_t)c->kRGBToV[1],
+ (int8_t)c->kRGBToV[2], (int8_t)c->kRGBToV[3]};
+ ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width,
+ uvconstants);
+}
+
+void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
+}
+
+#define ARGBTORGB565_SVE \
+ /* Inputs: \
+ * z0: rrrrrxxxbbbbbxxx \
+ * z1: xxxxxxxxggggggxx \
+ * z3: 0000000000000011 (3, 0, 3, 0, ...) \
+ * z4: 0000011111100000 \
+ */ \
+ "lsr z0.b, p0/m, z0.b, z3.b \n" \
+ "lsl z1.h, z1.h, #3 \n" \
+ "bsl z1.d, z1.d, z0.d, z4.d \n"
+
+void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ unsigned bsl_mask = 0x7e0;
+ uint64_t vl;
+ width *= 2;
+ asm volatile(
+ "mov z3.h, #3 \n"
+ "dup z4.h, %w[bsl_mask] \n"
+
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "incb %[src], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_rgb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [bsl_mask] "r"(bsl_mask) // %[bsl_mask]
+ : "cc", "memory", "z0", "z1", "z3", "z4", "p0");
+}
+
+void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ unsigned bsl_mask = 0x7e0;
+ uint64_t vl;
+ width *= 2;
+ asm volatile(
+ "mov z3.h, #3 \n"
+ "dup z4.h, %w[bsl_mask] \n"
+ "dup z2.s, %w[dither4] \n"
+ "zip1 z2.b, z2.b, z2.b \n"
+
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "incb %[src], all, mul #2 \n"
+ "uqadd z0.b, z0.b, z2.b \n"
+ "uqadd z1.b, z1.b, z2.b \n"
+ "subs %w[width], %w[width], %w[vl] \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "uqadd z0.b, z0.b, z2.b \n"
+ "uqadd z1.b, z1.b, z2.b \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_rgb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [bsl_mask] "r"(bsl_mask), // %[bsl_mask]
+ [dither4] "r"(dither4) // %[dither4]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+#define ARGB1555TOARGB \
+ /* Input: z1/z3.h = arrrrrgggggbbbbb */ \
+ "lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \
+ "lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \
+ "asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \
+ "asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \
+ "lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \
+ "lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \
+ "sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
+ "sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
+ "sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \
+ "sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */
+
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "mov z4.h, #0x0300 \n"
+ "ptrue p0.b \n"
+
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n" //
+ ARGB1555TOARGB
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st2h {z0.h, z1.h}, p0, [%[dst]] \n"
+ "st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n"
+ "incb %[dst], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.h, wzr, %w[width] \n"
+ "whilelt p2.h, %w[vl], %w[width] \n"
+ "ld1h {z1.h}, p1/z, [%[src]] \n"
+ "ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" //
+ ARGB1555TOARGB
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb1555), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
+}
+
+// clang-format off
+#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
+ "incb %[src0], all, mul #4 \n" \
+ "incb %[src1], all, mul #4 \n" \
+ "uaddlb z4.h, z0.b, z2.b \n" /* V */ \
+ "uaddlt z5.h, z0.b, z2.b \n" /* U */ \
+ "uaddlb z6.h, z1.b, z3.b \n" /* V */ \
+ "uaddlt z7.h, z1.b, z3.b \n" /* U */ \
+ "addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \
+ "addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \
+ "subs %w[width], %w[width], %w[vl] \n" \
+ "urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \
+ "urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \
+ "st1b {" #zU0 ".h}, p0, [%[dst]] \n" \
+ "st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \
+ "incb %[dst] \n"
+// clang-format on
+
+// Filter 2 rows of AYUV UV's (444) into UV (420).
+// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
+void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ // Output a row of UV values, filtering 2x2 rows of AYUV.
+ const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
+ int vl;
+ asm volatile (
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "ptrue p1.h \n"
+ "1: \n"
+ AYUVTOUV_SVE(z5, z4, z7, z6)
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n" // first row
+ "whilelt p1.h, %w[vl], %w[width] \n" // second row
+ AYUVTOUV_SVE(z5, z4, z7, z6)
+
+ "99: \n"
+ : [src0]"+r"(src_ayuv), // %[src0]
+ [src1]"+r"(src_ayuv1), // %[src1]
+ [dst]"+r"(dst_uv), // %[dst]
+ [width]"+r"(width), // %[width]
+ [vl]"=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+// Filter 2 rows of AYUV UV's (444) into VU (420).
+void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ // Output a row of VU values, filtering 2x2 rows of AYUV.
+ const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
+ int vl;
+ asm volatile (
+ "cntb %x[vl] \n"
+ "cmp %w[width], %w[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "ptrue p1.h \n"
+ "1: \n"
+ AYUVTOUV_SVE(z4, z5, z6, z7)
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n" // first row
+ "whilelt p1.h, %w[vl], %w[width] \n" // second row
+ AYUVTOUV_SVE(z4, z5, z6, z7)
+
+ "99: \n"
+ : [src0]"+r"(src_ayuv), // %[src0]
+ [src1]"+r"(src_ayuv1), // %[src1]
+ [dst]"+r"(dst_vu), // %[dst]
+ [width]"+r"(width), // %[width]
+ [vl]"=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
+}
+
+void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
+}
+
+static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw,
+ uint8_t* dst_wxyz,
+ int width,
+ uint32_t idx_start,
+ uint32_t idx_step,
+ uint32_t alpha) {
+ uint32_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ uint32_t vl_mul3 = vl * 3;
+ uint32_t rem_mul3;
+ asm volatile(
+ "index z31.s, %w[idx_start], %w[idx_step] \n"
+ "dup z30.s, %w[alpha] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead. We set up p1 to only load 3/4 of a vector.
+ "ptrue p0.s \n"
+ "whilelt p1.b, wzr, %w[vl_mul3] \n"
+ "1: \n"
+ "ld1b {z0.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "ld1b {z1.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "orr z0.d, z0.d, z30.d \n"
+ "orr z1.d, z1.d, z30.d \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate a pair of predicates for the final iteration to deal with
+ // the tail.
+ "3: \n"
+ "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.b, wzr, %w[rem_mul3] \n"
+ "ld1b {z0.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "orr z0.d, z0.d, z30.d \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src] "+r"(src_raw), // %[src]
+ [dst] "+r"(dst_wxyz), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl_mul3] "+r"(vl_mul3), // %[vl_mul3]
+ [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
+ : [idx_start] "r"(idx_start), // %[idx_start]
+ [idx_step] "r"(idx_step), // %[idx_step]
+ [alpha] "r"(alpha), // %[alpha]
+ [vl] "r"(vl) // %[vl]
+ : "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1");
+}
+
+void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U,
+ 0xff000000U);
+}
+
+void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U,
+ 0x000000ffU);
+}
+
+void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U,
+ 0xff000000U);
+}
+
+static const uint8_t kRAWToRGB24Indices[] = {
+ 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12,
+ 17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27,
+ 32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42,
+ 47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57,
+ 62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72,
+ 77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87,
+ 92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102,
+ 107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117,
+ 122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132,
+ 137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147,
+ 152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162,
+ 167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177,
+ 182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192,
+ 197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207,
+ 212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222,
+ 227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237,
+ 242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252};
+
+void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ // width is in elements, convert to bytes.
+ width *= 3;
+ // we use the mul3 predicate pattern throughout to use the largest multiple
+ // of three number of lanes, for instance with a vector length of 16 bytes
+ // only the first 15 bytes will be used for load/store instructions.
+ uint32_t vl;
+ asm volatile(
+ "cntb %x[vl], mul3 \n"
+ "ptrue p0.b, mul3 \n"
+ "ld1b {z31.b}, p0/z, [%[kIndices]] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicate to avoid predicate
+ // generation overhead.
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_raw), // %[src]
+ [dst] "+r"(dst_rgb24), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices]
+ : "cc", "memory", "z0", "z31", "p0");
+}
+
+static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_xyz,
+ int width,
+ const uint8_t* indices) {
+ uint32_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ uint32_t vl_mul3 = vl * 3;
+ uint32_t rem_mul3;
+ asm volatile(
+ "whilelt p1.b, wzr, %w[vl_mul3] \n"
+ "ld1b {z31.b}, p1/z, [%[indices]] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead. We set up p1 to only store 3/4 of a vector.
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1b {z0.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "st1b {z1.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.b, wzr, %w[rem_mul3] \n"
+ "whilelt p2.s, %w[vl], %w[width] \n"
+ "whilelt p3.b, %w[vl_mul3], %w[rem_mul3] \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "st1b {z0.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "st1b {z1.b}, p3, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_xyz), // %[dst]
+ [width] "+r"(width), // %[width]
+ [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
+ : [indices] "r"(indices), // %[indices]
+ [vl_mul3] "r"(vl_mul3), // %[vl_mul3]
+ [vl] "r"(vl) // %[vl]
+ : "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3");
+}
+
+static const uint8_t kARGBToRGB24RowIndices[] = {
+ 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18,
+ 20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38,
+ 40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58,
+ 60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78,
+ 80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98,
+ 100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118,
+ 120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138,
+ 140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158,
+ 160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178,
+ 180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198,
+ 200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218,
+ 220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238,
+ 240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254,
+};
+
+static const uint8_t kARGBToRAWRowIndices[] = {
+ 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16,
+ 22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36,
+ 42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56,
+ 62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76,
+ 82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96,
+ 102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116,
+ 122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136,
+ 142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156,
+ 162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176,
+ 182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196,
+ 202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216,
+ 222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236,
+ 242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252,
+};
+
+void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
+ ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices);
+}
+
+void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
+ ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices);
+}
+
+void DivideRow_16_SVE2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ uint64_t vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z0.h, %w[scale] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "umulh z1.h, z1.h, z0.h \n"
+ "umulh z2.h, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1h {z1.h}, p0, [%[dst]] \n"
+ "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate a pair of predicates for the final iteration to deal with
+ // the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n"
+ "umulh z1.h, z1.h, z0.h \n"
+ "umulh z2.h, z2.h, z0.h \n"
+ "st1h {z1.h}, p0, [%[dst]] \n"
+ "st1h {z2.h}, p1, [%[dst], #1, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale) // %[scale]
+ : "cc", "memory", "z0", "z1", "z2", "p0", "p1");
+}
+
+#define HALFFLOAT_SVE \
+ "scvtf z0.s, p0/m, z0.s \n" \
+ "scvtf z1.s, p0/m, z1.s \n" \
+ "scvtf z2.s, p0/m, z2.s \n" \
+ "scvtf z3.s, p0/m, z3.s \n" \
+ "fmul z0.s, z0.s, z4.s \n" \
+ "fmul z1.s, z1.s, z4.s \n" \
+ "fmul z2.s, z2.s, z4.s \n" \
+ "fmul z3.s, z3.s, z4.s \n" \
+ "fmul z0.s, z0.s, z5.s \n" \
+ "fmul z1.s, z1.s, z5.s \n" \
+ "fmul z2.s, z2.s, z5.s \n" \
+ "fmul z3.s, z3.s, z5.s \n" \
+ "uqshrnb z0.h, z0.s, #13 \n" \
+ "uqshrnb z1.h, z1.s, #13 \n" \
+ "uqshrnb z2.h, z2.s, #13 \n" \
+ "uqshrnb z3.h, z3.s, #13 \n"
+
+void HalfFloatRow_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ uint64_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ asm volatile(
+ "mov z4.s, %s[scale] \n"
+ "mov z5.s, %s[bias] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld1h {z0.s}, p0/z, [%[src]] \n"
+ "ld1h {z1.s}, p0/z, [%[src], #1, mul vl] \n"
+ "ld1h {z2.s}, p0/z, [%[src], #2, mul vl] \n"
+ "ld1h {z3.s}, p0/z, [%[src], #3, mul vl] \n"
+ "incb %[src], all, mul #2 \n" //
+ HALFFLOAT_SVE
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "st1h {z0.s}, p0, [%[dst]] \n"
+ "st1h {z1.s}, p0, [%[dst], #1, mul vl] \n"
+ "st1h {z2.s}, p0, [%[dst], #2, mul vl] \n"
+ "st1h {z3.s}, p0, [%[dst], #3, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.s, %w[vl], %w[width] \n"
+ "whilelt p2.s, %w[vl2], %w[width] \n"
+ "whilelt p3.s, %w[vl3], %w[width] \n"
+ "ld1h {z0.s}, p0/z, [%[src]] \n"
+ "ld1h {z1.s}, p1/z, [%[src], #1, mul vl] \n"
+ "ld1h {z2.s}, p2/z, [%[src], #2, mul vl] \n"
+ "ld1h {z3.s}, p3/z, [%[src], #3, mul vl] \n" //
+ HALFFLOAT_SVE
+ "st1h {z0.s}, p0, [%[dst]] \n"
+ "st1h {z1.s}, p1, [%[dst], #1, mul vl] \n"
+ "st1h {z2.s}, p2, [%[dst], #2, mul vl] \n"
+ "st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [vl2] "r"(vl * 2), // %[vl2]
+ [vl3] "r"(vl * 3), // %[vl3]
+ [scale] "w"(scale), // %[scale]
+ [bias] "w"(1.9259299444e-34f) // %[bias] 2^-112
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2",
+ "p3");
+}
+
+void HalfFloat1Row_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z0.h}, p0/z, [%[src]] \n"
+ "ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "ucvtf z0.h, p0/m, z0.h \n"
+ "ucvtf z1.h, p0/m, z1.h \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1h {z0.h}, p0, [%[dst]] \n"
+ "st1h {z1.h}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "ld1h {z0.h}, p0/z, [%[src]] \n"
+ "ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n"
+ "ucvtf z0.h, p0/m, z0.h \n"
+ "ucvtf z1.h, p0/m, z1.h \n"
+ "st1h {z0.h}, p0, [%[dst]] \n"
+ "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "p0", "p1");
+}
+
+void I210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void P210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void P210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+void I410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void P410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void P410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+void I212ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void I212ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void Convert8To8Row_SVE2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
+}
+
+void MergeUVRow_SVE2(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_v] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z1", "z2", "p0");
+}
+
+void SplitUVRow_SVE2(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z1.b}, p0, [%[dst_u]] \n"
+ "st1b {z2.b}, p0, [%[dst_v]] \n"
+ "incb %[dst_u] \n"
+ "incb %[dst_v] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
+ "st1b {z1.b}, p0, [%[dst_u]] \n"
+ "st1b {z2.b}, p0, [%[dst_v]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z1", "z2", "p0");
+}
+
+void HalfMergeUVRow_SVE2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ const uint8_t* src_u1 = src_u + src_stride_u;
+ const uint8_t* src_v1 = src_v + src_stride_v;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_u1]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_v]] \n"
+ "ld1b {z4.b}, p0/z, [%[src_v1]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_u1] \n"
+ "incb %[src_v] \n"
+ "incb %[src_v1] \n"
+ "uaddlb z5.h, z1.b, z2.b \n"
+ "uaddlb z6.h, z3.b, z4.b \n"
+ "uaddlt z1.h, z1.b, z2.b \n"
+ "uaddlt z3.h, z3.b, z4.b \n"
+ "add z1.h, z1.h, z5.h \n"
+ "add z3.h, z3.h, z6.h \n"
+ "rshrnb z1.b, z1.h, #2 \n"
+ "rshrnt z1.b, z3.h, #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z1.b}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_u1]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_v]] \n"
+ "ld1b {z4.b}, p0/z, [%[src_v1]] \n"
+ "uaddlb z5.h, z1.b, z2.b \n"
+ "uaddlb z6.h, z3.b, z4.b \n"
+ "uaddlt z1.h, z1.b, z2.b \n"
+ "uaddlt z3.h, z3.b, z4.b \n"
+ "add z1.h, z1.h, z5.h \n"
+ "add z3.h, z3.h, z6.h \n"
+ "rshrnb z1.b, z1.h, #2 \n"
+ "rshrnt z1.b, z3.h, #2 \n"
+ "st1b {z1.b}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_u1] "+r"(src_u1), // %[src_u1]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_v1] "+r"(src_v1), // %[src_v1]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0");
+}
+
+void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
+ int vl;
+ int vl2;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "lsl %w[vl2], %w[vl], #1 \n"
+ "subs %w[width], %w[width], %w[vl2] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n"
+ "ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl2] \n"
+ "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
+ "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl2] \n"
+ "b.eq 99f \n"
+
+ "3: \n"
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "incb %[src] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [vl] "=&r"(vl),
+ [vl2] "=&r"(vl2)
+ :
+ : "cc", "memory", "z0", "z1", "p0");
+}
+
+void InterpolateRow_SVE2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+
+ if (y0_fraction == 0) {
+ CopyRow_SVE2(src_ptr1, dst_ptr, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ CopyRow_SVE2(src_ptr, dst_ptr, width);
+ return;
+ }
+ if (y0_fraction == 128) {
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), [src_ptr1] "+r"(src_ptr1),
+ [dst_ptr] "+r"(dst_ptr), [width] "+r"(width), [vl] "=&r"(vl)
+ :
+ : "cc", "memory", "z2", "z3", "p0");
+ return;
+ }
+
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "dup z0.b, %w[y0_fraction] \n"
+ "dup z1.b, %w[y1_fraction] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction) // %[y1_fraction]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
+ "incb %[src_yuy2], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
+ "st1b {z0.b}, p0, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
+ "incb %[src_uyvy], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z1.b}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
+ "st1b {z1.b}, p0, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2;
+ int vl;
+ int vl_half;
+ int width_half;
+ if (src_stride_yuy2 == 0) {
+ src_yuy2_1 = src_yuy2;
+ }
+ asm volatile(
+ "cntb %x[vl] \n"
+ "lsr %w[vl_half], %w[vl], #1 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "whilelt p1.b, wzr, %w[vl_half] \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
+ "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
+ "incb %[src_yuy2], all, mul #2 \n"
+ "incb %[src_yuy2_1], all, mul #2 \n"
+ "urhadd z1.b, p0/m, z1.b, z5.b \n"
+ "uzp1 z0.b, z1.b, z1.b \n"
+ "uzp2 z2.b, z1.b, z1.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p1, [%[dst_u]] \n"
+ "st1b {z2.b}, p1, [%[dst_v]] \n"
+ "add %[dst_u], %[dst_u], %x[vl_half] \n"
+ "add %[dst_v], %[dst_v], %x[vl_half] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "add %w[width_half], %w[width], #1 \n"
+ "lsr %w[width_half], %w[width_half], #1 \n"
+ "whilelt p1.b, wzr, %w[width_half] \n"
+ "zip1 p0.b, p1.b, p1.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
+ "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
+ "urhadd z1.b, p0/m, z1.b, z5.b \n"
+ "uzp1 z0.b, z1.b, z1.b \n"
+ "uzp2 z2.b, z1.b, z1.b \n"
+ "st1b {z0.b}, p1, [%[dst_u]] \n"
+ "st1b {z2.b}, p1, [%[dst_v]] \n"
+
+ "99: \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl), // %[vl]
+ [vl_half] "=&r"(vl_half), // %[vl_half]
+ [width_half] "=&r"(width_half) // %[width_half]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
+}
+
+void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy;
+ int vl;
+ int vl_half;
+ int width_half;
+ if (src_stride_uyvy == 0) {
+ src_uyvy_1 = src_uyvy;
+ }
+ asm volatile(
+ "cntb %x[vl] \n"
+ "lsr %w[vl_half], %w[vl], #1 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "whilelt p1.b, wzr, %w[vl_half] \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
+ "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
+ "incb %[src_uyvy], all, mul #2 \n"
+ "incb %[src_uyvy_1], all, mul #2 \n"
+ "urhadd z0.b, p0/m, z0.b, z4.b \n"
+ "uzp1 z1.b, z0.b, z0.b \n"
+ "uzp2 z2.b, z0.b, z0.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z1.b}, p1, [%[dst_u]] \n"
+ "st1b {z2.b}, p1, [%[dst_v]] \n"
+ "add %[dst_u], %[dst_u], %x[vl_half] \n"
+ "add %[dst_v], %[dst_v], %x[vl_half] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "add %w[width_half], %w[width], #1 \n"
+ "lsr %w[width_half], %w[width_half], #1 \n"
+ "whilelt p1.b, wzr, %w[width_half] \n"
+ "zip1 p0.b, p1.b, p1.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
+ "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
+ "urhadd z0.b, p0/m, z0.b, z4.b \n"
+ "uzp1 z1.b, z0.b, z0.b \n"
+ "uzp2 z2.b, z0.b, z0.b \n"
+ "st1b {z1.b}, p1, [%[dst_u]] \n"
+ "st1b {z2.b}, p1, [%[dst_v]] \n"
+
+ "99: \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
+ [src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl), // %[vl]
+ [vl_half] "=&r"(vl_half), // %[vl_half]
+ [width_half] "=&r"(width_half) // %[width_half]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
+}
+
+#ifdef HAS_CONVERT16TO8ROW_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+void Convert16To8Row_SVE2(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_y), // %[src_ptr]
+ [dst_ptr] "+r"(dst_y), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z2", "z31", "p0");
+}
+#endif // HAS_CONVERT16TO8ROW_SVE2
+
+#ifdef HAS_HALFROW_16TO8_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+void HalfRow_16To8_SVE2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_uv), // %[src_ptr]
+ [src_ptr1] "+r"(src_uv1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_uv), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z2", "z3", "z31", "p0");
+}
+#endif // HAS_HALFROW_16TO8_SVE2
+
+#ifdef HAS_HALFWIDTHROW_16TO8_SVE2
+// Disabled: NEON version is faster for 128 bit vectors.
+void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const uint16_t* src_uv1 = src_uv + src_uv_stride;
+ const int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ int vl_input;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z1.h}, p0/z, [%[src_ptr], #1, mul vl] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n"
+ "incb %[src_ptr], all, mul #4 \n"
+ "incb %[src_ptr1], all, mul #4 \n"
+ "uzp1 z4.h, z0.h, z1.h \n"
+ "uzp2 z5.h, z0.h, z1.h \n"
+ "uzp1 z6.h, z2.h, z3.h \n"
+ "uzp2 z7.h, z2.h, z3.h \n"
+ "uaddlb z0.s, z4.h, z5.h \n"
+ "uaddlt z1.s, z4.h, z5.h \n"
+ "uaddlb z2.s, z6.h, z7.h \n"
+ "uaddlt z3.s, z6.h, z7.h \n"
+ "add z0.s, z0.s, z2.s \n"
+ "add z1.s, z1.s, z3.s \n"
+ "rshrnb z0.h, z0.s, #2 \n"
+ "rshrnt z0.h, z1.s, #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z0.h, p0/m, z0.h, z31.h \n"
+ "shrnb z0.b, z0.h, #8 \n"
+ "st1b {z0.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "lsl %w[vl_input], %w[width], #1 \n"
+ "whilelt p0.h, wzr, %w[vl_input] \n"
+ "whilelt p1.h, %w[vl], %w[vl_input] \n"
+ "whilelt p2.h, wzr, %w[width] \n"
+ "ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z1.h}, p1/z, [%[src_ptr], #1, mul vl] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
+ "ld1h {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n"
+ "uzp1 z4.h, z0.h, z1.h \n"
+ "uzp2 z5.h, z0.h, z1.h \n"
+ "uzp1 z6.h, z2.h, z3.h \n"
+ "uzp2 z7.h, z2.h, z3.h \n"
+ "uaddlb z0.s, z4.h, z5.h \n"
+ "uaddlt z1.s, z4.h, z5.h \n"
+ "uaddlb z2.s, z6.h, z7.h \n"
+ "uaddlt z3.s, z6.h, z7.h \n"
+ "add z0.s, z0.s, z2.s \n"
+ "add z1.s, z1.s, z3.s \n"
+ "rshrnb z0.h, z0.s, #2 \n"
+ "rshrnt z0.h, z1.s, #2 \n"
+ "uqshl z0.h, p2/m, z0.h, z31.h \n"
+ "shrnb z0.b, z0.h, #8 \n"
+ "st1b {z0.h}, p2, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_uv), // %[src_ptr]
+ [src_ptr1] "+r"(src_uv1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_uv), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl), // %[vl]
+ [vl_input] "=&r"(vl_input) // %[vl_input]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
+ "p0", "p1", "p2");
+}
+#endif // HAS_HALFWIDTHROW_16TO8_SVE2
+
+#ifdef HAS_SSIM4X4ROW_SVE2
+void Ssim4x4Row_SVE2(const uint8_t* src_a,
+ int stride_a,
+ const uint8_t* src_b,
+ int stride_b,
+ uint32_t* dst,
+ int width) {
+ Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
+}
+#endif // HAS_SSIM4X4ROW_SVE2
+
+#ifdef HAS_SSIMWINDOWROW_SVE2
+float SsimWindowRow_SVE2(const uint32_t* row0,
+ const uint32_t* row1,
+ int num_windows) {
+ return SsimWindowRow_SVE_SC(row0, row1, num_windows);
+}
+#endif // HAS_SSIMWINDOWROW_SVE2
+
+#ifdef HAS_ARGBORROW_SVE2
+void ARGBOrRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width);
+}
+#endif // HAS_ARGBORROW_SVE2
+
+#ifdef HAS_ARGBALPHABLENDROW_SVE2
+void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width);
+}
+#endif // HAS_ARGBALPHABLENDROW_SVE2
+
+#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/row_wasm.cc b/libs/libyuv/source/row_wasm.cc
new file mode 100644
index 0000000000..df9e3917a4
--- /dev/null
+++ b/libs/libyuv/source/row_wasm.cc
@@ -0,0 +1,2047 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/row.h"
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#include <wasm_simd128.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#ifdef __wasm_relaxed_simd__
+#define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle
+#else
+#define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle
+#endif
+
+// V8 materializes v128.const values and arbitrary i8x16.shuffle masks with
+// several instructions at every use, including inside loops. These helpers
+// make constants opaque to clang so they are built once, before the loop,
+// and kept in registers. Use swizzle with a LoadConst() table instead of
+// i8x16.shuffle unless the pattern maps to a native instruction (unpack,
+// 32-bit lane shuffle, concat/palignr, even/odd byte unzip).
+// Loops that access ptr + 16 etc. use asm("" : "+r"(ptr)) so LLVM keeps the
+// constant offsets in the load/store instructions instead of creating extra
+// pointer induction variables.
+static inline v128_t LoadConst(const void* p) {
+ asm("" : "+r"(p));
+ return wasm_v128_load(p);
+}
+static inline v128_t SplatConst8(int v) {
+ asm("" : "+r"(v));
+ return wasm_i8x16_splat(v);
+}
+static inline v128_t SplatConst16(int v) {
+ asm("" : "+r"(v));
+ return wasm_i16x8_splat(v);
+}
+static inline v128_t SplatConst32(int v) {
+ asm("" : "+r"(v));
+ return wasm_i32x4_splat(v);
+}
+
+#ifdef HAS_RAWTORGB24ROW_WASMSIMD
+static const uint8_t kRawToRGB24SwizzleTable[96] = {
+ 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12,
+ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 1, 0, 128, 4, 3, 2, 7, 6, 5, 10, 9, 8, 13, 12,
+ 11, 128, 15, 128, 1, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 2, 128, 128, 3, 2, 1, 6, 5, 4, 9, 8, 7, 12,
+ 11, 10, 15, 14, 13, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128};
+
+void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ const uint8_t* shuf = kRawToRGB24SwizzleTable;
+ // Load shuffle masks into Wasm locals outside the loop so Binaryen's
+ // precompute-propagate pass does not sink v128.const into the loop.
+ asm("" : "+r"(shuf));
+ v128_t kShuf0_0 = wasm_v128_load(shuf);
+ v128_t kShuf0_1 = wasm_v128_load(shuf + 16);
+ v128_t kShuf1_1 = wasm_v128_load(shuf + 32);
+ v128_t kShuf1_02 = wasm_v128_load(shuf + 48);
+ v128_t kShuf2_2 = wasm_v128_load(shuf + 64);
+ v128_t kShuf2_1 = wasm_v128_load(shuf + 80);
+
+ while (width >= 32) {
+ // Prevent LLVM LSR from splitting constant load/store offsets into separate
+ // pointer induction variables.
+ asm("" : "+r"(src_raw), "+r"(dst_rgb24));
+ v128_t s0 = wasm_v128_load(src_raw);
+ v128_t s1 = wasm_v128_load(src_raw + 16);
+ v128_t s2 = wasm_v128_load(src_raw + 32);
+ v128_t s3 = wasm_v128_load(src_raw + 48);
+ v128_t s4 = wasm_v128_load(src_raw + 64);
+ v128_t s5 = wasm_v128_load(src_raw + 80);
+
+ v128_t s02 = wasm_i8x16_shuffle(s0, s2, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+ 23, 24, 25, 26, 27, 28, 29);
+ v128_t s35 = wasm_i8x16_shuffle(s3, s5, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+ 23, 24, 25, 26, 27, 28, 29);
+
+ v128_t d0 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, kShuf0_0),
+ LIBYUV_WASM_SWIZZLE(s1, kShuf0_1));
+ v128_t d1 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, kShuf1_1),
+ LIBYUV_WASM_SWIZZLE(s02, kShuf1_02));
+ v128_t d2 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, kShuf2_2),
+ LIBYUV_WASM_SWIZZLE(s1, kShuf2_1));
+ v128_t d3 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s3, kShuf0_0),
+ LIBYUV_WASM_SWIZZLE(s4, kShuf0_1));
+ v128_t d4 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s4, kShuf1_1),
+ LIBYUV_WASM_SWIZZLE(s35, kShuf1_02));
+ v128_t d5 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s5, kShuf2_2),
+ LIBYUV_WASM_SWIZZLE(s4, kShuf2_1));
+
+ wasm_v128_store(dst_rgb24, d0);
+ wasm_v128_store(dst_rgb24 + 16, d1);
+ wasm_v128_store(dst_rgb24 + 32, d2);
+ wasm_v128_store(dst_rgb24 + 48, d3);
+ wasm_v128_store(dst_rgb24 + 64, d4);
+ wasm_v128_store(dst_rgb24 + 80, d5);
+
+ src_raw += 96;
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ if (width > 0) {
+ asm("" : "+r"(src_raw), "+r"(dst_rgb24));
+ v128_t s0 = wasm_v128_load(src_raw);
+ v128_t s1 = wasm_v128_load(src_raw + 16);
+ v128_t s2 = wasm_v128_load(src_raw + 32);
+
+ v128_t s02 = wasm_i8x16_shuffle(s0, s2, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+ 23, 24, 25, 26, 27, 28, 29);
+
+ v128_t d0 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, kShuf0_0),
+ LIBYUV_WASM_SWIZZLE(s1, kShuf0_1));
+ v128_t d1 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, kShuf1_1),
+ LIBYUV_WASM_SWIZZLE(s02, kShuf1_02));
+ v128_t d2 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, kShuf2_2),
+ LIBYUV_WASM_SWIZZLE(s1, kShuf2_1));
+
+ wasm_v128_store(dst_rgb24, d0);
+ wasm_v128_store(dst_rgb24 + 16, d1);
+ wasm_v128_store(dst_rgb24 + 32, d2);
+ }
+}
+#endif // HAS_RAWTORGB24ROW_WASMSIMD
+
+#if defined(HAS_RAWTOARGBROW_WASMSIMD) || defined(HAS_RGB24TOARGBROW_WASMSIMD)
+// 16 pixels of 3 bytes to 4 bytes. shuf holds 2 masks: one for loads at
+// offsets 0, 12, 24 and one for the load at offset 32 (avoids over-read).
+static void RGB3ToARGBRow_WASMSIMD(const uint8_t* src,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* shuf) {
+ v128_t m0 = LoadConst(shuf);
+ v128_t m1 = LoadConst(shuf + 16);
+ v128_t alpha = SplatConst32((int)0xff000000);
+ do {
+ asm("" : "+r"(src), "+r"(dst_argb));
+ v128_t d0 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), m0);
+ v128_t d1 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 12), m0);
+ v128_t d2 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 24), m0);
+ v128_t d3 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 32), m1);
+ wasm_v128_store(dst_argb, wasm_v128_or(d0, alpha));
+ wasm_v128_store(dst_argb + 16, wasm_v128_or(d1, alpha));
+ wasm_v128_store(dst_argb + 32, wasm_v128_or(d2, alpha));
+ wasm_v128_store(dst_argb + 48, wasm_v128_or(d3, alpha));
+ src += 48;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif
+
+#ifdef HAS_RAWTOARGBROW_WASMSIMD
+static const uint8_t kShuffleRAWToARGB[32] = {
+ 2, 1, 0, 128, 5, 4, 3, 128, 8, 7, 6, 128, 11, 10, 9, 128,
+ 6, 5, 4, 128, 9, 8, 7, 128, 12, 11, 10, 128, 15, 14, 13, 128};
+void RAWToARGBRow_WASMSIMD(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ int width) {
+ RGB3ToARGBRow_WASMSIMD(src_raw, dst_argb, width, kShuffleRAWToARGB);
+}
+#endif // HAS_RAWTOARGBROW_WASMSIMD
+
+#ifdef HAS_RGB24TOARGBROW_WASMSIMD
+static const uint8_t kShuffleRGB24ToARGB[32] = {
+ 0, 1, 2, 128, 3, 4, 5, 128, 6, 7, 8, 128, 9, 10, 11, 128,
+ 4, 5, 6, 128, 7, 8, 9, 128, 10, 11, 12, 128, 13, 14, 15, 128};
+void RGB24ToARGBRow_WASMSIMD(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ RGB3ToARGBRow_WASMSIMD(src_rgb24, dst_argb, width, kShuffleRGB24ToARGB);
+}
+#endif // HAS_RGB24TOARGBROW_WASMSIMD
+
+#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGB565TOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD)
+// RGB to YUV. Math is in 16 bit lanes, which wrap, but sum + bias fits in
+// 16 bits so the results match the C code. i32x4.dot_i16x8_s would be best
+// on x86 but is 3 instructions on Arm, so i16x8.mul (mul + mla on Arm) and
+// horizontal adds that are cheap on both are used instead.
+
+// lo * k0 + hi * k1.
+static inline v128_t MulAdd16(v128_t lo, v128_t hi, v128_t k0, v128_t k1) {
+ return wasm_i16x8_add(wasm_i16x8_mul(lo, k0), wasm_i16x8_mul(hi, k1));
+}
+
+// Sums adjacent 16 bit lanes of t0 and t1.
+static inline v128_t HAddPairs16(v128_t t0, v128_t t1) {
+ return wasm_i16x8_add(wasm_i16x8_shuffle(t0, t1, 0, 2, 4, 6, 8, 10, 12, 14),
+ wasm_i16x8_shuffle(t0, t1, 1, 3, 5, 7, 9, 11, 13, 15));
+}
+
+// Splats 16 bit {lo, hi} into each 32 bit lane.
+static inline v128_t SplatPair16(int lo, int hi) {
+ return SplatConst32((lo & 0xffff) | (hi << 16));
+}
+
+// (bias - sum) >> 8 for U and V.
+static inline v128_t SubShr8(v128_t bias, v128_t sum) {
+ return wasm_u16x8_shr(wasm_i16x8_sub(bias, sum), 8);
+}
+#endif
+
+#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD)
+// Swizzles 4 pixels to 16 bit {c0 x4, c2 x4} and {c1 x4, c3 x4}, so the 2
+// halves of the products are summed with a 64 bit interleave.
+static const uint8_t kShuffleARGBToPlanar[32] = {
+ 0, 128, 4, 128, 8, 128, 12, 128, 2, 128, 6, 128, 10, 128, 14, 128,
+ 1, 128, 5, 128, 9, 128, 13, 128, 3, 128, 7, 128, 11, 128, 15, 128};
+// RGB24 to {b x4, r x4} and {g x4, 0 x4}. The second pair of masks is for
+// the load at offset 32 (avoids over-read).
+static const uint8_t kShuffleRGBToPlanar[64] = {
+ 0, 128, 3, 128, 6, 128, 9, 128, 2, 128, 5, 128, 8, 128, 11, 128,
+ 1, 128, 4, 128, 7, 128, 10, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 4, 128, 7, 128, 10, 128, 13, 128, 6, 128, 9, 128, 12, 128, 15, 128,
+ 5, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128};
+
+// {a x4, b x4} as 16 bit lanes.
+static inline v128_t Splat4x2(int a, int b) {
+ asm("" : "+r"(a), "+r"(b));
+ return wasm_i16x8_make(a, a, a, a, b, b, b, b);
+}
+
+// Sums the 64 bit halves of t0 and t1.
+static inline v128_t HAddHalves64(v128_t t0, v128_t t1) {
+ return wasm_i16x8_add(wasm_i64x2_shuffle(t0, t1, 0, 2),
+ wasm_i64x2_shuffle(t0, t1, 1, 3));
+}
+
+// 16 U and V from 16 pixels swizzled by m (m[0], m[1] for the first 3
+// vectors, m[2], m[3] for the last).
+static inline void PlanarToUV16(const v128_t* s,
+ const v128_t* m,
+ v128_t ku02,
+ v128_t ku13,
+ v128_t kv02,
+ v128_t kv13,
+ v128_t bias_u,
+ v128_t bias_v,
+ uint8_t* dst_u,
+ uint8_t* dst_v) {
+ v128_t u[4], v[4];
+ for (int i = 0; i < 4; ++i) {
+ v128_t lo = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 2 : 0]);
+ v128_t hi = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 3 : 1]);
+ u[i] = MulAdd16(lo, hi, ku02, ku13);
+ v[i] = MulAdd16(lo, hi, kv02, kv13);
+ }
+ wasm_v128_store(dst_u, wasm_u8x16_narrow_i16x8(
+ SubShr8(bias_u, HAddHalves64(u[0], u[1])),
+ SubShr8(bias_u, HAddHalves64(u[2], u[3]))));
+ wasm_v128_store(dst_v, wasm_u8x16_narrow_i16x8(
+ SubShr8(bias_v, HAddHalves64(v[0], v[1])),
+ SubShr8(bias_v, HAddHalves64(v[2], v[3]))));
+}
+#endif
+
+#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_RGBTOUVMATRIXROW_WASMSIMD)
+// Rounded 2x2 average of 8 pixels held as 16 bit {c0, c2} or {c1, c3}
+// pairs. a = row 0, b = row 1, 4 pixels per vector. Returns 4 pixels.
+static inline v128_t Avg2x2Pairs(v128_t a0,
+ v128_t a1,
+ v128_t b0,
+ v128_t b1,
+ v128_t two) {
+ v128_t s0 = wasm_i16x8_add(a0, b0);
+ v128_t s1 = wasm_i16x8_add(a1, b1);
+ v128_t s = wasm_i16x8_add(wasm_i32x4_shuffle(s0, s1, 0, 2, 4, 6),
+ wasm_i32x4_shuffle(s0, s1, 1, 3, 5, 7));
+ return wasm_u16x8_shr(wasm_i16x8_add(s, two), 2);
+}
+
+// Stores 8 U and 8 V from 2 vectors each of {c0, c2} and {c1, c3} pairs.
+static inline void PairsToUV8(const v128_t* lo,
+ const v128_t* hi,
+ v128_t ku02,
+ v128_t ku13,
+ v128_t kv02,
+ v128_t kv13,
+ v128_t bias_u,
+ v128_t bias_v,
+ uint8_t* dst_u,
+ uint8_t* dst_v) {
+ v128_t u = HAddPairs16(MulAdd16(lo[0], hi[0], ku02, ku13),
+ MulAdd16(lo[1], hi[1], ku02, ku13));
+ v128_t v = HAddPairs16(MulAdd16(lo[0], hi[0], kv02, kv13),
+ MulAdd16(lo[1], hi[1], kv02, kv13));
+ v128_t uv = wasm_u8x16_narrow_i16x8(SubShr8(bias_u, u), SubShr8(bias_v, v));
+ wasm_v128_store64_lane(dst_u, uv, 0);
+ wasm_v128_store64_lane(dst_v, uv, 1);
+}
+#endif
+
+#ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD
+void ARGBToYMatrixRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t mask = SplatConst16(0xff);
+ v128_t k02 = SplatPair16(c->kRGBToY[0], c->kRGBToY[2]);
+ v128_t k13 = SplatPair16(c->kRGBToY[1], c->kRGBToY[3]);
+ v128_t bias = SplatConst16(c->kAddY[0]);
+ do {
+ v128_t t[4];
+ for (int i = 0; i < 4; ++i) {
+ v128_t s = wasm_v128_load(src_argb + i * 16);
+ t[i] = MulAdd16(wasm_v128_and(s, mask), wasm_u16x8_shr(s, 8), k02, k13);
+ }
+ v128_t y0 = wasm_i16x8_add(HAddPairs16(t[0], t[1]), bias);
+ v128_t y1 = wasm_i16x8_add(HAddPairs16(t[2], t[3]), bias);
+ wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(y0, 8),
+ wasm_u16x8_shr(y1, 8)));
+ src_argb += 64;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTOYMATRIXROW_WASMSIMD
+
+#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD
+// RGB24 is alpha 255 in the C version, so 255 * kRGBToY[3] is in the bias.
+void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m[4];
+ for (int i = 0; i < 4; ++i) {
+ m[i] = LoadConst(kShuffleRGBToPlanar + i * 16);
+ }
+ v128_t k02 = Splat4x2(c->kRGBToY[0], c->kRGBToY[2]);
+ v128_t k13 = Splat4x2(c->kRGBToY[1], 0);
+ v128_t bias = SplatConst16(c->kAddY[0] + 255 * c->kRGBToY[3]);
+ do {
+ v128_t t[4];
+ for (int i = 0; i < 4; ++i) {
+ v128_t s = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12));
+ t[i] = MulAdd16(LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 2 : 0]),
+ LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 3 : 1]), k02, k13);
+ }
+ v128_t y0 = wasm_i16x8_add(HAddHalves64(t[0], t[1]), bias);
+ v128_t y1 = wasm_i16x8_add(HAddHalves64(t[2], t[3]), bias);
+ wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(y0, 8),
+ wasm_u16x8_shr(y1, 8)));
+ src_rgb += 48;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGBTOYMATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD
+void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m[4];
+ m[0] = m[2] = LoadConst(kShuffleARGBToPlanar);
+ m[1] = m[3] = LoadConst(kShuffleARGBToPlanar + 16);
+ v128_t ku02 = Splat4x2(c->kRGBToU[0], c->kRGBToU[2]);
+ v128_t ku13 = Splat4x2(c->kRGBToU[1], c->kRGBToU[3]);
+ v128_t kv02 = Splat4x2(c->kRGBToV[0], c->kRGBToV[2]);
+ v128_t kv13 = Splat4x2(c->kRGBToV[1], c->kRGBToV[3]);
+ v128_t bias = SplatConst16(c->kAddUV[0]);
+ do {
+ v128_t s[4];
+ for (int i = 0; i < 4; ++i) {
+ s[i] = wasm_v128_load(src_argb + i * 16);
+ }
+ PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v);
+ src_argb += 64;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_WASMSIMD
+
+#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD
+void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m[4];
+ for (int i = 0; i < 4; ++i) {
+ m[i] = LoadConst(kShuffleRGBToPlanar + i * 16);
+ }
+ v128_t ku02 = Splat4x2(c->kRGBToU[0], c->kRGBToU[2]);
+ v128_t ku13 = Splat4x2(c->kRGBToU[1], 0);
+ v128_t kv02 = Splat4x2(c->kRGBToV[0], c->kRGBToV[2]);
+ v128_t kv13 = Splat4x2(c->kRGBToV[1], 0);
+ v128_t bias_u = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToU[3]);
+ v128_t bias_v = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToV[3]);
+ do {
+ v128_t s[4];
+ for (int i = 0; i < 4; ++i) {
+ s[i] = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12));
+ }
+ PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v);
+ src_rgb += 48;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGBTOUV444MATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGBTOUVMATRIXROW_WASMSIMD
+void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_argb1 = src_argb + src_stride_argb;
+ v128_t mask = SplatConst16(0xff);
+ v128_t two = SplatConst16(2);
+ v128_t ku02 = SplatPair16(c->kRGBToU[0], c->kRGBToU[2]);
+ v128_t ku13 = SplatPair16(c->kRGBToU[1], c->kRGBToU[3]);
+ v128_t kv02 = SplatPair16(c->kRGBToV[0], c->kRGBToV[2]);
+ v128_t kv13 = SplatPair16(c->kRGBToV[1], c->kRGBToV[3]);
+ v128_t bias = SplatConst16(c->kAddUV[0]);
+ do {
+ v128_t lo[2], hi[2];
+ for (int i = 0; i < 2; ++i) {
+ v128_t a0 = wasm_v128_load(src_argb + i * 32);
+ v128_t a1 = wasm_v128_load(src_argb + i * 32 + 16);
+ v128_t b0 = wasm_v128_load(src_argb1 + i * 32);
+ v128_t b1 = wasm_v128_load(src_argb1 + i * 32 + 16);
+ lo[i] =
+ Avg2x2Pairs(wasm_v128_and(a0, mask), wasm_v128_and(a1, mask),
+ wasm_v128_and(b0, mask), wasm_v128_and(b1, mask), two);
+ hi[i] = Avg2x2Pairs(wasm_u16x8_shr(a0, 8), wasm_u16x8_shr(a1, 8),
+ wasm_u16x8_shr(b0, 8), wasm_u16x8_shr(b1, 8), two);
+ }
+ PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v);
+ src_argb += 64;
+ src_argb1 += 64;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTOUVMATRIXROW_WASMSIMD
+
+#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD
+// Swizzles 4 RGB24 pixels to 16 bit {b, r} and {g, 0} pairs. The second
+// pair of masks is for the load at offset 32 (avoids over-read).
+static const uint8_t kShuffleRGBToPairs[64] = {
+ 0, 128, 2, 128, 3, 128, 5, 128, 6, 128, 8, 128, 9, 128, 11, 128,
+ 1, 128, 128, 128, 4, 128, 128, 128, 7, 128, 128, 128, 10, 128, 128, 128,
+ 4, 128, 6, 128, 7, 128, 9, 128, 10, 128, 12, 128, 13, 128, 15, 128,
+ 5, 128, 128, 128, 8, 128, 128, 128, 11, 128, 128, 128, 14, 128, 128, 128};
+
+void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_rgb1 = src_rgb + src_stride_rgb;
+ v128_t m[4];
+ for (int i = 0; i < 4; ++i) {
+ m[i] = LoadConst(kShuffleRGBToPairs + i * 16);
+ }
+ v128_t two = SplatConst16(2);
+ v128_t ku02 = SplatPair16(c->kRGBToU[0], c->kRGBToU[2]);
+ v128_t ku13 = SplatPair16(c->kRGBToU[1], 0);
+ v128_t kv02 = SplatPair16(c->kRGBToV[0], c->kRGBToV[2]);
+ v128_t kv13 = SplatPair16(c->kRGBToV[1], 0);
+ v128_t bias_u = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToU[3]);
+ v128_t bias_v = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToV[3]);
+ do {
+ v128_t lo[2], hi[2];
+ for (int i = 0; i < 2; ++i) {
+ // Pixels 8 * i to 8 * i + 3 at offset 24 * i, the rest at 12 or 32.
+ int off = i ? 32 : 12;
+ v128_t a0 = wasm_v128_load(src_rgb + i * 24);
+ v128_t a1 = wasm_v128_load(src_rgb + off);
+ v128_t b0 = wasm_v128_load(src_rgb1 + i * 24);
+ v128_t b1 = wasm_v128_load(src_rgb1 + off);
+ v128_t ml = m[i * 2];
+ v128_t mh = m[i * 2 + 1];
+ lo[i] = Avg2x2Pairs(
+ LIBYUV_WASM_SWIZZLE(a0, m[0]), LIBYUV_WASM_SWIZZLE(a1, ml),
+ LIBYUV_WASM_SWIZZLE(b0, m[0]), LIBYUV_WASM_SWIZZLE(b1, ml), two);
+ hi[i] = Avg2x2Pairs(
+ LIBYUV_WASM_SWIZZLE(a0, m[1]), LIBYUV_WASM_SWIZZLE(a1, mh),
+ LIBYUV_WASM_SWIZZLE(b0, m[1]), LIBYUV_WASM_SWIZZLE(b1, mh), two);
+ }
+ PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v);
+ src_rgb += 48;
+ src_rgb1 += 48;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGBTOUVMATRIXROW_WASMSIMD
+
+// 16 bit pixel formats keep each channel in its own 16 bit lane.
+// 5 and 6 bit channels expand as (v * 33) >> 2 and (v * 65) >> 4, and 4 bit
+// channels as v * 17, which match the C bit replication.
+
+#if defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \
+ defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD)
+// Rounded 2x2 average of 16 pixels of 1 channel. a = row 0, b = row 1.
+static inline v128_t Avg2x2Lanes(v128_t a0,
+ v128_t a1,
+ v128_t b0,
+ v128_t b1,
+ v128_t two) {
+ v128_t s = HAddPairs16(wasm_i16x8_add(a0, b0), wasm_i16x8_add(a1, b1));
+ return wasm_u16x8_shr(wasm_i16x8_add(s, two), 2);
+}
+
+// Stores 8 U and 8 V from 8 pixels of 16 bit channels. k holds U
+// coefficients for b, g, r, a, then V coefficients, then bias U and V.
+static inline void ChannelsToUV8(v128_t b,
+ v128_t g,
+ v128_t r,
+ v128_t a,
+ const v128_t* k,
+ uint8_t* dst_u,
+ uint8_t* dst_v) {
+ v128_t u =
+ wasm_i16x8_add(MulAdd16(b, g, k[0], k[1]), MulAdd16(r, a, k[2], k[3]));
+ v128_t v =
+ wasm_i16x8_add(MulAdd16(b, g, k[4], k[5]), MulAdd16(r, a, k[6], k[7]));
+ v128_t uv = wasm_u8x16_narrow_i16x8(SubShr8(k[8], u), SubShr8(k[9], v));
+ wasm_v128_store64_lane(dst_u, uv, 0);
+ wasm_v128_store64_lane(dst_v, uv, 1);
+}
+
+// U and V coefficients for ChannelsToUV8. The alpha term is folded into the
+// bias if alpha is always 255 (opaque).
+static inline void LoadUVChannelConstants(const struct ArgbConstants* c,
+ int opaque,
+ v128_t* k) {
+ for (int i = 0; i < 4; ++i) {
+ k[i] = SplatConst16(c->kRGBToU[i]);
+ k[i + 4] = SplatConst16(c->kRGBToV[i]);
+ }
+ k[8] = SplatConst16(c->kAddUV[0] - opaque * 255 * c->kRGBToU[3]);
+ k[9] = SplatConst16(c->kAddUV[0] - opaque * 255 * c->kRGBToV[3]);
+}
+#endif
+#ifdef HAS_RGB565TOYMATRIXROW_WASMSIMD
+void RGB565ToYMatrixRow_WASMSIMD(const uint8_t* src_rgb565,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m5 = SplatConst16(0x1f);
+ v128_t m6 = SplatConst16(0x3f);
+ v128_t k33 = SplatConst16(33);
+ v128_t k65 = SplatConst16(65);
+ v128_t kb = SplatConst16(c->kRGBToY[0]);
+ v128_t kg = SplatConst16(c->kRGBToY[1]);
+ v128_t kr = SplatConst16(c->kRGBToY[2]);
+ v128_t bias = SplatConst16(c->kAddY[0] + 255 * c->kRGBToY[3]);
+ do {
+ v128_t y[2];
+ for (int i = 0; i < 2; ++i) {
+ v128_t x = wasm_v128_load(src_rgb565 + i * 16);
+ v128_t b = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2);
+ v128_t g = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m6), k65), 4);
+ v128_t r = wasm_u16x8_shr(wasm_i16x8_mul(wasm_u16x8_shr(x, 11), k33), 2);
+ v128_t s = wasm_i16x8_add(wasm_i16x8_mul(b, kb), bias);
+ y[i] = wasm_u16x8_shr(wasm_i16x8_add(s, MulAdd16(g, r, kg, kr)), 8);
+ }
+ wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1]));
+ src_rgb565 += 32;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGB565TOYMATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGB1555TOYMATRIXROW_WASMSIMD
+void ARGB1555ToYMatrixRow_WASMSIMD(const uint8_t* src_argb1555,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m5 = SplatConst16(0x1f);
+ v128_t k33 = SplatConst16(33);
+ v128_t kb = SplatConst16(c->kRGBToY[0]);
+ v128_t kg = SplatConst16(c->kRGBToY[1]);
+ v128_t kr = SplatConst16(c->kRGBToY[2]);
+ v128_t ka = SplatConst16(255 * c->kRGBToY[3]);
+ v128_t bias = SplatConst16(c->kAddY[0]);
+ do {
+ v128_t y[2];
+ for (int i = 0; i < 2; ++i) {
+ v128_t x = wasm_v128_load(src_argb1555 + i * 16);
+ v128_t b = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2);
+ v128_t g = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m5), k33), 2);
+ v128_t r = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 10), m5), k33), 2);
+ // Alpha is 0 or 255.
+ v128_t s = wasm_i16x8_add(wasm_v128_and(wasm_i16x8_shr(x, 15), ka), bias);
+ s = wasm_i16x8_add(s, wasm_i16x8_mul(b, kb));
+ y[i] = wasm_u16x8_shr(wasm_i16x8_add(s, MulAdd16(g, r, kg, kr)), 8);
+ }
+ wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1]));
+ src_argb1555 += 32;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGB1555TOYMATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGB4444TOYMATRIXROW_WASMSIMD
+// 4 bit channels expand as v * 17, so the coefficients are scaled by 17.
+void ARGB4444ToYMatrixRow_WASMSIMD(const uint8_t* src_argb4444,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ v128_t m4 = SplatConst16(0x0f);
+ v128_t kb = SplatConst16(c->kRGBToY[0] * 17);
+ v128_t kg = SplatConst16(c->kRGBToY[1] * 17);
+ v128_t kr = SplatConst16(c->kRGBToY[2] * 17);
+ v128_t ka = SplatConst16(c->kRGBToY[3] * 17);
+ v128_t bias = SplatConst16(c->kAddY[0]);
+ do {
+ v128_t y[2];
+ for (int i = 0; i < 2; ++i) {
+ v128_t x = wasm_v128_load(src_argb4444 + i * 16);
+ v128_t s = wasm_i16x8_add(
+ MulAdd16(wasm_v128_and(x, m4), wasm_u16x8_shr(x, 12), kb, ka), bias);
+ s = wasm_i16x8_add(
+ s, MulAdd16(wasm_v128_and(wasm_u16x8_shr(x, 4), m4),
+ wasm_v128_and(wasm_u16x8_shr(x, 8), m4), kg, kr));
+ y[i] = wasm_u16x8_shr(s, 8);
+ }
+ wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1]));
+ src_argb4444 += 32;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGB4444TOYMATRIXROW_WASMSIMD
+
+#ifdef HAS_RGB565TOUVMATRIXROW_WASMSIMD
+void RGB565ToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb565,
+ int src_stride_rgb565,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_rgb565_1 = src_rgb565 + src_stride_rgb565;
+ v128_t m5 = SplatConst16(0x1f);
+ v128_t m6 = SplatConst16(0x3f);
+ v128_t k33 = SplatConst16(33);
+ v128_t k65 = SplatConst16(65);
+ v128_t two = SplatConst16(2);
+ v128_t k[10];
+ LoadUVChannelConstants(c, 1, k);
+ do {
+ v128_t b[4], g[4], r[4];
+ for (int i = 0; i < 4; ++i) {
+ v128_t x =
+ wasm_v128_load((i & 2 ? src_rgb565_1 : src_rgb565) + (i & 1) * 16);
+ b[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2);
+ g[i] = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m6), k65), 4);
+ r[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_u16x8_shr(x, 11), k33), 2);
+ }
+ ChannelsToUV8(Avg2x2Lanes(b[0], b[1], b[2], b[3], two),
+ Avg2x2Lanes(g[0], g[1], g[2], g[3], two),
+ Avg2x2Lanes(r[0], r[1], r[2], r[3], two),
+ wasm_i16x8_const_splat(0), k, dst_u, dst_v);
+ src_rgb565 += 32;
+ src_rgb565_1 += 32;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGB565TOUVMATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGB1555TOUVMATRIXROW_WASMSIMD
+void ARGB1555ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb1555,
+ int src_stride_argb1555,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_argb1555_1 = src_argb1555 + src_stride_argb1555;
+ v128_t m5 = SplatConst16(0x1f);
+ v128_t k33 = SplatConst16(33);
+ v128_t two = SplatConst16(2);
+ v128_t k[10];
+ LoadUVChannelConstants(c, 0, k);
+ do {
+ v128_t b[4], g[4], r[4], a[4];
+ for (int i = 0; i < 4; ++i) {
+ v128_t x = wasm_v128_load((i & 2 ? src_argb1555_1 : src_argb1555) +
+ (i & 1) * 16);
+ b[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2);
+ g[i] = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m5), k33), 2);
+ r[i] = wasm_u16x8_shr(
+ wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 10), m5), k33), 2);
+ a[i] = wasm_u16x8_shr(wasm_i16x8_shr(x, 15), 8);
+ }
+ ChannelsToUV8(Avg2x2Lanes(b[0], b[1], b[2], b[3], two),
+ Avg2x2Lanes(g[0], g[1], g[2], g[3], two),
+ Avg2x2Lanes(r[0], r[1], r[2], r[3], two),
+ Avg2x2Lanes(a[0], a[1], a[2], a[3], two), k, dst_u, dst_v);
+ src_argb1555 += 32;
+ src_argb1555_1 += 32;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGB1555TOUVMATRIXROW_WASMSIMD
+
+#ifdef HAS_ARGB4444TOUVMATRIXROW_WASMSIMD
+// Sums the nibble pairs {b, r} and {g, a} of 2x2 pixels in bytes, then
+// expands the sums by 17 before rounding.
+void ARGB4444ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb4444,
+ int src_stride_argb4444,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const uint8_t* src_argb4444_1 = src_argb4444 + src_stride_argb4444;
+ v128_t m4 = SplatConst16(0x0f0f);
+ v128_t m8 = SplatConst16(0xff);
+ v128_t k17 = SplatConst16(17);
+ v128_t two = SplatConst16(2);
+ v128_t k[10];
+ LoadUVChannelConstants(c, 0, k);
+ do {
+ v128_t s[2], t[2];
+ for (int i = 0; i < 2; ++i) {
+ v128_t a = wasm_v128_load(src_argb4444 + i * 16);
+ v128_t b = wasm_v128_load(src_argb4444_1 + i * 16);
+ s[i] = wasm_i16x8_add(wasm_v128_and(a, m4), wasm_v128_and(b, m4));
+ t[i] = wasm_i16x8_add(wasm_v128_and(wasm_u16x8_shr(a, 4), m4),
+ wasm_v128_and(wasm_u16x8_shr(b, 4), m4));
+ }
+ v128_t br = HAddPairs16(s[0], s[1]);
+ v128_t ga = HAddPairs16(t[0], t[1]);
+ v128_t ch[4] = {wasm_v128_and(br, m8), wasm_v128_and(ga, m8),
+ wasm_u16x8_shr(br, 8), wasm_u16x8_shr(ga, 8)};
+ for (int i = 0; i < 4; ++i) {
+ ch[i] =
+ wasm_u16x8_shr(wasm_i16x8_add(wasm_i16x8_mul(ch[i], k17), two), 2);
+ }
+ ChannelsToUV8(ch[0], ch[1], ch[2], ch[3], k, dst_u, dst_v);
+ src_argb4444 += 32;
+ src_argb4444_1 += 32;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGB4444TOUVMATRIXROW_WASMSIMD
+
+#ifdef HAS_CONVERT8TO16ROW_WASMSIMD
+// For bits 8 to 16, (y * 0x0101) >> (16 - bits) == p + (p >> 8) where
+// p = y << (bits - 8). This avoids a variable shift count.
+void Convert8To16Row_WASMSIMD(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ if (bits < 8) {
+ Convert8To16Row_C(src_y, dst_y, bits, width);
+ return;
+ }
+ v128_t scale = SplatConst16(1 << (bits - 8));
+ do {
+ asm("" : "+r"(dst_y));
+ v128_t y = wasm_v128_load(src_y);
+ v128_t y0 = wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(y), scale);
+ v128_t y1 = wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(y), scale);
+ wasm_v128_store(dst_y, wasm_i16x8_add(y0, wasm_u16x8_shr(y0, 8)));
+ wasm_v128_store(dst_y + 8, wasm_i16x8_add(y1, wasm_u16x8_shr(y1, 8)));
+ src_y += 16;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_CONVERT8TO16ROW_WASMSIMD
+
+#ifdef HAS_MULTIPLYROW_16_WASMSIMD
+void MultiplyRow_16_WASMSIMD(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ v128_t vscale = wasm_i16x8_splat(scale);
+ do {
+ asm("" : "+r"(src_y), "+r"(dst_y));
+ v128_t y0 = wasm_v128_load(src_y);
+ v128_t y1 = wasm_v128_load(src_y + 8);
+ y0 = wasm_i16x8_mul(y0, vscale);
+ y1 = wasm_i16x8_mul(y1, vscale);
+ wasm_v128_store(dst_y, y0);
+ wasm_v128_store(dst_y + 8, y1);
+ src_y += 16;
+ dst_y += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_MULTIPLYROW_16_WASMSIMD
+
+#ifdef HAS_MERGEUVROW_WASMSIMD
+void MergeUVRow_WASMSIMD(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ do {
+ asm("" : "+r"(dst_uv));
+ v128_t u = wasm_v128_load(src_u);
+ v128_t v = wasm_v128_load(src_v);
+ v128_t uv0 = wasm_i8x16_shuffle(u, v, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5,
+ 21, 6, 22, 7, 23);
+ v128_t uv1 = wasm_i8x16_shuffle(u, v, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28,
+ 13, 29, 14, 30, 15, 31);
+ wasm_v128_store(dst_uv, uv0);
+ wasm_v128_store(dst_uv + 16, uv1);
+ src_u += 16;
+ src_v += 16;
+ dst_uv += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_MERGEUVROW_WASMSIMD
+
+#ifdef HAS_MIRRORROW_WASMSIMD
+static const uint8_t kShuffleMirror[16] = {15, 14, 13, 12, 11, 10, 9, 8,
+ 7, 6, 5, 4, 3, 2, 1, 0};
+void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) {
+ v128_t shuf = LoadConst(kShuffleMirror);
+ src += width - 16;
+ do {
+ wasm_v128_store(dst, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), shuf));
+ src -= 16;
+ dst += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_MIRRORROW_WASMSIMD
+
+#ifdef HAS_MIRRORUVROW_WASMSIMD
+static const uint8_t kShuffleMirrorUV[16] = {14, 15, 12, 13, 10, 11, 8, 9,
+ 6, 7, 4, 5, 2, 3, 0, 1};
+void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ v128_t shuf = LoadConst(kShuffleMirrorUV);
+ src_uv += (width - 8) * 2;
+ do {
+ wasm_v128_store(dst_uv, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf));
+ src_uv -= 16;
+ dst_uv += 16;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_MIRRORUVROW_WASMSIMD
+
+#ifdef HAS_SWAPUVROW_WASMSIMD
+void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ do {
+ // Keep constant offsets in the load/store instructions.
+ asm("" : "+r"(src_uv), "+r"(dst_vu));
+ v128_t uv0 = wasm_v128_load(src_uv);
+ v128_t uv1 = wasm_v128_load(src_uv + 16);
+ // 16 bit byte swap: rev16 on Arm, shifts on x86.
+ wasm_v128_store(
+ dst_vu, wasm_v128_or(wasm_i16x8_shl(uv0, 8), wasm_u16x8_shr(uv0, 8)));
+ wasm_v128_store(dst_vu + 16, wasm_v128_or(wasm_i16x8_shl(uv1, 8),
+ wasm_u16x8_shr(uv1, 8)));
+ src_uv += 32;
+ dst_vu += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_SWAPUVROW_WASMSIMD
+
+#ifdef HAS_MIRRORSPLITUVROW_WASMSIMD
+// Reverses 8 UV pairs into 8 U then 8 V.
+static const uint8_t kShuffleMirrorSplitUV[16] = {14, 12, 10, 8, 6, 4, 2, 0,
+ 15, 13, 11, 9, 7, 5, 3, 1};
+void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ v128_t shuf = LoadConst(kShuffleMirrorSplitUV);
+ src_uv += (width - 16) * 2;
+ do {
+ asm("" : "+r"(src_uv));
+ v128_t a = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf);
+ v128_t b = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv + 16), shuf);
+ wasm_v128_store(dst_u, wasm_i64x2_shuffle(b, a, 0, 2));
+ wasm_v128_store(dst_v, wasm_i64x2_shuffle(b, a, 1, 3));
+ src_uv -= 32;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_MIRRORSPLITUVROW_WASMSIMD
+
+#ifdef HAS_RGB24MIRRORROW_WASMSIMD
+// Each 16 byte output needs 18 source bytes, so it is the OR of 2 swizzles.
+static const uint8_t kShuffleRGB24Mirror[96] = {
+ 13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2,
+ 3, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 14, 15, 128, 11, 12, 13, 8, 9, 10, 5, 6,
+ 7, 2, 3, 4, 128, 0, 128, 1, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 0, 128, 128, 12, 13, 14, 9, 10,
+ 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, 1, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128};
+void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ const uint8_t* shuf = kShuffleRGB24Mirror;
+ v128_t m0a = LoadConst(shuf);
+ v128_t m0b = LoadConst(shuf + 16);
+ v128_t m1a = LoadConst(shuf + 32);
+ v128_t m1b = LoadConst(shuf + 48);
+ v128_t m2a = LoadConst(shuf + 64);
+ v128_t m2b = LoadConst(shuf + 80);
+ src_rgb24 += (width - 16) * 3;
+ do {
+ asm("" : "+r"(src_rgb24), "+r"(dst_rgb24));
+ v128_t s0 = wasm_v128_load(src_rgb24);
+ v128_t s1 = wasm_v128_load(src_rgb24 + 16);
+ v128_t s2 = wasm_v128_load(src_rgb24 + 32);
+ // Bytes 15 and 32 of the 48 source bytes.
+ v128_t s02 = wasm_i8x16_shuffle(s0, s2, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30);
+ wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, m0a),
+ LIBYUV_WASM_SWIZZLE(s1, m0b)));
+ wasm_v128_store(dst_rgb24 + 16,
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, m1a),
+ LIBYUV_WASM_SWIZZLE(s02, m1b)));
+ wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, m2a),
+ LIBYUV_WASM_SWIZZLE(s1, m2b)));
+ src_rgb24 -= 48;
+ dst_rgb24 += 48;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_RGB24MIRRORROW_WASMSIMD
+
+#ifdef HAS_ARGBMIRRORROW_WASMSIMD
+void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) {
+ src += (width - 4) * 4;
+ do {
+ v128_t v = wasm_v128_load(src);
+ v = wasm_i32x4_shuffle(v, v, 3, 2, 1, 0);
+ wasm_v128_store(dst, v);
+ src -= 16;
+ dst += 16;
+ width -= 4;
+ } while (width > 0);
+}
+#endif // HAS_ARGBMIRRORROW_WASMSIMD
+
+#ifdef HAS_INTERPOLATEROW_WASMSIMD
+void InterpolateRow_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ if (source_y_fraction == 0) {
+ do {
+ wasm_v128_store(dst_ptr, wasm_v128_load(src_ptr));
+ src_ptr += 16;
+ dst_ptr += 16;
+ width -= 16;
+ } while (width > 0);
+ return;
+ }
+ if (source_y_fraction == 128) {
+ do {
+ v128_t s0 = wasm_v128_load(src_ptr);
+ v128_t s1 = wasm_v128_load(src_ptr1);
+ wasm_v128_store(dst_ptr, wasm_u8x16_avgr(s0, s1));
+ src_ptr += 16;
+ src_ptr1 += 16;
+ dst_ptr += 16;
+ width -= 16;
+ } while (width > 0);
+ return;
+ }
+ // s0 * (256 - f) + s1 * f + 128 fits in 16 bits. Both weights fit in 8
+ // bits since f is 1 to 255. The high bytes are the result (uzp2 on Arm).
+ v128_t w0 = SplatConst8(256 - source_y_fraction);
+ v128_t w1 = SplatConst8(source_y_fraction);
+ v128_t round = SplatConst16(128);
+ do {
+ v128_t s0 = wasm_v128_load(src_ptr);
+ v128_t s1 = wasm_v128_load(src_ptr1);
+ v128_t lo = wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s0, w0),
+ wasm_u16x8_extmul_low_u8x16(s1, w1));
+ v128_t hi = wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s0, w0),
+ wasm_u16x8_extmul_high_u8x16(s1, w1));
+ lo = wasm_i16x8_add(lo, round);
+ hi = wasm_i16x8_add(hi, round);
+ wasm_v128_store(
+ dst_ptr, wasm_i8x16_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19,
+ 21, 23, 25, 27, 29, 31));
+ src_ptr += 16;
+ src_ptr1 += 16;
+ dst_ptr += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_INTERPOLATEROW_WASMSIMD
+
+#ifdef HAS_INTERPOLATEROW_16_WASMSIMD
+void InterpolateRow_16_WASMSIMD(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ if (source_y_fraction == 0) {
+ do {
+ wasm_v128_store(dst_ptr, wasm_v128_load(src_ptr));
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width > 0);
+ return;
+ }
+ if (source_y_fraction == 128) {
+ do {
+ v128_t s0 = wasm_v128_load(src_ptr);
+ v128_t s1 = wasm_v128_load(src_ptr1);
+ wasm_v128_store(dst_ptr, wasm_u16x8_avgr(s0, s1));
+ src_ptr += 8;
+ src_ptr1 += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width > 0);
+ return;
+ }
+ // s0 * (256 - f) + s1 * f + 128 fits in 24 bits. Shift the result into
+ // the high 16 bits of each lane, then take the odd 16 bit lanes.
+ v128_t w0 = SplatConst16(256 - source_y_fraction);
+ v128_t w1 = SplatConst16(source_y_fraction);
+ v128_t round = SplatConst32(128);
+ do {
+ v128_t s0 = wasm_v128_load(src_ptr);
+ v128_t s1 = wasm_v128_load(src_ptr1);
+ v128_t lo = wasm_i32x4_add(wasm_u32x4_extmul_low_u16x8(s0, w0),
+ wasm_u32x4_extmul_low_u16x8(s1, w1));
+ v128_t hi = wasm_i32x4_add(wasm_u32x4_extmul_high_u16x8(s0, w0),
+ wasm_u32x4_extmul_high_u16x8(s1, w1));
+ lo = wasm_i32x4_shl(wasm_i32x4_add(lo, round), 8);
+ hi = wasm_i32x4_shl(wasm_i32x4_add(hi, round), 8);
+ wasm_v128_store(dst_ptr,
+ wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15));
+ src_ptr += 8;
+ src_ptr1 += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_INTERPOLATEROW_16_WASMSIMD
+
+#ifdef HAS_J400TOARGBROW_WASMSIMD
+static const uint8_t kShuffleJ400ToARGB[64] = {
+ 0, 0, 0, 128, 1, 1, 1, 128, 2, 2, 2, 128, 3, 3, 3, 128,
+ 4, 4, 4, 128, 5, 5, 5, 128, 6, 6, 6, 128, 7, 7, 7, 128,
+ 8, 8, 8, 128, 9, 9, 9, 128, 10, 10, 10, 128, 11, 11, 11, 128,
+ 12, 12, 12, 128, 13, 13, 13, 128, 14, 14, 14, 128, 15, 15, 15, 128};
+void J400ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t shuf0 = LoadConst(kShuffleJ400ToARGB);
+ v128_t shuf1 = LoadConst(kShuffleJ400ToARGB + 16);
+ v128_t shuf2 = LoadConst(kShuffleJ400ToARGB + 32);
+ v128_t shuf3 = LoadConst(kShuffleJ400ToARGB + 48);
+ v128_t alpha = SplatConst32((int)0xff000000);
+ do {
+ asm("" : "+r"(dst_argb));
+ v128_t y = wasm_v128_load(src_y);
+ wasm_v128_store(dst_argb,
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf0), alpha));
+ wasm_v128_store(dst_argb + 16,
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf1), alpha));
+ wasm_v128_store(dst_argb + 32,
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf2), alpha));
+ wasm_v128_store(dst_argb + 48,
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf3), alpha));
+ src_y += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_J400TOARGBROW_WASMSIMD
+
+#ifdef HAS_RGB565TOARGBROW_WASMSIMD
+void RGB565ToARGBRow_WASMSIMD(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t mask_b5 = SplatConst16(0x001f);
+ v128_t mask_g6 = SplatConst16(0x07e0);
+ v128_t mask_r5 = SplatConst16(0xf800);
+ v128_t mask_hi = SplatConst16(0xff00);
+ do {
+ asm("" : "+r"(dst_argb));
+ v128_t p = wasm_v128_load(src_rgb565);
+ v128_t b5 = wasm_v128_and(p, mask_b5);
+ v128_t b8 = wasm_v128_or(wasm_i16x8_shl(b5, 3), wasm_u16x8_shr(b5, 2));
+ v128_t g6 = wasm_v128_and(p, mask_g6);
+ v128_t g8_hi = wasm_v128_and(
+ wasm_v128_or(wasm_i16x8_shl(g6, 5), wasm_u16x8_shr(g6, 1)), mask_hi);
+ v128_t bg = wasm_v128_or(b8, g8_hi);
+
+ v128_t r8 = wasm_v128_or(wasm_u16x8_shr(wasm_v128_and(p, mask_r5), 8),
+ wasm_u16x8_shr(p, 13));
+ v128_t ra = wasm_v128_or(r8, mask_hi);
+
+ v128_t d0 = wasm_i16x8_shuffle(bg, ra, 0, 8, 1, 9, 2, 10, 3, 11);
+ v128_t d1 = wasm_i16x8_shuffle(bg, ra, 4, 12, 5, 13, 6, 14, 7, 15);
+ wasm_v128_store(dst_argb, d0);
+ wasm_v128_store(dst_argb + 16, d1);
+ src_rgb565 += 16;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_RGB565TOARGBROW_WASMSIMD
+
+#ifdef HAS_ARGB1555TOARGBROW_WASMSIMD
+// 5 to 8 bit replication with multiplies: (v * 0x108) >> 5 == v << 3 | v >> 2.
+void ARGB1555ToARGBRow_WASMSIMD(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t mask_b5 = SplatConst16(0x001f);
+ v128_t mask_g5 = SplatConst16(0x03e0);
+ v128_t mask_hi = SplatConst16(0xff00);
+ v128_t mul_br = SplatConst16(0x0108);
+ v128_t mul_g = SplatConst16(0x0042);
+ do {
+ asm("" : "+r"(dst_argb));
+ v128_t p = wasm_v128_load(src_argb1555);
+ v128_t b8 =
+ wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(p, mask_b5), mul_br), 5);
+ // g5 << 5 times 0x42 is g5 << 11 | g5 << 6; the high byte is g8.
+ v128_t g8_hi = wasm_v128_and(
+ wasm_i16x8_mul(wasm_v128_and(p, mask_g5), mul_g), mask_hi);
+ v128_t bg = wasm_v128_or(b8, g8_hi);
+ v128_t r5 = wasm_u16x8_shr(wasm_i16x8_shl(p, 1), 11);
+ v128_t r8 = wasm_u16x8_shr(wasm_i16x8_mul(r5, mul_br), 5);
+ v128_t a8_hi = wasm_v128_and(wasm_i16x8_shr(p, 15), mask_hi);
+ v128_t ra = wasm_v128_or(r8, a8_hi);
+
+ v128_t d0 = wasm_i16x8_shuffle(bg, ra, 0, 8, 1, 9, 2, 10, 3, 11);
+ v128_t d1 = wasm_i16x8_shuffle(bg, ra, 4, 12, 5, 13, 6, 14, 7, 15);
+ wasm_v128_store(dst_argb, d0);
+ wasm_v128_store(dst_argb + 16, d1);
+ src_argb1555 += 16;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGB1555TOARGBROW_WASMSIMD
+
+#ifdef HAS_ARGB4444TOARGBROW_WASMSIMD
+void ARGB4444ToARGBRow_WASMSIMD(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t mask_lo = SplatConst8(0x0f);
+ v128_t mask_hi = SplatConst8(0xf0);
+ do {
+ asm("" : "+r"(dst_argb));
+ v128_t v = wasm_v128_load(src_argb4444);
+ v128_t v_lo = wasm_v128_and(v, mask_lo);
+ v_lo = wasm_v128_or(v_lo, wasm_i16x8_shl(v_lo, 4));
+ v128_t v_hi = wasm_v128_and(v, mask_hi);
+ v_hi = wasm_v128_or(v_hi, wasm_u16x8_shr(v_hi, 4));
+
+ v128_t d0 = wasm_i8x16_shuffle(v_lo, v_hi, 0, 16, 1, 17, 2, 18, 3, 19, 4,
+ 20, 5, 21, 6, 22, 7, 23);
+ v128_t d1 = wasm_i8x16_shuffle(v_lo, v_hi, 8, 24, 9, 25, 10, 26, 11, 27, 12,
+ 28, 13, 29, 14, 30, 15, 31);
+ wasm_v128_store(dst_argb, d0);
+ wasm_v128_store(dst_argb + 16, d1);
+ src_argb4444 += 16;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGB4444TOARGBROW_WASMSIMD
+
+#ifdef HAS_ARGBTORGB565DITHERROW_WASMSIMD
+void ARGBToRGB565DitherRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ v128_t dither = wasm_i32x4_splat((int32_t)dither4);
+ dither = wasm_i8x16_shuffle(dither, dither, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2,
+ 2, 3, 3, 3, 3);
+ v128_t mask_b = SplatConst32(0x0000001f);
+ v128_t mask_g = SplatConst32(0x000007e0);
+ v128_t mask_r = SplatConst32(0x0000f800);
+ do {
+ asm("" : "+r"(src_argb));
+ v128_t s0 = wasm_u8x16_add_sat(wasm_v128_load(src_argb), dither);
+ v128_t s1 = wasm_u8x16_add_sat(wasm_v128_load(src_argb + 16), dither);
+ v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b);
+ v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 5), mask_g);
+ v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 8), mask_r);
+ v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b);
+ v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 5), mask_g);
+ v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 8), mask_r);
+ v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), r0);
+ v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), r1);
+ wasm_v128_store(dst_rgb, wasm_u16x8_narrow_i32x4(p0, p1));
+ src_argb += 32;
+ dst_rgb += 16;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTORGB565DITHERROW_WASMSIMD
+
+#ifdef HAS_ARGBTORGB565ROW_WASMSIMD
+void ARGBToRGB565Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ v128_t mask_b = SplatConst32(0x0000001f);
+ v128_t mask_g = SplatConst32(0x000007e0);
+ v128_t mask_r = SplatConst32(0x0000f800);
+ do {
+ asm("" : "+r"(src_argb));
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb + 16);
+ v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b);
+ v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 5), mask_g);
+ v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 8), mask_r);
+ v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b);
+ v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 5), mask_g);
+ v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 8), mask_r);
+ v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), r0);
+ v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), r1);
+ wasm_v128_store(dst_rgb, wasm_u16x8_narrow_i32x4(p0, p1));
+ src_argb += 32;
+ dst_rgb += 16;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTORGB565ROW_WASMSIMD
+
+#ifdef HAS_ARGBTOARGB1555ROW_WASMSIMD
+void ARGBToARGB1555Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb1555,
+ int width) {
+ v128_t mask_b = SplatConst32(0x0000001f);
+ v128_t mask_g = SplatConst32(0x000003e0);
+ v128_t mask_r = SplatConst32(0x00007c00);
+ v128_t mask_a = SplatConst32(0x00008000);
+ do {
+ asm("" : "+r"(src_argb));
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb + 16);
+ v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b);
+ v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 6), mask_g);
+ v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 9), mask_r);
+ v128_t a0 = wasm_v128_and(wasm_u32x4_shr(s0, 16), mask_a);
+ v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b);
+ v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 6), mask_g);
+ v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 9), mask_r);
+ v128_t a1 = wasm_v128_and(wasm_u32x4_shr(s1, 16), mask_a);
+ v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), wasm_v128_or(r0, a0));
+ v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), wasm_v128_or(r1, a1));
+ wasm_v128_store(dst_argb1555, wasm_u16x8_narrow_i32x4(p0, p1));
+ src_argb += 32;
+ dst_argb1555 += 16;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTOARGB1555ROW_WASMSIMD
+
+#ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD
+void ARGBToARGB4444Row_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb4444,
+ int width) {
+ v128_t mask_lo = SplatConst16(0x000f);
+ v128_t mask_hi = SplatConst16(0x00f0);
+ do {
+ asm("" : "+r"(src_argb));
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb + 16);
+ v128_t n0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(s0, 4), mask_lo),
+ wasm_v128_and(wasm_u16x8_shr(s0, 8), mask_hi));
+ v128_t n1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(s1, 4), mask_lo),
+ wasm_v128_and(wasm_u16x8_shr(s1, 8), mask_hi));
+ wasm_v128_store(dst_argb4444, wasm_u8x16_narrow_i16x8(n0, n1));
+ src_argb += 32;
+ dst_argb4444 += 16;
+ width -= 8;
+ } while (width > 0);
+}
+#endif // HAS_ARGBTOARGB4444ROW_WASMSIMD
+
+#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD
+void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ v128_t shuf = wasm_v128_load(shuffler);
+ while (width >= 8) {
+ asm("" : "+r"(src_argb), "+r"(dst_argb));
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb + 16);
+ wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf));
+ wasm_v128_store(dst_argb + 16, LIBYUV_WASM_SWIZZLE(s1, shuf));
+ src_argb += 32;
+ dst_argb += 32;
+ width -= 8;
+ }
+ if (width > 0) {
+ v128_t s0 = wasm_v128_load(src_argb);
+ wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf));
+ }
+}
+#endif // HAS_ARGBSHUFFLEROW_WASMSIMD
+
+#if defined(HAS_I422TOARGBROW_WASMSIMD) || \
+ defined(HAS_I422TOAR30ROW_WASMSIMD) || \
+ defined(HAS_NV12TOARGBROW_WASMSIMD) || \
+ defined(HAS_NV21TOARGBROW_WASMSIMD) || \
+ defined(HAS_I422TORGB24ROW_WASMSIMD) || \
+ defined(HAS_I422TORGB565ROW_WASMSIMD) || \
+ defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \
+ defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \
+ defined(HAS_NV12TORGB565ROW_WASMSIMD)
+// Interleaves map to punpck on x64 and zip on arm64 when the two inputs
+// differ. V8 arm64 lowers shuffle(x, x) to tbl with a rebuilt mask.
+static __inline v128_t ZipLo8_WASMSIMD(v128_t a, v128_t b) {
+ return wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6,
+ 22, 7, 23);
+}
+static __inline v128_t ZipHi8_WASMSIMD(v128_t a, v128_t b) {
+ return wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29,
+ 14, 30, 15, 31);
+}
+
+// YuvConstants splatted to 16-bit lanes, built once per row.
+struct YuvCoefs_WASMSIMD {
+ v128_t ub, ug, vg, vr, yg, yb, x80, k00ff, k0101;
+};
+
+static __inline void LoadYuvCoefs_WASMSIMD(const struct YuvConstants* yc,
+ int yb_adjust,
+ struct YuvCoefs_WASMSIMD* c) {
+ c->ub = wasm_i16x8_splat(yc->kUVToB[0]);
+ c->ug = wasm_i16x8_splat(yc->kUVToG[0]);
+ c->vg = wasm_i16x8_splat(yc->kUVToG[1]);
+ c->vr = wasm_i16x8_splat(yc->kUVToR[1]);
+ c->yg = wasm_i16x8_splat(yc->kYToRgb[0]);
+ c->yb = wasm_i16x8_splat(yc->kYBiasToRgb[0] - yb_adjust);
+ c->x80 = SplatConst8(0x80);
+ c->k00ff = SplatConst16(0x00ff);
+ c->k0101 = SplatConst16(0x0101);
+}
+
+// Returns ((y * 0x0101 * yg) >> 16) + yb for 8 Y in 16-bit lanes.
+static __inline v128_t YToRgb_WASMSIMD(v128_t y,
+ const struct YuvCoefs_WASMSIMD* c) {
+ // The barrier keeps clang from widening yg outside the loop, which would
+ // turn extmul (umull on arm64) into extend + i32x4.mul.
+ v128_t yg = c->yg;
+ asm volatile("" : "+r"(yg));
+ v128_t y16 = wasm_i16x8_mul(y, c->k0101);
+ v128_t lo = wasm_u32x4_extmul_low_u16x8(y16, yg);
+ v128_t hi = wasm_u32x4_extmul_high_u16x8(y16, yg);
+ return wasm_i16x8_add(wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15),
+ c->yb);
+}
+
+// Converts 16 pixels to B, G, R with 6 fractional bits (unclamped int16).
+// y = 16 Y. uv = 8 U/V pairs, U in the low byte of each 16-bit lane (V if vu).
+// Even and odd pixels are kept apart so each pair shares its chroma terms:
+// rgb[0], rgb[2], rgb[4] = even B, G, R and rgb[1], rgb[3], rgb[5] = odd.
+static __inline void YuvToRgb16_WASMSIMD(v128_t y,
+ v128_t uv,
+ int vu,
+ const struct YuvCoefs_WASMSIMD* c,
+ v128_t* rgb) {
+ v128_t ye = YToRgb_WASMSIMD(wasm_v128_and(y, c->k00ff), c);
+ v128_t yo = YToRgb_WASMSIMD(wasm_u16x8_shr(y, 8), c);
+ // u - 128 and v - 128 as int8, sign extended to int16.
+ v128_t uvs = wasm_v128_xor(uv, c->x80);
+ v128_t lo = wasm_i16x8_shr(wasm_i16x8_shl(uvs, 8), 8);
+ v128_t hi = wasm_i16x8_shr(uvs, 8);
+ v128_t ui = vu ? hi : lo;
+ v128_t vi = vu ? lo : hi;
+ v128_t b = wasm_i16x8_mul(ui, c->ub);
+ v128_t g =
+ wasm_i16x8_add(wasm_i16x8_mul(ui, c->ug), wasm_i16x8_mul(vi, c->vg));
+ v128_t r = wasm_i16x8_mul(vi, c->vr);
+ rgb[0] = wasm_i16x8_add_sat(ye, b);
+ rgb[1] = wasm_i16x8_add_sat(yo, b);
+ rgb[2] = wasm_i16x8_sub_sat(ye, g);
+ rgb[3] = wasm_i16x8_sub_sat(yo, g);
+ rgb[4] = wasm_i16x8_add_sat(ye, r);
+ rgb[5] = wasm_i16x8_add_sat(yo, r);
+}
+
+// Loads 16 Y and 8 U/V, or 8 Y and 4 U/V when width < 16.
+static __inline void ReadI422_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ int width,
+ v128_t* y,
+ v128_t* uv) {
+ if (width >= 16) {
+ *y = wasm_v128_load(src_y);
+ *uv = ZipLo8_WASMSIMD(wasm_v128_load64_zero(src_u),
+ wasm_v128_load64_zero(src_v));
+ } else {
+ *y = wasm_v128_load64_zero(src_y);
+ *uv = ZipLo8_WASMSIMD(wasm_v128_load32_zero(src_u),
+ wasm_v128_load32_zero(src_v));
+ }
+}
+
+// Loads 16 Y and 8 UV pairs, or 8 Y and 4 UV pairs when width < 16.
+static __inline void ReadNV12_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ int width,
+ v128_t* y,
+ v128_t* uv) {
+ if (width >= 16) {
+ *y = wasm_v128_load(src_y);
+ *uv = wasm_v128_load(src_uv);
+ } else {
+ *y = wasm_v128_load64_zero(src_y);
+ *uv = wasm_v128_load64_zero(src_uv);
+ }
+}
+#endif
+
+#if defined(HAS_I422TOARGBROW_WASMSIMD) || \
+ defined(HAS_NV12TOARGBROW_WASMSIMD) || defined(HAS_NV21TOARGBROW_WASMSIMD)
+// Clamps even/odd B, G, R to 8 bits and stores 16 ARGB pixels, or 8 when
+// width < 16.
+static __inline void StoreARGB_WASMSIMD(const v128_t* rgb,
+ v128_t ff,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6),
+ wasm_i16x8_shr(rgb[2], 6));
+ v128_t gb_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[3], 6),
+ wasm_i16x8_shr(rgb[1], 6));
+ v128_t rr = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6),
+ wasm_i16x8_shr(rgb[5], 6));
+ v128_t br_e = ZipLo8_WASMSIMD(bg_e, rr);
+ v128_t ga_e = ZipHi8_WASMSIMD(bg_e, ff);
+ v128_t br_o = ZipHi8_WASMSIMD(gb_o, rr);
+ v128_t ga_o = ZipLo8_WASMSIMD(gb_o, ff);
+ v128_t e = ZipLo8_WASMSIMD(br_e, ga_e); // Pixels 0, 2, 4, 6.
+ v128_t o = ZipLo8_WASMSIMD(br_o, ga_o); // Pixels 1, 3, 5, 7.
+ wasm_v128_store(dst_argb, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5));
+ wasm_v128_store(dst_argb + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7));
+ if (width >= 16) {
+ e = ZipHi8_WASMSIMD(br_e, ga_e);
+ o = ZipHi8_WASMSIMD(br_o, ga_o);
+ wasm_v128_store(dst_argb + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5));
+ wasm_v128_store(dst_argb + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7));
+ }
+}
+#endif
+
+#ifdef HAS_I422TOARGBROW_WASMSIMD
+void I422ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t ff = SplatConst8(0xff);
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ StoreARGB_WASMSIMD(rgb, ff, dst_argb, width);
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TOARGBROW_WASMSIMD
+
+#ifdef HAS_I422TOAR30ROW_WASMSIMD
+// Clamps B, G, R with 6 fractional bits to 10 bits and packs them into the
+// low and high 16 bits of AR30, without the alpha bits.
+static __inline void RgbToAR30_WASMSIMD(v128_t b,
+ v128_t g,
+ v128_t r,
+ v128_t zero,
+ v128_t k3ff0,
+ v128_t* lo,
+ v128_t* hi) {
+ b = wasm_i16x8_max(wasm_i16x8_min(b, k3ff0), zero);
+ g = wasm_i16x8_max(wasm_i16x8_min(g, k3ff0), zero);
+ r = wasm_i16x8_max(wasm_i16x8_min(r, k3ff0), zero);
+ *lo = wasm_v128_or(wasm_u16x8_shr(b, 4),
+ wasm_i16x8_shl(wasm_v128_and(g, k3ff0), 6));
+ *hi = wasm_v128_or(wasm_v128_and(r, k3ff0), wasm_u16x8_shr(g, 10));
+}
+
+void I422ToAR30Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t zero = SplatConst16(0);
+ v128_t k3ff0 = SplatConst16(0x3ff0);
+ v128_t kc000 = SplatConst16(0xc000);
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 24, &c); // AR30 bias.
+ do {
+ v128_t y, uv, rgb[6], lo_e, hi_e, lo_o, hi_o;
+ ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ RgbToAR30_WASMSIMD(rgb[0], rgb[2], rgb[4], zero, k3ff0, &lo_e, &hi_e);
+ RgbToAR30_WASMSIMD(rgb[1], rgb[3], rgb[5], zero, k3ff0, &lo_o, &hi_o);
+ hi_e = wasm_v128_or(hi_e, kc000);
+ hi_o = wasm_v128_or(hi_o, kc000);
+ // Even pixels 0, 2, 4, 6 and odd pixels 1, 3, 5, 7 as 32 bits.
+ v128_t e = wasm_i16x8_shuffle(lo_e, hi_e, 0, 8, 1, 9, 2, 10, 3, 11);
+ v128_t o = wasm_i16x8_shuffle(lo_o, hi_o, 0, 8, 1, 9, 2, 10, 3, 11);
+ wasm_v128_store(dst_ar30, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5));
+ wasm_v128_store(dst_ar30 + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7));
+ if (width >= 16) {
+ e = wasm_i16x8_shuffle(lo_e, hi_e, 4, 12, 5, 13, 6, 14, 7, 15);
+ o = wasm_i16x8_shuffle(lo_o, hi_o, 4, 12, 5, 13, 6, 14, 7, 15);
+ wasm_v128_store(dst_ar30 + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5));
+ wasm_v128_store(dst_ar30 + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7));
+ }
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_ar30 += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TOAR30ROW_WASMSIMD
+
+#ifdef HAS_NV12TOARGBROW_WASMSIMD
+void NV12ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t ff = SplatConst8(0xff);
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadNV12_WASMSIMD(src_y, src_uv, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ StoreARGB_WASMSIMD(rgb, ff, dst_argb, width);
+ src_y += 16;
+ src_uv += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_NV12TOARGBROW_WASMSIMD
+
+#ifdef HAS_NV21TOARGBROW_WASMSIMD
+void NV21ToARGBRow_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t ff = SplatConst8(0xff);
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, vu, rgb[6];
+ ReadNV12_WASMSIMD(src_y, src_vu, width, &y, &vu);
+ YuvToRgb16_WASMSIMD(y, vu, 1, &c, rgb);
+ StoreARGB_WASMSIMD(rgb, ff, dst_argb, width);
+ src_y += 16;
+ src_vu += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_NV21TOARGBROW_WASMSIMD
+
+#ifdef HAS_I422TORGB24ROW_WASMSIMD
+// Swizzles that interleave B/G pairs and R into 3 vectors of RGB24. R is
+// 8 even pixels followed by 8 odd pixels.
+static const uint8_t kI422ToRGB24Swizzle[96] = {
+ 0, 1, 128, 2, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9,
+ 128, 10, 128, 128, 0, 128, 128, 8, 128, 128, 1, 128, 128, 9,
+ 128, 128, 2, 128, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9,
+ 128, 10, 11, 128, 12, 13, 128, 10, 128, 128, 3, 128, 128, 11,
+ 128, 128, 4, 128, 128, 12, 128, 128, 128, 6, 7, 128, 8, 9,
+ 128, 10, 11, 128, 12, 13, 128, 14, 15, 128, 5, 128, 128, 13,
+ 128, 128, 6, 128, 128, 14, 128, 128, 7, 128, 128, 15};
+
+void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t m0 = LoadConst(kI422ToRGB24Swizzle);
+ v128_t m1 = LoadConst(kI422ToRGB24Swizzle + 16);
+ v128_t m2 = LoadConst(kI422ToRGB24Swizzle + 32);
+ v128_t m3 = LoadConst(kI422ToRGB24Swizzle + 48);
+ v128_t m4 = LoadConst(kI422ToRGB24Swizzle + 64);
+ v128_t m5 = LoadConst(kI422ToRGB24Swizzle + 80);
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadI422_WASMSIMD(src_y, src_u, src_v, 16, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6),
+ wasm_i16x8_shr(rgb[2], 6));
+ v128_t bg_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[1], 6),
+ wasm_i16x8_shr(rgb[3], 6));
+ v128_t r = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6),
+ wasm_i16x8_shr(rgb[5], 6));
+ v128_t b = ZipLo8_WASMSIMD(bg_e, bg_o);
+ v128_t g = ZipHi8_WASMSIMD(bg_e, bg_o);
+ v128_t bg0 = ZipLo8_WASMSIMD(b, g);
+ v128_t bg2 = ZipHi8_WASMSIMD(b, g);
+ v128_t bg1 = wasm_i8x16_shuffle(bg0, bg2, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+ 17, 18, 19, 20, 21, 22, 23);
+ wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg0, m0),
+ LIBYUV_WASM_SWIZZLE(r, m1)));
+ wasm_v128_store(dst_rgb24 + 16, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg1, m2),
+ LIBYUV_WASM_SWIZZLE(r, m3)));
+ wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg2, m4),
+ LIBYUV_WASM_SWIZZLE(r, m5)));
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_rgb24 += 48;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TORGB24ROW_WASMSIMD
+
+#if defined(HAS_I422TORGB565ROW_WASMSIMD) || \
+ defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \
+ defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \
+ defined(HAS_NV12TORGB565ROW_WASMSIMD)
+// Clamps x with 6 fractional bits to 8 bits, keeps the top 'bits' and moves
+// them to bit 'pos'.
+static __inline v128_t PackChannel_WASMSIMD(v128_t x,
+ v128_t zero,
+ v128_t k3fff,
+ int bits,
+ int pos) {
+ x = wasm_i16x8_max(wasm_i16x8_min(x, k3fff), zero);
+ return wasm_i16x8_shl(wasm_u16x8_shr(x, 14 - bits), pos);
+}
+
+// Packs even/odd B, G, R into 16 bit pixels with the given channel sizes and
+// stores 16 pixels, or 8 when width < 16. k = {0, 0x3fff, alpha} in 16-bit
+// lanes. Alpha is set if the channels use fewer than 16 bits.
+static __inline void Store16Bit_WASMSIMD(const v128_t* rgb,
+ int bbits,
+ int gbits,
+ int rbits,
+ const v128_t* k,
+ uint8_t* dst,
+ int width) {
+ v128_t p[2];
+ int i;
+ for (i = 0; i < 2; ++i) {
+ p[i] = wasm_v128_or(
+ PackChannel_WASMSIMD(rgb[i], k[0], k[1], bbits, 0),
+ wasm_v128_or(PackChannel_WASMSIMD(rgb[2 + i], k[0], k[1], gbits, bbits),
+ PackChannel_WASMSIMD(rgb[4 + i], k[0], k[1], rbits,
+ bbits + gbits)));
+ if (bbits + gbits + rbits < 16) {
+ p[i] = wasm_v128_or(p[i], k[2]);
+ }
+ }
+ wasm_v128_store(dst,
+ wasm_i16x8_shuffle(p[0], p[1], 0, 8, 1, 9, 2, 10, 3, 11));
+ if (width >= 16) {
+ wasm_v128_store(dst + 16,
+ wasm_i16x8_shuffle(p[0], p[1], 4, 12, 5, 13, 6, 14, 7, 15));
+ }
+}
+#endif
+
+#ifdef HAS_I422TORGB565ROW_WASMSIMD
+void I422ToRGB565Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t k[2] = {SplatConst16(0), SplatConst16(0x3fff)};
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ Store16Bit_WASMSIMD(rgb, 5, 6, 5, k, dst_rgb565, width);
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_rgb565 += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TORGB565ROW_WASMSIMD
+
+#ifdef HAS_I422TOARGB1555ROW_WASMSIMD
+void I422ToARGB1555Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t k[3] = {SplatConst16(0), SplatConst16(0x3fff), SplatConst16(0x8000)};
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ Store16Bit_WASMSIMD(rgb, 5, 5, 5, k, dst_argb1555, width);
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_argb1555 += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TOARGB1555ROW_WASMSIMD
+
+#ifdef HAS_I422TOARGB4444ROW_WASMSIMD
+void I422ToARGB4444Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t k[3] = {SplatConst16(0), SplatConst16(0x3fff), SplatConst16(0xf000)};
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ Store16Bit_WASMSIMD(rgb, 4, 4, 4, k, dst_argb4444, width);
+ src_y += 16;
+ src_u += 8;
+ src_v += 8;
+ dst_argb4444 += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_I422TOARGB4444ROW_WASMSIMD
+
+#ifdef HAS_NV12TORGB565ROW_WASMSIMD
+void NV12ToRGB565Row_WASMSIMD(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ struct YuvCoefs_WASMSIMD c;
+ v128_t k[2] = {SplatConst16(0), SplatConst16(0x3fff)};
+ LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c);
+ do {
+ v128_t y, uv, rgb[6];
+ ReadNV12_WASMSIMD(src_y, src_uv, width, &y, &uv);
+ YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb);
+ Store16Bit_WASMSIMD(rgb, 5, 6, 5, k, dst_rgb565, width);
+ src_y += 16;
+ src_uv += 16;
+ dst_rgb565 += 32;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_NV12TORGB565ROW_WASMSIMD
+
+#ifdef HAS_ARGBBLENDROW_WASMSIMD
+// Broadcast 255 - alpha (from byte 3 of each pixel) to 16 bit lanes.
+static const uint8_t kShuffleAlpha16[16] = {3, 128, 3, 128, 7, 128, 7, 128,
+ 11, 128, 11, 128, 15, 128, 15, 128};
+void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t shuf_a = LoadConst(kShuffleAlpha16);
+ v128_t one = SplatConst16(1);
+ v128_t mask_lo = SplatConst16(0x00ff);
+ v128_t mask_alpha = SplatConst32((int)0xff000000);
+
+ while (width >= 4) {
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb1);
+ src_argb += 16;
+ src_argb1 += 16;
+
+ // a = 256 - alpha.
+ v128_t a = wasm_i16x8_add(
+ LIBYUV_WASM_SWIZZLE(wasm_v128_xor(s0, mask_alpha), shuf_a), one);
+ v128_t s1_lo =
+ wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(s1, mask_lo), a), 8);
+ v128_t s1_hi =
+ wasm_v128_andnot(wasm_i16x8_mul(wasm_u16x8_shr(s1, 8), a), mask_lo);
+
+ v128_t res = wasm_v128_or(s0, mask_alpha);
+ res = wasm_u8x16_add_sat(res, s1_lo);
+ res = wasm_u8x16_add_sat(res, s1_hi);
+
+ wasm_v128_store(dst_argb, res);
+ dst_argb += 16;
+ width -= 4;
+ }
+
+ while (width > 0) {
+ uint32_t s0 = *(const uint32_t*)src_argb;
+ uint32_t s1 = *(const uint32_t*)src_argb1;
+ src_argb += 4;
+ src_argb1 += 4;
+
+ uint32_t a = 256 - (s0 >> 24);
+ uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff);
+ uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00);
+ uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000);
+
+ b = (b > 255) ? 255 : b;
+ g = (g > 0xff00) ? 0xff00 : (g & 0xff00);
+ r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000);
+
+ *(uint32_t*)dst_argb = 0xff000000u | r | g | b;
+ dst_argb += 4;
+ --width;
+ }
+}
+#endif // HAS_ARGBBLENDROW_WASMSIMD
+
+#ifdef HAS_BLENDPLANEROW_WASMSIMD
+void BlendPlaneRow_WASMSIMD(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ // src0 * a + src1 * (255 - a) + 255 fits in 16 bits. 255 - a is a ^ 255.
+ // The high bytes are the result (uzp2 on Arm).
+ v128_t c255 = SplatConst16(255);
+ v128_t ff = SplatConst8(255);
+ do {
+ v128_t s0 = wasm_v128_load(src0);
+ v128_t s1 = wasm_v128_load(src1);
+ v128_t a = wasm_v128_load(alpha);
+ v128_t ia = wasm_v128_xor(a, ff);
+ src0 += 16;
+ src1 += 16;
+ alpha += 16;
+ v128_t lo = wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s0, a),
+ wasm_u16x8_extmul_low_u8x16(s1, ia));
+ v128_t hi = wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s0, a),
+ wasm_u16x8_extmul_high_u8x16(s1, ia));
+ lo = wasm_i16x8_add(lo, c255);
+ hi = wasm_i16x8_add(hi, c255);
+ wasm_v128_store(dst, wasm_i8x16_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15,
+ 17, 19, 21, 23, 25, 27, 29, 31));
+ dst += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_BLENDPLANEROW_WASMSIMD
+
+#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD
+void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ do {
+ asm("" : "+r"(src_argb), "+r"(dst_a));
+ v128_t s0 = wasm_v128_load(src_argb);
+ v128_t s1 = wasm_v128_load(src_argb + 16);
+ v128_t s2 = wasm_v128_load(src_argb + 32);
+ v128_t s3 = wasm_v128_load(src_argb + 48);
+ s0 = wasm_u32x4_shr(s0, 24);
+ s1 = wasm_u32x4_shr(s1, 24);
+ s2 = wasm_u32x4_shr(s2, 24);
+ s3 = wasm_u32x4_shr(s3, 24);
+ v128_t a = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(s0, s1),
+ wasm_i16x8_narrow_i32x4(s2, s3));
+ wasm_v128_store(dst_a, a);
+ src_argb += 64;
+ dst_a += 16;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGBEXTRACTALPHAROW_WASMSIMD
+
+#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
+// Swizzle Y[i] to the alpha byte of 32 bit lane i; 128 indices give 0.
+static const uint8_t kYToAlphaSwizzleTable[64] = {
+ 128, 128, 128, 0, 128, 128, 128, 1, 128, 128, 128, 2, 128, 128, 128, 3,
+ 128, 128, 128, 4, 128, 128, 128, 5, 128, 128, 128, 6, 128, 128, 128, 7,
+ 128, 128, 128, 8, 128, 128, 128, 9, 128, 128, 128, 10, 128, 128, 128, 11,
+ 128, 128, 128, 12, 128, 128, 128, 13, 128, 128, 128, 14, 128, 128, 128, 15};
+
+void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) {
+ v128_t rgb_mask = SplatConst32(0x00ffffff);
+ v128_t idx0 = LoadConst(kYToAlphaSwizzleTable);
+ v128_t idx1 = LoadConst(kYToAlphaSwizzleTable + 16);
+ v128_t idx2 = LoadConst(kYToAlphaSwizzleTable + 32);
+ v128_t idx3 = LoadConst(kYToAlphaSwizzleTable + 48);
+ do {
+ asm("" : "+r"(src), "+r"(dst));
+ v128_t y = wasm_v128_load(src);
+ v128_t d0 = wasm_v128_and(wasm_v128_load(dst), rgb_mask);
+ v128_t d1 = wasm_v128_and(wasm_v128_load(dst + 16), rgb_mask);
+ v128_t d2 = wasm_v128_and(wasm_v128_load(dst + 32), rgb_mask);
+ v128_t d3 = wasm_v128_and(wasm_v128_load(dst + 48), rgb_mask);
+ wasm_v128_store(dst, wasm_v128_or(d0, LIBYUV_WASM_SWIZZLE(y, idx0)));
+ wasm_v128_store(dst + 16, wasm_v128_or(d1, LIBYUV_WASM_SWIZZLE(y, idx1)));
+ wasm_v128_store(dst + 32, wasm_v128_or(d2, LIBYUV_WASM_SWIZZLE(y, idx2)));
+ wasm_v128_store(dst + 48, wasm_v128_or(d3, LIBYUV_WASM_SWIZZLE(y, idx3)));
+ src += 16;
+ dst += 64;
+ width -= 16;
+ } while (width > 0);
+}
+#endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
+
+#ifdef HAS_ARGBORROW_WASMSIMD
+void ARGBOrRow_WASMSIMD(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ v128_t val = wasm_i32x4_splat((int32_t)value);
+ do {
+ asm("" : "+r"(src_argb), "+r"(dst_argb));
+ v128_t s = wasm_v128_load(src_argb);
+ wasm_v128_store(dst_argb, wasm_v128_or(s, val));
+ src_argb += 16;
+ dst_argb += 16;
+ width -= 4;
+ } while (width > 0);
+}
+#endif // HAS_ARGBORROW_WASMSIMD
+
+#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD
+void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ v128_t c1 = SplatConst16(1);
+ v128_t c00ffffff = SplatConst32(0x00ffffff);
+ do {
+ asm("" : "+r"(src_argb0), "+r"(src_argb1), "+r"(dst_argb));
+ v128_t s = wasm_v128_load(src_argb0);
+ v128_t d = wasm_v128_load(src_argb1);
+ src_argb0 += 16;
+ src_argb1 += 16;
+ v128_t sa = wasm_u32x4_shr(s, 24);
+ v128_t da = wasm_u32x4_shr(d, 24);
+ v128_t prod16 = wasm_i16x8_add(wasm_i16x8_mul(da, sa), c1);
+ v128_t da_sa =
+ wasm_u16x8_shr(wasm_i16x8_add(prod16, wasm_u16x8_shr(prod16, 8)), 8);
+ v128_t da2 = wasm_i32x4_add(da, wasm_i32x4_sub(sa, da_sa));
+ v128_t t = wasm_u32x4_shr(wasm_i32x4_eq(da, wasm_i32x4_const_splat(0)), 24);
+ v128_t sa_eff = wasm_v128_or(sa, t);
+ v128_t den_eff = wasm_v128_or(da2, t);
+ v128_t num = wasm_i32x4_sub(wasm_i32x4_shl(sa_eff, 8), sa_eff);
+ v128_t ratio = wasm_u32x4_trunc_sat_f32x4(wasm_f32x4_div(
+ wasm_f32x4_convert_u32x4(num), wasm_f32x4_convert_u32x4(den_eff)));
+ v128_t r8 = wasm_i8x16_shuffle(ratio, ratio, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8,
+ 8, 8, 12, 12, 12, 12);
+ v128_t inv_r8 = wasm_v128_not(r8);
+ v128_t acc_lo =
+ wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(d, inv_r8),
+ wasm_u16x8_extmul_low_u8x16(s, r8)),
+ c1);
+ v128_t acc_hi =
+ wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(d, inv_r8),
+ wasm_u16x8_extmul_high_u8x16(s, r8)),
+ c1);
+ acc_lo =
+ wasm_u16x8_shr(wasm_i16x8_add(acc_lo, wasm_u16x8_shr(acc_lo, 8)), 8);
+ acc_hi =
+ wasm_u16x8_shr(wasm_i16x8_add(acc_hi, wasm_u16x8_shr(acc_hi, 8)), 8);
+ v128_t rgb = wasm_u8x16_narrow_i16x8(acc_lo, acc_hi);
+ v128_t out = wasm_v128_or(wasm_v128_and(rgb, c00ffffff),
+ wasm_i32x4_shl(da2, 24));
+ wasm_v128_store(dst_argb, out);
+ dst_argb += 16;
+ width -= 4;
+ } while (width > 0);
+}
+#endif // HAS_ARGBALPHABLENDROW_WASMSIMD
+
+#undef LIBYUV_WASM_SWIZZLE
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
diff --git a/libs/libyuv/source/row_win.cc b/libs/libyuv/source/row_win.cc
index f976d40260..ba6f7b56e3 100644
--- a/libs/libyuv/source/row_win.cc
+++ b/libs/libyuv/source/row_win.cc
@@ -8,15 +8,22 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "libyuv/convert_from_argb.h" // For ArgbConstants
#include "libyuv/row.h"
-// This module is for Visual C 32/64 bit and clangcl 32 bit
-#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \
- (defined(_M_IX86) || (defined(_M_X64) && !defined(__clang__)))
+// This module is for Visual C 32/64 bit
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
-#if defined(_M_X64)
#include <emmintrin.h>
+#include <immintrin.h> // For AVX2 intrinsics
+#include <string.h> // For memcpy
#include <tmmintrin.h> // For _mm_maddubs_epi16
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
#endif
#ifdef __cplusplus
@@ -24,15 +31,170 @@ namespace libyuv {
extern "C" {
#endif
-// 64 bit
-#if defined(_M_X64)
+#if defined(__clang__) || defined(__GNUC__)
+#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3")))
+#define LIBYUV_TARGET_AVX __attribute__((target("avx")))
+#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
+#define LIBYUV_TARGET_AVX2_FMA __attribute__((target("avx2,fma")))
+#define LIBYUV_TARGET_AVX512BW \
+ __attribute__((target("avx512bw,avx512vl,avx512f")))
+#define LIBYUV_TARGET_AVX512VBMI \
+ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
+#define LIBYUV_TARGET_AVX512VNNI \
+ __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f")))
+#define LIBYUV_TARGET_AVX512VNNIVBMI \
+ __attribute__((target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f")))
+#else
+#define LIBYUV_TARGET_SSSE3
+#define LIBYUV_TARGET_AVX
+#define LIBYUV_TARGET_AVX2
+#define LIBYUV_TARGET_AVX2_FMA
+#define LIBYUV_TARGET_AVX512BW
+#define LIBYUV_TARGET_AVX512VBMI
+#define LIBYUV_TARGET_AVX512VNNI
+#define LIBYUV_TARGET_AVX512VNNIVBMI
+#endif // defined(__clang__) || defined(__GNUC__)
+
+#if defined(__x86_64__) || defined(_M_X64)
+// Under MemorySanitizer, use portable lane-by-lane C loops for single-source
+// variable permutes and 512-bit float-to-int truncation so shadow propagation
+// relies only on standard C memory/array semantics rather than target-specific
+// LLVM intrinsic modeling. In non-MSan builds, call the native AVX-512
+// intrinsics directly.
+LIBYUV_TARGET_AVX512VBMI
+static inline __m512i PermutexvarEpi8(__m512i idx, __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint8_t idx_arr[64];
+ uint8_t v_arr[64];
+ uint8_t out_arr[64];
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 64; ++i) {
+ out_arr[i] = v_arr[idx_arr[i] & 63];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_permutexvar_epi8(idx, v);
+#endif
+}
+
+LIBYUV_TARGET_AVX512VBMI
+static inline __m512i MaskPermutexvarEpi8(__m512i src,
+ __mmask64 k,
+ __m512i idx,
+ __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint8_t src_arr[64];
+ uint8_t idx_arr[64];
+ uint8_t v_arr[64];
+ uint8_t out_arr[64];
+ _mm512_storeu_si512((__m512i*)src_arr, src);
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 64; ++i) {
+ out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 63] : src_arr[i];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_mask_permutexvar_epi8(src, k, idx, v);
+#endif
+}
+
+LIBYUV_TARGET_AVX512BW
+static inline __m512i PermutexvarEpi16(__m512i idx, __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint16_t idx_arr[32];
+ uint16_t v_arr[32];
+ uint16_t out_arr[32];
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 32; ++i) {
+ out_arr[i] = v_arr[idx_arr[i] & 31];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_permutexvar_epi16(idx, v);
+#endif
+}
+
+LIBYUV_TARGET_AVX512BW
+static inline __m512i PermutexvarEpi32(__m512i idx, __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint32_t idx_arr[16];
+ uint32_t v_arr[16];
+ uint32_t out_arr[16];
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 16; ++i) {
+ out_arr[i] = v_arr[idx_arr[i] & 15];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_permutexvar_epi32(idx, v);
+#endif
+}
+
+LIBYUV_TARGET_AVX512BW
+static inline __m512i PermutexvarEpi64(__m512i idx, __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint64_t idx_arr[8];
+ uint64_t v_arr[8];
+ uint64_t out_arr[8];
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 8; ++i) {
+ out_arr[i] = v_arr[idx_arr[i] & 7];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_permutexvar_epi64(idx, v);
+#endif
+}
+
+LIBYUV_TARGET_AVX512BW
+static inline __m512i CvttpsEpi32(__m512 v) {
+#if defined(LIBYUV_MSAN)
+ float v_arr[16];
+ int32_t out_arr[16];
+ _mm512_storeu_ps(v_arr, v);
+ for (int i = 0; i < 16; ++i) {
+ out_arr[i] = (int32_t)v_arr[i];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_cvttps_epi32(v);
+#endif
+}
+#endif // defined(__x86_64__) || defined(_M_X64)
+
+// Read 8 UV from 444
+#define READYUV444 \
+ xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
+ xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
+ xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
+ u_buf += 8; \
+ xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
+ xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
+ y_buf += 8;
+
+// Read 8 UV from 444, With 8 Alpha.
+#define READYUVA444 \
+ xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
+ xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
+ xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
+ u_buf += 8; \
+ xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
+ xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
+ y_buf += 8; \
+ xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
+ a_buf += 8;
// Read 4 UV from 422, upsample to 8 UV.
#define READYUV422 \
- xmm0 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
+ xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
- xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
- xmm0 = _mm_unpacklo_epi16(xmm0, xmm0); \
+ xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
+ xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
u_buf += 4; \
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
@@ -40,10 +202,10 @@ extern "C" {
// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
#define READYUVA422 \
- xmm0 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
+ xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
- xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
- xmm0 = _mm_unpacklo_epi16(xmm0, xmm0); \
+ xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
+ xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
u_buf += 4; \
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
@@ -52,24 +214,21 @@ extern "C" {
a_buf += 8;
// Convert 8 pixels: 8 UV and 8 Y.
-#define YUVTORGB(yuvconstants) \
- xmm1 = _mm_loadu_si128(&xmm0); \
- xmm2 = _mm_loadu_si128(&xmm0); \
- xmm0 = _mm_maddubs_epi16(xmm0, *(__m128i*)yuvconstants->kUVToB); \
- xmm1 = _mm_maddubs_epi16(xmm1, *(__m128i*)yuvconstants->kUVToG); \
- xmm2 = _mm_maddubs_epi16(xmm2, *(__m128i*)yuvconstants->kUVToR); \
- xmm0 = _mm_sub_epi16(*(__m128i*)yuvconstants->kUVBiasB, xmm0); \
- xmm1 = _mm_sub_epi16(*(__m128i*)yuvconstants->kUVBiasG, xmm1); \
- xmm2 = _mm_sub_epi16(*(__m128i*)yuvconstants->kUVBiasR, xmm2); \
- xmm4 = _mm_mulhi_epu16(xmm4, *(__m128i*)yuvconstants->kYToRgb); \
- xmm0 = _mm_adds_epi16(xmm0, xmm4); \
- xmm1 = _mm_adds_epi16(xmm1, xmm4); \
- xmm2 = _mm_adds_epi16(xmm2, xmm4); \
- xmm0 = _mm_srai_epi16(xmm0, 6); \
- xmm1 = _mm_srai_epi16(xmm1, 6); \
- xmm2 = _mm_srai_epi16(xmm2, 6); \
- xmm0 = _mm_packus_epi16(xmm0, xmm0); \
- xmm1 = _mm_packus_epi16(xmm1, xmm1); \
+#define YUVTORGB(yuvconstants) \
+ xmm3 = _mm_sub_epi8(xmm3, _mm_set1_epi8((char)0x80)); \
+ xmm4 = _mm_mulhi_epu16(xmm4, *(__m128i*)yuvconstants->kYToRgb); \
+ xmm4 = _mm_add_epi16(xmm4, *(__m128i*)yuvconstants->kYBiasToRgb); \
+ xmm0 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToB, xmm3); \
+ xmm1 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToG, xmm3); \
+ xmm2 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToR, xmm3); \
+ xmm0 = _mm_adds_epi16(xmm4, xmm0); \
+ xmm1 = _mm_subs_epi16(xmm4, xmm1); \
+ xmm2 = _mm_adds_epi16(xmm4, xmm2); \
+ xmm0 = _mm_srai_epi16(xmm0, 6); \
+ xmm1 = _mm_srai_epi16(xmm1, 6); \
+ xmm2 = _mm_srai_epi16(xmm2, 6); \
+ xmm0 = _mm_packus_epi16(xmm0, xmm0); \
+ xmm1 = _mm_packus_epi16(xmm1, xmm1); \
xmm2 = _mm_packus_epi16(xmm2, xmm2);
// Store 8 ARGB values.
@@ -83,6152 +242,10616 @@ extern "C" {
_mm_storeu_si128((__m128i*)(dst_argb + 16), xmm1); \
dst_argb += 32;
-#if defined(HAS_I422TOARGBROW_SSSE3)
-void I422ToARGBRow_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __m128i xmm0, xmm1, xmm2, xmm4;
- const __m128i xmm5 = _mm_set1_epi8(-1);
- const ptrdiff_t offset = (uint8_t*)v_buf - (uint8_t*)u_buf;
+// Store 8 AR30 values.
+#define STOREAR30 \
+ xmm1 = _mm_and_si128(xmm1, xmm7); \
+ xmm2 = _mm_and_si128(xmm2, xmm7); \
+ xmm0 = _mm_srli_epi16(xmm0, 4); \
+ xmm3 = _mm_slli_epi16(xmm1, 6); \
+ xmm0 = _mm_or_si128(xmm0, xmm3); \
+ xmm1 = _mm_srli_epi16(xmm1, 10); \
+ xmm2 = _mm_or_si128(xmm2, xmm6); \
+ xmm2 = _mm_or_si128(xmm2, xmm1); \
+ xmm1 = _mm_unpackhi_epi16(xmm0, xmm2); \
+ xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
+ _mm_storeu_si128((__m128i*)dst_ar30, xmm0); \
+ _mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \
+ dst_ar30 += 32;
+
+// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
+LIBYUV_TARGET_AVX2
+void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ __m256i ymm_u =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m256i ymm_v =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ __m256i ymm5 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ __m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
+
while (width > 0) {
- READYUV422
- YUVTORGB(yuvconstants)
- STOREARGB
- width -= 8;
- }
-}
-#endif
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
+ __m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
+ __m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
+ src_argb += 128;
+
+ __m256i ymm0_u = _mm256_maddubs_epi16(ymm0, ymm_u);
+ __m256i ymm1_u = _mm256_maddubs_epi16(ymm1, ymm_u);
+ __m256i ymm2_u = _mm256_maddubs_epi16(ymm2, ymm_u);
+ __m256i ymm3_u = _mm256_maddubs_epi16(ymm3, ymm_u);
+
+ __m256i ymm0_v = _mm256_maddubs_epi16(ymm0, ymm_v);
+ __m256i ymm1_v = _mm256_maddubs_epi16(ymm1, ymm_v);
+ __m256i ymm2_v = _mm256_maddubs_epi16(ymm2, ymm_v);
+ __m256i ymm3_v = _mm256_maddubs_epi16(ymm3, ymm_v);
+
+ ymm0_u = _mm256_hadd_epi16(ymm0_u, ymm1_u);
+ ymm2_u = _mm256_hadd_epi16(ymm2_u, ymm3_u);
+
+ ymm0_v = _mm256_hadd_epi16(ymm0_v, ymm1_v);
+ ymm2_v = _mm256_hadd_epi16(ymm2_v, ymm3_v);
+
+ ymm0_u = _mm256_sub_epi16(ymm5, ymm0_u);
+ ymm2_u = _mm256_sub_epi16(ymm5, ymm2_u);
+
+ ymm0_v = _mm256_sub_epi16(ymm5, ymm0_v);
+ ymm2_v = _mm256_sub_epi16(ymm5, ymm2_v);
+
+ ymm0_u = _mm256_srli_epi16(ymm0_u, 8);
+ ymm2_u = _mm256_srli_epi16(ymm2_u, 8);
+
+ ymm0_v = _mm256_srli_epi16(ymm0_v, 8);
+ ymm2_v = _mm256_srli_epi16(ymm2_v, 8);
+
+ ymm0_u = _mm256_packus_epi16(ymm0_u, ymm2_u);
+ ymm0_u = _mm256_permutevar8x32_epi32(ymm0_u, perm_mask);
+
+ ymm0_v = _mm256_packus_epi16(ymm0_v, ymm2_v);
+ ymm0_v = _mm256_permutevar8x32_epi32(ymm0_v, perm_mask);
+
+ _mm256_storeu_si256((__m256i*)dst_u, ymm0_u);
+ _mm256_storeu_si256((__m256i*)dst_v, ymm0_v);
+ dst_u += 32;
+ dst_v += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
+ // defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
+LIBYUV_TARGET_AVX2
+void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ __m256i ymm5 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm4 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToY));
+ __m256i ymm7 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddY));
+ __m256i ymm6 = _mm256_maddubs_epi16(ymm4, ymm5);
+ ymm6 = _mm256_hadd_epi16(ymm6, ymm6);
+ ymm7 = _mm256_sub_epi16(ymm7, ymm6);
+ __m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
-#if defined(HAS_I422ALPHATOARGBROW_SSSE3)
-void I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __m128i xmm0, xmm1, xmm2, xmm4, xmm5;
- const ptrdiff_t offset = (uint8_t*)v_buf - (uint8_t*)u_buf;
while (width > 0) {
- READYUVA422
- YUVTORGB(yuvconstants)
- STOREARGB
- width -= 8;
- }
-}
-#endif
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
+ __m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
+ __m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
+ src_argb += 128;
-// 32 bit
-#else // defined(_M_X64)
-#ifdef HAS_ARGBTOYROW_SSSE3
+ ymm0 = _mm256_sub_epi8(ymm0, ymm5);
+ ymm1 = _mm256_sub_epi8(ymm1, ymm5);
+ ymm2 = _mm256_sub_epi8(ymm2, ymm5);
+ ymm3 = _mm256_sub_epi8(ymm3, ymm5);
-// Constants for ARGB.
-static const vec8 kARGBToY = {13, 65, 33, 0, 13, 65, 33, 0,
- 13, 65, 33, 0, 13, 65, 33, 0};
+ ymm0 = _mm256_maddubs_epi16(ymm4, ymm0);
+ ymm1 = _mm256_maddubs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_maddubs_epi16(ymm4, ymm2);
+ ymm3 = _mm256_maddubs_epi16(ymm4, ymm3);
-// JPeg full range.
-static const vec8 kARGBToYJ = {15, 75, 38, 0, 15, 75, 38, 0,
- 15, 75, 38, 0, 15, 75, 38, 0};
+ ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
+ ymm2 = _mm256_hadd_epi16(ymm2, ymm3);
-static const vec8 kARGBToU = {112, -74, -38, 0, 112, -74, -38, 0,
- 112, -74, -38, 0, 112, -74, -38, 0};
+ ymm0 = _mm256_add_epi16(ymm0, ymm7);
+ ymm2 = _mm256_add_epi16(ymm2, ymm7);
-static const vec8 kARGBToUJ = {127, -84, -43, 0, 127, -84, -43, 0,
- 127, -84, -43, 0, 127, -84, -43, 0};
+ ymm0 = _mm256_srli_epi16(ymm0, 8);
+ ymm2 = _mm256_srli_epi16(ymm2, 8);
-static const vec8 kARGBToV = {
- -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0,
-};
+ ymm0 = _mm256_packus_epi16(ymm0, ymm2);
+ ymm0 = _mm256_permutevar8x32_epi32(ymm0, perm_mask);
-static const vec8 kARGBToVJ = {-20, -107, 127, 0, -20, -107, 127, 0,
- -20, -107, 127, 0, -20, -107, 127, 0};
+ _mm256_storeu_si256((__m256i*)dst_y, ymm0);
+ dst_y += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2)
-// vpshufb for vphaddw + vpackuswb packed to shorts.
-static const lvec8 kShufARGBToUV_AVX = {
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15};
+#ifdef HAS_RAWTOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ __m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
+ __m128i shuf_low =
+ _mm_set_epi8(-1, 9, 10, 11, -1, 6, 7, 8, -1, 3, 4, 5, -1, 0, 1, 2);
+ __m128i shuf_high =
+ _mm_set_epi8(-1, 13, 14, 15, -1, 10, 11, 12, -1, 7, 8, 9, -1, 4, 5, 6);
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(shuf_low);
+ __m256i ymm_shuf2 = _mm256_broadcastsi128_si256(shuf_high);
-// Constants for BGRA.
-static const vec8 kBGRAToY = {0, 33, 65, 13, 0, 33, 65, 13,
- 0, 33, 65, 13, 0, 33, 65, 13};
+ while (width > 0) {
+ __m128i xmm0 = _mm_loadu_si128((const __m128i*)src_raw);
+ __m256i ymm0 = _mm256_castsi128_si256(xmm0);
+ ymm0 = _mm256_inserti128_si256(
+ ymm0, _mm_loadu_si128((const __m128i*)(src_raw + 12)), 1);
+
+ __m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_raw + 24));
+ __m256i ymm1 = _mm256_castsi128_si256(xmm1);
+ ymm1 = _mm256_inserti128_si256(
+ ymm1, _mm_loadu_si128((const __m128i*)(src_raw + 36)), 1);
+
+ __m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_raw + 48));
+ __m256i ymm2 = _mm256_castsi128_si256(xmm2);
+ ymm2 = _mm256_inserti128_si256(
+ ymm2, _mm_loadu_si128((const __m128i*)(src_raw + 60)), 1);
+
+ __m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_raw + 68));
+ __m256i ymm3 = _mm256_castsi128_si256(xmm3);
+ ymm3 = _mm256_inserti128_si256(
+ ymm3, _mm_loadu_si128((const __m128i*)(src_raw + 80)), 1);
+
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
+ ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
+ ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
+
+ ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
+ ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
+ ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
+ ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
+
+ src_raw += 96;
+ dst_argb += 128;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTOARGBROW_AVX2
+
+#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
+ defined(HAS_RGB24TOARGBROW_AVX512BW) || \
+ defined(HAS_RAWTORGB24ROW_AVX512BW) || \
+ defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, 3, 4, 4, 5};
+static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
+ 9, 10, 10, 11};
+static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, 7, 8, 8, 9};
+static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7};
+#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
+ // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
+ // defined(HAS_RAWTORGB24ROW_AVX512BW) ||
+ // defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+
+#if defined(HAS_RGB24TOARGBROW_AVX512BW) || \
+ defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = {
+ 0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u,
+ 128u, 9u, 10u, 11u, 128u, 4u, 5u, 6u, 128u, 7u, 8u,
+ 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u};
+#endif // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
+ // defined(HAS_RGBTOYMATRIXROW_AVX512BW)
+
+#if defined(HAS_RAWTOARGBROW_AVX512BW)
+static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = {
+ 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u,
+ 128u, 11u, 10u, 9u, 128u, 6u, 5u, 4u, 128u, 9u, 8u,
+ 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u};
+#endif // defined(HAS_RAWTOARGBROW_AVX512BW)
+
+#if defined(HAS_RAWTORGB24ROW_SSSE3)
+// Shuffle table for converting RAW to RGB24. First 8.
+static const uint8_t kShuffleMaskRAWToRGB24_0[16] = {
+ 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u,
+ 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
-static const vec8 kBGRAToU = {0, -38, -74, 112, 0, -38, -74, 112,
- 0, -38, -74, 112, 0, -38, -74, 112};
+// Shuffle table for converting RAW to RGB24. Middle 8.
+static const uint8_t kShuffleMaskRAWToRGB24_1[16] = {
+ 2u, 7u, 6u, 5u, 10u, 9u, 8u, 13u,
+ 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
-static const vec8 kBGRAToV = {0, 112, -94, -18, 0, 112, -94, -18,
- 0, 112, -94, -18, 0, 112, -94, -18};
+// Shuffle table for converting RAW to RGB24. Last 8.
+static const uint8_t kShuffleMaskRAWToRGB24_2[16] = {
+ 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u,
+ 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+#endif // defined(HAS_RAWTORGB24ROW_SSSE3)
+
+#if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
+static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = {
+ 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u,
+ 9u, 128u, 128u, 128u, 128u, 6u, 5u, 4u, 9u, 8u, 7u,
+ 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u};
+#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
+ // defined(HAS_RAWTORGB24ROW_AVX512BW)
+
+#if defined(HAS_RAWTORGB24ROW_AVX2)
+static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 3, 7};
+static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1};
+static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4};
+#endif // defined(HAS_RAWTORGB24ROW_AVX2)
+
+#if defined(HAS_RAWTORGB24ROW_AVX512BW) || \
+ defined(HAS_ARGBTORGB24ROW_AVX512BW) || \
+ defined(HAS_ARGBTORAWROW_AVX512BW)
+static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
+ 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
+static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
+ 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
+static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
+ 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
+#endif
+#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
+static const uint8_t kPermRAWToRGB24_0[64] = {
+ 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a,
+ 0x09, 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17,
+ 0x16, 0x15, 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e,
+ 0x23, 0x22, 0x21, 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b,
+ 0x2a, 0x2f, 0x2e, 0x2d, 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38,
+ 0x37, 0x36, 0x3b, 0x3a, 0x39, 0x3e, 0x3d, 0x3c, 0x41};
+static const uint8_t kPermRAWToRGB24_1[64] = {
+ 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08,
+ 0x0d, 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15,
+ 0x14, 0x19, 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22,
+ 0x21, 0x20, 0x25, 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29,
+ 0x2e, 0x2d, 0x2c, 0x31, 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36,
+ 0x35, 0x3a, 0x39, 0x38, 0x3d, 0x3c, 0x3b, 0x00, 0x3f};
+static const uint8_t kPermRAWToRGB24_2[64] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
+ 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20,
+ 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
+ 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36,
+ 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x40, 0x3f};
+static const uint8_t kPermRAWToRGB24_3[64] = {
+ 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c,
+ 0x4b, 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53,
+ 0x58, 0x57, 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60,
+ 0x5f, 0x64, 0x63, 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d,
+ 0x6c, 0x6b, 0x70, 0x6f, 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74,
+ 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d};
+#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
+
+#if defined(HAS_RAWTOARGBROW_AVX512BW) || defined(HAS_RGB24TOARGBROW_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ const uint32_t* shuffler,
+ int width) {
+ __m512i zmm_alpha = _mm512_set1_epi32(0xff000000);
+ __m512i zmm_q0 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
+ __m512i zmm_q1 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
+ __m512i zmm_q2 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
+ __m512i zmm_q3 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
+ __m512i zmm_shuf =
+ _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler));
+
+ while (width >= 64) {
+ __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
+ __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
+ __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
+ zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
+ zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
+ zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
+
+ _mm512_storeu_si512(dst_argb, zmm0);
+ _mm512_storeu_si512(dst_argb + 64, zmm1);
+ _mm512_storeu_si512(dst_argb + 128, zmm2);
+ _mm512_storeu_si512(dst_argb + 192, zmm3);
+
+ src_raw += 192;
+ dst_argb += 256;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t dst_mask = (1ULL << width) - 1ULL;
+ int src_bytes = width * 3;
+ __mmask64 src_mask0 =
+ (src_bytes >= 64) ? ~0ULL : ((1ULL << src_bytes) - 1ULL);
+ __mmask64 src_mask1 =
+ (src_bytes >= 128)
+ ? ~0ULL
+ : ((src_bytes > 64) ? ((1ULL << (src_bytes - 64)) - 1ULL) : 0ULL);
+ __mmask64 src_mask2 =
+ (src_bytes > 128) ? ((1ULL << (src_bytes - 128)) - 1ULL) : 0ULL;
+
+ __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(src_mask0, src_raw);
+ __m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64);
+ __m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128);
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
+ zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
+ zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
+ zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
+
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)dst_mask, zmm0);
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(dst_mask >> 16), zmm1);
+ _mm512_mask_storeu_epi32(dst_argb + 128, (__mmask16)(dst_mask >> 32), zmm2);
+ _mm512_mask_storeu_epi32(dst_argb + 192, (__mmask16)(dst_mask >> 48), zmm3);
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
+ // defined(HAS_RGB24TOARGBROW_AVX512BW)
+
+#ifdef HAS_RAWTOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_argb,
+ int width) {
+ RGBToARGBRow_AVX512BW(src_raw, dst_argb,
+ (const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width);
+}
+#endif // HAS_RAWTOARGBROW_AVX512BW
+
+#ifdef HAS_RGB24TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ RGBToARGBRow_AVX512BW(src_rgb24, dst_argb,
+ (const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW,
+ width);
+}
+#endif // HAS_RGB24TOARGBROW_AVX512BW
+
+#ifdef HAS_RAWTORGB24ROW_SSSE3
+LIBYUV_TARGET_SSSE3
+void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ __m128i shuf0 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_0);
+ __m128i shuf1 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_1);
+ __m128i shuf2 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_2);
-// Constants for ABGR.
-static const vec8 kABGRToY = {33, 65, 13, 0, 33, 65, 13, 0,
- 33, 65, 13, 0, 33, 65, 13, 0};
+ while (width > 0) {
+ __m128i raw0 = _mm_loadu_si128((const __m128i*)src_raw);
+ __m128i raw1 = _mm_loadu_si128((const __m128i*)(src_raw + 4));
+ __m128i raw2 = _mm_loadu_si128((const __m128i*)(src_raw + 8));
-static const vec8 kABGRToU = {-38, -74, 112, 0, -38, -74, 112, 0,
- -38, -74, 112, 0, -38, -74, 112, 0};
+ raw0 = _mm_shuffle_epi8(raw0, shuf0);
+ raw1 = _mm_shuffle_epi8(raw1, shuf1);
+ raw2 = _mm_shuffle_epi8(raw2, shuf2);
-static const vec8 kABGRToV = {112, -94, -18, 0, 112, -94, -18, 0,
- 112, -94, -18, 0, 112, -94, -18, 0};
+ _mm_storel_epi64((__m128i*)dst_rgb24, raw0);
+ _mm_storel_epi64((__m128i*)(dst_rgb24 + 8), raw1);
+ _mm_storel_epi64((__m128i*)(dst_rgb24 + 16), raw2);
-// Constants for RGBA.
-static const vec8 kRGBAToY = {0, 13, 65, 33, 0, 13, 65, 33,
- 0, 13, 65, 33, 0, 13, 65, 33};
+ src_raw += 24;
+ dst_rgb24 += 24;
+ width -= 8;
+ }
+}
+#endif // HAS_RAWTORGB24ROW_SSSE3
-static const vec8 kRGBAToU = {0, 112, -74, -38, 0, 112, -74, -38,
- 0, 112, -74, -38, 0, 112, -74, -38};
+#ifdef HAS_RAWTORGB24ROW_AVX2
+LIBYUV_TARGET_AVX2
+void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ __m256i ymm_shuf =
+ _mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2);
+ __m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2);
+ __m256i ymm_p1 = _mm256_loadu_si256((const __m256i*)kPermd1_AVX2);
+ __m256i ymm_p2 = _mm256_loadu_si256((const __m256i*)kPermd2_AVX2);
-static const vec8 kRGBAToV = {0, -18, -94, 112, 0, -18, -94, 112,
- 0, -18, -94, 112, 0, -18, -94, 112};
+ while (width > 0) {
+ __m256i raw0 = _mm256_loadu_si256((const __m256i*)src_raw);
+ __m256i raw1 = _mm256_loadu_si256((const __m256i*)(src_raw + 32));
+ __m256i raw2 = _mm256_loadu_si256((const __m256i*)(src_raw + 64));
+
+ __m256i m01 = _mm256_permute2x128_si256(raw0, raw1, 0x21);
+ __m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94);
+ __m256i m12 = _mm256_permute2x128_si256(raw1, raw2, 0x21);
+ __m256i b3 = _mm256_permute4x64_epi64(raw2, 0xe9);
+ __m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9);
+ __m256i b2 = _mm256_permute4x64_epi64(m12, 0x94);
+
+ b0 = _mm256_shuffle_epi8(b0, ymm_shuf);
+ b1 = _mm256_shuffle_epi8(b1, ymm_shuf);
+ b2 = _mm256_shuffle_epi8(b2, ymm_shuf);
+ b3 = _mm256_shuffle_epi8(b3, ymm_shuf);
+
+ __m256i d0 = _mm256_permutevar8x32_epi32(b0, ymm_p0);
+ __m256i d1 = _mm256_permutevar8x32_epi32(b1, ymm_p1);
+ __m256i d2 = _mm256_permutevar8x32_epi32(b2, ymm_p2);
+ __m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p0);
+
+ // Dst0 = blend(d0, d1)
+ __m256i dst0 = _mm256_blend_epi32(d0, d1, 0xc0);
+ _mm256_storeu_si256((__m256i*)dst_rgb24, dst0);
+
+ // Dst1 = blend(d1, d2)
+ __m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0);
+ d3 = _mm256_permute4x64_epi64(d3, 0x90);
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 32), dst1);
+
+ // Dst2 = blend(d2, d3)
+ __m256i dst2 = _mm256_blend_epi32(d2, d3, 0xfc);
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), dst2);
+
+ src_raw += 96;
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTORGB24ROW_AVX2
+
+#ifdef HAS_RAWTORGB24ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ __m512i zmm_q0 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
+ __m512i zmm_q1 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
+ __m512i zmm_q2 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
+ __m512i zmm_q3 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
+ __m512i zmm_shuf = _mm512_broadcast_i64x4(
+ _mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2));
+ __m512i zmm_d0 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
+ __m512i zmm_d1 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
+ __m512i zmm_d2 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
+
+ while (width >= 64) {
+ __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
+ __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
+ __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ __m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
+ __m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
+ __m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
+
+ _mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
+
+ src_raw += 192;
+ dst_rgb24 += 192;
+ width -= 64;
+ }
+ if (width > 0) {
+ int nbytes = width * 3;
+ __mmask64 k0 =
+ (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
+ __mmask64 k1 = (nbytes <= 64) ? 0
+ : (nbytes >= 128)
+ ? ~(__mmask64)0
+ : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
+ __mmask64 k2 =
+ (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
+ __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw);
+ __m512i zmm_raw1 =
+ k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512();
+ __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128)
+ : _mm512_setzero_si512();
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ __m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
+ __m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
+ __m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
+
+ _mm512_mask_storeu_epi8(dst_rgb24, k0, dst0);
+ if (k1) {
+ _mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1);
+ }
+ if (k2) {
+ _mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2);
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTORGB24ROW_AVX512BW
+
+#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgb24,
+ int width) {
+ __m512i zmm_p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_0);
+ __m512i zmm_p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_1);
+ __m512i zmm_p2 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_2);
+ __m512i zmm_p3 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_3);
+
+ while (width >= 64) {
+ __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
+ __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
+ __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
+
+ __m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
+ __m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
+ __m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
+ __m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
+
+ _mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
+
+ src_raw += 192;
+ dst_rgb24 += 192;
+ width -= 64;
+ }
+ if (width > 0) {
+ int nbytes = width * 3;
+ __mmask64 k0 =
+ (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
+ __mmask64 k1 = (nbytes <= 64) ? 0
+ : (nbytes >= 128)
+ ? ~(__mmask64)0
+ : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
+ __mmask64 k2 =
+ (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
+ __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw);
+ __m512i zmm_raw1 =
+ k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512();
+ __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128)
+ : _mm512_setzero_si512();
+
+ __m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
+ __m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
+ __m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
+ __m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
+
+ _mm512_mask_storeu_epi8(dst_rgb24, k0, dst0);
+ if (k1) {
+ _mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1);
+ }
+ if (k2) {
+ _mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2);
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTORGB24ROW_AVX512VBMI
+
+#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \
+ defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI)
+static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+#endif
-static const uvec8 kAddY16 = {16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u,
- 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u};
+#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ __m512i zmm_q0 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
+ __m512i zmm_q1 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
+ __m512i zmm_q2 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
+ __m512i zmm_q3 =
+ _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
+ __m512i zmm_shuf = _mm512_broadcast_i64x4(
+ _mm256_loadu_si256((const __m256i*)kShuffleMaskRGB24ToARGB_AVX512BW));
+ __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+
+ __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
+ __m512i zmm_one = _mm512_set1_epi16(1);
+ __m512i zmm_coeff =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
+ __m512i zmm_add =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY));
+
+ // Compute bias adjusted for 0x80 subtraction:
+ // bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1)
+ __m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
+ zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
+ zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
+ zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
+
+ while (width >= 64) {
+ __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb);
+ __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64));
+ __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128));
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_xor_si512(zmm0, zmm_80);
+ zmm1 = _mm512_xor_si512(zmm1, zmm_80);
+ zmm2 = _mm512_xor_si512(zmm2, zmm_80);
+ zmm3 = _mm512_xor_si512(zmm3, zmm_80);
+
+ zmm0 = _mm512_maddubs_epi16(zmm_coeff, zmm0);
+ zmm1 = _mm512_maddubs_epi16(zmm_coeff, zmm1);
+ zmm2 = _mm512_maddubs_epi16(zmm_coeff, zmm2);
+ zmm3 = _mm512_maddubs_epi16(zmm_coeff, zmm3);
+
+ zmm0 = _mm512_madd_epi16(zmm0, zmm_one);
+ zmm1 = _mm512_madd_epi16(zmm1, zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+
+ zmm2 = _mm512_madd_epi16(zmm2, zmm_one);
+ zmm3 = _mm512_madd_epi16(zmm3, zmm_one);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm_add);
+ zmm2 = _mm512_add_epi16(zmm2, zmm_add);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm2 = _mm512_srli_epi16(zmm2, 8);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm2);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+
+ _mm512_storeu_si512((__m512i*)dst_y, zmm0);
+
+ src_rgb += 192;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 ky = (__mmask64)((1ULL << width) - 1ULL);
+ int nbytes = width * 3;
+ __mmask64 k0 =
+ (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
+ __mmask64 k1 = (nbytes <= 64) ? 0
+ : (nbytes >= 128)
+ ? ~(__mmask64)0
+ : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
+ __mmask64 k2 =
+ (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
+ __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_rgb);
+ __m512i zmm_raw1 =
+ k1 ? _mm512_maskz_loadu_epi8(k1, src_rgb + 64) : _mm512_setzero_si512();
+ __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_rgb + 128)
+ : _mm512_setzero_si512();
+
+ __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
+ __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
+ __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
+ __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
+
+ zmm0 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm0, zmm_shuf), zmm_80);
+ zmm1 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm1, zmm_shuf), zmm_80);
+ zmm2 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm2, zmm_shuf), zmm_80);
+ zmm3 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm3, zmm_shuf), zmm_80);
+
+ zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
+ zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+
+ zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
+ zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+
+ zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
+ zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_mask_storeu_epi8(dst_y, ky, zmm0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RGBTOYMATRIXROW_AVX512BW
+
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+ __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
+ __m512i zmm_coeff =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
+ __m512i zmm_add =
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY));
+
+ __m512i zmm_bias =
+ _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80);
+ zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias);
+
+ while (width >= 64) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
+ __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
+
+ zmm0 = _mm512_xor_si512(zmm0, zmm_80);
+ zmm1 = _mm512_xor_si512(zmm1, zmm_80);
+ zmm2 = _mm512_xor_si512(zmm2, zmm_80);
+ zmm3 = _mm512_xor_si512(zmm3, zmm_80);
+
+ zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
+ zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
+
+ zmm0 = _mm512_packus_epi32(zmm0, zmm1);
+ zmm2 = _mm512_packus_epi32(zmm2, zmm3);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm2 = _mm512_srli_epi16(zmm2, 8);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm2);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+
+ _mm512_storeu_si512((__m512i*)dst_y, zmm0);
+
+ src_argb += 256;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i zmm0 =
+ _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
+ __m512i zmm1 =
+ _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm2 =
+ _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm3 =
+ _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512(),
+ zmm_80);
+
+ zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
+ zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
+
+ zmm0 = _mm512_srli_epi16(_mm512_packus_epi32(zmm0, zmm1), 8);
+ zmm2 = _mm512_srli_epi16(_mm512_packus_epi32(zmm2, zmm3), 8);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI
+
+#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) || \
+ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) || \
+ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI)
+// AVX512VNNIVBMI kernels use both AVX512VNNI (vpdpbusd) and AVX512VBMI
+// (vpermt2b) and require both CPU features. There are no AVX512VBMI
+// versions without AVX512VNNI.
+
+// vpermt2b indices selecting byte 1 of each int32 from 2 vectors. The 32
+// byte pattern is broadcast so each 256 bit half holds the same 32 results.
+// Clang constant folds the indices and lowers _mm512_permutex2var_epi8 to a
+// longer vpshufb + vpor + vpermd sequence, so hide the indices from it.
+#if defined(__clang__)
+#define LIBYUV_HIDE_CONSTANT(v) __asm__("" : "+v"(v))
+#else
+#define LIBYUV_HIDE_CONSTANT(v)
+#endif
+static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = {
+ 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61,
+ 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125};
+#endif
-// 7 bit fixed point 0.5.
-static const vec16 kAddYJ64 = {64, 64, 64, 64, 64, 64, 64, 64};
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+// Requires AVX512VNNI and AVX512VBMI.
+// The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in
+// [0, 65535], so Y is byte 1 of each int32 and no saturation is needed.
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ __m512i zmm_perm = _mm512_broadcast_i64x4(
+ _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
+ LIBYUV_HIDE_CONSTANT(zmm_perm);
+ __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
+ __m512i zmm_coeff =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
+ __m512i zmm_add =
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY));
+
+ __m512i zmm_bias =
+ _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80);
+ zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias);
+
+ while (width >= 64) {
+ __m512i zmm0 =
+ _mm512_xor_si512(_mm512_loadu_si512((const __m512i*)src_argb), zmm_80);
+ __m512i zmm1 = _mm512_xor_si512(
+ _mm512_loadu_si512((const __m512i*)(src_argb + 64)), zmm_80);
+ __m512i zmm2 = _mm512_xor_si512(
+ _mm512_loadu_si512((const __m512i*)(src_argb + 128)), zmm_80);
+ __m512i zmm3 = _mm512_xor_si512(
+ _mm512_loadu_si512((const __m512i*)(src_argb + 192)), zmm_80);
+
+ zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
+ zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
+
+ zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // Y 0..31
+ zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // Y 32..63
+
+ _mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(zmm0));
+ _mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_castsi512_si256(zmm2));
+
+ src_argb += 256;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i zmm0 =
+ _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
+ __m512i zmm1 =
+ _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm2 =
+ _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm3 =
+ _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512(),
+ zmm_80);
+
+ zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0);
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1);
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2);
+ zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3);
+
+ zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
+ zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3);
+
+ _mm256_mask_storeu_epi8(dst_y, (__mmask32)mask,
+ _mm512_castsi512_si256(zmm0));
+ if (mask >> 32) {
+ _mm256_mask_storeu_epi8(dst_y + 32, (__mmask32)(mask >> 32),
+ _mm512_castsi512_si256(zmm2));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ __m512i zmm_8000 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+
+ while (width >= 64) {
+ __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
+ __m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
+
+ __m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u);
+ __m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u);
+ __m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u);
+ __m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u);
+
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+ zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
+ zmm2 = _mm512_sub_epi16(zmm_8000, zmm2);
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm2 = _mm512_srli_epi16(zmm2, 8);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm2);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+ _mm512_storeu_si512((__m512i*)dst_u, zmm0);
+
+ zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v);
+ zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v);
+ zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v);
+ zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v);
+
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+ zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
+ zmm2 = _mm512_sub_epi16(zmm_8000, zmm2);
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm2 = _mm512_srli_epi16(zmm2, 8);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm2);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+ _mm512_storeu_si512((__m512i*)dst_v, zmm0);
+
+ src_argb += 256;
+ dst_u += 64;
+ dst_v += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
+ __m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512();
+ __m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512();
+ __m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512();
+
+ __m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u);
+ __m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u);
+ __m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u);
+ __m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u);
+
+ zmm0 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8);
+ zmm2 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0);
+
+ zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v);
+ zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v);
+ zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v);
+ zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v);
+
+ zmm0 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8);
+ zmm2 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+// Requires AVX512VNNI and AVX512VBMI.
+// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at
+// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each
+// int32.
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ __m512i zmm_perm = _mm512_broadcast_i64x4(
+ _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
+ LIBYUV_HIDE_CONSTANT(zmm_perm);
+ __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ const __m512i zmm_neg1 = _mm512_set1_epi32(-1);
+ __m512i zmm_add = _mm512_xor_si512(
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)),
+ zmm_neg1); // ~kAddUV
+
+ while (width >= 64) {
+ __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
+ __m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
+
+ __m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u);
+ __m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u);
+ __m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u);
+ __m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u);
+ __m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v);
+ __m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v);
+ __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v);
+ __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v);
+
+ zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31
+ zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63
+ zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31
+ zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63
+ // Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan; see GEMINI.md).
+ zmm0 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm0, zmm2, 0x44),
+ zmm_neg1); // U
+ zmm4 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm4, zmm6, 0x44),
+ zmm_neg1); // V
+
+ _mm512_storeu_si512((__m512i*)dst_u, zmm0);
+ _mm512_storeu_si512((__m512i*)dst_v, zmm4);
+
+ src_argb += 256;
+ dst_u += 64;
+ dst_v += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
+ __m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512();
+ __m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512();
+ __m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512();
+
+ __m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u);
+ __m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u);
+ __m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u);
+ __m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u);
+ __m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v);
+ __m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v);
+ __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v);
+ __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v);
+
+ zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
+ zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3);
+ zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5);
+ zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7);
+ zmm0 = _mm512_xor_si512(
+ _mm512_inserti64x4(zmm0, _mm512_castsi512_si256(zmm2), 1), zmm_neg1);
+ zmm4 = _mm512_xor_si512(
+ _mm512_inserti64x4(zmm4, _mm512_castsi512_si256(zmm6), 1), zmm_neg1);
+
+ _mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0);
+ _mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm4);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
+ 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
+ static const uint8_t kShuffleAARRGGBB_AVX512[16] = {
+ 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
+ __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
+ __m512i zmm_8000 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ __m512i zmm_zero = _mm512_setzero_si512();
+ __m512i zmm_shuf = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512));
+ __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+ __m512i zmm_perm_uv =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
+
+ while (width >= 32) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm2 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
+ __m512i zmm3 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 1);
+ zmm1 = _mm512_srli_epi16(zmm1, 1);
+ zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
+ zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm1);
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
+
+ zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
+ zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
+ zmm0 = _mm512_packs_epi32(zmm1, zmm2);
+ zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+
+ _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
+ _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
+
+ src_argb += 128;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)mask;
+ __mmask16 k2 = (__mmask16)(mask >> 16);
+ __mmask16 k3 = ~k1;
+ __mmask16 k4 = ~k2;
+ __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
+ __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
+ __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
+ __m512i zmm3 =
+ _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
+
+ zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
+ zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
+ zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
+ zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 1);
+ zmm1 = _mm512_srli_epi16(zmm1, 1);
+ zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
+ zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm1);
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
+
+ zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
+ zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
+ zmm0 = _mm512_packs_epi32(zmm1, zmm2);
+ zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
+ zmm0 = _mm512_srli_epi16(zmm0, 8);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
+
+ int halfwidth = (width + 1) >> 1;
+ __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
+ _mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0));
+ _mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNI
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
+// Requires AVX512VNNI and AVX512VBMI.
+LIBYUV_TARGET_AVX512VNNIVBMI
+void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
+ 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
+ static const uint8_t kShuffleAARRGGBB_AVX512[16] = {
+ 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
+ __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
+ __m512i zmm_add = _mm512_xor_si512(
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)),
+ _mm512_set1_epi32(-1)); // ~kAddUV
+ __m512i zmm_zero = _mm512_setzero_si512();
+ __m512i zmm_shuf = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512));
+ __m512i zmm_perm = _mm512_broadcast_i64x4(
+ _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI));
+ LIBYUV_HIDE_CONSTANT(zmm_perm);
+ __m512i zmm_perm_uv =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
+
+ while (width >= 32) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm2 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
+ __m512i zmm3 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 1);
+ zmm1 = _mm512_srli_epi16(zmm1, 1);
+ zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
+ zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm1);
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
+
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
+ zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2);
+ __m256i ymm_uv =
+ _mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1));
+
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm_uv));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm_uv, 1));
+
+ src_argb += 128;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)mask;
+ __mmask16 k2 = (__mmask16)(mask >> 16);
+ __mmask16 k3 = ~k1;
+ __mmask16 k4 = ~k2;
+ __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
+ __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
+ __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
+ __m512i zmm3 =
+ _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
+
+ zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
+ zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
+ zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
+ zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 1);
+ zmm1 = _mm512_srli_epi16(zmm1, 1);
+ zmm0 = _mm512_avg_epu16(zmm0, zmm_zero);
+ zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
+
+ zmm0 = _mm512_packus_epi16(zmm0, zmm1);
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
+
+ zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
+ zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
+ zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2);
+ __m256i ymm_uv =
+ _mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1));
+
+ int halfwidth = (width + 1) >> 1;
+ __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
+ _mm_mask_storeu_epi8(dst_u, k_out, _mm256_castsi256_si128(ymm_uv));
+ _mm_mask_storeu_epi8(dst_v, k_out, _mm256_extracti128_si256(ymm_uv, 1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ __m256i ymm_u =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ __m256i ymm_v =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ __m256i ymm_0101 = _mm256_set1_epi16(0x0101);
+ __m256i ymm_shuf =
+ _mm256_setr_epi8(0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15, 0,
+ 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15);
+ __m256i ymm_8000 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ __m256i ymm_zero = _mm256_setzero_si256();
-static const uvec8 kAddUV128 = {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
+ __m256i ymm2 =
+ _mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb));
+ __m256i ymm3 =
+ _mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb + 32));
+
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
+ ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
+ ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf);
+
+ ymm0 = _mm256_maddubs_epi16(ymm0, ymm_0101);
+ ymm1 = _mm256_maddubs_epi16(ymm1, ymm_0101);
+ ymm2 = _mm256_maddubs_epi16(ymm2, ymm_0101);
+ ymm3 = _mm256_maddubs_epi16(ymm3, ymm_0101);
+
+ ymm0 = _mm256_add_epi16(ymm0, ymm2);
+ ymm1 = _mm256_add_epi16(ymm1, ymm3);
+
+ ymm0 = _mm256_srli_epi16(ymm0, 1);
+ ymm1 = _mm256_srli_epi16(ymm1, 1);
+ ymm0 = _mm256_avg_epu16(ymm0, ymm_zero);
+ ymm1 = _mm256_avg_epu16(ymm1, ymm_zero);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm1);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+
+ ymm1 = _mm256_maddubs_epi16(ymm0, ymm_v);
+ ymm0 = _mm256_maddubs_epi16(ymm0, ymm_u);
+
+ ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ ymm0 = _mm256_sub_epi16(ymm_8000, ymm0);
+ ymm0 = _mm256_srli_epi16(ymm0, 8);
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+
+ __m128i xmm_u = _mm256_castsi256_si128(ymm0);
+ __m128i xmm_v = _mm256_extracti128_si256(ymm0, 1);
+
+ _mm_storel_epi64((__m128i*)dst_u, xmm_u);
+ _mm_storel_epi64((__m128i*)dst_v, xmm_v);
+
+ src_argb += 64;
+ dst_u += 8;
+ dst_v += 8;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX2
+
+#ifdef HAS_CONVERT8TO16ROW_AVX2
+LIBYUV_TARGET_AVX2
+void Convert8To16Row_AVX2(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ const int shift = 16 - bits;
+ __m128i shift128 = _mm_cvtsi32_si128(shift);
+ do {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ __m256i row0 = _mm256_unpacklo_epi8(ymm0, ymm0);
+ __m256i row1 = _mm256_unpackhi_epi8(ymm0, ymm0);
+ row0 = _mm256_srl_epi16(row0, shift128);
+ row1 = _mm256_srl_epi16(row1, shift128);
+ _mm256_storeu_si256((__m256i*)dst_y, row0);
+ _mm256_storeu_si256((__m256i*)(dst_y + 16), row1);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT8TO16ROW_AVX2
+
+#ifdef HAS_MULTIPLYROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void MultiplyRow_16_AVX2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ __m256i scale256 = _mm256_set1_epi16((short)scale);
+ do {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
+ ymm0 = _mm256_mullo_epi16(ymm0, scale256);
+ ymm1 = _mm256_mullo_epi16(ymm1, scale256);
+ _mm256_storeu_si256((__m256i*)dst_y, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_y + 16), ymm1);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MULTIPLYROW_16_AVX2
-static const uvec16 kAddUVJ128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u,
- 0x8080u, 0x8080u, 0x8080u, 0x8080u};
+#ifdef HAS_MERGEUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeUVRow_AVX2(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ while (width > 0) {
+ __m256i ymm0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_u));
+ __m256i ymm1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_v));
-// Shuffle table for converting RGB24 to ARGB.
-static const uvec8 kShuffleMaskRGB24ToARGB = {
- 0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u};
+ ymm1 = _mm256_slli_epi16(ymm1, 8);
+ ymm0 = _mm256_or_si256(ymm0, ymm1);
-// Shuffle table for converting RAW to ARGB.
-static const uvec8 kShuffleMaskRAWToARGB = {2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u,
- 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u};
+ _mm256_storeu_si256((__m256i*)dst_uv, ymm0);
-// Shuffle table for converting RAW to RGB24. First 8.
-static const uvec8 kShuffleMaskRAWToRGB24_0 = {
- 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+ src_u += 16;
+ src_v += 16;
+ dst_uv += 32;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEUVROW_AVX2
-// Shuffle table for converting RAW to RGB24. Middle 8.
-static const uvec8 kShuffleMaskRAWToRGB24_1 = {
- 2u, 7u, 6u, 5u, 10u, 9u, 8u, 13u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+#ifdef HAS_MIRRORROW_AVX2
+LIBYUV_TARGET_AVX2
+void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
+ src += width;
+ while (width > 0) {
+ src -= 32;
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
+ _mm256_storeu_si256((__m256i*)dst, ymm0);
+ dst += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORROW_AVX2
-// Shuffle table for converting RAW to RGB24. Last 8.
-static const uvec8 kShuffleMaskRAWToRGB24_2 = {
- 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+#ifdef HAS_MIRRORUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1));
+ src_uv += width * 2;
+ while (width > 0) {
+ src_uv -= 32;
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
+ _mm256_storeu_si256((__m256i*)dst_uv, ymm0);
+ dst_uv += 32;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORUVROW_AVX2
+
+#ifdef HAS_MIRRORROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ const __m512i zmm_shuf = _mm512_broadcast_i32x4(
+ _mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
+ const __m512i zmm_rev = _mm512_set_epi16(
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
+ const uint8_t* s = src + width;
+ while (width >= 64) {
+ s -= 64;
+ __m512i v = _mm512_loadu_si512((const __m512i*)s);
+ v = _mm512_shuffle_epi8(v, zmm_shuf);
+ v = _mm512_shuffle_i64x2(v, v, 0x1b);
+ _mm512_storeu_si512((__m512i*)dst, v);
+ dst += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 k = (1ULL << width) - 1ULL;
+ __m512i v = _mm512_maskz_loadu_epi8(k, src);
+ v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8));
+ __m512i idx =
+ _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1)));
+ v = PermutexvarEpi16(idx, v);
+ if (width & 1) {
+ __m512i v_hi = _mm512_alignr_epi64(v, v, 2);
+ v = _mm512_alignr_epi8(v_hi, v, 1);
+ }
+ _mm512_mask_storeu_epi8(dst, k, v);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORROW_AVX512BW
+
+#ifdef HAS_MIRRORSPLITUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m512i zmm_shuf = _mm512_broadcast_i32x4(
+ _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1));
+ const __m512i zmm_perm = _mm512_setr_epi64(6, 4, 2, 0, 7, 5, 3, 1);
+ const __m512i zmm_rev = _mm512_set_epi16(
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
+ const uint8_t* s = src_uv + (ptrdiff_t)width * 2;
+ while (width >= 32) {
+ s -= 64;
+ __m512i v = _mm512_loadu_si512((const __m512i*)s);
+ v = _mm512_shuffle_epi8(v, zmm_shuf);
+ v = PermutexvarEpi64(zmm_perm, v);
+ _mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v));
+ _mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1));
+ dst_u += 32;
+ dst_v += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i idx =
+ _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width)));
+ __m512i v = _mm512_maskz_loadu_epi16(k, src_uv);
+ v = PermutexvarEpi16(idx, v);
+ _mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v));
+ _mm256_mask_storeu_epi8(dst_v, k,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORSPLITUVROW_AVX512BW
+
+#ifdef HAS_MIRRORUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
+ const __m512i zmm_idx = _mm512_set_epi16(
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
+ const uint8_t* s = src_uv + (ptrdiff_t)width * 2;
+ while (width >= 32) {
+ s -= 64;
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)s);
+ zmm0 = PermutexvarEpi16(zmm_idx, zmm0);
+ _mm512_storeu_si512((__m512i*)dst_uv, zmm0);
+ dst_uv += 64;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i idx =
+ _mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width)));
+ __m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv);
+ _mm512_mask_storeu_epi16(dst_uv, k, PermutexvarEpi16(idx, zmm0));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORUVROW_AVX512BW
+
+#ifdef HAS_SWAPUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14));
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
+ _mm256_storeu_si256((__m256i*)dst_vu, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_vu + 32), ymm1);
+ src_uv += 64;
+ dst_vu += 64;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SWAPUVROW_AVX2
+
+#ifdef HAS_MIRRORSPLITUVROW_SSSE3
+LIBYUV_TARGET_SSSE3
+void MirrorSplitUVRow_SSSE3(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m128i xmm_shuf =
+ _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1);
+ src_uv += (ptrdiff_t)width * 2;
+ while (width > 0) {
+ src_uv -= 16;
+ __m128i xmm0 =
+ _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), xmm_shuf);
+ _mm_storel_epi64((__m128i*)dst_u, xmm0);
+ _mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(xmm0, xmm0));
+ dst_u += 8;
+ dst_v += 8;
+ width -= 8;
+ }
+}
+#endif // HAS_MIRRORSPLITUVROW_SSSE3
+
+#ifdef HAS_MIRRORSPLITUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void MirrorSplitUVRow_AVX2(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1));
+ src_uv += (ptrdiff_t)width * 2;
+ while (width > 0) {
+ src_uv -= 32;
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0x72);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm0));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm0, 1));
+ dst_u += 16;
+ dst_v += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORSPLITUVROW_AVX2
+
+#ifdef HAS_RGB24MIRRORROW_AVX2
+LIBYUV_TARGET_AVX2
+void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ __m256i shuf0 =
+ _mm256_setr_epi8(-1, 12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, -1,
+ 12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2);
+ __m128i shuf1 =
+ _mm_setr_epi8(13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, 3, -1);
-// Shuffle table for converting ARGB to RGB24.
-static const uvec8 kShuffleMaskARGBToRGB24 = {
- 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u};
-
-// Shuffle table for converting ARGB to RAW.
-static const uvec8 kShuffleMaskARGBToRAW = {
- 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u};
-
-// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4
-static const uvec8 kShuffleMaskARGBToRGB24_0 = {
- 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
-
-// YUY2 shuf 16 Y to 32 Y.
-static const lvec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10,
- 10, 12, 12, 14, 14, 0, 0, 2, 2, 4, 4,
- 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
-
-// YUY2 shuf 8 UV to 16 UV.
-static const lvec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7, 9, 11, 9,
- 11, 13, 15, 13, 15, 1, 3, 1, 3, 5, 7,
- 5, 7, 9, 11, 9, 11, 13, 15, 13, 15};
-
-// UYVY shuf 16 Y to 32 Y.
-static const lvec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11,
- 11, 13, 13, 15, 15, 1, 1, 3, 3, 5, 5,
- 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
-
-// UYVY shuf 8 UV to 16 UV.
-static const lvec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6, 8, 10, 8,
- 10, 12, 14, 12, 14, 0, 2, 0, 2, 4, 6,
- 4, 6, 8, 10, 8, 10, 12, 14, 12, 14};
-
-// NV21 shuf 8 VU to 16 UV.
-static const lvec8 kShuffleNV21 = {
- 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
- 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
-};
-
-// Duplicates gray value 3 times and fills in alpha opaque.
-__declspec(naked) void J400ToARGBRow_SSE2(const uint8_t* src_y,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_y
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm5, xmm5 // generate mask 0xff000000
- pslld xmm5, 24
-
- convertloop:
- movq xmm0, qword ptr [eax]
- lea eax, [eax + 8]
- punpcklbw xmm0, xmm0
- movdqa xmm1, xmm0
- punpcklwd xmm0, xmm0
- punpckhwd xmm1, xmm1
- por xmm0, xmm5
- por xmm1, xmm5
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
- ret
+ src_rgb24 += (ptrdiff_t)width * 3 - 96;
+ while (width > 0) {
+ __m128i v0_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 0));
+ __m128i v0_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 15));
+ __m256i v0 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(v0_lo), v0_hi, 1);
+
+ __m128i v1_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 30));
+ __m128i v1_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 45));
+ __m256i v1 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(v1_lo), v1_hi, 1);
+
+ __m128i v2_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 60));
+ __m128i v2_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 75));
+ __m256i v2 =
+ _mm256_inserti128_si256(_mm256_castsi128_si256(v2_lo), v2_hi, 1);
+
+ __m128i v3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 80));
+
+ v0 = _mm256_shuffle_epi8(v0, shuf0);
+ v1 = _mm256_shuffle_epi8(v1, shuf0);
+ v2 = _mm256_shuffle_epi8(v2, shuf0);
+ v3 = _mm_shuffle_epi8(v3, shuf1);
+
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 80), _mm256_castsi256_si128(v0));
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 65),
+ _mm256_extracti128_si256(v0, 1));
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 50), _mm256_castsi256_si128(v1));
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 35),
+ _mm256_extracti128_si256(v1, 1));
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 20), _mm256_castsi256_si128(v2));
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 5),
+ _mm256_extracti128_si256(v2, 1));
+ _mm_storel_epi64((__m128i*)(dst_rgb24 + 0), v3);
+
+ src_rgb24 -= 96;
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RGB24MIRRORROW_AVX2
+
+#ifdef HAS_INTERPOLATEROW_AVX2
+LIBYUV_TARGET_AVX2
+void InterpolateRow_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1 = source_y_fraction;
+ int y0 = 256 - y1;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ __m256i ymm_y = _mm256_set1_epi16((y1 << 8) | y0);
+ __m256i ymm_8080 = _mm256_set1_epi16(0x8080);
+ int i;
+
+ if (y1 == 0) {
+ for (i = 0; i < width; i += 32) {
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + i)));
+ }
+ } else if (y1 == 128) {
+ for (i = 0; i < width; i += 32) {
+ __m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
+ __m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i), _mm256_avg_epu8(row0, row1));
+ }
+ } else {
+ for (i = 0; i < width; i += 32) {
+ __m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
+ __m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
+ __m256i low = _mm256_unpacklo_epi8(row0, row1);
+ __m256i high = _mm256_unpackhi_epi8(row0, row1);
+ low = _mm256_sub_epi8(low, ymm_8080);
+ high = _mm256_sub_epi8(high, ymm_8080);
+ low = _mm256_maddubs_epi16(ymm_y, low);
+ high = _mm256_maddubs_epi16(ymm_y, high);
+ low = _mm256_add_epi16(low, ymm_8080);
+ high = _mm256_add_epi16(high, ymm_8080);
+ low = _mm256_srli_epi16(low, 8);
+ high = _mm256_srli_epi16(high, 8);
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i),
+ _mm256_packus_epi16(low, high));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_INTERPOLATEROW_AVX2
+
+#ifdef HAS_INTERPOLATEROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1 = source_y_fraction;
+ int y0 = 256 - y1;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ __m512i zmm_y = _mm512_set1_epi16((short)((y1 << 8) | y0));
+ __m512i zmm_8080 = _mm512_set1_epi16((short)0x8080);
+ int i;
+
+ if (y1 == 0) {
+ for (i = 0; i <= width - 64; i += 64) {
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i),
+ _mm512_loadu_si512((const __m512i*)(src_ptr + i)));
+ }
+ if (i < width) {
+ __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
+ _mm512_mask_storeu_epi8(dst_ptr + i, mask,
+ _mm512_maskz_loadu_epi8(mask, src_ptr + i));
+ }
+ } else if (y1 == 128) {
+ for (i = 0; i <= width - 64; i += 64) {
+ __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
+ __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i), _mm512_avg_epu8(row0, row1));
+ }
+ if (i < width) {
+ __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
+ __m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
+ __m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
+ _mm512_mask_storeu_epi8(dst_ptr + i, mask, _mm512_avg_epu8(row0, row1));
+ }
+ } else {
+ for (i = 0; i <= width - 64; i += 64) {
+ __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
+ __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
+ __m512i low = _mm512_unpacklo_epi8(row0, row1);
+ __m512i high = _mm512_unpackhi_epi8(row0, row1);
+ low = _mm512_sub_epi8(low, zmm_8080);
+ high = _mm512_sub_epi8(high, zmm_8080);
+ low = _mm512_maddubs_epi16(zmm_y, low);
+ high = _mm512_maddubs_epi16(zmm_y, high);
+ low = _mm512_add_epi16(low, zmm_8080);
+ high = _mm512_add_epi16(high, zmm_8080);
+ low = _mm512_srli_epi16(low, 8);
+ high = _mm512_srli_epi16(high, 8);
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i),
+ _mm512_packus_epi16(low, high));
+ }
+ if (i < width) {
+ __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
+ __m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
+ __m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
+ __m512i low = _mm512_unpacklo_epi8(row0, row1);
+ __m512i high = _mm512_unpackhi_epi8(row0, row1);
+ low = _mm512_sub_epi8(low, zmm_8080);
+ high = _mm512_sub_epi8(high, zmm_8080);
+ low = _mm512_maddubs_epi16(zmm_y, low);
+ high = _mm512_maddubs_epi16(zmm_y, high);
+ low = _mm512_add_epi16(low, zmm_8080);
+ high = _mm512_add_epi16(high, zmm_8080);
+ low = _mm512_srli_epi16(low, 8);
+ high = _mm512_srli_epi16(high, 8);
+ _mm512_mask_storeu_epi8(dst_ptr + i, mask,
+ _mm512_packus_epi16(low, high));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_INTERPOLATEROW_AVX512BW
+
+#ifdef HAS_INTERPOLATEROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1 = source_y_fraction;
+ int y0 = 256 - y1;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ __m256i ymm_y = _mm256_set1_epi32((y1 << 16) | y0);
+ __m256i ymm_8000 = _mm256_set1_epi16((short)0x8000);
+ __m256i ymm_round = _mm256_set1_epi32(8388736); // 0x800000 + 128
+ int i;
+
+ if (y1 == 0) {
+ for (i = 0; i < width; i += 16) {
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + i)));
+ }
+ } else if (y1 == 128) {
+ for (i = 0; i < width; i += 16) {
+ __m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
+ __m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i),
+ _mm256_avg_epu16(row0, row1));
+ }
+ } else {
+ for (i = 0; i < width; i += 16) {
+ __m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
+ __m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
+ __m256i row0l = _mm256_unpacklo_epi16(row0, row1);
+ __m256i row0h = _mm256_unpackhi_epi16(row0, row1);
+ row0l = _mm256_sub_epi16(row0l, ymm_8000);
+ row0h = _mm256_sub_epi16(row0h, ymm_8000);
+ __m256i resl = _mm256_madd_epi16(row0l, ymm_y);
+ __m256i resh = _mm256_madd_epi16(row0h, ymm_y);
+ resl = _mm256_add_epi32(resl, ymm_round);
+ resh = _mm256_add_epi32(resh, ymm_round);
+ resl = _mm256_srai_epi32(resl, 8);
+ resh = _mm256_srai_epi32(resh, 8);
+ _mm256_storeu_si256((__m256i*)(dst_ptr + i),
+ _mm256_packus_epi32(resl, resh));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_INTERPOLATEROW_16_AVX2
+
+#ifdef HAS_ARGBMIRRORROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ __m256i ymm_shuf = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0);
+ src += width * 4;
+ while (width > 0) {
+ src -= 32;
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
+ ymm0 = _mm256_permutevar8x32_epi32(ymm0, ymm_shuf);
+ _mm256_storeu_si256((__m256i*)dst, ymm0);
+ dst += 32;
+ width -= 8;
}
+ _mm256_zeroupper();
}
+#endif // HAS_ARGBMIRRORROW_AVX2
#ifdef HAS_J400TOARGBROW_AVX2
-// Duplicates gray value 3 times and fills in alpha opaque.
-__declspec(naked) void J400ToARGBRow_AVX2(const uint8_t* src_y,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_y
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xff000000
- vpslld ymm5, ymm5, 24
-
- convertloop:
- vmovdqu xmm0, [eax]
- lea eax, [eax + 16]
- vpermq ymm0, ymm0, 0xd8
- vpunpcklbw ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8
- vpunpckhwd ymm1, ymm0, ymm0
- vpunpcklwd ymm0, ymm0, ymm0
- vpor ymm0, ymm0, ymm5
- vpor ymm1, ymm1, ymm5
- vmovdqu [edx], ymm0
- vmovdqu [edx + 32], ymm1
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
+static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
+ 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u,
+ 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u};
+static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
+ 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u,
+ 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u,
+ 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
+
+LIBYUV_TARGET_AVX2
+void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ __m256i ymm_mask0 =
+ _mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
+ __m256i ymm_mask1 =
+ _mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
+ __m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u);
+
+ while (width > 0) {
+ __m256i ymm0 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)src_y));
+
+ __m256i ymm1 = _mm256_shuffle_epi8(ymm0, ymm_mask0);
+ __m256i ymm2 = _mm256_shuffle_epi8(ymm0, ymm_mask1);
+
+ ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
+ ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm1);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm2);
+
+ src_y += 16;
+ dst_argb += 64;
+ width -= 16;
}
+ _mm256_zeroupper();
}
#endif // HAS_J400TOARGBROW_AVX2
-__declspec(naked) void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_rgb24
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm5, xmm5 // generate mask 0xff000000
- pslld xmm5, 24
- movdqa xmm4, xmmword ptr kShuffleMaskRGB24ToARGB
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm3, [eax + 32]
- lea eax, [eax + 48]
- movdqa xmm2, xmm3
- palignr xmm2, xmm1, 8 // xmm2 = { xmm3[0:3] xmm1[8:15]}
- pshufb xmm2, xmm4
- por xmm2, xmm5
- palignr xmm1, xmm0, 12 // xmm1 = { xmm3[0:7] xmm0[12:15]}
- pshufb xmm0, xmm4
- movdqu [edx + 32], xmm2
- por xmm0, xmm5
- pshufb xmm1, xmm4
- movdqu [edx], xmm0
- por xmm1, xmm5
- palignr xmm3, xmm3, 4 // xmm3 = { xmm3[4:15]}
- pshufb xmm3, xmm4
- movdqu [edx + 16], xmm1
- por xmm3, xmm5
- movdqu [edx + 48], xmm3
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void RAWToARGBRow_SSSE3(const uint8_t* src_raw,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_raw
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm5, xmm5 // generate mask 0xff000000
- pslld xmm5, 24
- movdqa xmm4, xmmword ptr kShuffleMaskRAWToARGB
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm3, [eax + 32]
- lea eax, [eax + 48]
- movdqa xmm2, xmm3
- palignr xmm2, xmm1, 8 // xmm2 = { xmm3[0:3] xmm1[8:15]}
- pshufb xmm2, xmm4
- por xmm2, xmm5
- palignr xmm1, xmm0, 12 // xmm1 = { xmm3[0:7] xmm0[12:15]}
- pshufb xmm0, xmm4
- movdqu [edx + 32], xmm2
- por xmm0, xmm5
- pshufb xmm1, xmm4
- movdqu [edx], xmm0
- por xmm1, xmm5
- palignr xmm3, xmm3, 4 // xmm3 = { xmm3[4:15]}
- pshufb xmm3, xmm4
- movdqu [edx + 16], xmm1
- por xmm3, xmm5
- movdqu [edx + 48], xmm3
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
- uint8_t* dst_rgb24,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_raw
- mov edx, [esp + 8] // dst_rgb24
- mov ecx, [esp + 12] // width
- movdqa xmm3, xmmword ptr kShuffleMaskRAWToRGB24_0
- movdqa xmm4, xmmword ptr kShuffleMaskRAWToRGB24_1
- movdqa xmm5, xmmword ptr kShuffleMaskRAWToRGB24_2
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 4]
- movdqu xmm2, [eax + 8]
- lea eax, [eax + 24]
- pshufb xmm0, xmm3
- pshufb xmm1, xmm4
- pshufb xmm2, xmm5
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + 8], xmm1
- movq qword ptr [edx + 16], xmm2
- lea edx, [edx + 24]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-
-// pmul method to replicate bits.
-// Math to replicate bits:
-// (v << 8) | (v << 3)
-// v * 256 + v * 8
-// v * (256 + 8)
-// G shift of 5 is incorporated, so shift is 5 + 8 and 5 + 3
-// 20 instructions.
-__declspec(naked) void RGB565ToARGBRow_SSE2(const uint8_t* src_rgb565,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x01080108 // generate multiplier to repeat 5 bits
- movd xmm5, eax
- pshufd xmm5, xmm5, 0
- mov eax, 0x20802080 // multiplier shift by 5 and then repeat 6 bits
- movd xmm6, eax
- pshufd xmm6, xmm6, 0
- pcmpeqb xmm3, xmm3 // generate mask 0xf800f800 for Red
- psllw xmm3, 11
- pcmpeqb xmm4, xmm4 // generate mask 0x07e007e0 for Green
- psllw xmm4, 10
- psrlw xmm4, 5
- pcmpeqb xmm7, xmm7 // generate mask 0xff00ff00 for Alpha
- psllw xmm7, 8
-
- mov eax, [esp + 4] // src_rgb565
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- movdqu xmm0, [eax] // fetch 8 pixels of bgr565
- movdqa xmm1, xmm0
- movdqa xmm2, xmm0
- pand xmm1, xmm3 // R in upper 5 bits
- psllw xmm2, 11 // B in upper 5 bits
- pmulhuw xmm1, xmm5 // * (256 + 8)
- pmulhuw xmm2, xmm5 // * (256 + 8)
- psllw xmm1, 8
- por xmm1, xmm2 // RB
- pand xmm0, xmm4 // G in middle 6 bits
- pmulhuw xmm0, xmm6 // << 5 * (256 + 4)
- por xmm0, xmm7 // AG
- movdqa xmm2, xmm1
- punpcklbw xmm1, xmm0
- punpckhbw xmm2, xmm0
- movdqu [eax * 2 + edx], xmm1 // store 4 pixels of ARGB
- movdqu [eax * 2 + edx + 16], xmm2 // store next 4 pixels of ARGB
- lea eax, [eax + 16]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
+#ifdef HAS_RGB24TOARGBROW_AVX2
+static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
+ {0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
+ {4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
+ 128u}};
+#endif // HAS_RGB24TOARGBROW_AVX2
#ifdef HAS_RGB565TOARGBROW_AVX2
-// pmul method to replicate bits.
-// Math to replicate bits:
-// (v << 8) | (v << 3)
-// v * 256 + v * 8
-// v * (256 + 8)
-// G shift of 5 is incorporated, so shift is 5 + 8 and 5 + 3
-__declspec(naked) void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x01080108 // generate multiplier to repeat 5 bits
- vmovd xmm5, eax
- vbroadcastss ymm5, xmm5
- mov eax, 0x20802080 // multiplier shift by 5 and then repeat 6 bits
- vmovd xmm6, eax
- vbroadcastss ymm6, xmm6
- vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0xf800f800 for Red
- vpsllw ymm3, ymm3, 11
- vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x07e007e0 for Green
- vpsllw ymm4, ymm4, 10
- vpsrlw ymm4, ymm4, 5
- vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xff00ff00 for Alpha
- vpsllw ymm7, ymm7, 8
-
- mov eax, [esp + 4] // src_rgb565
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 16 pixels of bgr565
- vpand ymm1, ymm0, ymm3 // R in upper 5 bits
- vpsllw ymm2, ymm0, 11 // B in upper 5 bits
- vpmulhuw ymm1, ymm1, ymm5 // * (256 + 8)
- vpmulhuw ymm2, ymm2, ymm5 // * (256 + 8)
- vpsllw ymm1, ymm1, 8
- vpor ymm1, ymm1, ymm2 // RB
- vpand ymm0, ymm0, ymm4 // G in middle 6 bits
- vpmulhuw ymm0, ymm0, ymm6 // << 5 * (256 + 4)
- vpor ymm0, ymm0, ymm7 // AG
- vpermq ymm0, ymm0, 0xd8 // mutate for unpack
- vpermq ymm1, ymm1, 0xd8
- vpunpckhbw ymm2, ymm1, ymm0
- vpunpcklbw ymm1, ymm1, ymm0
- vmovdqu [eax * 2 + edx], ymm1 // store 4 pixels of ARGB
- vmovdqu [eax * 2 + edx + 32], ymm2 // store next 4 pixels of ARGB
- lea eax, [eax + 32]
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
+LIBYUV_TARGET_AVX2
+void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
+ uint8_t* dst_argb,
+ int width) {
+ __m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
+ __m256i ymm_scale_g = _mm256_set1_epi32(0x20802080);
+ __m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
+ __m256i ymm_mask_g = _mm256_set1_epi16(0x07e0);
+ __m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
+
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_rgb565);
+ __m256i ymm1 = ymm0;
+ __m256i ymm2 = ymm0;
+
+ ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
+ ymm2 = _mm256_slli_epi16(ymm2, 11);
+ ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
+ ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
+ ymm1 = _mm256_slli_epi16(ymm1, 8);
+ ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
+
+ ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
+ ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
+ ymm0 = _mm256_or_si256(ymm0, ymm_mask_a); // GA
+
+ ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
+ ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
+
+ ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
+ ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
+
+ src_rgb565 += 32;
+ dst_argb += 64;
+ width -= 16;
}
+ _mm256_zeroupper();
}
#endif // HAS_RGB565TOARGBROW_AVX2
#ifdef HAS_ARGB1555TOARGBROW_AVX2
-__declspec(naked) void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x01080108 // generate multiplier to repeat 5 bits
- vmovd xmm5, eax
- vbroadcastss ymm5, xmm5
- mov eax, 0x42004200 // multiplier shift by 6 and then repeat 5 bits
- vmovd xmm6, eax
- vbroadcastss ymm6, xmm6
- vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0xf800f800 for Red
- vpsllw ymm3, ymm3, 11
- vpsrlw ymm4, ymm3, 6 // generate mask 0x03e003e0 for Green
- vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xff00ff00 for Alpha
- vpsllw ymm7, ymm7, 8
-
- mov eax, [esp + 4] // src_argb1555
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 16 pixels of 1555
- vpsllw ymm1, ymm0, 1 // R in upper 5 bits
- vpsllw ymm2, ymm0, 11 // B in upper 5 bits
- vpand ymm1, ymm1, ymm3
- vpmulhuw ymm2, ymm2, ymm5 // * (256 + 8)
- vpmulhuw ymm1, ymm1, ymm5 // * (256 + 8)
- vpsllw ymm1, ymm1, 8
- vpor ymm1, ymm1, ymm2 // RB
- vpsraw ymm2, ymm0, 8 // A
- vpand ymm0, ymm0, ymm4 // G in middle 5 bits
- vpmulhuw ymm0, ymm0, ymm6 // << 6 * (256 + 8)
- vpand ymm2, ymm2, ymm7
- vpor ymm0, ymm0, ymm2 // AG
- vpermq ymm0, ymm0, 0xd8 // mutate for unpack
- vpermq ymm1, ymm1, 0xd8
- vpunpckhbw ymm2, ymm1, ymm0
- vpunpcklbw ymm1, ymm1, ymm0
- vmovdqu [eax * 2 + edx], ymm1 // store 8 pixels of ARGB
- vmovdqu [eax * 2 + edx + 32], ymm2 // store next 8 pixels of ARGB
- lea eax, [eax + 32]
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
+LIBYUV_TARGET_AVX2
+void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ __m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
+ __m256i ymm_scale_g = _mm256_set1_epi32(0x42004200);
+ __m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
+ __m256i ymm_mask_g = _mm256_set1_epi16(0x03e0);
+ __m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
+
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb1555);
+ __m256i ymm1 = ymm0;
+ __m256i ymm2 = ymm0;
+
+ ymm1 = _mm256_slli_epi16(ymm1, 1);
+ ymm2 = _mm256_slli_epi16(ymm2, 11);
+ ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
+ ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
+ ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
+ ymm1 = _mm256_slli_epi16(ymm1, 8);
+ ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
+
+ ymm2 = ymm0;
+ ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
+ ymm2 = _mm256_srai_epi16(ymm2, 8);
+ ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
+ ymm2 = _mm256_and_si256(ymm2, ymm_mask_a);
+ ymm0 = _mm256_or_si256(ymm0, ymm2); // GA
+
+ ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
+ ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
+
+ ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
+ ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
+
+ src_argb1555 += 32;
+ dst_argb += 64;
+ width -= 16;
}
+ _mm256_zeroupper();
}
#endif // HAS_ARGB1555TOARGBROW_AVX2
#ifdef HAS_ARGB4444TOARGBROW_AVX2
-__declspec(naked) void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x0f0f0f0f // generate mask 0x0f0f0f0f
- vmovd xmm4, eax
- vbroadcastss ymm4, xmm4
- vpslld ymm5, ymm4, 4 // 0xf0f0f0f0 for high nibbles
- mov eax, [esp + 4] // src_argb4444
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 16 pixels of bgra4444
- vpand ymm2, ymm0, ymm5 // mask high nibbles
- vpand ymm0, ymm0, ymm4 // mask low nibbles
- vpsrlw ymm3, ymm2, 4
- vpsllw ymm1, ymm0, 4
- vpor ymm2, ymm2, ymm3
- vpor ymm0, ymm0, ymm1
- vpermq ymm0, ymm0, 0xd8 // mutate for unpack
- vpermq ymm2, ymm2, 0xd8
- vpunpckhbw ymm1, ymm0, ymm2
- vpunpcklbw ymm0, ymm0, ymm2
- vmovdqu [eax * 2 + edx], ymm0 // store 8 pixels of ARGB
- vmovdqu [eax * 2 + edx + 32], ymm1 // store next 8 pixels of ARGB
- lea eax, [eax + 32]
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGB4444TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
+ uint8_t* dst_argb,
+ int width) {
+ __m256i ymm_mask = _mm256_set1_epi32(0x0f0f0f0f);
+ __m256i ymm_mask2 = _mm256_slli_epi32(ymm_mask, 4);
-// 24 instructions
-__declspec(naked) void ARGB1555ToARGBRow_SSE2(const uint8_t* src_argb1555,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x01080108 // generate multiplier to repeat 5 bits
- movd xmm5, eax
- pshufd xmm5, xmm5, 0
- mov eax, 0x42004200 // multiplier shift by 6 and then repeat 5 bits
- movd xmm6, eax
- pshufd xmm6, xmm6, 0
- pcmpeqb xmm3, xmm3 // generate mask 0xf800f800 for Red
- psllw xmm3, 11
- movdqa xmm4, xmm3 // generate mask 0x03e003e0 for Green
- psrlw xmm4, 6
- pcmpeqb xmm7, xmm7 // generate mask 0xff00ff00 for Alpha
- psllw xmm7, 8
-
- mov eax, [esp + 4] // src_argb1555
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- movdqu xmm0, [eax] // fetch 8 pixels of 1555
- movdqa xmm1, xmm0
- movdqa xmm2, xmm0
- psllw xmm1, 1 // R in upper 5 bits
- psllw xmm2, 11 // B in upper 5 bits
- pand xmm1, xmm3
- pmulhuw xmm2, xmm5 // * (256 + 8)
- pmulhuw xmm1, xmm5 // * (256 + 8)
- psllw xmm1, 8
- por xmm1, xmm2 // RB
- movdqa xmm2, xmm0
- pand xmm0, xmm4 // G in middle 5 bits
- psraw xmm2, 8 // A
- pmulhuw xmm0, xmm6 // << 6 * (256 + 8)
- pand xmm2, xmm7
- por xmm0, xmm2 // AG
- movdqa xmm2, xmm1
- punpcklbw xmm1, xmm0
- punpckhbw xmm2, xmm0
- movdqu [eax * 2 + edx], xmm1 // store 4 pixels of ARGB
- movdqu [eax * 2 + edx + 16], xmm2 // store next 4 pixels of ARGB
- lea eax, [eax + 16]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-
-// 18 instructions.
-__declspec(naked) void ARGB4444ToARGBRow_SSE2(const uint8_t* src_argb4444,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, 0x0f0f0f0f // generate mask 0x0f0f0f0f
- movd xmm4, eax
- pshufd xmm4, xmm4, 0
- movdqa xmm5, xmm4 // 0xf0f0f0f0 for high nibbles
- pslld xmm5, 4
- mov eax, [esp + 4] // src_argb4444
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- sub edx, eax
-
- convertloop:
- movdqu xmm0, [eax] // fetch 8 pixels of bgra4444
- movdqa xmm2, xmm0
- pand xmm0, xmm4 // mask low nibbles
- pand xmm2, xmm5 // mask high nibbles
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- psllw xmm1, 4
- psrlw xmm3, 4
- por xmm0, xmm1
- por xmm2, xmm3
- movdqa xmm1, xmm0
- punpcklbw xmm0, xmm2
- punpckhbw xmm1, xmm2
- movdqu [eax * 2 + edx], xmm0 // store 4 pixels of ARGB
- movdqu [eax * 2 + edx + 16], xmm1 // store next 4 pixels of ARGB
- lea eax, [eax + 16]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToRGB24Row_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- movdqa xmm6, xmmword ptr kShuffleMaskARGBToRGB24
-
- convertloop:
- movdqu xmm0, [eax] // fetch 16 pixels of argb
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- lea eax, [eax + 64]
- pshufb xmm0, xmm6 // pack 16 bytes of ARGB to 12 bytes of RGB
- pshufb xmm1, xmm6
- pshufb xmm2, xmm6
- pshufb xmm3, xmm6
- movdqa xmm4, xmm1 // 4 bytes from 1 for 0
- psrldq xmm1, 4 // 8 bytes from 1
- pslldq xmm4, 12 // 4 bytes from 1 for 0
- movdqa xmm5, xmm2 // 8 bytes from 2 for 1
- por xmm0, xmm4 // 4 bytes from 1 for 0
- pslldq xmm5, 8 // 8 bytes from 2 for 1
- movdqu [edx], xmm0 // store 0
- por xmm1, xmm5 // 8 bytes from 2 for 1
- psrldq xmm2, 8 // 4 bytes from 2
- pslldq xmm3, 4 // 12 bytes from 3 for 2
- por xmm2, xmm3 // 12 bytes from 3 for 2
- movdqu [edx + 16], xmm1 // store 1
- movdqu [edx + 32], xmm2 // store 2
- lea edx, [edx + 48]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToRAWRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- movdqa xmm6, xmmword ptr kShuffleMaskARGBToRAW
-
- convertloop:
- movdqu xmm0, [eax] // fetch 16 pixels of argb
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- lea eax, [eax + 64]
- pshufb xmm0, xmm6 // pack 16 bytes of ARGB to 12 bytes of RGB
- pshufb xmm1, xmm6
- pshufb xmm2, xmm6
- pshufb xmm3, xmm6
- movdqa xmm4, xmm1 // 4 bytes from 1 for 0
- psrldq xmm1, 4 // 8 bytes from 1
- pslldq xmm4, 12 // 4 bytes from 1 for 0
- movdqa xmm5, xmm2 // 8 bytes from 2 for 1
- por xmm0, xmm4 // 4 bytes from 1 for 0
- pslldq xmm5, 8 // 8 bytes from 2 for 1
- movdqu [edx], xmm0 // store 0
- por xmm1, xmm5 // 8 bytes from 2 for 1
- psrldq xmm2, 8 // 4 bytes from 2
- pslldq xmm3, 4 // 12 bytes from 3 for 2
- por xmm2, xmm3 // 12 bytes from 3 for 2
- movdqu [edx + 16], xmm1 // store 1
- movdqu [edx + 32], xmm2 // store 2
- lea edx, [edx + 48]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToRGB565Row_SSE2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm3, xmm3 // generate mask 0x0000001f
- psrld xmm3, 27
- pcmpeqb xmm4, xmm4 // generate mask 0x000007e0
- psrld xmm4, 26
- pslld xmm4, 5
- pcmpeqb xmm5, xmm5 // generate mask 0xfffff800
- pslld xmm5, 11
-
- convertloop:
- movdqu xmm0, [eax] // fetch 4 pixels of argb
- movdqa xmm1, xmm0 // B
- movdqa xmm2, xmm0 // G
- pslld xmm0, 8 // R
- psrld xmm1, 3 // B
- psrld xmm2, 5 // G
- psrad xmm0, 16 // R
- pand xmm1, xmm3 // B
- pand xmm2, xmm4 // G
- pand xmm0, xmm5 // R
- por xmm1, xmm2 // BG
- por xmm0, xmm1 // BGR
- packssdw xmm0, xmm0
- lea eax, [eax + 16]
- movq qword ptr [edx], xmm0 // store 4 pixels of RGB565
- lea edx, [edx + 8]
- sub ecx, 4
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToRGB565DitherRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- const uint32_t dither4,
- int width) {
- __asm {
-
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- movd xmm6, [esp + 12] // dither4
- mov ecx, [esp + 16] // width
- punpcklbw xmm6, xmm6 // make dither 16 bytes
- movdqa xmm7, xmm6
- punpcklwd xmm6, xmm6
- punpckhwd xmm7, xmm7
- pcmpeqb xmm3, xmm3 // generate mask 0x0000001f
- psrld xmm3, 27
- pcmpeqb xmm4, xmm4 // generate mask 0x000007e0
- psrld xmm4, 26
- pslld xmm4, 5
- pcmpeqb xmm5, xmm5 // generate mask 0xfffff800
- pslld xmm5, 11
-
- convertloop:
- movdqu xmm0, [eax] // fetch 4 pixels of argb
- paddusb xmm0, xmm6 // add dither
- movdqa xmm1, xmm0 // B
- movdqa xmm2, xmm0 // G
- pslld xmm0, 8 // R
- psrld xmm1, 3 // B
- psrld xmm2, 5 // G
- psrad xmm0, 16 // R
- pand xmm1, xmm3 // B
- pand xmm2, xmm4 // G
- pand xmm0, xmm5 // R
- por xmm1, xmm2 // BG
- por xmm0, xmm1 // BGR
- packssdw xmm0, xmm0
- lea eax, [eax + 16]
- movq qword ptr [edx], xmm0 // store 4 pixels of RGB565
- lea edx, [edx + 8]
- sub ecx, 4
- jg convertloop
- ret
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb4444);
+ __m256i ymm2 = ymm0;
+
+ ymm0 = _mm256_and_si256(ymm0, ymm_mask);
+ ymm2 = _mm256_and_si256(ymm2, ymm_mask2);
+
+ __m256i ymm1 = ymm0;
+ __m256i ymm3 = ymm2;
+
+ ymm1 = _mm256_slli_epi16(ymm1, 4);
+ ymm3 = _mm256_srli_epi16(ymm3, 4);
+
+ ymm0 = _mm256_or_si256(ymm0, ymm1);
+ ymm2 = _mm256_or_si256(ymm2, ymm3);
+
+ ymm1 = ymm0;
+ ymm0 = _mm256_unpacklo_epi8(ymm0, ymm2);
+ ymm1 = _mm256_unpackhi_epi8(ymm1, ymm2);
+
+ ymm2 = _mm256_permute2x128_si256(ymm0, ymm1, 0x20);
+ ymm1 = _mm256_permute2x128_si256(ymm0, ymm1, 0x31);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm2);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
+
+ src_argb4444 += 32;
+ dst_argb += 64;
+ width -= 16;
}
+ _mm256_zeroupper();
}
+#endif // HAS_ARGB4444TOARGBROW_AVX2
#ifdef HAS_ARGBTORGB565DITHERROW_AVX2
-__declspec(naked) void ARGBToRGB565DitherRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- const uint32_t dither4,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- vbroadcastss xmm6, [esp + 12] // dither4
- mov ecx, [esp + 16] // width
- vpunpcklbw xmm6, xmm6, xmm6 // make dither 32 bytes
- vpermq ymm6, ymm6, 0xd8
- vpunpcklwd ymm6, ymm6, ymm6
- vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0x0000001f
- vpsrld ymm3, ymm3, 27
- vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x000007e0
- vpsrld ymm4, ymm4, 26
- vpslld ymm4, ymm4, 5
- vpslld ymm5, ymm3, 11 // generate mask 0x0000f800
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 8 pixels of argb
- vpaddusb ymm0, ymm0, ymm6 // add dither
- vpsrld ymm2, ymm0, 5 // G
- vpsrld ymm1, ymm0, 3 // B
- vpsrld ymm0, ymm0, 8 // R
- vpand ymm2, ymm2, ymm4 // G
- vpand ymm1, ymm1, ymm3 // B
- vpand ymm0, ymm0, ymm5 // R
- vpor ymm1, ymm1, ymm2 // BG
- vpor ymm0, ymm0, ymm1 // BGR
- vpackusdw ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8
- lea eax, [eax + 32]
- vmovdqu [edx], xmm0 // store 8 pixels of RGB565
- lea edx, [edx + 16]
- sub ecx, 8
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBTORGB565DITHERROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToRGB565DitherRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ __m128i dither_shuf =
+ _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
+ __m128i d128 = _mm_shuffle_epi8(_mm_cvtsi32_si128((int)dither4), dither_shuf);
+ __m256i ymm_dither = _mm256_broadcastsi128_si256(d128);
+ __m256i mask_b = _mm256_set1_epi32(0x0000001f);
+ __m256i mask_g = _mm256_set1_epi32(0x000007e0);
+ __m256i mask_r = _mm256_set1_epi32(0x0000f800);
-// TODO(fbarchard): Improve sign extension/packing.
-__declspec(naked) void ARGBToARGB1555Row_SSE2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm4, xmm4 // generate mask 0x0000001f
- psrld xmm4, 27
- movdqa xmm5, xmm4 // generate mask 0x000003e0
- pslld xmm5, 5
- movdqa xmm6, xmm4 // generate mask 0x00007c00
- pslld xmm6, 10
- pcmpeqb xmm7, xmm7 // generate mask 0xffff8000
- pslld xmm7, 15
-
- convertloop:
- movdqu xmm0, [eax] // fetch 4 pixels of argb
- movdqa xmm1, xmm0 // B
- movdqa xmm2, xmm0 // G
- movdqa xmm3, xmm0 // R
- psrad xmm0, 16 // A
- psrld xmm1, 3 // B
- psrld xmm2, 6 // G
- psrld xmm3, 9 // R
- pand xmm0, xmm7 // A
- pand xmm1, xmm4 // B
- pand xmm2, xmm5 // G
- pand xmm3, xmm6 // R
- por xmm0, xmm1 // BA
- por xmm2, xmm3 // GR
- por xmm0, xmm2 // BGRA
- packssdw xmm0, xmm0
- lea eax, [eax + 16]
- movq qword ptr [edx], xmm0 // store 4 pixels of ARGB1555
- lea edx, [edx + 8]
- sub ecx, 4
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToARGB4444Row_SSE2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm4, xmm4 // generate mask 0xf000f000
- psllw xmm4, 12
- movdqa xmm3, xmm4 // generate mask 0x00f000f0
- psrlw xmm3, 8
-
- convertloop:
- movdqu xmm0, [eax] // fetch 4 pixels of argb
- movdqa xmm1, xmm0
- pand xmm0, xmm3 // low nibble
- pand xmm1, xmm4 // high nibble
- psrld xmm0, 4
- psrld xmm1, 8
- por xmm0, xmm1
- packuswb xmm0, xmm0
- lea eax, [eax + 16]
- movq qword ptr [edx], xmm0 // store 4 pixels of ARGB4444
- lea edx, [edx + 8]
- sub ecx, 4
- jg convertloop
- ret
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ ymm0 = _mm256_adds_epu8(ymm0, ymm_dither);
+ __m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
+ __m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
+ ymm0 = _mm256_srli_epi32(ymm0, 8);
+ ymm2 = _mm256_and_si256(ymm2, mask_g);
+ ymm1 = _mm256_and_si256(ymm1, mask_b);
+ ymm0 = _mm256_and_si256(ymm0, mask_r);
+ ymm1 = _mm256_or_si256(ymm2, ymm1);
+ ymm0 = _mm256_or_si256(ymm1, ymm0);
+ ymm0 = _mm256_packus_epi32(ymm0, ymm0);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ _mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
+ src_argb += 32;
+ dst_rgb += 16;
+ width -= 8;
}
+ _mm256_zeroupper();
}
+#endif // HAS_ARGBTORGB565DITHERROW_AVX2
#ifdef HAS_ARGBTORGB565ROW_AVX2
-__declspec(naked) void ARGBToRGB565Row_AVX2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm3, ymm3, ymm3 // generate mask 0x0000001f
- vpsrld ymm3, ymm3, 27
- vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0x000007e0
- vpsrld ymm4, ymm4, 26
- vpslld ymm4, ymm4, 5
- vpslld ymm5, ymm3, 11 // generate mask 0x0000f800
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 8 pixels of argb
- vpsrld ymm2, ymm0, 5 // G
- vpsrld ymm1, ymm0, 3 // B
- vpsrld ymm0, ymm0, 8 // R
- vpand ymm2, ymm2, ymm4 // G
- vpand ymm1, ymm1, ymm3 // B
- vpand ymm0, ymm0, ymm5 // R
- vpor ymm1, ymm1, ymm2 // BG
- vpor ymm0, ymm0, ymm1 // BGR
- vpackusdw ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8
- lea eax, [eax + 32]
- vmovdqu [edx], xmm0 // store 8 pixels of RGB565
- lea edx, [edx + 16]
- sub ecx, 8
- jg convertloop
- vzeroupper
- ret
+LIBYUV_TARGET_AVX2
+void ARGBToRGB565Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ __m256i mask_b = _mm256_set1_epi32(0x0000001f);
+ __m256i mask_g = _mm256_set1_epi32(0x000007e0);
+ __m256i mask_r = _mm256_set1_epi32(0x0000f800);
+
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
+ __m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
+ ymm0 = _mm256_srli_epi32(ymm0, 8);
+ ymm2 = _mm256_and_si256(ymm2, mask_g);
+ ymm1 = _mm256_and_si256(ymm1, mask_b);
+ ymm0 = _mm256_and_si256(ymm0, mask_r);
+ ymm1 = _mm256_or_si256(ymm2, ymm1);
+ ymm0 = _mm256_or_si256(ymm1, ymm0);
+ ymm0 = _mm256_packus_epi32(ymm0, ymm0);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ _mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
+ src_argb += 32;
+ dst_rgb += 16;
+ width -= 8;
}
+ _mm256_zeroupper();
}
#endif // HAS_ARGBTORGB565ROW_AVX2
#ifdef HAS_ARGBTOARGB1555ROW_AVX2
-__declspec(naked) void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm4, ymm4, ymm4
- vpsrld ymm4, ymm4, 27 // generate mask 0x0000001f
- vpslld ymm5, ymm4, 5 // generate mask 0x000003e0
- vpslld ymm6, ymm4, 10 // generate mask 0x00007c00
- vpcmpeqb ymm7, ymm7, ymm7 // generate mask 0xffff8000
- vpslld ymm7, ymm7, 15
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 8 pixels of argb
- vpsrld ymm3, ymm0, 9 // R
- vpsrld ymm2, ymm0, 6 // G
- vpsrld ymm1, ymm0, 3 // B
- vpsrad ymm0, ymm0, 16 // A
- vpand ymm3, ymm3, ymm6 // R
- vpand ymm2, ymm2, ymm5 // G
- vpand ymm1, ymm1, ymm4 // B
- vpand ymm0, ymm0, ymm7 // A
- vpor ymm0, ymm0, ymm1 // BA
- vpor ymm2, ymm2, ymm3 // GR
- vpor ymm0, ymm0, ymm2 // BGRA
- vpackssdw ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8
- lea eax, [eax + 32]
- vmovdqu [edx], xmm0 // store 8 pixels of ARGB1555
- lea edx, [edx + 16]
- sub ecx, 8
- jg convertloop
- vzeroupper
- ret
+LIBYUV_TARGET_AVX2
+void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb1555,
+ int width) {
+ __m256i mask_b = _mm256_set1_epi32(0x0000001f);
+ __m256i mask_g = _mm256_set1_epi32(0x000003e0);
+ __m256i mask_r = _mm256_set1_epi32(0x00007c00);
+ __m256i mask_a = _mm256_set1_epi32(0x00008000);
+
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm2 = _mm256_srli_epi32(ymm0, 6);
+ __m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
+ __m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
+ ymm0 = _mm256_srli_epi32(ymm0, 9);
+ ymm2 = _mm256_and_si256(ymm2, mask_g);
+ ymm1 = _mm256_and_si256(ymm1, mask_b);
+ ymm7 = _mm256_and_si256(ymm7, mask_a);
+ ymm0 = _mm256_and_si256(ymm0, mask_r);
+ ymm1 = _mm256_or_si256(ymm2, ymm1);
+ ymm0 = _mm256_or_si256(ymm7, ymm0);
+ ymm0 = _mm256_or_si256(ymm1, ymm0);
+ ymm0 = _mm256_packus_epi32(ymm0, ymm0);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ _mm_storeu_si128((__m128i*)dst_argb1555, _mm256_castsi256_si128(ymm0));
+ src_argb += 32;
+ dst_argb1555 += 16;
+ width -= 8;
}
+ _mm256_zeroupper();
}
#endif // HAS_ARGBTOARGB1555ROW_AVX2
#ifdef HAS_ARGBTOARGB4444ROW_AVX2
-__declspec(naked) void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
- uint8_t* dst_rgb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_rgb
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0xf000f000
- vpsllw ymm4, ymm4, 12
- vpsrlw ymm3, ymm4, 8 // generate mask 0x00f000f0
-
- convertloop:
- vmovdqu ymm0, [eax] // fetch 8 pixels of argb
- vpand ymm1, ymm0, ymm4 // high nibble
- vpand ymm0, ymm0, ymm3 // low nibble
- vpsrld ymm1, ymm1, 8
- vpsrld ymm0, ymm0, 4
- vpor ymm0, ymm0, ymm1
- vpackuswb ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8
- lea eax, [eax + 32]
- vmovdqu [edx], xmm0 // store 8 pixels of ARGB4444
- lea edx, [edx + 16]
- sub ecx, 8
- jg convertloop
- vzeroupper
- ret
+LIBYUV_TARGET_AVX2
+void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb4444,
+ int width) {
+ __m256i mask_b = _mm256_set1_epi32(0x0000000f);
+ __m256i mask_g = _mm256_set1_epi32(0x000000f0);
+ __m256i mask_r = _mm256_set1_epi32(0x00000f00);
+ __m256i mask_a = _mm256_set1_epi32(0x0000f000);
+
+ while (width > 0) {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm2 = _mm256_srli_epi32(ymm0, 8);
+ __m256i ymm1 = _mm256_srli_epi32(ymm0, 4);
+ __m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
+ ymm0 = _mm256_srli_epi32(ymm0, 12);
+ ymm2 = _mm256_and_si256(ymm2, mask_g);
+ ymm1 = _mm256_and_si256(ymm1, mask_b);
+ ymm7 = _mm256_and_si256(ymm7, mask_a);
+ ymm0 = _mm256_and_si256(ymm0, mask_r);
+ ymm1 = _mm256_or_si256(ymm2, ymm1);
+ ymm0 = _mm256_or_si256(ymm7, ymm0);
+ ymm0 = _mm256_or_si256(ymm1, ymm0);
+ ymm0 = _mm256_packus_epi32(ymm0, ymm0);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ _mm_storeu_si128((__m128i*)dst_argb4444, _mm256_castsi256_si128(ymm0));
+ src_argb += 32;
+ dst_argb4444 += 16;
+ width -= 8;
}
+ _mm256_zeroupper();
}
#endif // HAS_ARGBTOARGB4444ROW_AVX2
-// Convert 16 ARGB pixels (64 bytes) to 16 Y values.
-__declspec(naked) void ARGBToYRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kARGBToY
- movdqa xmm5, xmmword ptr kAddY16
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm2, xmm4
- pmaddubsw xmm3, xmm4
- lea eax, [eax + 64]
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psrlw xmm0, 7
- psrlw xmm2, 7
- packuswb xmm0, xmm2
- paddb xmm0, xmm5
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-// Convert 16 ARGB pixels (64 bytes) to 16 YJ values.
-// Same as ARGBToYRow but different coefficients, no add 16, but do rounding.
-__declspec(naked) void ARGBToYJRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kARGBToYJ
- movdqa xmm5, xmmword ptr kAddYJ64
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm2, xmm4
- pmaddubsw xmm3, xmm4
- lea eax, [eax + 64]
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- paddw xmm0, xmm5 // Add .5 for rounding.
- paddw xmm2, xmm5
- psrlw xmm0, 7
- psrlw xmm2, 7
- packuswb xmm0, xmm2
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-#ifdef HAS_ARGBTOYROW_AVX2
-// vpermd for vphaddw + vpackuswb vpermd.
-static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7};
-
-// Convert 32 ARGB pixels (128 bytes) to 32 Y values.
-__declspec(naked) void ARGBToYRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- vbroadcastf128 ymm4, xmmword ptr kARGBToY
- vbroadcastf128 ymm5, xmmword ptr kAddY16
- vmovdqu ymm6, ymmword ptr kPermdARGBToY_AVX
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + 64]
- vmovdqu ymm3, [eax + 96]
- vpmaddubsw ymm0, ymm0, ymm4
- vpmaddubsw ymm1, ymm1, ymm4
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- lea eax, [eax + 128]
- vphaddw ymm0, ymm0, ymm1 // mutates.
- vphaddw ymm2, ymm2, ymm3
- vpsrlw ymm0, ymm0, 7
- vpsrlw ymm2, ymm2, 7
- vpackuswb ymm0, ymm0, ymm2 // mutates.
- vpermd ymm0, ymm6, ymm0 // For vphaddw + vpackuswb mutation.
- vpaddb ymm0, ymm0, ymm5 // add 16 for Y
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBTOYROW_AVX2
-
-#ifdef HAS_ARGBTOYJROW_AVX2
-// Convert 32 ARGB pixels (128 bytes) to 32 Y values.
-__declspec(naked) void ARGBToYJRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- vbroadcastf128 ymm4, xmmword ptr kARGBToYJ
- vbroadcastf128 ymm5, xmmword ptr kAddYJ64
- vmovdqu ymm6, ymmword ptr kPermdARGBToY_AVX
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + 64]
- vmovdqu ymm3, [eax + 96]
- vpmaddubsw ymm0, ymm0, ymm4
- vpmaddubsw ymm1, ymm1, ymm4
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- lea eax, [eax + 128]
- vphaddw ymm0, ymm0, ymm1 // mutates.
- vphaddw ymm2, ymm2, ymm3
- vpaddw ymm0, ymm0, ymm5 // Add .5 for rounding.
- vpaddw ymm2, ymm2, ymm5
- vpsrlw ymm0, ymm0, 7
- vpsrlw ymm2, ymm2, 7
- vpackuswb ymm0, ymm0, ymm2 // mutates.
- vpermd ymm0, ymm6, ymm0 // For vphaddw + vpackuswb mutation.
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
-
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBTOYJROW_AVX2
-
-__declspec(naked) void BGRAToYRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kBGRAToY
- movdqa xmm5, xmmword ptr kAddY16
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm2, xmm4
- pmaddubsw xmm3, xmm4
- lea eax, [eax + 64]
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psrlw xmm0, 7
- psrlw xmm2, 7
- packuswb xmm0, xmm2
- paddb xmm0, xmm5
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ABGRToYRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kABGRToY
- movdqa xmm5, xmmword ptr kAddY16
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm2, xmm4
- pmaddubsw xmm3, xmm4
- lea eax, [eax + 64]
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psrlw xmm0, 7
- psrlw xmm2, 7
- packuswb xmm0, xmm2
- paddb xmm0, xmm5
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void RGBAToYRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_y */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kRGBAToY
- movdqa xmm5, xmmword ptr kAddY16
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm2, xmm4
- pmaddubsw xmm3, xmm4
- lea eax, [eax + 64]
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psrlw xmm0, 7
- psrlw xmm2, 7
- packuswb xmm0, xmm2
- paddb xmm0, xmm5
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void ARGBToUVRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- movdqa xmm5, xmmword ptr kAddUV128
- movdqa xmm6, xmmword ptr kARGBToV
- movdqa xmm7, xmmword ptr kARGBToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 16x2 argb pixels to 8x1 */
- movdqu xmm0, [eax]
- movdqu xmm4, [eax + esi]
- pavgb xmm0, xmm4
- movdqu xmm1, [eax + 16]
- movdqu xmm4, [eax + esi + 16]
- pavgb xmm1, xmm4
- movdqu xmm2, [eax + 32]
- movdqu xmm4, [eax + esi + 32]
- pavgb xmm2, xmm4
- movdqu xmm3, [eax + 48]
- movdqu xmm4, [eax + esi + 48]
- pavgb xmm3, xmm4
-
- lea eax, [eax + 64]
- movdqa xmm4, xmm0
- shufps xmm0, xmm1, 0x88
- shufps xmm4, xmm1, 0xdd
- pavgb xmm0, xmm4
- movdqa xmm4, xmm2
- shufps xmm2, xmm3, 0x88
- shufps xmm4, xmm3, 0xdd
- pavgb xmm2, xmm4
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 16 different pixels, its 8 pixels of U and 8 of V
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- pmaddubsw xmm0, xmm7 // U
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm1, xmm6 // V
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm2
- phaddw xmm1, xmm3
- psraw xmm0, 8
- psraw xmm1, 8
- packsswb xmm0, xmm1
- paddb xmm0, xmm5 // -> unsigned
-
- // step 3 - store 8 U and 8 V values
- movlps qword ptr [edx], xmm0 // U
- movhps qword ptr [edx + edi], xmm0 // V
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void ARGBToUVJRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- movdqa xmm5, xmmword ptr kAddUVJ128
- movdqa xmm6, xmmword ptr kARGBToVJ
- movdqa xmm7, xmmword ptr kARGBToUJ
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 16x2 argb pixels to 8x1 */
- movdqu xmm0, [eax]
- movdqu xmm4, [eax + esi]
- pavgb xmm0, xmm4
- movdqu xmm1, [eax + 16]
- movdqu xmm4, [eax + esi + 16]
- pavgb xmm1, xmm4
- movdqu xmm2, [eax + 32]
- movdqu xmm4, [eax + esi + 32]
- pavgb xmm2, xmm4
- movdqu xmm3, [eax + 48]
- movdqu xmm4, [eax + esi + 48]
- pavgb xmm3, xmm4
-
- lea eax, [eax + 64]
- movdqa xmm4, xmm0
- shufps xmm0, xmm1, 0x88
- shufps xmm4, xmm1, 0xdd
- pavgb xmm0, xmm4
- movdqa xmm4, xmm2
- shufps xmm2, xmm3, 0x88
- shufps xmm4, xmm3, 0xdd
- pavgb xmm2, xmm4
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 16 different pixels, its 8 pixels of U and 8 of V
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- pmaddubsw xmm0, xmm7 // U
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm1, xmm6 // V
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm2
- phaddw xmm1, xmm3
- paddw xmm0, xmm5 // +.5 rounding -> unsigned
- paddw xmm1, xmm5
- psraw xmm0, 8
- psraw xmm1, 8
- packsswb xmm0, xmm1
-
- // step 3 - store 8 U and 8 V values
- movlps qword ptr [edx], xmm0 // U
- movhps qword ptr [edx + edi], xmm0 // V
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-#ifdef HAS_ARGBTOUVROW_AVX2
-__declspec(naked) void ARGBToUVRow_AVX2(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- vbroadcastf128 ymm5, xmmword ptr kAddUV128
- vbroadcastf128 ymm6, xmmword ptr kARGBToV
- vbroadcastf128 ymm7, xmmword ptr kARGBToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 32x2 argb pixels to 16x1 */
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + 64]
- vmovdqu ymm3, [eax + 96]
- vpavgb ymm0, ymm0, [eax + esi]
- vpavgb ymm1, ymm1, [eax + esi + 32]
- vpavgb ymm2, ymm2, [eax + esi + 64]
- vpavgb ymm3, ymm3, [eax + esi + 96]
- lea eax, [eax + 128]
- vshufps ymm4, ymm0, ymm1, 0x88
- vshufps ymm0, ymm0, ymm1, 0xdd
- vpavgb ymm0, ymm0, ymm4 // mutated by vshufps
- vshufps ymm4, ymm2, ymm3, 0x88
- vshufps ymm2, ymm2, ymm3, 0xdd
- vpavgb ymm2, ymm2, ymm4 // mutated by vshufps
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 32 different pixels, its 16 pixels of U and 16 of V
- vpmaddubsw ymm1, ymm0, ymm7 // U
- vpmaddubsw ymm3, ymm2, ymm7
- vpmaddubsw ymm0, ymm0, ymm6 // V
- vpmaddubsw ymm2, ymm2, ymm6
- vphaddw ymm1, ymm1, ymm3 // mutates
- vphaddw ymm0, ymm0, ymm2
- vpsraw ymm1, ymm1, 8
- vpsraw ymm0, ymm0, 8
- vpacksswb ymm0, ymm1, ymm0 // mutates
- vpermq ymm0, ymm0, 0xd8 // For vpacksswb
- vpshufb ymm0, ymm0, ymmword ptr kShufARGBToUV_AVX // for vshufps/vphaddw
- vpaddb ymm0, ymm0, ymm5 // -> unsigned
-
- // step 3 - store 16 U and 16 V values
- vextractf128 [edx], ymm0, 0 // U
- vextractf128 [edx + edi], ymm0, 1 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBTOUVROW_AVX2
-
-#ifdef HAS_ARGBTOUVJROW_AVX2
-__declspec(naked) void ARGBToUVJRow_AVX2(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- vbroadcastf128 ymm5, xmmword ptr kAddUVJ128
- vbroadcastf128 ymm6, xmmword ptr kARGBToVJ
- vbroadcastf128 ymm7, xmmword ptr kARGBToUJ
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 32x2 argb pixels to 16x1 */
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + 64]
- vmovdqu ymm3, [eax + 96]
- vpavgb ymm0, ymm0, [eax + esi]
- vpavgb ymm1, ymm1, [eax + esi + 32]
- vpavgb ymm2, ymm2, [eax + esi + 64]
- vpavgb ymm3, ymm3, [eax + esi + 96]
- lea eax, [eax + 128]
- vshufps ymm4, ymm0, ymm1, 0x88
- vshufps ymm0, ymm0, ymm1, 0xdd
- vpavgb ymm0, ymm0, ymm4 // mutated by vshufps
- vshufps ymm4, ymm2, ymm3, 0x88
- vshufps ymm2, ymm2, ymm3, 0xdd
- vpavgb ymm2, ymm2, ymm4 // mutated by vshufps
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 32 different pixels, its 16 pixels of U and 16 of V
- vpmaddubsw ymm1, ymm0, ymm7 // U
- vpmaddubsw ymm3, ymm2, ymm7
- vpmaddubsw ymm0, ymm0, ymm6 // V
- vpmaddubsw ymm2, ymm2, ymm6
- vphaddw ymm1, ymm1, ymm3 // mutates
- vphaddw ymm0, ymm0, ymm2
- vpaddw ymm1, ymm1, ymm5 // +.5 rounding -> unsigned
- vpaddw ymm0, ymm0, ymm5
- vpsraw ymm1, ymm1, 8
- vpsraw ymm0, ymm0, 8
- vpacksswb ymm0, ymm1, ymm0 // mutates
- vpermq ymm0, ymm0, 0xd8 // For vpacksswb
- vpshufb ymm0, ymm0, ymmword ptr kShufARGBToUV_AVX // for vshufps/vphaddw
-
- // step 3 - store 16 U and 16 V values
- vextractf128 [edx], ymm0, 0 // U
- vextractf128 [edx + edi], ymm0, 1 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBTOUVJROW_AVX2
-
-__declspec(naked) void ARGBToUV444Row_SSSE3(const uint8_t* src_argb0,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_argb
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- movdqa xmm5, xmmword ptr kAddUV128
- movdqa xmm6, xmmword ptr kARGBToV
- movdqa xmm7, xmmword ptr kARGBToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* convert to U and V */
- movdqu xmm0, [eax] // U
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm7
- pmaddubsw xmm1, xmm7
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm3, xmm7
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psraw xmm0, 8
- psraw xmm2, 8
- packsswb xmm0, xmm2
- paddb xmm0, xmm5
- movdqu [edx], xmm0
-
- movdqu xmm0, [eax] // V
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
- pmaddubsw xmm0, xmm6
- pmaddubsw xmm1, xmm6
- pmaddubsw xmm2, xmm6
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm1
- phaddw xmm2, xmm3
- psraw xmm0, 8
- psraw xmm2, 8
- packsswb xmm0, xmm2
- paddb xmm0, xmm5
- lea eax, [eax + 64]
- movdqu [edx + edi], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
-
- pop edi
- ret
- }
-}
-
-__declspec(naked) void BGRAToUVRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- movdqa xmm5, xmmword ptr kAddUV128
- movdqa xmm6, xmmword ptr kBGRAToV
- movdqa xmm7, xmmword ptr kBGRAToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 16x2 argb pixels to 8x1 */
- movdqu xmm0, [eax]
- movdqu xmm4, [eax + esi]
- pavgb xmm0, xmm4
- movdqu xmm1, [eax + 16]
- movdqu xmm4, [eax + esi + 16]
- pavgb xmm1, xmm4
- movdqu xmm2, [eax + 32]
- movdqu xmm4, [eax + esi + 32]
- pavgb xmm2, xmm4
- movdqu xmm3, [eax + 48]
- movdqu xmm4, [eax + esi + 48]
- pavgb xmm3, xmm4
-
- lea eax, [eax + 64]
- movdqa xmm4, xmm0
- shufps xmm0, xmm1, 0x88
- shufps xmm4, xmm1, 0xdd
- pavgb xmm0, xmm4
- movdqa xmm4, xmm2
- shufps xmm2, xmm3, 0x88
- shufps xmm4, xmm3, 0xdd
- pavgb xmm2, xmm4
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 16 different pixels, its 8 pixels of U and 8 of V
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- pmaddubsw xmm0, xmm7 // U
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm1, xmm6 // V
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm2
- phaddw xmm1, xmm3
- psraw xmm0, 8
- psraw xmm1, 8
- packsswb xmm0, xmm1
- paddb xmm0, xmm5 // -> unsigned
-
- // step 3 - store 8 U and 8 V values
- movlps qword ptr [edx], xmm0 // U
- movhps qword ptr [edx + edi], xmm0 // V
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void ABGRToUVRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- movdqa xmm5, xmmword ptr kAddUV128
- movdqa xmm6, xmmword ptr kABGRToV
- movdqa xmm7, xmmword ptr kABGRToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 16x2 argb pixels to 8x1 */
- movdqu xmm0, [eax]
- movdqu xmm4, [eax + esi]
- pavgb xmm0, xmm4
- movdqu xmm1, [eax + 16]
- movdqu xmm4, [eax + esi + 16]
- pavgb xmm1, xmm4
- movdqu xmm2, [eax + 32]
- movdqu xmm4, [eax + esi + 32]
- pavgb xmm2, xmm4
- movdqu xmm3, [eax + 48]
- movdqu xmm4, [eax + esi + 48]
- pavgb xmm3, xmm4
-
- lea eax, [eax + 64]
- movdqa xmm4, xmm0
- shufps xmm0, xmm1, 0x88
- shufps xmm4, xmm1, 0xdd
- pavgb xmm0, xmm4
- movdqa xmm4, xmm2
- shufps xmm2, xmm3, 0x88
- shufps xmm4, xmm3, 0xdd
- pavgb xmm2, xmm4
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 16 different pixels, its 8 pixels of U and 8 of V
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- pmaddubsw xmm0, xmm7 // U
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm1, xmm6 // V
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm2
- phaddw xmm1, xmm3
- psraw xmm0, 8
- psraw xmm1, 8
- packsswb xmm0, xmm1
- paddb xmm0, xmm5 // -> unsigned
-
- // step 3 - store 8 U and 8 V values
- movlps qword ptr [edx], xmm0 // U
- movhps qword ptr [edx + edi], xmm0 // V
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void RGBAToUVRow_SSSE3(const uint8_t* src_argb0,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- mov esi, [esp + 8 + 8] // src_stride_argb
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- movdqa xmm5, xmmword ptr kAddUV128
- movdqa xmm6, xmmword ptr kRGBAToV
- movdqa xmm7, xmmword ptr kRGBAToU
- sub edi, edx // stride from u to v
-
- convertloop:
- /* step 1 - subsample 16x2 argb pixels to 8x1 */
- movdqu xmm0, [eax]
- movdqu xmm4, [eax + esi]
- pavgb xmm0, xmm4
- movdqu xmm1, [eax + 16]
- movdqu xmm4, [eax + esi + 16]
- pavgb xmm1, xmm4
- movdqu xmm2, [eax + 32]
- movdqu xmm4, [eax + esi + 32]
- pavgb xmm2, xmm4
- movdqu xmm3, [eax + 48]
- movdqu xmm4, [eax + esi + 48]
- pavgb xmm3, xmm4
-
- lea eax, [eax + 64]
- movdqa xmm4, xmm0
- shufps xmm0, xmm1, 0x88
- shufps xmm4, xmm1, 0xdd
- pavgb xmm0, xmm4
- movdqa xmm4, xmm2
- shufps xmm2, xmm3, 0x88
- shufps xmm4, xmm3, 0xdd
- pavgb xmm2, xmm4
-
- // step 2 - convert to U and V
- // from here down is very similar to Y code except
- // instead of 16 different pixels, its 8 pixels of U and 8 of V
- movdqa xmm1, xmm0
- movdqa xmm3, xmm2
- pmaddubsw xmm0, xmm7 // U
- pmaddubsw xmm2, xmm7
- pmaddubsw xmm1, xmm6 // V
- pmaddubsw xmm3, xmm6
- phaddw xmm0, xmm2
- phaddw xmm1, xmm3
- psraw xmm0, 8
- psraw xmm1, 8
- packsswb xmm0, xmm1
- paddb xmm0, xmm5 // -> unsigned
-
- // step 3 - store 8 U and 8 V values
- movlps qword ptr [edx], xmm0 // U
- movhps qword ptr [edx + edi], xmm0 // V
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBTOYROW_SSSE3
-
-// Read 16 UV from 444
-#define READYUV444_AVX2 \
- __asm { \
- __asm vmovdqu xmm0, [esi] /* U */ \
- __asm vmovdqu xmm1, [esi + edi] /* V */ \
- __asm lea esi, [esi + 16] \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpermq ymm1, ymm1, 0xd8 \
- __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \
- __asm vmovdqu xmm4, [eax] /* Y */ \
- __asm vpermq ymm4, ymm4, 0xd8 \
- __asm vpunpcklbw ymm4, ymm4, ymm4 \
- __asm lea eax, [eax + 16]}
-
-// Read 8 UV from 422, upsample to 16 UV.
-#define READYUV422_AVX2 \
- __asm { \
- __asm vmovq xmm0, qword ptr [esi] /* U */ \
- __asm vmovq xmm1, qword ptr [esi + edi] /* V */ \
- __asm lea esi, [esi + 8] \
- __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \
- __asm vmovdqu xmm4, [eax] /* Y */ \
- __asm vpermq ymm4, ymm4, 0xd8 \
- __asm vpunpcklbw ymm4, ymm4, ymm4 \
- __asm lea eax, [eax + 16]}
-
-// Read 8 UV from 422, upsample to 16 UV. With 16 Alpha.
-#define READYUVA422_AVX2 \
- __asm { \
- __asm vmovq xmm0, qword ptr [esi] /* U */ \
- __asm vmovq xmm1, qword ptr [esi + edi] /* V */ \
- __asm lea esi, [esi + 8] \
- __asm vpunpcklbw ymm0, ymm0, ymm1 /* UV */ \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \
- __asm vmovdqu xmm4, [eax] /* Y */ \
- __asm vpermq ymm4, ymm4, 0xd8 \
- __asm vpunpcklbw ymm4, ymm4, ymm4 \
- __asm lea eax, [eax + 16] \
- __asm vmovdqu xmm5, [ebp] /* A */ \
- __asm vpermq ymm5, ymm5, 0xd8 \
- __asm lea ebp, [ebp + 16]}
-
-// Read 8 UV from NV12, upsample to 16 UV.
-#define READNV12_AVX2 \
- __asm { \
- __asm vmovdqu xmm0, [esi] /* UV */ \
- __asm lea esi, [esi + 16] \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpunpcklwd ymm0, ymm0, ymm0 /* UVUV (upsample) */ \
- __asm vmovdqu xmm4, [eax] /* Y */ \
- __asm vpermq ymm4, ymm4, 0xd8 \
- __asm vpunpcklbw ymm4, ymm4, ymm4 \
- __asm lea eax, [eax + 16]}
-
-// Read 8 UV from NV21, upsample to 16 UV.
-#define READNV21_AVX2 \
- __asm { \
- __asm vmovdqu xmm0, [esi] /* UV */ \
- __asm lea esi, [esi + 16] \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpshufb ymm0, ymm0, ymmword ptr kShuffleNV21 \
- __asm vmovdqu xmm4, [eax] /* Y */ \
- __asm vpermq ymm4, ymm4, 0xd8 \
- __asm vpunpcklbw ymm4, ymm4, ymm4 \
- __asm lea eax, [eax + 16]}
-
-// Read 8 YUY2 with 16 Y and upsample 8 UV to 16 UV.
-#define READYUY2_AVX2 \
- __asm { \
- __asm vmovdqu ymm4, [eax] /* YUY2 */ \
- __asm vpshufb ymm4, ymm4, ymmword ptr kShuffleYUY2Y \
- __asm vmovdqu ymm0, [eax] /* UV */ \
- __asm vpshufb ymm0, ymm0, ymmword ptr kShuffleYUY2UV \
- __asm lea eax, [eax + 32]}
-
-// Read 8 UYVY with 16 Y and upsample 8 UV to 16 UV.
-#define READUYVY_AVX2 \
- __asm { \
- __asm vmovdqu ymm4, [eax] /* UYVY */ \
- __asm vpshufb ymm4, ymm4, ymmword ptr kShuffleUYVYY \
- __asm vmovdqu ymm0, [eax] /* UV */ \
- __asm vpshufb ymm0, ymm0, ymmword ptr kShuffleUYVYUV \
- __asm lea eax, [eax + 32]}
-
-// Convert 16 pixels: 16 UV and 16 Y.
-#define YUVTORGB_AVX2(YuvConstants) \
- __asm { \
- __asm vpmaddubsw ymm2, ymm0, ymmword ptr [YuvConstants + KUVTOR] /* R UV */\
- __asm vpmaddubsw ymm1, ymm0, ymmword ptr [YuvConstants + KUVTOG] /* G UV */\
- __asm vpmaddubsw ymm0, ymm0, ymmword ptr [YuvConstants + KUVTOB] /* B UV */\
- __asm vmovdqu ymm3, ymmword ptr [YuvConstants + KUVBIASR] \
- __asm vpsubw ymm2, ymm3, ymm2 \
- __asm vmovdqu ymm3, ymmword ptr [YuvConstants + KUVBIASG] \
- __asm vpsubw ymm1, ymm3, ymm1 \
- __asm vmovdqu ymm3, ymmword ptr [YuvConstants + KUVBIASB] \
- __asm vpsubw ymm0, ymm3, ymm0 /* Step 2: Find Y contribution to 16 R,G,B values */ \
- __asm vpmulhuw ymm4, ymm4, ymmword ptr [YuvConstants + KYTORGB] \
- __asm vpaddsw ymm0, ymm0, ymm4 /* B += Y */ \
- __asm vpaddsw ymm1, ymm1, ymm4 /* G += Y */ \
- __asm vpaddsw ymm2, ymm2, ymm4 /* R += Y */ \
- __asm vpsraw ymm0, ymm0, 6 \
- __asm vpsraw ymm1, ymm1, 6 \
- __asm vpsraw ymm2, ymm2, 6 \
- __asm vpackuswb ymm0, ymm0, ymm0 /* B */ \
- __asm vpackuswb ymm1, ymm1, ymm1 /* G */ \
- __asm vpackuswb ymm2, ymm2, ymm2 /* R */ \
- }
-
-// Store 16 ARGB values.
-#define STOREARGB_AVX2 \
- __asm { \
- __asm vpunpcklbw ymm0, ymm0, ymm1 /* BG */ \
- __asm vpermq ymm0, ymm0, 0xd8 \
- __asm vpunpcklbw ymm2, ymm2, ymm5 /* RA */ \
- __asm vpermq ymm2, ymm2, 0xd8 \
- __asm vpunpcklwd ymm1, ymm0, ymm2 /* BGRA first 8 pixels */ \
- __asm vpunpckhwd ymm0, ymm0, ymm2 /* BGRA next 8 pixels */ \
- __asm vmovdqu 0[edx], ymm1 \
- __asm vmovdqu 32[edx], ymm0 \
- __asm lea edx, [edx + 64]}
-
-// Store 16 RGBA values.
-#define STORERGBA_AVX2 \
- __asm { \
- __asm vpunpcklbw ymm1, ymm1, ymm2 /* GR */ \
- __asm vpermq ymm1, ymm1, 0xd8 \
- __asm vpunpcklbw ymm2, ymm5, ymm0 /* AB */ \
- __asm vpermq ymm2, ymm2, 0xd8 \
- __asm vpunpcklwd ymm0, ymm2, ymm1 /* ABGR first 8 pixels */ \
- __asm vpunpckhwd ymm1, ymm2, ymm1 /* ABGR next 8 pixels */ \
- __asm vmovdqu [edx], ymm0 \
- __asm vmovdqu [edx + 32], ymm1 \
- __asm lea edx, [edx + 64]}
+#ifdef HAS_RGB24TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ __m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
+ __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
+ __m256i ymm_shuf2 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
-#ifdef HAS_I422TOARGBROW_AVX2
-// 16 pixels
-// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
-__declspec(naked) void I422ToARGBRow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READYUV422_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- vzeroupper
- ret
+ while (width > 0) {
+ __m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24);
+ __m256i ymm0 = _mm256_castsi128_si256(xmm0);
+ ymm0 = _mm256_inserti128_si256(
+ ymm0, _mm_loadu_si128((const __m128i*)(src_rgb24 + 12)), 1);
+
+ __m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 24));
+ __m256i ymm1 = _mm256_castsi128_si256(xmm1);
+ ymm1 = _mm256_inserti128_si256(
+ ymm1, _mm_loadu_si128((const __m128i*)(src_rgb24 + 36)), 1);
+
+ __m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 48));
+ __m256i ymm2 = _mm256_castsi128_si256(xmm2);
+ ymm2 = _mm256_inserti128_si256(
+ ymm2, _mm_loadu_si128((const __m128i*)(src_rgb24 + 60)), 1);
+
+ __m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 68));
+ __m256i ymm3 = _mm256_castsi128_si256(xmm3);
+ ymm3 = _mm256_inserti128_si256(
+ ymm3, _mm_loadu_si128((const __m128i*)(src_rgb24 + 80)), 1);
+
+ ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
+ ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
+ ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
+ ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
+
+ ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
+ ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
+ ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
+ ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, ymm0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
+
+ src_rgb24 += 96;
+ dst_argb += 128;
+ width -= 32;
}
+ _mm256_zeroupper();
}
-#endif // HAS_I422TOARGBROW_AVX2
+#endif // HAS_RGB24TOARGBROW_AVX2
+
+#ifdef HAS_ARGBSHUFFLEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ __m256i control =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler));
+ while (width >= 16) {
+ __m256i row = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i row1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
+ row = _mm256_shuffle_epi8(row, control);
+ row1 = _mm256_shuffle_epi8(row1, control);
+ _mm256_storeu_si256((__m256i*)dst_argb, row);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), row1);
+ src_argb += 64;
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBSHUFFLEROW_AVX2
-#ifdef HAS_I422ALPHATOARGBROW_AVX2
-// 16 pixels
-// 8 UV values upsampled to 16 UV, mixed with 16 Y and 16 A producing 16 ARGB.
-__declspec(naked) void I422AlphaToARGBRow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- push ebp
- mov eax, [esp + 16 + 4] // Y
- mov esi, [esp + 16 + 8] // U
- mov edi, [esp + 16 + 12] // V
- mov ebp, [esp + 16 + 16] // A
- mov edx, [esp + 16 + 20] // argb
- mov ebx, [esp + 16 + 24] // yuvconstants
- mov ecx, [esp + 16 + 28] // width
- sub edi, esi
-
- convertloop:
- READYUVA422_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebp
- pop ebx
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_I422ALPHATOARGBROW_AVX2
+#ifdef HAS_ARGBSHUFFLEROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const uint8_t* shuffler,
+ int width) {
+ __m512i control =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler));
+ while (width >= 32) {
+ __m512i row = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ row = _mm512_shuffle_epi8(row, control);
+ row1 = _mm512_shuffle_epi8(row1, control);
+ _mm512_storeu_si512((__m512i*)dst_argb, row);
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64), row1);
+ src_argb += 128;
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 mask0 = (__mmask16)mask;
+ __mmask16 mask1 = (__mmask16)(mask >> 16);
+ __m512i row = _mm512_maskz_loadu_epi32(mask0, src_argb);
+ __m512i row1 = _mm512_maskz_loadu_epi32(mask1, src_argb + 64);
+ row = _mm512_shuffle_epi8(row, control);
+ row1 = _mm512_shuffle_epi8(row1, control);
+ _mm512_mask_storeu_epi32(dst_argb, mask0, row);
+ _mm512_mask_storeu_epi32(dst_argb + 64, mask1, row1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBSHUFFLEROW_AVX512BW
-#ifdef HAS_I444TOARGBROW_AVX2
-// 16 pixels
-// 16 UV values with 16 Y producing 16 ARGB (64 bytes).
-__declspec(naked) void I444ToARGBRow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
- convertloop:
- READYUV444_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- vzeroupper
- ret
- }
+#ifdef HAS_I422TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void I422ToARGBRow_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ __m256i ymm_kYBiasToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
+ __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
+ __m256i ymm_u_zero = _mm256_setzero_si256();
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 16) {
+ __m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
+ __m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
+ src_u += 8;
+
+ __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
+ __m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
+
+ ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
+ ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
+ ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
+
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
+ src_y += 16;
+ __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
+ ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
+ ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
+
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_adds_epi16(ymm0, ymm4);
+ ymm1 = _mm256_subs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_adds_epi16(ymm2, ymm4);
+
+ ymm0 = _mm256_srai_epi16(ymm0, 6);
+ ymm1 = _mm256_srai_epi16(ymm1, 6);
+ ymm2 = _mm256_srai_epi16(ymm2, 6);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+ ymm1 = _mm256_packus_epi16(ymm1, ymm1);
+ ymm2 = _mm256_packus_epi16(ymm2, ymm2);
+
+ ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
+ ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
+
+ __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
+ __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, argb0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
+
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
}
-#endif // HAS_I444TOARGBROW_AVX2
+#endif // HAS_I422TOARGBROW_AVX2
+
+#ifdef HAS_I422TOAR30ROW_AVX2
+LIBYUV_TARGET_AVX2
+void I422ToAR30Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ __m256i ymm_kYBiasToRgb = _mm256_sub_epi16(
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb),
+ _mm256_set1_epi16(24));
+ __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm_3ff0 = _mm256_set1_epi16((short)0x3ff0);
+ __m256i ymm_c000 = _mm256_set1_epi16((short)0xc000);
+ __m256i ymm_u_zero = _mm256_setzero_si256();
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 16) {
+ __m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
+ __m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
+ src_u += 8;
+
+ __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
+ __m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
+
+ ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
+ ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
+ ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
+
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
+ src_y += 16;
+ __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
+ ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
+ ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
+
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_adds_epi16(ymm0, ymm4);
+ ymm1 = _mm256_subs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_adds_epi16(ymm2, ymm4);
+
+ ymm0 = _mm256_min_epi16(ymm0, ymm_3ff0);
+ ymm1 = _mm256_min_epi16(ymm1, ymm_3ff0);
+ ymm2 = _mm256_min_epi16(ymm2, ymm_3ff0);
+ ymm0 = _mm256_max_epi16(ymm0, ymm_u_zero);
+ ymm1 = _mm256_max_epi16(ymm1, ymm_u_zero);
+ ymm2 = _mm256_max_epi16(ymm2, ymm_u_zero);
+
+ __m256i ymm_g_masked = _mm256_and_si256(ymm1, ymm_3ff0);
+ __m256i ymm_r_masked = _mm256_and_si256(ymm2, ymm_3ff0);
+ ymm0 = _mm256_srli_epi16(ymm0, 4);
+ ymm3 = _mm256_slli_epi16(ymm_g_masked, 6);
+ ymm0 = _mm256_or_si256(ymm0, ymm3);
+ ymm1 = _mm256_srli_epi16(ymm1, 10);
+ ymm2 = _mm256_or_si256(ymm_r_masked, ymm_c000);
+ ymm2 = _mm256_or_si256(ymm2, ymm1);
+
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
+
+ __m256i ar30_lo = _mm256_unpacklo_epi16(ymm0, ymm2);
+ __m256i ar30_hi = _mm256_unpackhi_epi16(ymm0, ymm2);
+
+ _mm256_storeu_si256((__m256i*)dst_ar30, ar30_lo);
+ _mm256_storeu_si256((__m256i*)(dst_ar30 + 32), ar30_hi);
+
+ dst_ar30 += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TOAR30ROW_AVX2
#ifdef HAS_NV12TOARGBROW_AVX2
-// 16 pixels.
-// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
-__declspec(naked) void NV12ToARGBRow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push ebx
- mov eax, [esp + 8 + 4] // Y
- mov esi, [esp + 8 + 8] // UV
- mov edx, [esp + 8 + 12] // argb
- mov ebx, [esp + 8 + 16] // yuvconstants
- mov ecx, [esp + 8 + 20] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READNV12_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- pop esi
- vzeroupper
- ret
- }
+LIBYUV_TARGET_AVX2
+void NV12ToARGBRow_AVX2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ __m256i ymm_kYBiasToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
+ __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
+ __m256i ymm_u_zero = _mm256_setzero_si256();
+
+ while (width >= 16) {
+ __m128i xmm_uv = _mm_loadu_si128((const __m128i*)src_uv);
+ src_uv += 16;
+
+ __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_uv, 0);
+ ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
+ ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
+
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
+ src_y += 16;
+ __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
+ ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
+ ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
+
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_adds_epi16(ymm0, ymm4);
+ ymm1 = _mm256_subs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_adds_epi16(ymm2, ymm4);
+
+ ymm0 = _mm256_srai_epi16(ymm0, 6);
+ ymm1 = _mm256_srai_epi16(ymm1, 6);
+ ymm2 = _mm256_srai_epi16(ymm2, 6);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+ ymm1 = _mm256_packus_epi16(ymm1, ymm1);
+ ymm2 = _mm256_packus_epi16(ymm2, ymm2);
+
+ ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
+ ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
+
+ __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
+ __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, argb0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
+
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
}
#endif // HAS_NV12TOARGBROW_AVX2
#ifdef HAS_NV21TOARGBROW_AVX2
-// 16 pixels.
-// 8 VU values upsampled to 16 UV, mixed with 16 Y producing 16 ARGB (64 bytes).
-__declspec(naked) void NV21ToARGBRow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* vu_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push ebx
- mov eax, [esp + 8 + 4] // Y
- mov esi, [esp + 8 + 8] // VU
- mov edx, [esp + 8 + 12] // argb
- mov ebx, [esp + 8 + 16] // yuvconstants
- mov ecx, [esp + 8 + 20] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READNV21_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- pop esi
- vzeroupper
- ret
- }
+static const uint8_t kShuffleNV21_AVX2[32] = {
+ 1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u,
+ 1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
+
+LIBYUV_TARGET_AVX2
+void NV21ToARGBRow_AVX2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ __m256i ymm_kYBiasToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
+ __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
+ __m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2);
+ __m256i ymm_u_zero = _mm256_setzero_si256();
+
+ while (width >= 16) {
+ __m128i xmm_vu = _mm_loadu_si128((const __m128i*)src_vu);
+ src_vu += 16;
+
+ __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_vu, 0);
+ ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
+ ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf_nv21);
+ ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
+
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
+ src_y += 16;
+ __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
+ ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
+ ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
+
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_adds_epi16(ymm0, ymm4);
+ ymm1 = _mm256_subs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_adds_epi16(ymm2, ymm4);
+
+ ymm0 = _mm256_srai_epi16(ymm0, 6);
+ ymm1 = _mm256_srai_epi16(ymm1, 6);
+ ymm2 = _mm256_srai_epi16(ymm2, 6);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+ ymm1 = _mm256_packus_epi16(ymm1, ymm1);
+ ymm2 = _mm256_packus_epi16(ymm2, ymm2);
+
+ ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
+ ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
+ ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
+ ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
+
+ __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
+ __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, argb0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
+
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
}
#endif // HAS_NV21TOARGBROW_AVX2
-#ifdef HAS_YUY2TOARGBROW_AVX2
-// 16 pixels.
-// 8 YUY2 values with 16 Y and 8 UV producing 16 ARGB (64 bytes).
-__declspec(naked) void YUY2ToARGBRow_AVX2(
- const uint8_t* src_yuy2,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push ebx
- mov eax, [esp + 4 + 4] // yuy2
- mov edx, [esp + 4 + 8] // argb
- mov ebx, [esp + 4 + 12] // yuvconstants
- mov ecx, [esp + 4 + 16] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READYUY2_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- vzeroupper
- ret
- }
-}
-#endif // HAS_YUY2TOARGBROW_AVX2
-
-#ifdef HAS_UYVYTOARGBROW_AVX2
-// 16 pixels.
-// 8 UYVY values with 16 Y and 8 UV producing 16 ARGB (64 bytes).
-__declspec(naked) void UYVYToARGBRow_AVX2(
- const uint8_t* src_uyvy,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push ebx
- mov eax, [esp + 4 + 4] // uyvy
- mov edx, [esp + 4 + 8] // argb
- mov ebx, [esp + 4 + 12] // yuvconstants
- mov ecx, [esp + 4 + 16] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READUYVY_AVX2
- YUVTORGB_AVX2(ebx)
- STOREARGB_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- vzeroupper
- ret
- }
-}
-#endif // HAS_UYVYTOARGBROW_AVX2
-
-#ifdef HAS_I422TORGBAROW_AVX2
-// 16 pixels
-// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGBA (64 bytes).
-__declspec(naked) void I422ToRGBARow_AVX2(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // abgr
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- vpcmpeqb ymm5, ymm5, ymm5 // generate 0xffffffffffffffff for alpha
-
- convertloop:
- READYUV422_AVX2
- YUVTORGB_AVX2(ebx)
- STORERGBA_AVX2
-
- sub ecx, 16
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_I422TORGBAROW_AVX2
-
-#if defined(HAS_I422TOARGBROW_SSSE3)
-// TODO(fbarchard): Read that does half size on Y and treats 420 as 444.
-// Allows a conversion with half size scaling.
-
-// Read 8 UV from 444.
-#define READYUV444 \
- __asm { \
- __asm movq xmm0, qword ptr [esi] /* U */ \
- __asm movq xmm1, qword ptr [esi + edi] /* V */ \
- __asm lea esi, [esi + 8] \
- __asm punpcklbw xmm0, xmm1 /* UV */ \
- __asm movq xmm4, qword ptr [eax] \
- __asm punpcklbw xmm4, xmm4 \
- __asm lea eax, [eax + 8]}
-
-// Read 4 UV from 422, upsample to 8 UV.
-#define READYUV422 \
- __asm { \
- __asm movd xmm0, [esi] /* U */ \
- __asm movd xmm1, [esi + edi] /* V */ \
- __asm lea esi, [esi + 4] \
- __asm punpcklbw xmm0, xmm1 /* UV */ \
- __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \
- __asm movq xmm4, qword ptr [eax] \
- __asm punpcklbw xmm4, xmm4 \
- __asm lea eax, [eax + 8]}
+#if defined(HAS_I422TORGB24ROW_AVX2) || \
+ defined(HAS_I422TORGB24ROW_AVX512BW) || \
+ defined(HAS_ARGBTORGB24ROW_AVX2) || \
+ defined(HAS_ARGBTORGB24ROW_AVX512BW)
+static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
+ {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
+ 128u},
+ {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
+ 14u}};
+#endif
-// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
-#define READYUVA422 \
- __asm { \
- __asm movd xmm0, [esi] /* U */ \
- __asm movd xmm1, [esi + edi] /* V */ \
- __asm lea esi, [esi + 4] \
- __asm punpcklbw xmm0, xmm1 /* UV */ \
- __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \
- __asm movq xmm4, qword ptr [eax] /* Y */ \
- __asm punpcklbw xmm4, xmm4 \
- __asm lea eax, [eax + 8] \
- __asm movq xmm5, qword ptr [ebp] /* A */ \
- __asm lea ebp, [ebp + 8]}
-
-// Read 4 UV from NV12, upsample to 8 UV.
-#define READNV12 \
- __asm { \
- __asm movq xmm0, qword ptr [esi] /* UV */ \
- __asm lea esi, [esi + 8] \
- __asm punpcklwd xmm0, xmm0 /* UVUV (upsample) */ \
- __asm movq xmm4, qword ptr [eax] \
- __asm punpcklbw xmm4, xmm4 \
- __asm lea eax, [eax + 8]}
-
-// Read 4 VU from NV21, upsample to 8 UV.
-#define READNV21 \
- __asm { \
- __asm movq xmm0, qword ptr [esi] /* UV */ \
- __asm lea esi, [esi + 8] \
- __asm pshufb xmm0, xmmword ptr kShuffleNV21 \
- __asm movq xmm4, qword ptr [eax] \
- __asm punpcklbw xmm4, xmm4 \
- __asm lea eax, [eax + 8]}
-
-// Read 4 YUY2 with 8 Y and upsample 4 UV to 8 UV.
-#define READYUY2 \
- __asm { \
- __asm movdqu xmm4, [eax] /* YUY2 */ \
- __asm pshufb xmm4, xmmword ptr kShuffleYUY2Y \
- __asm movdqu xmm0, [eax] /* UV */ \
- __asm pshufb xmm0, xmmword ptr kShuffleYUY2UV \
- __asm lea eax, [eax + 16]}
-
-// Read 4 UYVY with 8 Y and upsample 4 UV to 8 UV.
-#define READUYVY \
- __asm { \
- __asm movdqu xmm4, [eax] /* UYVY */ \
- __asm pshufb xmm4, xmmword ptr kShuffleUYVYY \
- __asm movdqu xmm0, [eax] /* UV */ \
- __asm pshufb xmm0, xmmword ptr kShuffleUYVYUV \
- __asm lea eax, [eax + 16]}
+#if defined(HAS_I422TORGB24ROW_AVX2)
+LIBYUV_TARGET_AVX2
+void I422ToRGB24Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Constants
+ __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ __m256i ymm_kYBiasToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
+ __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+
+ __m256i ymm_shuf0 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
+ __m256i ymm_shuf1 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
+ __m256i ymm_u_zero = _mm256_setzero_si256();
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 16) {
+ // READYUV422_AVX2
+ __m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
+ __m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
+ src_u += 8;
+
+ __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
+ __m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
+
+ ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
+ ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
+ ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
+
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
+ src_y += 16;
+ __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
+ ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
+ ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
+
+ // YUVTORGB_AVX2
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_adds_epi16(ymm0, ymm4);
+ ymm1 = _mm256_subs_epi16(ymm4, ymm1);
+ ymm2 = _mm256_adds_epi16(ymm2, ymm4);
+
+ ymm0 = _mm256_srai_epi16(ymm0, 6);
+ ymm1 = _mm256_srai_epi16(ymm1, 6);
+ ymm2 = _mm256_srai_epi16(ymm2, 6);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+ ymm1 = _mm256_packus_epi16(ymm1, ymm1);
+ ymm2 = _mm256_packus_epi16(ymm2, ymm2);
+
+ // STORERGB24_AVX2
+ __m256i ymm0_packed = _mm256_unpacklo_epi8(ymm0, ymm1);
+ __m256i ymm2_packed = _mm256_unpacklo_epi8(ymm2, ymm2);
+ __m256i ymm1_packed = ymm0_packed;
+
+ ymm0_packed = _mm256_unpacklo_epi16(ymm0_packed, ymm2_packed);
+ ymm1_packed = _mm256_unpackhi_epi16(ymm1_packed, ymm2_packed);
+
+ ymm0_packed = _mm256_shuffle_epi8(ymm0_packed, ymm_shuf0);
+ ymm1_packed = _mm256_shuffle_epi8(ymm1_packed, ymm_shuf1);
+
+ ymm1_packed = _mm256_alignr_epi8(ymm1_packed, ymm0_packed, 0xc);
+
+ __m128i xmm0_store = _mm256_castsi256_si128(ymm0_packed);
+ __m128i xmm1_store = _mm256_castsi256_si128(ymm1_packed);
+ __m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
+ __m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
+
+ xmm0_store = _mm_unpacklo_epi64(xmm0_store, xmm1_store);
+ xmm2_store = _mm_alignr_epi8(xmm2_store, xmm1_store, 8);
+
+ _mm_storeu_si128((__m128i*)dst_rgb24, xmm0_store);
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 16), xmm2_store);
+ _mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
+
+ dst_rgb24 += 48;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_I422TORGB24ROW_AVX2) ||
+ // defined(HAS_I422TORGB24ROW_AVX512BW)
+
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I422TORGB24ROW_AVX512BW)
+#define READYUV422_AVX512BW \
+ __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); \
+ __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); \
+ src_u += 16; \
+ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
+ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
+ zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
+ zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
+ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
+ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
+ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
+ __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \
+ src_y += 32; \
+ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
+ zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
+ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
+ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
+
+#define READYUV422_MASK_AVX512BW \
+ __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \
+ __mmask32 mask_y = (__mmask32)((1u << width) - 1u); \
+ __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \
+ __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \
+ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
+ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
+ zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
+ zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
+ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
+ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
+ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
+ __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \
+ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
+ zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
+ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
+ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
+
+#define YUVTORGB_AVX512BW \
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \
+ zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \
+ __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \
+ __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \
+ __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \
+ zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \
+ zmm0 = _mm512_adds_epi16(zmm0, zmm4); \
+ zmm1 = _mm512_subs_epi16(zmm4, zmm1); \
+ zmm2 = _mm512_adds_epi16(zmm2, zmm4); \
+ zmm0 = _mm512_srai_epi16(zmm0, 6); \
+ zmm1 = _mm512_srai_epi16(zmm1, 6); \
+ zmm2 = _mm512_srai_epi16(zmm2, 6); \
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0); \
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1); \
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+
+#define STORERGB24_AVX512VBMI \
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
+ _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
+ _mm512_castsi512_si256(zmm_dst1)); \
+ dst_rgb24 += 96;
+
+#define STORERGB24_MASK_AVX512VBMI \
+ int dst_bytes = width * 3; \
+ __mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
+ __mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
+ _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
+ _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
+ _mm512_castsi512_si256(zmm_dst1));
+
+#define STORERGB24_AVX512BW \
+ __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
+ __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
+ __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
+ __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
+ zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
+ zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
+ zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
+ __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
+ __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
+ _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
+ _mm512_castsi512_si256(zmm_dst1)); \
+ dst_rgb24 += 96;
+
+#define STORERGB24_MASK_AVX512BW \
+ int dst_bytes = width * 3; \
+ __mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
+ __mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
+ __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
+ __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
+ __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
+ __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
+ zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
+ zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
+ zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
+ __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
+ __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
+ _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
+ _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
+ _mm512_castsi512_si256(zmm_dst1));
+#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) ||
+ // defined(HAS_I422TORGB24ROW_AVX512BW)
+
+#ifdef HAS_I422TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Masks
+ static const uint8_t kMaskBG[64] = {
+ 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
+ 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
+ 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
+ 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
+ 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
+ 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
+ static const uint8_t kMaskDST0[64] = {
+ 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
+ 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
+ 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
+ 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
+ 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
+ 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
+ static const uint8_t kMaskDST1[64] = {
+ 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
+ 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
+ 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+ static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
+ static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
+
+ // Constants
+ __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+
+ __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG);
+ __m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0);
+ __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1);
+ __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
+ __m512i zmm_split_y =
+ _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 32) {
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW
+ STORERGB24_AVX512VBMI
+ width -= 32;
+ }
+ if (width > 0) {
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW
+ STORERGB24_MASK_AVX512VBMI
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_I422TORGB24ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
+ static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
+ static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
+ static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
+
+ // Constants
+ __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+
+ __m512i zmm_shuf0 = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
+ __m512i zmm_shuf1 = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
+ __m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0);
+ __m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1);
+ __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
+ __m512i zmm_split_y =
+ _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 32) {
+ READYUV422_AVX512BW
+ YUVTORGB_AVX512BW
+ STORERGB24_AVX512BW
+ width -= 32;
+ }
+ if (width > 0) {
+ READYUV422_MASK_AVX512BW
+ YUVTORGB_AVX512BW
+ STORERGB24_MASK_AVX512BW
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TORGB24ROW_AVX512BW
+
+#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I422TORGB24ROW_AVX512BW)
+#undef READYUV422_AVX512BW
+#undef READYUV422_MASK_AVX512BW
+#undef YUVTORGB_AVX512BW
+#undef STORERGB24_AVX512VBMI
+#undef STORERGB24_MASK_AVX512VBMI
+#undef STORERGB24_AVX512BW
+#undef STORERGB24_MASK_AVX512BW
+#endif
-// Convert 8 pixels: 8 UV and 8 Y.
-#define YUVTORGB(YuvConstants) \
- __asm { \
- __asm movdqa xmm1, xmm0 \
- __asm movdqa xmm2, xmm0 \
- __asm movdqa xmm3, xmm0 \
- __asm movdqa xmm0, xmmword ptr [YuvConstants + KUVBIASB] \
- __asm pmaddubsw xmm1, xmmword ptr [YuvConstants + KUVTOB] \
- __asm psubw xmm0, xmm1 \
- __asm movdqa xmm1, xmmword ptr [YuvConstants + KUVBIASG] \
- __asm pmaddubsw xmm2, xmmword ptr [YuvConstants + KUVTOG] \
- __asm psubw xmm1, xmm2 \
- __asm movdqa xmm2, xmmword ptr [YuvConstants + KUVBIASR] \
- __asm pmaddubsw xmm3, xmmword ptr [YuvConstants + KUVTOR] \
- __asm psubw xmm2, xmm3 \
- __asm pmulhuw xmm4, xmmword ptr [YuvConstants + KYTORGB] \
- __asm paddsw xmm0, xmm4 /* B += Y */ \
- __asm paddsw xmm1, xmm4 /* G += Y */ \
- __asm paddsw xmm2, xmm4 /* R += Y */ \
- __asm psraw xmm0, 6 \
- __asm psraw xmm1, 6 \
- __asm psraw xmm2, 6 \
- __asm packuswb xmm0, xmm0 /* B */ \
- __asm packuswb xmm1, xmm1 /* G */ \
- __asm packuswb xmm2, xmm2 /* R */ \
+#ifdef HAS_I422TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
+ static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
+ static const uint16_t kPermAR30_0[32] = {
+ 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
+ 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
+ static const uint16_t kPermAR30_1[32] = {
+ 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
+ 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
+
+ // Constants
+ __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ __m512i zmm_kYBiasToRgb = _mm512_sub_epi16(
+ _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)),
+ _mm512_set1_epi16(24));
+ __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ __m512i zmm_3ff0 = _mm512_set1_epi16((short)0x3ff0);
+ __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ __m512i zmm_zero = _mm512_setzero_si512();
+
+ __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0);
+ __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1);
+ __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
+ __m512i zmm_split_y =
+ _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
+
+ ptrdiff_t offset = src_v - src_u;
+
+ while (width >= 32) {
+ // READYUV422_AVX512BW
+ __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
+ __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
+ src_u += 16;
+
+ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
+ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
+
+ zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
+ zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
+
+ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
+ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
+ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
+
+ __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
+ src_y += 32;
+ __m512i zmm4 = _mm512_castsi256_si512(ymm_y);
+ zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
+ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
+ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
+
+ // YUVTORGB_AVX512BW
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
+
+ __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
+ __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
+ __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
+
+ zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
+
+ zmm0 = _mm512_adds_epi16(zmm0, zmm4);
+ zmm1 = _mm512_subs_epi16(zmm4, zmm1);
+ zmm2 = _mm512_adds_epi16(zmm2, zmm4);
+
+ // Clamp to [0, 0x3ff0] (1023 * 16)
+ zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
+ zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
+ zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
+ zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
+ zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
+ zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
+
+ // STOREAR30_AVX512BW:
+ zmm0 = _mm512_srli_epi16(zmm0, 4);
+ zmm3 = _mm512_srli_epi16(zmm1, 10);
+ zmm1 = _mm512_slli_epi16(zmm1, 6);
+ zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
+ zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
+ _mm512_andnot_si512(zmm_3ff0, zmm3));
+ zmm2 = _mm512_or_si512(zmm2, zmm_c000);
+
+ __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
+
+ _mm512_storeu_si512((__m512i*)dst_ar30, zmm_dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), zmm_dst1);
+
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __mmask32 mask_y = (__mmask32)((1u << width) - 1u);
+ __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u);
+ __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset);
+
+ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
+ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
+
+ zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
+ zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
+
+ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
+ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
+ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
+
+ __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y);
+ __m512i zmm4 = _mm512_castsi256_si512(ymm_y);
+ zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
+ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
+ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
+
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
+
+ __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
+ __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
+ __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
+
+ zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
+
+ zmm0 = _mm512_adds_epi16(zmm0, zmm4);
+ zmm1 = _mm512_subs_epi16(zmm4, zmm1);
+ zmm2 = _mm512_adds_epi16(zmm2, zmm4);
+
+ zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
+ zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
+ zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
+ zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
+ zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
+ zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
+
+ zmm0 = _mm512_srli_epi16(zmm0, 4);
+ zmm3 = _mm512_srli_epi16(zmm1, 10);
+ zmm1 = _mm512_slli_epi16(zmm1, 6);
+ zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
+ zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
+ _mm512_andnot_si512(zmm_3ff0, zmm3));
+ zmm2 = _mm512_or_si512(zmm2, zmm_c000);
+
+ __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
+
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)mask_y, zmm_dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(mask_y >> 16),
+ zmm_dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TOAR30ROW_AVX512BW
+
+#ifdef HAS_ARGBBLENDROW_AVX2
+static const uint8_t kShuffleAlpha[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80,
+ 7u, 0x80, 11u, 0x80, 11u, 0x80,
+ 15u, 0x80, 15u, 0x80};
+
+LIBYUV_TARGET_AVX2
+void ARGBBlendRow_AVX2(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ __m256i shuff =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShuffleAlpha));
+ __m256i ymm_1 = _mm256_set1_epi16(1);
+ __m256i mask_lo = _mm256_set1_epi16(0x00ff);
+ __m256i mask_hi = _mm256_set1_epi16((short)0xff00);
+ __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000);
+
+ while (width >= 8) {
+ __m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src_argb0);
+ __m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
+ src_argb0 += 32;
+ src_argb1 += 32;
+
+ __m256i ymm_a = _mm256_xor_si256(ymm_s0, mask_alpha);
+ ymm_a = _mm256_shuffle_epi8(ymm_a, shuff);
+ ymm_a = _mm256_add_epi16(ymm_a, ymm_1);
+
+ __m256i ymm_s1_lo = _mm256_and_si256(ymm_s1, mask_lo);
+ __m256i ymm_s1_hi = _mm256_srli_epi16(ymm_s1, 8);
+
+ ymm_s1_lo = _mm256_mullo_epi16(ymm_s1_lo, ymm_a);
+ ymm_s1_hi = _mm256_mullo_epi16(ymm_s1_hi, ymm_a);
+
+ ymm_s1_lo = _mm256_srli_epi16(ymm_s1_lo, 8);
+ ymm_s1_hi = _mm256_and_si256(ymm_s1_hi, mask_hi);
+
+ __m256i res = _mm256_or_si256(ymm_s0, mask_alpha);
+ res = _mm256_adds_epu8(res, ymm_s1_lo);
+ res = _mm256_adds_epu8(res, ymm_s1_hi);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, res);
+ dst_argb += 32;
+ width -= 8;
}
+ _mm256_zeroupper();
-// Store 8 ARGB values.
-#define STOREARGB \
- __asm { \
- __asm punpcklbw xmm0, xmm1 /* BG */ \
- __asm punpcklbw xmm2, xmm5 /* RA */ \
- __asm movdqa xmm1, xmm0 \
- __asm punpcklwd xmm0, xmm2 /* BGRA first 4 pixels */ \
- __asm punpckhwd xmm1, xmm2 /* BGRA next 4 pixels */ \
- __asm movdqu 0[edx], xmm0 \
- __asm movdqu 16[edx], xmm1 \
- __asm lea edx, [edx + 32]}
-
-// Store 8 BGRA values.
-#define STOREBGRA \
- __asm { \
- __asm pcmpeqb xmm5, xmm5 /* generate 0xffffffff for alpha */ \
- __asm punpcklbw xmm1, xmm0 /* GB */ \
- __asm punpcklbw xmm5, xmm2 /* AR */ \
- __asm movdqa xmm0, xmm5 \
- __asm punpcklwd xmm5, xmm1 /* BGRA first 4 pixels */ \
- __asm punpckhwd xmm0, xmm1 /* BGRA next 4 pixels */ \
- __asm movdqu 0[edx], xmm5 \
- __asm movdqu 16[edx], xmm0 \
- __asm lea edx, [edx + 32]}
-
-// Store 8 RGBA values.
-#define STORERGBA \
- __asm { \
- __asm pcmpeqb xmm5, xmm5 /* generate 0xffffffff for alpha */ \
- __asm punpcklbw xmm1, xmm2 /* GR */ \
- __asm punpcklbw xmm5, xmm0 /* AB */ \
- __asm movdqa xmm0, xmm5 \
- __asm punpcklwd xmm5, xmm1 /* RGBA first 4 pixels */ \
- __asm punpckhwd xmm0, xmm1 /* RGBA next 4 pixels */ \
- __asm movdqu 0[edx], xmm5 \
- __asm movdqu 16[edx], xmm0 \
- __asm lea edx, [edx + 32]}
-
-// Store 8 RGB24 values.
-#define STORERGB24 \
- __asm {/* Weave into RRGB */ \
- __asm punpcklbw xmm0, xmm1 /* BG */ \
- __asm punpcklbw xmm2, xmm2 /* RR */ \
- __asm movdqa xmm1, xmm0 \
- __asm punpcklwd xmm0, xmm2 /* BGRR first 4 pixels */ \
- __asm punpckhwd xmm1, xmm2 /* BGRR next 4 pixels */ /* RRGB -> RGB24 */ \
- __asm pshufb xmm0, xmm5 /* Pack first 8 and last 4 bytes. */ \
- __asm pshufb xmm1, xmm6 /* Pack first 12 bytes. */ \
- __asm palignr xmm1, xmm0, 12 /* last 4 bytes of xmm0 + 12 xmm1 */ \
- __asm movq qword ptr 0[edx], xmm0 /* First 8 bytes */ \
- __asm movdqu 8[edx], xmm1 /* Last 16 bytes */ \
- __asm lea edx, [edx + 24]}
-
-// Store 8 RGB565 values.
-#define STORERGB565 \
- __asm {/* Weave into RRGB */ \
- __asm punpcklbw xmm0, xmm1 /* BG */ \
- __asm punpcklbw xmm2, xmm2 /* RR */ \
- __asm movdqa xmm1, xmm0 \
- __asm punpcklwd xmm0, xmm2 /* BGRR first 4 pixels */ \
- __asm punpckhwd xmm1, xmm2 /* BGRR next 4 pixels */ /* RRGB -> RGB565 */ \
- __asm movdqa xmm3, xmm0 /* B first 4 pixels of argb */ \
- __asm movdqa xmm2, xmm0 /* G */ \
- __asm pslld xmm0, 8 /* R */ \
- __asm psrld xmm3, 3 /* B */ \
- __asm psrld xmm2, 5 /* G */ \
- __asm psrad xmm0, 16 /* R */ \
- __asm pand xmm3, xmm5 /* B */ \
- __asm pand xmm2, xmm6 /* G */ \
- __asm pand xmm0, xmm7 /* R */ \
- __asm por xmm3, xmm2 /* BG */ \
- __asm por xmm0, xmm3 /* BGR */ \
- __asm movdqa xmm3, xmm1 /* B next 4 pixels of argb */ \
- __asm movdqa xmm2, xmm1 /* G */ \
- __asm pslld xmm1, 8 /* R */ \
- __asm psrld xmm3, 3 /* B */ \
- __asm psrld xmm2, 5 /* G */ \
- __asm psrad xmm1, 16 /* R */ \
- __asm pand xmm3, xmm5 /* B */ \
- __asm pand xmm2, xmm6 /* G */ \
- __asm pand xmm1, xmm7 /* R */ \
- __asm por xmm3, xmm2 /* BG */ \
- __asm por xmm1, xmm3 /* BGR */ \
- __asm packssdw xmm0, xmm1 \
- __asm movdqu 0[edx], xmm0 /* store 8 pixels of RGB565 */ \
- __asm lea edx, [edx + 16]}
-
-// 8 pixels.
-// 8 UV values, mixed with 8 Y producing 8 ARGB (32 bytes).
-__declspec(naked) void I444ToARGBRow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READYUV444
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 RGB24 (24 bytes).
-__declspec(naked) void I422ToRGB24Row_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_rgb24,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- movdqa xmm5, xmmword ptr kShuffleMaskARGBToRGB24_0
- movdqa xmm6, xmmword ptr kShuffleMaskARGBToRGB24
-
- convertloop:
- READYUV422
- YUVTORGB(ebx)
- STORERGB24
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-
-// 8 pixels
-// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 RGB565 (16 bytes).
-__declspec(naked) void I422ToRGB565Row_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* rgb565_buf,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- pcmpeqb xmm5, xmm5 // generate mask 0x0000001f
- psrld xmm5, 27
- pcmpeqb xmm6, xmm6 // generate mask 0x000007e0
- psrld xmm6, 26
- pslld xmm6, 5
- pcmpeqb xmm7, xmm7 // generate mask 0xfffff800
- pslld xmm7, 11
-
- convertloop:
- READYUV422
- YUVTORGB(ebx)
- STORERGB565
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes).
-__declspec(naked) void I422ToARGBRow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READYUV422
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 UV values upsampled to 8 UV, mixed with 8 Y and 8 A producing 8 ARGB.
-__declspec(naked) void I422AlphaToARGBRow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- const uint8_t* a_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- push ebp
- mov eax, [esp + 16 + 4] // Y
- mov esi, [esp + 16 + 8] // U
- mov edi, [esp + 16 + 12] // V
- mov ebp, [esp + 16 + 16] // A
- mov edx, [esp + 16 + 20] // argb
- mov ebx, [esp + 16 + 24] // yuvconstants
- mov ecx, [esp + 16 + 28] // width
- sub edi, esi
-
- convertloop:
- READYUVA422
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebp
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes).
-__declspec(naked) void NV12ToARGBRow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* uv_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push ebx
- mov eax, [esp + 8 + 4] // Y
- mov esi, [esp + 8 + 8] // UV
- mov edx, [esp + 8 + 12] // argb
- mov ebx, [esp + 8 + 16] // yuvconstants
- mov ecx, [esp + 8 + 20] // width
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READNV12
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 UV values upsampled to 8 UV, mixed with 8 Y producing 8 ARGB (32 bytes).
-__declspec(naked) void NV21ToARGBRow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* vu_buf,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push ebx
- mov eax, [esp + 8 + 4] // Y
- mov esi, [esp + 8 + 8] // VU
- mov edx, [esp + 8 + 12] // argb
- mov ebx, [esp + 8 + 16] // yuvconstants
- mov ecx, [esp + 8 + 20] // width
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READNV21
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop esi
- ret
- }
-}
-
-// 8 pixels.
-// 4 YUY2 values with 8 Y and 4 UV producing 8 ARGB (32 bytes).
-__declspec(naked) void YUY2ToARGBRow_SSSE3(
- const uint8_t* src_yuy2,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push ebx
- mov eax, [esp + 4 + 4] // yuy2
- mov edx, [esp + 4 + 8] // argb
- mov ebx, [esp + 4 + 12] // yuvconstants
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READYUY2
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- ret
- }
-}
-
-// 8 pixels.
-// 4 UYVY values with 8 Y and 4 UV producing 8 ARGB (32 bytes).
-__declspec(naked) void UYVYToARGBRow_SSSE3(
- const uint8_t* src_uyvy,
- uint8_t* dst_argb,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push ebx
- mov eax, [esp + 4 + 4] // uyvy
- mov edx, [esp + 4 + 8] // argb
- mov ebx, [esp + 4 + 12] // yuvconstants
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm5, xmm5 // generate 0xffffffff for alpha
-
- convertloop:
- READUYVY
- YUVTORGB(ebx)
- STOREARGB
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- ret
- }
-}
-
-__declspec(naked) void I422ToRGBARow_SSSE3(
- const uint8_t* y_buf,
- const uint8_t* u_buf,
- const uint8_t* v_buf,
- uint8_t* dst_rgba,
- const struct YuvConstants* yuvconstants,
- int width) {
- __asm {
- push esi
- push edi
- push ebx
- mov eax, [esp + 12 + 4] // Y
- mov esi, [esp + 12 + 8] // U
- mov edi, [esp + 12 + 12] // V
- mov edx, [esp + 12 + 16] // argb
- mov ebx, [esp + 12 + 20] // yuvconstants
- mov ecx, [esp + 12 + 24] // width
- sub edi, esi
-
- convertloop:
- READYUV422
- YUVTORGB(ebx)
- STORERGBA
-
- sub ecx, 8
- jg convertloop
-
- pop ebx
- pop edi
- pop esi
- ret
- }
-}
-#endif // HAS_I422TOARGBROW_SSSE3
-
-#ifdef HAS_I400TOARGBROW_SSE2
-// 8 pixels of Y converted to 8 pixels of ARGB (32 bytes).
-__declspec(naked) void I400ToARGBRow_SSE2(const uint8_t* y_buf,
- uint8_t* rgb_buf,
- int width) {
- __asm {
- mov eax, 0x4a354a35 // 4a35 = 18997 = round(1.164 * 64 * 256)
- movd xmm2, eax
- pshufd xmm2, xmm2,0
- mov eax, 0x04880488 // 0488 = 1160 = round(1.164 * 64 * 16)
- movd xmm3, eax
- pshufd xmm3, xmm3, 0
- pcmpeqb xmm4, xmm4 // generate mask 0xff000000
- pslld xmm4, 24
-
- mov eax, [esp + 4] // Y
- mov edx, [esp + 8] // rgb
- mov ecx, [esp + 12] // width
-
- convertloop:
- // Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164
- movq xmm0, qword ptr [eax]
- lea eax, [eax + 8]
- punpcklbw xmm0, xmm0 // Y.Y
- pmulhuw xmm0, xmm2
- psubusw xmm0, xmm3
- psrlw xmm0, 6
- packuswb xmm0, xmm0 // G
-
- // Step 2: Weave into ARGB
- punpcklbw xmm0, xmm0 // GG
- movdqa xmm1, xmm0
- punpcklwd xmm0, xmm0 // BGRA first 4 pixels
- punpckhwd xmm1, xmm1 // BGRA next 4 pixels
- por xmm0, xmm4
- por xmm1, xmm4
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-#endif // HAS_I400TOARGBROW_SSE2
-
-#ifdef HAS_I400TOARGBROW_AVX2
-// 16 pixels of Y converted to 16 pixels of ARGB (64 bytes).
-// note: vpunpcklbw mutates and vpackuswb unmutates.
-__declspec(naked) void I400ToARGBRow_AVX2(const uint8_t* y_buf,
- uint8_t* rgb_buf,
- int width) {
- __asm {
- mov eax, 0x4a354a35 // 4a35 = 18997 = round(1.164 * 64 * 256)
- vmovd xmm2, eax
- vbroadcastss ymm2, xmm2
- mov eax, 0x04880488 // 0488 = 1160 = round(1.164 * 64 * 16)
- vmovd xmm3, eax
- vbroadcastss ymm3, xmm3
- vpcmpeqb ymm4, ymm4, ymm4 // generate mask 0xff000000
- vpslld ymm4, ymm4, 24
-
- mov eax, [esp + 4] // Y
- mov edx, [esp + 8] // rgb
- mov ecx, [esp + 12] // width
-
- convertloop:
- // Step 1: Scale Y contriportbution to 16 G values. G = (y - 16) * 1.164
- vmovdqu xmm0, [eax]
- lea eax, [eax + 16]
- vpermq ymm0, ymm0, 0xd8 // vpunpcklbw mutates
- vpunpcklbw ymm0, ymm0, ymm0 // Y.Y
- vpmulhuw ymm0, ymm0, ymm2
- vpsubusw ymm0, ymm0, ymm3
- vpsrlw ymm0, ymm0, 6
- vpackuswb ymm0, ymm0, ymm0 // G. still mutated: 3120
-
- // TODO(fbarchard): Weave alpha with unpack.
- // Step 2: Weave into ARGB
- vpunpcklbw ymm1, ymm0, ymm0 // GG - mutates
- vpermq ymm1, ymm1, 0xd8
- vpunpcklwd ymm0, ymm1, ymm1 // GGGG first 8 pixels
- vpunpckhwd ymm1, ymm1, ymm1 // GGGG next 8 pixels
- vpor ymm0, ymm0, ymm4
- vpor ymm1, ymm1, ymm4
- vmovdqu [edx], ymm0
- vmovdqu [edx + 32], ymm1
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_I400TOARGBROW_AVX2
-
-#ifdef HAS_MIRRORROW_SSSE3
-// Shuffle table for reversing the bytes.
-static const uvec8 kShuffleMirror = {15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u,
- 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
-
-// TODO(fbarchard): Replace lea with -16 offset.
-__declspec(naked) void MirrorRow_SSSE3(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- movdqa xmm5, xmmword ptr kShuffleMirror
-
- convertloop:
- movdqu xmm0, [eax - 16 + ecx]
- pshufb xmm0, xmm5
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-#endif // HAS_MIRRORROW_SSSE3
-
-#ifdef HAS_MIRRORROW_AVX2
-__declspec(naked) void MirrorRow_AVX2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- vbroadcastf128 ymm5, xmmword ptr kShuffleMirror
-
- convertloop:
- vmovdqu ymm0, [eax - 32 + ecx]
- vpshufb ymm0, ymm0, ymm5
- vpermq ymm0, ymm0, 0x4e // swap high and low halfs
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_MIRRORROW_AVX2
+ while (width > 0) {
+ uint32_t s0 = *(const uint32_t*)src_argb0;
+ uint32_t s1 = *(const uint32_t*)src_argb1;
+ src_argb0 += 4;
+ src_argb1 += 4;
-#ifdef HAS_MIRRORUVROW_SSSE3
-// Shuffle table for reversing the bytes of UV channels.
-static const uvec8 kShuffleMirrorUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u,
- 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u};
+ uint32_t a = 256 - (s0 >> 24);
+ uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff);
+ uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00);
+ uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000);
-__declspec(naked) void MirrorUVRow_SSSE3(const uint8_t* src,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- movdqa xmm1, xmmword ptr kShuffleMirrorUV
- lea eax, [eax + ecx * 2 - 16]
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- lea eax, [eax - 16]
- pshufb xmm0, xmm1
- movlpd qword ptr [edx], xmm0
- movhpd qword ptr [edx + edi], xmm0
- lea edx, [edx + 8]
- sub ecx, 8
- jg convertloop
-
- pop edi
- ret
- }
-}
-#endif // HAS_MIRRORUVROW_SSSE3
-
-#ifdef HAS_ARGBMIRRORROW_SSE2
-__declspec(naked) void ARGBMirrorRow_SSE2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- lea eax, [eax - 16 + ecx * 4] // last 4 pixels.
-
- convertloop:
- movdqu xmm0, [eax]
- lea eax, [eax - 16]
- pshufd xmm0, xmm0, 0x1b
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
- ret
- }
-}
-#endif // HAS_ARGBMIRRORROW_SSE2
+ b = (b > 255) ? 255 : b;
+ g = (g > 0xff00) ? 0xff00 : (g & 0xff00);
+ r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000);
-#ifdef HAS_ARGBMIRRORROW_AVX2
-// Shuffle table for reversing the bytes.
-static const ulvec32 kARGBShuffleMirror_AVX2 = {7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
-
-__declspec(naked) void ARGBMirrorRow_AVX2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- vmovdqu ymm5, ymmword ptr kARGBShuffleMirror_AVX2
-
- convertloop:
- vpermd ymm0, ymm5, [eax - 32 + ecx * 4] // permute dword order
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
- vzeroupper
- ret
+ *(uint32_t*)dst_argb = 0xff000000 | r | g | b;
+ dst_argb += 4;
+ --width;
}
}
-#endif // HAS_ARGBMIRRORROW_AVX2
+#endif // HAS_ARGBBLENDROW_AVX2
-#ifdef HAS_SPLITUVROW_SSE2
-__declspec(naked) void SplitUVRow_SSE2(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_uv
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- movdqa xmm2, xmm0
- movdqa xmm3, xmm1
- pand xmm0, xmm5 // even bytes
- pand xmm1, xmm5
- packuswb xmm0, xmm1
- psrlw xmm2, 8 // odd bytes
- psrlw xmm3, 8
- packuswb xmm2, xmm3
- movdqu [edx], xmm0
- movdqu [edx + edi], xmm2
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
-
- pop edi
- ret
- }
-}
-
-#endif // HAS_SPLITUVROW_SSE2
+#ifdef HAS_BLENDPLANEROW_SSSE3
+void BlendPlaneRow_SSSE3(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ __m128i xmm_ff00 = _mm_set1_epi16((short)0xff00);
+ __m128i xmm_80 = _mm_set1_epi8((char)0x80);
+ __m128i xmm_round = _mm_set1_epi16((short)0x807f);
-#ifdef HAS_SPLITUVROW_AVX2
-__declspec(naked) void SplitUVRow_AVX2(const uint8_t* src_uv,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_uv
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
- sub edi, edx
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpsrlw ymm2, ymm0, 8 // odd bytes
- vpsrlw ymm3, ymm1, 8
- vpand ymm0, ymm0, ymm5 // even bytes
- vpand ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1
- vpackuswb ymm2, ymm2, ymm3
- vpermq ymm0, ymm0, 0xd8
- vpermq ymm2, ymm2, 0xd8
- vmovdqu [edx], ymm0
- vmovdqu [edx + edi], ymm2
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
-
- pop edi
- vzeroupper
- ret
+ while (width > 0) {
+ __m128i xmm_a = _mm_loadl_epi64((const __m128i*)alpha);
+ __m128i xmm_s0 = _mm_loadl_epi64((const __m128i*)src0);
+ __m128i xmm_s1 = _mm_loadl_epi64((const __m128i*)src1);
+ alpha += 8;
+ src0 += 8;
+ src1 += 8;
+
+ xmm_a = _mm_unpacklo_epi8(xmm_a, xmm_a);
+ xmm_a = _mm_xor_si128(xmm_a, xmm_ff00);
+
+ __m128i xmm_s = _mm_unpacklo_epi8(xmm_s0, xmm_s1);
+ xmm_s = _mm_sub_epi8(xmm_s, xmm_80);
+
+ __m128i xmm_p = _mm_maddubs_epi16(xmm_a, xmm_s);
+ xmm_p = _mm_add_epi16(xmm_p, xmm_round);
+ xmm_p = _mm_srli_epi16(xmm_p, 8);
+
+ __m128i xmm_d = _mm_packus_epi16(xmm_p, xmm_p);
+ _mm_storel_epi64((__m128i*)dst, xmm_d);
+ dst += 8;
+ width -= 8;
}
}
-#endif // HAS_SPLITUVROW_AVX2
+#endif // HAS_BLENDPLANEROW_SSSE3
-#ifdef HAS_MERGEUVROW_SSE2
-__declspec(naked) void MergeUVRow_SSE2(const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uv,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_u
- mov edx, [esp + 4 + 8] // src_v
- mov edi, [esp + 4 + 12] // dst_uv
- mov ecx, [esp + 4 + 16] // width
- sub edx, eax
-
- convertloop:
- movdqu xmm0, [eax] // read 16 U's
- movdqu xmm1, [eax + edx] // and 16 V's
- lea eax, [eax + 16]
- movdqa xmm2, xmm0
- punpcklbw xmm0, xmm1 // first 8 UV pairs
- punpckhbw xmm2, xmm1 // next 8 UV pairs
- movdqu [edi], xmm0
- movdqu [edi + 16], xmm2
- lea edi, [edi + 32]
- sub ecx, 16
- jg convertloop
-
- pop edi
- ret
- }
-}
-#endif // HAS_MERGEUVROW_SSE2
+#ifdef HAS_BLENDPLANEROW_AVX2
+LIBYUV_TARGET_AVX2
+void BlendPlaneRow_AVX2(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ __m256i ymm_ff00 = _mm256_set1_epi16((short)0xff00);
+ __m256i ymm_80 = _mm256_set1_epi8((char)0x80);
+ __m256i ymm_round = _mm256_set1_epi16((short)0x807f);
-#ifdef HAS_MERGEUVROW_AVX2
-__declspec(naked) void MergeUVRow_AVX2(const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_uv,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_u
- mov edx, [esp + 4 + 8] // src_v
- mov edi, [esp + 4 + 12] // dst_uv
- mov ecx, [esp + 4 + 16] // width
- sub edx, eax
-
- convertloop:
- vmovdqu ymm0, [eax] // read 32 U's
- vmovdqu ymm1, [eax + edx] // and 32 V's
- lea eax, [eax + 32]
- vpunpcklbw ymm2, ymm0, ymm1 // low 16 UV pairs. mutated qqword 0,2
- vpunpckhbw ymm0, ymm0, ymm1 // high 16 UV pairs. mutated qqword 1,3
- vextractf128 [edi], ymm2, 0 // bytes 0..15
- vextractf128 [edi + 16], ymm0, 0 // bytes 16..31
- vextractf128 [edi + 32], ymm2, 1 // bytes 32..47
- vextractf128 [edi + 48], ymm0, 1 // bytes 47..63
- lea edi, [edi + 64]
- sub ecx, 32
- jg convertloop
-
- pop edi
- vzeroupper
- ret
- }
-}
-#endif // HAS_MERGEUVROW_AVX2
-
-#ifdef HAS_COPYROW_SSE2
-// CopyRow copys 'width' bytes using a 16 byte load/store, 32 bytes at time.
-__declspec(naked) void CopyRow_SSE2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- test eax, 15
- jne convertloopu
- test edx, 15
- jne convertloopu
-
- convertloopa:
- movdqa xmm0, [eax]
- movdqa xmm1, [eax + 16]
- lea eax, [eax + 32]
- movdqa [edx], xmm0
- movdqa [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloopa
- ret
-
- convertloopu:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloopu
- ret
- }
-}
-#endif // HAS_COPYROW_SSE2
+ while (width > 0) {
+ __m256i ymm_a = _mm256_loadu_si256((const __m256i*)alpha);
+ __m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src0);
+ __m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src1);
+ alpha += 32;
+ src0 += 32;
+ src1 += 32;
-#ifdef HAS_COPYROW_AVX
-// CopyRow copys 'width' bytes using a 32 byte load/store, 64 bytes at time.
-__declspec(naked) void CopyRow_AVX(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
+ __m256i ymm_a_lo = _mm256_unpacklo_epi8(ymm_a, ymm_a);
+ __m256i ymm_a_hi = _mm256_unpackhi_epi8(ymm_a, ymm_a);
+ ymm_a_lo = _mm256_xor_si256(ymm_a_lo, ymm_ff00);
+ ymm_a_hi = _mm256_xor_si256(ymm_a_hi, ymm_ff00);
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vmovdqu [edx], ymm0
- vmovdqu [edx + 32], ymm1
- lea edx, [edx + 64]
- sub ecx, 64
- jg convertloop
+ __m256i ymm_s_lo = _mm256_unpacklo_epi8(ymm_s0, ymm_s1);
+ __m256i ymm_s_hi = _mm256_unpackhi_epi8(ymm_s0, ymm_s1);
+ ymm_s_lo = _mm256_sub_epi8(ymm_s_lo, ymm_80);
+ ymm_s_hi = _mm256_sub_epi8(ymm_s_hi, ymm_80);
- vzeroupper
- ret
- }
-}
-#endif // HAS_COPYROW_AVX
+ __m256i ymm_p_lo = _mm256_maddubs_epi16(ymm_a_lo, ymm_s_lo);
+ __m256i ymm_p_hi = _mm256_maddubs_epi16(ymm_a_hi, ymm_s_hi);
-// Multiple of 1.
-__declspec(naked) void CopyRow_ERMS(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, esi
- mov edx, edi
- mov esi, [esp + 4] // src
- mov edi, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- rep movsb
- mov edi, edx
- mov esi, eax
- ret
- }
-}
-
-#ifdef HAS_ARGBCOPYALPHAROW_SSE2
-// width in pixels
-__declspec(naked) void ARGBCopyAlphaRow_SSE2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- pcmpeqb xmm0, xmm0 // generate mask 0xff000000
- pslld xmm0, 24
- pcmpeqb xmm1, xmm1 // generate mask 0x00ffffff
- psrld xmm1, 8
-
- convertloop:
- movdqu xmm2, [eax]
- movdqu xmm3, [eax + 16]
- lea eax, [eax + 32]
- movdqu xmm4, [edx]
- movdqu xmm5, [edx + 16]
- pand xmm2, xmm0
- pand xmm3, xmm0
- pand xmm4, xmm1
- pand xmm5, xmm1
- por xmm2, xmm4
- por xmm3, xmm5
- movdqu [edx], xmm2
- movdqu [edx + 16], xmm3
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
-
- ret
- }
-}
-#endif // HAS_ARGBCOPYALPHAROW_SSE2
+ ymm_p_lo = _mm256_add_epi16(ymm_p_lo, ymm_round);
+ ymm_p_hi = _mm256_add_epi16(ymm_p_hi, ymm_round);
-#ifdef HAS_ARGBCOPYALPHAROW_AVX2
-// width in pixels
-__declspec(naked) void ARGBCopyAlphaRow_AVX2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm0, ymm0, ymm0
- vpsrld ymm0, ymm0, 8 // generate mask 0x00ffffff
-
- convertloop:
- vmovdqu ymm1, [eax]
- vmovdqu ymm2, [eax + 32]
- lea eax, [eax + 64]
- vpblendvb ymm1, ymm1, [edx], ymm0
- vpblendvb ymm2, ymm2, [edx + 32], ymm0
- vmovdqu [edx], ymm1
- vmovdqu [edx + 32], ymm2
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
-
- vzeroupper
- ret
+ ymm_p_lo = _mm256_srli_epi16(ymm_p_lo, 8);
+ ymm_p_hi = _mm256_srli_epi16(ymm_p_hi, 8);
+
+ __m256i ymm_d = _mm256_packus_epi16(ymm_p_lo, ymm_p_hi);
+ _mm256_storeu_si256((__m256i*)dst, ymm_d);
+ dst += 32;
+ width -= 32;
}
+ _mm256_zeroupper();
}
-#endif // HAS_ARGBCOPYALPHAROW_AVX2
+#endif // HAS_BLENDPLANEROW_AVX2
-#ifdef HAS_ARGBEXTRACTALPHAROW_SSE2
-// width in pixels
-__declspec(naked) void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_a,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_a
- mov ecx, [esp + 12] // width
-
- extractloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- psrld xmm0, 24
- psrld xmm1, 24
- packssdw xmm0, xmm1
- packuswb xmm0, xmm0
- movq qword ptr [edx], xmm0
- lea edx, [edx + 8]
- sub ecx, 8
- jg extractloop
-
- ret
- }
-}
-#endif // HAS_ARGBEXTRACTALPHAROW_SSE2
-
-#ifdef HAS_ARGBEXTRACTALPHAROW_AVX2
-// width in pixels
-__declspec(naked) void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_a,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_a
- mov ecx, [esp + 12] // width
- vmovdqa ymm4, ymmword ptr kPermdARGBToY_AVX
-
- extractloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vpsrld ymm0, ymm0, 24
- vpsrld ymm1, ymm1, 24
- vmovdqu ymm2, [eax + 64]
- vmovdqu ymm3, [eax + 96]
- lea eax, [eax + 128]
- vpackssdw ymm0, ymm0, ymm1 // mutates
- vpsrld ymm2, ymm2, 24
- vpsrld ymm3, ymm3, 24
- vpackssdw ymm2, ymm2, ymm3 // mutates
- vpackuswb ymm0, ymm0, ymm2 // mutates
- vpermd ymm0, ymm4, ymm0 // unmutate
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg extractloop
-
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBEXTRACTALPHAROW_AVX2
-
-#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2
-// width in pixels
-__declspec(naked) void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- pcmpeqb xmm0, xmm0 // generate mask 0xff000000
- pslld xmm0, 24
- pcmpeqb xmm1, xmm1 // generate mask 0x00ffffff
- psrld xmm1, 8
-
- convertloop:
- movq xmm2, qword ptr [eax] // 8 Y's
- lea eax, [eax + 8]
- punpcklbw xmm2, xmm2
- punpckhwd xmm3, xmm2
- punpcklwd xmm2, xmm2
- movdqu xmm4, [edx]
- movdqu xmm5, [edx + 16]
- pand xmm2, xmm0
- pand xmm3, xmm0
- pand xmm4, xmm1
- pand xmm5, xmm1
- por xmm2, xmm4
- por xmm3, xmm5
- movdqu [edx], xmm2
- movdqu [edx + 16], xmm3
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
-
- ret
- }
-}
-#endif // HAS_ARGBCOPYYTOALPHAROW_SSE2
-
-#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2
-// width in pixels
-__declspec(naked) void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src,
- uint8_t* dst,
- int width) {
- __asm {
- mov eax, [esp + 4] // src
- mov edx, [esp + 8] // dst
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm0, ymm0, ymm0
- vpsrld ymm0, ymm0, 8 // generate mask 0x00ffffff
-
- convertloop:
- vpmovzxbd ymm1, qword ptr [eax]
- vpmovzxbd ymm2, qword ptr [eax + 8]
- lea eax, [eax + 16]
- vpslld ymm1, ymm1, 24
- vpslld ymm2, ymm2, 24
- vpblendvb ymm1, ymm1, [edx], ymm0
- vpblendvb ymm2, ymm2, [edx + 32], ymm0
- vmovdqu [edx], ymm1
- vmovdqu [edx + 32], ymm2
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
-
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBCOPYYTOALPHAROW_AVX2
-
-#ifdef HAS_SETROW_X86
-// Write 'width' bytes using an 8 bit value repeated.
-// width should be multiple of 4.
-__declspec(naked) void SetRow_X86(uint8_t* dst, uint8_t v8, int width) {
- __asm {
- movzx eax, byte ptr [esp + 8] // v8
- mov edx, 0x01010101 // Duplicate byte to all bytes.
- mul edx // overwrites edx with upper part of result.
- mov edx, edi
- mov edi, [esp + 4] // dst
- mov ecx, [esp + 12] // width
- shr ecx, 2
- rep stosd
- mov edi, edx
- ret
- }
-}
-
-// Write 'width' bytes using an 8 bit value repeated.
-__declspec(naked) void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) {
- __asm {
- mov edx, edi
- mov edi, [esp + 4] // dst
- mov eax, [esp + 8] // v8
- mov ecx, [esp + 12] // width
- rep stosb
- mov edi, edx
- ret
- }
-}
-
-// Write 'width' 32 bit values.
-__declspec(naked) void ARGBSetRow_X86(uint8_t* dst_argb,
- uint32_t v32,
- int width) {
- __asm {
- mov edx, edi
- mov edi, [esp + 4] // dst
- mov eax, [esp + 8] // v32
- mov ecx, [esp + 12] // width
- rep stosd
- mov edi, edx
- ret
- }
-}
-#endif // HAS_SETROW_X86
+#ifdef HAS_BLENDPLANEROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void BlendPlaneRow_AVX512BW(const uint8_t* src0,
+ const uint8_t* src1,
+ const uint8_t* alpha,
+ uint8_t* dst,
+ int width) {
+ __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00);
+ __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
+ __m512i zmm_round = _mm512_set1_epi16((short)0x807f);
+
+ while (width >= 64) {
+ __m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha);
+ __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0);
+ __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1);
+ alpha += 64;
+ src0 += 64;
+ src1 += 64;
+
+ __m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
+ __m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
+ zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
+ zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
+
+ __m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
+ __m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
+ zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
+ zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
+
+ __m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
+ __m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
+
+ zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
+ zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
+
+ zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
+ zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
+
+ __m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
+ _mm512_storeu_si512((__m512i*)dst, zmm_d);
+ dst += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 mask = (1ULL << width) - 1ULL;
+ __m512i zmm_a = _mm512_maskz_loadu_epi8(mask, alpha);
+ __m512i zmm_s0 = _mm512_maskz_loadu_epi8(mask, src0);
+ __m512i zmm_s1 = _mm512_maskz_loadu_epi8(mask, src1);
+
+ __m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
+ __m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
+ zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
+ zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
+
+ __m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
+ __m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
+ zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
+ zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
+
+ __m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
+ __m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
+
+ zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
+ zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
+
+ zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
+ zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
+
+ __m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
+ _mm512_mask_storeu_epi8(dst, mask, zmm_d);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_BLENDPLANEROW_AVX512BW
+
+#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ while (width >= 32) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
+ __m128i a1 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z1, 24));
+ _mm_storeu_si128((__m128i*)dst_a, a0);
+ _mm_storeu_si128((__m128i*)(dst_a + 16), a1);
+ src_argb += 128;
+ dst_a += 32;
+ width -= 32;
+ }
+ if (width >= 16) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
+ _mm_storeu_si128((__m128i*)dst_a, a0);
+ src_argb += 64;
+ dst_a += 16;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 mask = (__mmask16)((1u << width) - 1u);
+ __m512i z0 = _mm512_maskz_loadu_epi32(mask, src_argb);
+ __m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
+ // Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
+ // because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
+ // to unpoison destination shadow memory.
+ _mm_mask_storeu_epi8(dst_a, mask, a0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW
+
+#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ // Store only the alpha byte (every 4th byte) using a byte mask instead of
+ // loading dst and blending with _mm512_ternarylogic_epi32. This avoids
+ // reading uninitialized destination bytes and avoids
+ // @llvm.x86.avx512.pternlog which LLVM MSan does not model bitwise.
+ const __mmask64 alpha_mask = 0x8888888888888888ull;
+ while (width >= 32) {
+ __m512i a0 = _mm512_slli_epi32(
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
+ __m512i a1 = _mm512_slli_epi32(
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)(src + 16))), 24);
+ _mm512_mask_storeu_epi8(dst, alpha_mask, a0);
+ _mm512_mask_storeu_epi8(dst + 64, alpha_mask, a1);
+ src += 32;
+ dst += 128;
+ width -= 32;
+ }
+ if (width >= 16) {
+ __m512i a0 = _mm512_slli_epi32(
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
+ _mm512_mask_storeu_epi8(dst, alpha_mask, a0);
+ src += 16;
+ dst += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 mask = (__mmask16)((1u << width) - 1u);
+ __mmask64 tail_alpha_mask = alpha_mask >> (64 - width * 4);
+ __m512i a0 = _mm512_slli_epi32(
+ _mm512_maskz_cvtepu8_epi32(mask, _mm_maskz_loadu_epi8(mask, src)), 24);
+ _mm512_mask_storeu_epi8(dst, tail_alpha_mask, a0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW
+
+#ifdef HAS_ARGBMIRRORROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ const __m512i rev =
+ _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+ const uint8_t* s = src + (ptrdiff_t)width * 4;
+ while (width >= 16) {
+ s -= 64;
+ _mm512_storeu_si512(
+ (__m512i*)dst,
+ PermutexvarEpi32(rev, _mm512_loadu_si512((const __m512i*)s)));
+ dst += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ // Remaining 1 to 15 pixels are at the start of src.
+ __mmask16 mask = (__mmask16)((1u << width) - 1u);
+ __m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width));
+ __m512i v = _mm512_maskz_loadu_epi32(mask, src);
+ _mm512_mask_storeu_epi32(dst, mask, PermutexvarEpi32(idx, v));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBMIRRORROW_AVX512BW
+
+#ifdef HAS_RGB24MIRRORROW_AVX512VBMI
+static const uint8_t kPermMirrorRGB24_0[64] = {
+ 125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u,
+ 118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u,
+ 105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u,
+ 92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u,
+ 85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u,
+ 72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u};
+static const uint8_t kPermMirrorRGB24_1[64] = {
+ 127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u,
+ 114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u,
+ 107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u,
+ 94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u,
+ 81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u,
+ 74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u};
+static const uint8_t kPermMirrorRGB24_2[64] = {
+ 65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u,
+ 48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u,
+ 37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u,
+ 26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u,
+ 9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
+
+// Byte mask with the low n bits set, clamped to [0, 64].
+static inline __mmask64 RGB24MirrorMask(int n) {
+ return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull;
+}
+
+// Mirror 64 RGB24 pixels (3 full zmm vectors) per loop. The remaining 1 to
+// 63 pixels use masked loads and stores that access only valid bytes.
+LIBYUV_TARGET_AVX512VBMI
+void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
+ uint8_t* dst_rgb24,
+ int width) {
+ const __m512i idx0 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_0);
+ const __m512i idx1 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_1);
+ const __m512i idx2 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_2);
+ int r = width & 63;
+ int w = width - r;
+ if (w > 0) {
+ const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192;
+ do {
+ __m512i a = _mm512_loadu_si512((const __m512i*)src);
+ __m512i b = _mm512_loadu_si512((const __m512i*)(src + 64));
+ __m512i c = _mm512_loadu_si512((const __m512i*)(src + 128));
+ __m512i d0 = _mm512_permutex2var_epi8(b, idx0, c);
+ __m512i d1 = _mm512_permutex2var_epi8(a, idx1, b);
+ __m512i d2 = _mm512_permutex2var_epi8(a, idx2, b);
+ d1 = _mm512_mask_broadcastb_epi8(d1, (__mmask64)2,
+ _mm512_castsi512_si128(c));
+ _mm512_storeu_si512((__m512i*)dst_rgb24, d0);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), d1);
+ _mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), d2);
+ src -= 192;
+ dst_rgb24 += 192;
+ w -= 64;
+ } while (w > 0);
+ }
+ if (r) {
+ int nb = r * 3;
+ int off = 192 - nb;
+ __mmask64 m0 = RGB24MirrorMask(nb);
+ __mmask64 m1 = RGB24MirrorMask(nb - 64);
+ __mmask64 m2 = RGB24MirrorMask(nb - 128);
+ __m512i a = _mm512_maskz_loadu_epi8(m0, src_rgb24);
+ __m512i b = _mm512_maskz_loadu_epi8(m1, src_rgb24 + 64);
+ __m512i c = _mm512_maskz_loadu_epi8(m2, src_rgb24 + 128);
+ __m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64)));
+ __m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off));
+ __m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off));
+ __m512i d0 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t0, b),
+ _mm512_movepi8_mask(t0), t0, c);
+ __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t1, b),
+ _mm512_movepi8_mask(t1), t1, c);
+ __m512i d2 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t2, b),
+ _mm512_movepi8_mask(t2), t2, c);
+ _mm512_mask_storeu_epi8(dst_rgb24, m0, d0);
+ _mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1);
+ _mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RGB24MIRRORROW_AVX512VBMI
+
+// ===========================================================================
+// Module 1: AYUV to Planar Y / UV / VU (AVX2 / AVX512VBMI)
+// ===========================================================================
+
+#ifdef HAS_AYUVTOYROW_AVX2
+LIBYUV_TARGET_AVX2
+void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
+ const __m256i ymm_mask = _mm256_set1_epi32(0x000000ff);
+ while (width >= 16) {
+ __m256i ymm0 = _mm256_and_si256(
+ _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_ayuv), 16),
+ ymm_mask);
+ __m256i ymm1 = _mm256_and_si256(
+ _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)(src_ayuv + 32)),
+ 16),
+ ymm_mask);
+ src_ayuv += 64;
+ ymm0 = _mm256_permute4x64_epi64(_mm256_packus_epi32(ymm0, ymm1), 0xd8);
+ __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
+ _mm256_extracti128_si256(ymm0, 1));
+ _mm_storeu_si128((__m128i*)dst_y, out);
+ dst_y += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AYUVTOYROW_AVX2
+
+#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2)
+static const uint8_t kShufAYUVToUV16_Win[16] = {
+ 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+
+static const uint8_t kShufAYUVToVU16_Win[16] = {
+ 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+
+LIBYUV_TARGET_AVX2
+static inline void AYUVToUVOrVURow_Win_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ const uint8_t* shuf_table,
+ int width) {
+ const __m256i ymm_shuf =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuf_table));
+ const __m256i ymm_0101 = _mm256_set1_epi16(0x0101);
+ const __m256i ymm_zero = _mm256_setzero_si256();
+ while (width >= 16) {
+ __m256i s00 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)src_ayuv), ymm_shuf);
+ __m256i s01 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), ymm_shuf);
+ __m256i s10 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv)),
+ ymm_shuf);
+ __m256i s11 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv + 32)),
+ ymm_shuf);
+ src_ayuv += 64;
+ __m256i p0 =
+ _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s00, s01), ymm_0101);
+ __m256i p1 =
+ _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s10, s11), ymm_0101);
+ p0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(p0, p1), 1),
+ ymm_zero);
+ p0 = _mm256_permute4x64_epi64(p0, 0xd8);
+ __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(p0),
+ _mm256_extracti128_si256(p0, 1));
+ _mm_storeu_si128((__m128i*)dst_uv, out);
+ dst_uv += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif
-#ifdef HAS_YUY2TOYROW_AVX2
-__declspec(naked) void YUY2ToYRow_AVX2(const uint8_t* src_yuy2,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_yuy2
- mov edx, [esp + 8] // dst_y
- mov ecx, [esp + 12] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpand ymm0, ymm0, ymm5 // even bytes are Y
- vpand ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
- vzeroupper
- ret
- }
-}
-
-__declspec(naked) void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
- int stride_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_yuy2
- mov esi, [esp + 8 + 8] // stride_yuy2
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
- sub edi, edx
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vpavgb ymm0, ymm0, [eax + esi]
- vpavgb ymm1, ymm1, [eax + esi + 32]
- lea eax, [eax + 64]
- vpsrlw ymm0, ymm0, 8 // YUYV -> UVUV
- vpsrlw ymm1, ymm1, 8
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vpand ymm1, ymm0, ymm5 // U
- vpsrlw ymm0, ymm0, 8 // V
- vpackuswb ymm1, ymm1, ymm1 // mutates.
- vpackuswb ymm0, ymm0, ymm0 // mutates.
- vpermq ymm1, ymm1, 0xd8
- vpermq ymm0, ymm0, 0xd8
- vextractf128 [edx], ymm1, 0 // U
- vextractf128 [edx + edi], ymm0, 0 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-
-__declspec(naked) void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_yuy2
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
- sub edi, edx
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpsrlw ymm0, ymm0, 8 // YUYV -> UVUV
- vpsrlw ymm1, ymm1, 8
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vpand ymm1, ymm0, ymm5 // U
- vpsrlw ymm0, ymm0, 8 // V
- vpackuswb ymm1, ymm1, ymm1 // mutates.
- vpackuswb ymm0, ymm0, ymm0 // mutates.
- vpermq ymm1, ymm1, 0xd8
- vpermq ymm0, ymm0, 0xd8
- vextractf128 [edx], ymm1, 0 // U
- vextractf128 [edx + edi], ymm0, 0 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- vzeroupper
- ret
- }
-}
-
-__declspec(naked) void UYVYToYRow_AVX2(const uint8_t* src_uyvy,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_uyvy
- mov edx, [esp + 8] // dst_y
- mov ecx, [esp + 12] // width
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpsrlw ymm0, ymm0, 8 // odd bytes are Y
- vpsrlw ymm1, ymm1, 8
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg convertloop
- vzeroupper
- ret
- }
-}
-
-__declspec(naked) void UYVYToUVRow_AVX2(const uint8_t* src_uyvy,
- int stride_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_yuy2
- mov esi, [esp + 8 + 8] // stride_yuy2
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
- sub edi, edx
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vpavgb ymm0, ymm0, [eax + esi]
- vpavgb ymm1, ymm1, [eax + esi + 32]
- lea eax, [eax + 64]
- vpand ymm0, ymm0, ymm5 // UYVY -> UVUV
- vpand ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vpand ymm1, ymm0, ymm5 // U
- vpsrlw ymm0, ymm0, 8 // V
- vpackuswb ymm1, ymm1, ymm1 // mutates.
- vpackuswb ymm0, ymm0, ymm0 // mutates.
- vpermq ymm1, ymm1, 0xd8
- vpermq ymm0, ymm0, 0xd8
- vextractf128 [edx], ymm1, 0 // U
- vextractf128 [edx + edi], ymm0, 0 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- pop esi
- vzeroupper
- ret
- }
-}
-
-__declspec(naked) void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_yuy2
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff00ff
- vpsrlw ymm5, ymm5, 8
- sub edi, edx
-
- convertloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpand ymm0, ymm0, ymm5 // UYVY -> UVUV
- vpand ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1 // mutates.
- vpermq ymm0, ymm0, 0xd8
- vpand ymm1, ymm0, ymm5 // U
- vpsrlw ymm0, ymm0, 8 // V
- vpackuswb ymm1, ymm1, ymm1 // mutates.
- vpackuswb ymm0, ymm0, ymm0 // mutates.
- vpermq ymm1, ymm1, 0xd8
- vpermq ymm0, ymm0, 0xd8
- vextractf128 [edx], ymm1, 0 // U
- vextractf128 [edx + edi], ymm0, 0 // V
- lea edx, [edx + 16]
- sub ecx, 32
- jg convertloop
-
- pop edi
- vzeroupper
- ret
+#ifdef HAS_AYUVTOUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void AYUVToUVRow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_uv,
+ kShufAYUVToUV16_Win, width);
+}
+#endif // HAS_AYUVTOUVROW_AVX2
+
+#ifdef HAS_AYUVTOVUROW_AVX2
+LIBYUV_TARGET_AVX2
+void AYUVToVURow_AVX2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_vu,
+ kShufAYUVToVU16_Win, width);
+}
+#endif // HAS_AYUVTOVUROW_AVX2
+
+#ifdef HAS_AYUVTOYROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
+ static const uint8_t kPermAYUVToY32[64] = {
+ 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62,
+ 66, 70, 74, 78, 82, 86, 90, 94, 98, 102, 106, 110, 114, 118, 122, 126,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)kPermAYUVToY32);
+ while (width >= 32) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_ayuv);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64));
+ src_ayuv += 128;
+ __m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
+ _mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(out));
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)m32;
+ __mmask16 k2 = (__mmask16)(m32 >> 16);
+ __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_ayuv);
+ __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64);
+ __m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1);
+ _mm256_mask_storeu_epi8(dst_y, (__mmask32)m32, _mm512_castsi512_si256(out));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AYUVTOYROW_AVX512VBMI
+
+#if defined(HAS_AYUVTOUVROW_AVX512VBMI) || defined(HAS_AYUVTOVUROW_AVX512VBMI)
+static const uint8_t kPermAYUVToUV_Win[64] = {
+ 1, 5, 0, 4, 9, 13, 8, 12, 17, 21, 16, 20, 25,
+ 29, 24, 28, 33, 37, 32, 36, 41, 45, 40, 44, 49, 53,
+ 48, 52, 57, 61, 56, 60, 65, 69, 64, 68, 73, 77, 72,
+ 76, 81, 85, 80, 84, 89, 93, 88, 92, 97, 101, 96, 100,
+ 105, 109, 104, 108, 113, 117, 112, 116, 121, 125, 120, 124};
+
+static const uint8_t kPermAYUVToVU_Win[64] = {
+ 0, 4, 1, 5, 8, 12, 9, 13, 16, 20, 17, 21, 24,
+ 28, 25, 29, 32, 36, 33, 37, 40, 44, 41, 45, 48, 52,
+ 49, 53, 56, 60, 57, 61, 64, 68, 65, 69, 72, 76, 73,
+ 77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101,
+ 104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125};
+
+LIBYUV_TARGET_AVX512VBMI
+static inline void AYUVToUVOrVURow_Win_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ const uint8_t* perm_table,
+ int width) {
+ const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)perm_table);
+ const __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ while (width >= 32) {
+ __m512i s00 = _mm512_loadu_si512((const __m512i*)src_ayuv);
+ __m512i s01 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64));
+ __m512i s10 =
+ _mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv));
+ __m512i s11 =
+ _mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv + 64));
+ src_ayuv += 128;
+ __m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01);
+ __m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11);
+ p0 = _mm512_maddubs_epi16(p0, zmm_0101);
+ p1 = _mm512_maddubs_epi16(p1, zmm_0101);
+ p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1),
+ zmm_zero);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(p0));
+ dst_uv += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)m32;
+ __mmask16 k2 = (__mmask16)(m32 >> 16);
+ __m512i s00 = _mm512_maskz_loadu_epi32(k1, src_ayuv);
+ __m512i s01 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64);
+ __m512i s10 = _mm512_maskz_loadu_epi32(k1, src_ayuv + src_stride_ayuv);
+ __m512i s11 = _mm512_maskz_loadu_epi32(k2, src_ayuv + src_stride_ayuv + 64);
+ __m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01);
+ __m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11);
+ __mmask16 k_odd = (__mmask16)((width & 1) << (width >> 1));
+ p0 = _mm512_mask_or_epi32(p0, k_odd, p0, _mm512_slli_epi32(p0, 8));
+ p1 = _mm512_mask_or_epi32(p1, k_odd, p1, _mm512_slli_epi32(p1, 8));
+ p0 = _mm512_maddubs_epi16(p0, zmm_0101);
+ p1 = _mm512_maddubs_epi16(p1, zmm_0101);
+ p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1),
+ zmm_zero);
+ uint32_t out_bytes = (uint32_t)((width + 1) & ~1);
+ __mmask32 k_out =
+ out_bytes >= 32 ? ~0u : (__mmask32)((1u << out_bytes) - 1u);
+ _mm256_mask_storeu_epi8(dst_uv, k_out, _mm512_cvtepi16_epi8(p0));
+ }
+ _mm256_zeroupper();
+}
+#endif
+
+#ifdef HAS_AYUVTOUVROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_uv,
+ kPermAYUVToUV_Win, width);
+}
+#endif // HAS_AYUVTOUVROW_AVX512VBMI
+
+#ifdef HAS_AYUVTOVUROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_vu,
+ kPermAYUVToVU_Win, width);
+}
+#endif // HAS_AYUVTOVUROW_AVX512VBMI
+
+// ===========================================================================
+// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW)
+// ===========================================================================
+
+#ifdef HAS_AR30TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void AR30ToARGBRow_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_argb,
+ int width) {
+ const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u);
+ const __m256i m1 = _mm256_set1_epi32(0x300000ff);
+ const __m256i m2 = _mm256_set1_epi32(0x0c00ff00);
+ const __m256i m3 = _mm256_set1_epi32(0x03ff0000);
+ while (width >= 8) {
+ __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
+ src_ar30 += 32;
+ __m256i out = _mm256_or_si256(
+ _mm256_or_si256(_mm256_and_si256(v, m0),
+ _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)),
+ _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2),
+ _mm256_and_si256(_mm256_srli_epi32(v, 6), m3)));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOARGBROW_AVX2
+
+#ifdef HAS_AR30TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_argb,
+ int width) {
+ const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u);
+ const __m512i m1 = _mm512_set1_epi32(0x300000ff);
+ const __m512i m2 = _mm512_set1_epi32(0x0c00ff00);
+ const __m512i m3 = _mm512_set1_epi32(0x03ff0000);
+ while (width >= 16) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
+ src_ar30 += 64;
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_and_si512(v, m0),
+ _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
+ _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
+ _mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_and_si512(v, m0),
+ _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
+ _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
+ _mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOARGBROW_AVX512BW
+
+#ifdef HAS_AR30TOABGRROW_AVX2
+LIBYUV_TARGET_AVX2
+void AR30ToABGRRow_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_abgr,
+ int width) {
+ static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+ const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u);
+ const __m256i m1 = _mm256_set1_epi32(0x300000ff);
+ const __m256i m2 = _mm256_set1_epi32(0x0c00ff00);
+ const __m256i m3 = _mm256_set1_epi32(0x03ff0000);
+ const __m256i shuf =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShufSwapRB16));
+ while (width >= 8) {
+ __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
+ src_ar30 += 32;
+ __m256i out = _mm256_or_si256(
+ _mm256_or_si256(_mm256_and_si256(v, m0),
+ _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)),
+ _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2),
+ _mm256_and_si256(_mm256_srli_epi32(v, 6), m3)));
+ out = _mm256_shuffle_epi8(out, shuf);
+ _mm256_storeu_si256((__m256i*)dst_abgr, out);
+ dst_abgr += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOABGRROW_AVX2
+
+#ifdef HAS_AR30TOABGRROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_abgr,
+ int width) {
+ static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7,
+ 10, 9, 8, 11, 14, 13, 12, 15};
+ const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u);
+ const __m512i m1 = _mm512_set1_epi32(0x300000ff);
+ const __m512i m2 = _mm512_set1_epi32(0x0c00ff00);
+ const __m512i m3 = _mm512_set1_epi32(0x03ff0000);
+ const __m512i shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufSwapRB16));
+ while (width >= 16) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
+ src_ar30 += 64;
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_and_si512(v, m0),
+ _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
+ _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
+ _mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
+ out = _mm512_shuffle_epi8(out, shuf);
+ _mm512_storeu_si512((__m512i*)dst_abgr, out);
+ dst_abgr += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_and_si512(v, m0),
+ _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)),
+ _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2),
+ _mm512_and_si512(_mm512_srli_epi32(v, 6), m3)));
+ out = _mm512_shuffle_epi8(out, shuf);
+ _mm512_mask_storeu_epi32(dst_abgr, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOABGRROW_AVX512BW
+
+#ifdef HAS_AR30TOAB30ROW_AVX2
+LIBYUV_TARGET_AVX2
+void AR30ToAB30Row_AVX2(const uint8_t* src_ar30,
+ uint8_t* dst_ab30,
+ int width) {
+ const __m256i m0 = _mm256_set1_epi32((int)0xc00ffc00u);
+ const __m256i m1 = _mm256_set1_epi32(0x3ff00000);
+ const __m256i m2 = _mm256_set1_epi32(0x000003ff);
+ while (width >= 8) {
+ __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30);
+ src_ar30 += 32;
+ __m256i out = _mm256_or_si256(
+ _mm256_and_si256(v, m0),
+ _mm256_or_si256(_mm256_and_si256(_mm256_slli_epi32(v, 20), m1),
+ _mm256_and_si256(_mm256_srli_epi32(v, 20), m2)));
+ _mm256_storeu_si256((__m256i*)dst_ab30, out);
+ dst_ab30 += 32;
+ width -= 8;
}
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOAB30ROW_AVX2
+
+#ifdef HAS_AR30TOAB30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
+ uint8_t* dst_ab30,
+ int width) {
+ const __m512i m0 = _mm512_set1_epi32((int)0xc00ffc00u);
+ const __m512i m1 = _mm512_set1_epi32(0x3ff00000);
+ const __m512i m2 = _mm512_set1_epi32(0x000003ff);
+ while (width >= 16) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30);
+ src_ar30 += 64;
+ __m512i out = _mm512_or_si512(
+ _mm512_and_si512(v, m0),
+ _mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1),
+ _mm512_and_si512(_mm512_srli_epi32(v, 20), m2)));
+ _mm512_storeu_si512((__m512i*)dst_ab30, out);
+ dst_ab30 += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30);
+ __m512i out = _mm512_or_si512(
+ _mm512_and_si512(v, m0),
+ _mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1),
+ _mm512_and_si512(_mm512_srli_epi32(v, 20), m2)));
+ _mm512_mask_storeu_epi32(dst_ab30, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_AR30TOAB30ROW_AVX512BW
+
+#ifdef HAS_MERGEXR30ROW_10_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MergeXR30Row_10_AVX512BW(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ (void)depth;
+ const __m512i max10 = _mm512_set1_epi32(1023);
+ const __m512i alpha = _mm512_set1_epi32((int)0xc0000000u);
+ while (width >= 16) {
+ __m512i r = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_r)),
+ max10);
+ __m512i g = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_g)),
+ max10);
+ __m512i b = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_b)),
+ max10);
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)),
+ _mm512_or_si512(b, alpha));
+ _mm512_storeu_si512((__m512i*)dst_ar30, out);
+ dst_ar30 += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i r = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_r)), max10);
+ __m512i g = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_g)), max10);
+ __m512i b = _mm512_min_epu32(
+ _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_b)), max10);
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)),
+ _mm512_or_si512(b, alpha));
+ _mm512_mask_storeu_epi32(dst_ar30, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEXR30ROW_10_AVX512BW
+
+#if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW)
+static const uint8_t kShuffleRB30_Win[16] = {128u, 0u, 128u, 2u, 128u, 4u,
+ 128u, 6u, 128u, 8u, 128u, 10u,
+ 128u, 12u, 128u, 14u};
+
+static const uint8_t kShuffleBR30_Win[16] = {128u, 2u, 128u, 0u, 128u, 6u,
+ 128u, 4u, 128u, 10u, 128u, 8u,
+ 128u, 14u, 128u, 12u};
+
+LIBYUV_TARGET_AVX512BW
+static inline void ARGBOrABGRToAR30Row_Win_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ const uint8_t* shuf_rb,
+ int width) {
+ const __m512i zmm_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuf_rb));
+ const __m512i zmm_mul_rb = _mm512_set1_epi32(1028 * 16 * 65536 + 1028);
+ const __m512i zmm_mask_rb = _mm512_set1_epi32(0x3ff003ff);
+ const __m512i zmm_mask_ag = _mm512_set1_epi32((int)0xc000ff00u);
+ const __m512i zmm_mul_ag = _mm512_set1_epi32(64 * 65536 + 1028);
+ while (width >= 16) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src);
+ src += 64;
+ __m512i rb = _mm512_and_si512(
+ _mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb),
+ zmm_mask_rb);
+ __m512i ag = _mm512_slli_epi32(
+ _mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10);
+ _mm512_storeu_si512((__m512i*)dst, _mm512_or_si512(ag, rb));
+ dst += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi32(k, src);
+ __m512i rb = _mm512_and_si512(
+ _mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb),
+ zmm_mask_rb);
+ __m512i ag = _mm512_slli_epi32(
+ _mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10);
+ _mm512_mask_storeu_epi32(dst, k, _mm512_or_si512(ag, rb));
+ }
+ _mm256_zeroupper();
}
-#endif // HAS_YUY2TOYROW_AVX2
+#endif
-#ifdef HAS_YUY2TOYROW_SSE2
-__declspec(naked) void YUY2ToYRow_SSE2(const uint8_t* src_yuy2,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_yuy2
- mov edx, [esp + 8] // dst_y
- mov ecx, [esp + 12] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- pand xmm0, xmm5 // even bytes are Y
- pand xmm1, xmm5
- packuswb xmm0, xmm1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void YUY2ToUVRow_SSE2(const uint8_t* src_yuy2,
- int stride_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_yuy2
- mov esi, [esp + 8 + 8] // stride_yuy2
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + esi]
- movdqu xmm3, [eax + esi + 16]
- lea eax, [eax + 32]
- pavgb xmm0, xmm2
- pavgb xmm1, xmm3
- psrlw xmm0, 8 // YUYV -> UVUV
- psrlw xmm1, 8
- packuswb xmm0, xmm1
- movdqa xmm1, xmm0
- pand xmm0, xmm5 // U
- packuswb xmm0, xmm0
- psrlw xmm1, 8 // V
- packuswb xmm1, xmm1
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + edi], xmm1
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_yuy2
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- psrlw xmm0, 8 // YUYV -> UVUV
- psrlw xmm1, 8
- packuswb xmm0, xmm1
- movdqa xmm1, xmm0
- pand xmm0, xmm5 // U
- packuswb xmm0, xmm0
- psrlw xmm1, 8 // V
- packuswb xmm1, xmm1
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + edi], xmm1
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- ret
- }
-}
-
-__declspec(naked) void UYVYToYRow_SSE2(const uint8_t* src_uyvy,
- uint8_t* dst_y,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_uyvy
- mov edx, [esp + 8] // dst_y
- mov ecx, [esp + 12] // width
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- psrlw xmm0, 8 // odd bytes are Y
- psrlw xmm1, 8
- packuswb xmm0, xmm1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
- ret
- }
-}
-
-__declspec(naked) void UYVYToUVRow_SSE2(const uint8_t* src_uyvy,
- int stride_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_yuy2
- mov esi, [esp + 8 + 8] // stride_yuy2
- mov edx, [esp + 8 + 12] // dst_u
- mov edi, [esp + 8 + 16] // dst_v
- mov ecx, [esp + 8 + 20] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + esi]
- movdqu xmm3, [eax + esi + 16]
- lea eax, [eax + 32]
- pavgb xmm0, xmm2
- pavgb xmm1, xmm3
- pand xmm0, xmm5 // UYVY -> UVUV
- pand xmm1, xmm5
- packuswb xmm0, xmm1
- movdqa xmm1, xmm0
- pand xmm0, xmm5 // U
- packuswb xmm0, xmm0
- psrlw xmm1, 8 // V
- packuswb xmm1, xmm1
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + edi], xmm1
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void UYVYToUV422Row_SSE2(const uint8_t* src_uyvy,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- __asm {
- push edi
- mov eax, [esp + 4 + 4] // src_yuy2
- mov edx, [esp + 4 + 8] // dst_u
- mov edi, [esp + 4 + 12] // dst_v
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff00ff
- psrlw xmm5, 8
- sub edi, edx
-
- convertloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- pand xmm0, xmm5 // UYVY -> UVUV
- pand xmm1, xmm5
- packuswb xmm0, xmm1
- movdqa xmm1, xmm0
- pand xmm0, xmm5 // U
- packuswb xmm0, xmm0
- psrlw xmm1, 8 // V
- packuswb xmm1, xmm1
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + edi], xmm1
- lea edx, [edx + 8]
- sub ecx, 16
- jg convertloop
-
- pop edi
- ret
- }
-}
-#endif // HAS_YUY2TOYROW_SSE2
+#ifdef HAS_ARGBTOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleRB30_Win, width);
+}
+#endif // HAS_ARGBTOAR30ROW_AVX512BW
+
+#ifdef HAS_ABGRTOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleBR30_Win, width);
+}
+#endif // HAS_ABGRTOAR30ROW_AVX512BW
+
+// ===========================================================================
+// Module 3: Planar / Channel Pack & Mirror / Interpolate_16 (AVX512VBMI / BW)
+// ===========================================================================
+
+#ifdef HAS_MERGERGBROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void MergeRGBRow_AVX512VBMI(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ static const uint8_t kPermMergeRGB64_0[64] = {
+ 0x00, 0x20, 0x40, 0x01, 0x21, 0x41, 0x02, 0x22, 0x42, 0x03, 0x23,
+ 0x43, 0x04, 0x24, 0x44, 0x05, 0x25, 0x45, 0x06, 0x26, 0x46, 0x07,
+ 0x27, 0x47, 0x08, 0x28, 0x48, 0x09, 0x29, 0x49, 0x0a, 0x2a, 0x4a,
+ 0x0b, 0x2b, 0x4b, 0x0c, 0x2c, 0x4c, 0x0d, 0x2d, 0x4d, 0x0e, 0x2e,
+ 0x4e, 0x0f, 0x2f, 0x4f, 0x10, 0x30, 0x50, 0x11, 0x31, 0x51, 0x12,
+ 0x32, 0x52, 0x13, 0x33, 0x53, 0x14, 0x34, 0x54, 0x15};
+ static const uint8_t kPermMergeRGB64_1[64] = {
+ 0x35, 0x55, 0x16, 0x36, 0x56, 0x17, 0x37, 0x57, 0x18, 0x38, 0x58,
+ 0x19, 0x39, 0x59, 0x1a, 0x3a, 0x5a, 0x1b, 0x3b, 0x5b, 0x1c, 0x3c,
+ 0x5c, 0x1d, 0x3d, 0x5d, 0x1e, 0x3e, 0x5e, 0x1f, 0x3f, 0x5f, 0x60,
+ 0x00, 0x20, 0x61, 0x01, 0x21, 0x62, 0x02, 0x22, 0x63, 0x03, 0x23,
+ 0x64, 0x04, 0x24, 0x65, 0x05, 0x25, 0x66, 0x06, 0x26, 0x67, 0x07,
+ 0x27, 0x68, 0x08, 0x28, 0x69, 0x09, 0x29, 0x6a, 0x0a};
+ static const uint8_t kPermMergeRGB64_2[64] = {
+ 0x6a, 0x2b, 0x4b, 0x6b, 0x2c, 0x4c, 0x6c, 0x2d, 0x4d, 0x6d, 0x2e,
+ 0x4e, 0x6e, 0x2f, 0x4f, 0x6f, 0x30, 0x50, 0x70, 0x31, 0x51, 0x71,
+ 0x32, 0x52, 0x72, 0x33, 0x53, 0x73, 0x34, 0x54, 0x74, 0x35, 0x55,
+ 0x75, 0x36, 0x56, 0x76, 0x37, 0x57, 0x77, 0x38, 0x58, 0x78, 0x39,
+ 0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c,
+ 0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f};
+ const __m512i m_rgb0 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_0);
+ const __m512i m_rgb1 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_1);
+ const __m512i m_rgb2 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_2);
+ while (width >= 64) {
+ __m512i rg0 = _mm512_inserti64x4(
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)),
+ _mm256_loadu_si256((const __m256i*)src_g), 1);
+ __m512i br = _mm512_inserti64x4(
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b)),
+ _mm256_loadu_si256((const __m256i*)(src_r + 32)), 1);
+ __m512i gb1 = _mm512_inserti64x4(
+ _mm512_castsi256_si512(
+ _mm256_loadu_si256((const __m256i*)(src_g + 32))),
+ _mm256_loadu_si256((const __m256i*)(src_b + 32)), 1);
+ src_r += 64;
+ src_g += 64;
+ src_b += 64;
+ __m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br);
+ __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br),
+ 0xb6db6db600000000ull, m_rgb1, gb1);
+ __m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1);
+ _mm512_storeu_si512((__m512i*)dst_rgb, d0);
+ _mm512_storeu_si512((__m512i*)(dst_rgb + 64), d1);
+ _mm512_storeu_si512((__m512i*)(dst_rgb + 128), d2);
+ dst_rgb += 192;
+ width -= 64;
+ }
+ if (width >= 32) {
+ __m512i rg = _mm512_inserti64x4(
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)),
+ _mm256_loadu_si256((const __m256i*)src_g), 1);
+ __m512i b =
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b));
+ src_r += 32;
+ src_g += 32;
+ src_b += 32;
+ __m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b);
+ __m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b);
+ _mm512_storeu_si512((__m512i*)dst_rgb, d0);
+ _mm256_storeu_si256((__m256i*)(dst_rgb + 64), _mm512_castsi512_si256(d1));
+ dst_rgb += 96;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k_in = (__mmask32)((1u << width) - 1u);
+ __m512i rg = _mm512_inserti64x4(
+ _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_r)),
+ _mm256_maskz_loadu_epi8(k_in, src_g), 1);
+ __m512i b = _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_b));
+ __m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b);
+ __m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b);
+ int dst_bytes = width * 3;
+ __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
+ __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
+ _mm512_mask_storeu_epi8(dst_rgb, k0, d0);
+ _mm256_mask_storeu_epi8(dst_rgb + 64, k1, _mm512_castsi512_si256(d1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGERGBROW_AVX512VBMI
+
+#ifdef HAS_RAWTORGBAROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
+ uint8_t* dst_rgba,
+ int width) {
+ static const uint8_t kPermRAWToRGBA0[64] = {
+ 0, 2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, 0, 11, 10, 9,
+ 0, 14, 13, 12, 0, 17, 16, 15, 0, 20, 19, 18, 0, 23, 22, 21,
+ 0, 26, 25, 24, 0, 29, 28, 27, 0, 32, 31, 30, 0, 35, 34, 33,
+ 0, 38, 37, 36, 0, 41, 40, 39, 0, 44, 43, 42, 0, 47, 46, 45};
+ static const uint8_t kPermRAWToRGBA1[64] = {
+ 0, 50, 49, 48, 0, 53, 52, 51, 0, 56, 55, 54, 0, 59, 58, 57,
+ 0, 62, 61, 60, 0, 65, 64, 63, 0, 68, 67, 66, 0, 71, 70, 69,
+ 0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81,
+ 0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93};
+ const __m512i p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA0);
+ const __m512i p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA1);
+ const __m512i alpha = _mm512_set1_epi32(0x000000ff);
+ while (width >= 32) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_raw);
+ __m512i s1 = _mm512_castsi256_si512(
+ _mm256_loadu_si256((const __m256i*)(src_raw + 64)));
+ src_raw += 96;
+ __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
+ __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
+ _mm512_storeu_si512((__m512i*)dst_rgba, d0);
+ _mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1);
+ dst_rgba += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ int src_bytes = width * 3;
+ __mmask64 ks0 = src_bytes >= 64 ? ~0ull : (1ull << src_bytes) - 1ull;
+ __mmask32 ks1 = src_bytes > 64 ? (1u << (src_bytes - 64)) - 1u : 0u;
+ __m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw);
+ __m512i s1 =
+ _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64));
+ __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
+ __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
+ uint32_t m32 = (1u << width) - 1u;
+ _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0);
+ _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), d1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTORGBAROW_AVX512VBMI
+
+#ifdef HAS_SWAPUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
+ static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6,
+ 9, 8, 11, 10, 13, 12, 15, 14};
+ const __m512i swap =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kSwapUV16));
+ while (width >= 32) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_uv);
+ src_uv += 64;
+ _mm512_storeu_si512((__m512i*)dst_vu, _mm512_shuffle_epi8(v, swap));
+ dst_vu += 64;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi16(k, src_uv);
+ _mm512_mask_storeu_epi16(dst_vu, k, _mm512_shuffle_epi8(v, swap));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SWAPUVROW_AVX512BW
+
+#ifdef HAS_MIRRORROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) {
+ static const uint16_t kMirror16Idx[32] = {
+ 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
+ 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
+ const __m512i rev = _mm512_loadu_si512((const __m512i*)kMirror16Idx);
+ src += width;
+ while (width >= 32) {
+ src -= 32;
+ __m512i v = _mm512_loadu_si512((const __m512i*)src);
+ _mm512_storeu_si512((__m512i*)dst, PermutexvarEpi16(rev, v));
+ dst += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width)));
+ __m512i v = _mm512_maskz_loadu_epi16(k, src - width);
+ _mm512_mask_storeu_epi16(dst, k, PermutexvarEpi16(idx, v));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MIRRORROW_16_AVX512BW
+
+#ifdef HAS_HALFMERGEUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfMergeUVRow_AVX512BW(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ const __m512i c0101 = _mm512_set1_epi16(0x0101);
+ const __m512i czero = _mm512_setzero_si512();
+ const __m512i c00ff = _mm512_set1_epi16(0x00ff);
+ while (width >= 64) {
+ __m512i u0 = _mm512_loadu_si512((const __m512i*)src_u);
+ __m512i u1 = _mm512_loadu_si512((const __m512i*)(src_u + src_stride_u));
+ __m512i v0 = _mm512_loadu_si512((const __m512i*)src_v);
+ __m512i v1 = _mm512_loadu_si512((const __m512i*)(src_v + src_stride_v));
+ src_u += 64;
+ src_v += 64;
+ u0 = _mm512_avg_epu16(
+ _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101),
+ _mm512_maddubs_epi16(u1, c0101)),
+ 1),
+ czero);
+ v0 = _mm512_avg_epu16(
+ _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101),
+ _mm512_maddubs_epi16(v1, c0101)),
+ 1),
+ czero);
+ _mm512_storeu_si512((__m512i*)dst_uv,
+ _mm512_or_si512(u0, _mm512_slli_epi16(v0, 8)));
+ dst_uv += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 k = width >= 64 ? ~0ull : (1ull << width) - 1ull;
+ __m512i u0 = _mm512_maskz_loadu_epi8(k, src_u);
+ __m512i u1 = _mm512_maskz_loadu_epi8(k, src_u + src_stride_u);
+ __m512i v0 = _mm512_maskz_loadu_epi8(k, src_v);
+ __m512i v1 = _mm512_maskz_loadu_epi8(k, src_v + src_stride_v);
+ __mmask32 k_odd = (__mmask32)((uint32_t)(width & 1) << (width >> 1));
+ u0 = _mm512_mask_mullo_epi16(u0, k_odd, _mm512_and_si512(u0, c00ff), c0101);
+ u1 = _mm512_mask_mullo_epi16(u1, k_odd, _mm512_and_si512(u1, c00ff), c0101);
+ v0 = _mm512_mask_mullo_epi16(v0, k_odd, _mm512_and_si512(v0, c00ff), c0101);
+ v1 = _mm512_mask_mullo_epi16(v1, k_odd, _mm512_and_si512(v1, c00ff), c0101);
+ u0 = _mm512_avg_epu16(
+ _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101),
+ _mm512_maddubs_epi16(u1, c0101)),
+ 1),
+ czero);
+ v0 = _mm512_avg_epu16(
+ _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101),
+ _mm512_maddubs_epi16(v1, c0101)),
+ 1),
+ czero);
+ int out_words = (width + 1) >> 1;
+ __mmask32 k_out =
+ out_words >= 32 ? ~0u : (__mmask32)((1u << out_words) - 1u);
+ _mm512_mask_storeu_epi16(dst_uv, k_out,
+ _mm512_or_si512(u0, _mm512_slli_epi16(v0, 8)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFMERGEUVROW_AVX512BW
+
+#ifdef HAS_INTERPOLATEROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void InterpolateRow_16_AVX512BW(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1 = source_y_fraction;
+ int y0 = 256 - y1;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ __m512i zmm_y = _mm512_set1_epi32((y1 << 16) | y0);
+ __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000);
+ __m512i zmm_round = _mm512_set1_epi32(8388736); // 0x800000 + 128
+ int i;
+
+ if (y1 == 0) {
+ for (i = 0; i <= width - 32; i += 32) {
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i),
+ _mm512_loadu_si512((const __m512i*)(src_ptr + i)));
+ }
+ if (i < width) {
+ __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
+ _mm512_mask_storeu_epi16(dst_ptr + i, mask,
+ _mm512_maskz_loadu_epi16(mask, src_ptr + i));
+ }
+ } else if (y1 == 128) {
+ for (i = 0; i <= width - 32; i += 32) {
+ __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
+ __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i),
+ _mm512_avg_epu16(row0, row1));
+ }
+ if (i < width) {
+ __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
+ __m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i);
+ __m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i);
+ _mm512_mask_storeu_epi16(dst_ptr + i, mask, _mm512_avg_epu16(row0, row1));
+ }
+ } else {
+ for (i = 0; i <= width - 32; i += 32) {
+ __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
+ __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
+ __m512i row0l = _mm512_unpacklo_epi16(row0, row1);
+ __m512i row0h = _mm512_unpackhi_epi16(row0, row1);
+ row0l = _mm512_sub_epi16(row0l, zmm_8000);
+ row0h = _mm512_sub_epi16(row0h, zmm_8000);
+ __m512i resl = _mm512_madd_epi16(row0l, zmm_y);
+ __m512i resh = _mm512_madd_epi16(row0h, zmm_y);
+ resl = _mm512_add_epi32(resl, zmm_round);
+ resh = _mm512_add_epi32(resh, zmm_round);
+ resl = _mm512_srai_epi32(resl, 8);
+ resh = _mm512_srai_epi32(resh, 8);
+ _mm512_storeu_si512((__m512i*)(dst_ptr + i),
+ _mm512_packus_epi32(resl, resh));
+ }
+ if (i < width) {
+ __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u);
+ __m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i);
+ __m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i);
+ __m512i row0l = _mm512_unpacklo_epi16(row0, row1);
+ __m512i row0h = _mm512_unpackhi_epi16(row0, row1);
+ row0l = _mm512_sub_epi16(row0l, zmm_8000);
+ row0h = _mm512_sub_epi16(row0h, zmm_8000);
+ __m512i resl = _mm512_madd_epi16(row0l, zmm_y);
+ __m512i resh = _mm512_madd_epi16(row0h, zmm_y);
+ resl = _mm512_add_epi32(resl, zmm_round);
+ resh = _mm512_add_epi32(resh, zmm_round);
+ resl = _mm512_srai_epi32(resl, 8);
+ resh = _mm512_srai_epi32(resh, 8);
+ _mm512_mask_storeu_epi16(dst_ptr + i, mask,
+ _mm512_packus_epi32(resl, resh));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_INTERPOLATEROW_16_AVX512BW
+
+// ===========================================================================
+// Module 4: NV12 / NV21 & I422Alpha to ARGB / RGB24 (AVX512BW / AVX512VBMI)
+// ===========================================================================
+
+#if defined(HAS_NV12TOARGBROW_AVX512BW) || \
+ defined(HAS_NV21TOARGBROW_AVX512BW) || \
+ defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_NV21TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_I422ALPHATOARGBROW_AVX512BW)
+static const uint16_t kSplitQuadWords_Win[32] = {
+ 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+ 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+static const uint64_t kUnpermuteAVX512_Win[8] = {0, 4, 1, 5, 2, 6, 3, 7};
+#endif
-#ifdef HAS_BLENDPLANEROW_SSSE3
-// Blend 8 pixels at a time.
-// unsigned version of math
-// =((A2*C2)+(B2*(255-C2))+255)/256
-// signed version of math
-// =(((A2-128)*C2)+((B2-128)*(255-C2))+32768+127)/256
-__declspec(naked) void BlendPlaneRow_SSSE3(const uint8_t* src0,
- const uint8_t* src1,
- const uint8_t* alpha,
- uint8_t* dst,
- int width) {
- __asm {
- push esi
- push edi
- pcmpeqb xmm5, xmm5 // generate mask 0xff00ff00
- psllw xmm5, 8
- mov eax, 0x80808080 // 128 for biasing image to signed.
- movd xmm6, eax
- pshufd xmm6, xmm6, 0x00
-
- mov eax, 0x807f807f // 32768 + 127 for unbias and round.
- movd xmm7, eax
- pshufd xmm7, xmm7, 0x00
- mov eax, [esp + 8 + 4] // src0
- mov edx, [esp + 8 + 8] // src1
- mov esi, [esp + 8 + 12] // alpha
- mov edi, [esp + 8 + 16] // dst
- mov ecx, [esp + 8 + 20] // width
- sub eax, esi
- sub edx, esi
- sub edi, esi
-
- // 8 pixel loop.
- convertloop8:
- movq xmm0, qword ptr [esi] // alpha
- punpcklbw xmm0, xmm0
- pxor xmm0, xmm5 // a, 255-a
- movq xmm1, qword ptr [eax + esi] // src0
- movq xmm2, qword ptr [edx + esi] // src1
- punpcklbw xmm1, xmm2
- psubb xmm1, xmm6 // bias src0/1 - 128
- pmaddubsw xmm0, xmm1
- paddw xmm0, xmm7 // unbias result - 32768 and round.
- psrlw xmm0, 8
- packuswb xmm0, xmm0
- movq qword ptr [edi + esi], xmm0
- lea esi, [esi + 8]
- sub ecx, 8
- jg convertloop8
-
- pop edi
- pop esi
- ret
+#ifdef HAS_NV12TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ uv_buf += 32;
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64),
+ _mm512_unpackhi_epi16(bg, ra));
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
+ _mm512_unpacklo_epi16(bg, ra));
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
+ _mm512_unpackhi_epi16(bg, ra));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_NV12TOARGBROW_AVX512BW
+
+#ifdef HAS_NV21TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_or_si256(
+ _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8))));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ vu_buf += 32;
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64),
+ _mm512_unpackhi_epi16(bg, ra));
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_or_si256(
+ _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
+ _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8))));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
+ _mm512_unpacklo_epi16(bg, ra));
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
+ _mm512_unpackhi_epi16(bg, ra));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_NV21TOARGBROW_AVX512BW
+
+#if defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \
+ defined(HAS_NV21TORGB24ROW_AVX512VBMI)
+static const uint8_t kMaskBG_NV[64] = {
+ 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
+ 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
+ 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
+ 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
+ 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
+ 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
+static const uint8_t kMaskDST0_NV[64] = {
+ 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
+ 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
+ 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
+ 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
+ 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
+ 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
+static const uint8_t kMaskDST1_NV[64] = {
+ 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
+ 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
+ 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+#endif
+
+#ifdef HAS_NV12TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* uv_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
+ const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV);
+ const __m512i zmm_mask_DST0 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST0_NV);
+ const __m512i zmm_mask_DST1 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
+ while (width >= 32) {
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ uv_buf += 32;
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
+ __m512i zmm1 = _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
+ __m512i zmm2 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
+ _mm512_castsi512_si256(zmm_dst1));
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
+ __m512i zmm1 = _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
+ __m512i zmm2 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ int dst_bytes = width * 3;
+ __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
+ __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
+ _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
+ _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
+ _mm512_castsi512_si256(zmm_dst1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_NV12TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_NV21TORGB24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* vu_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
+ const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV);
+ const __m512i zmm_mask_DST0 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST0_NV);
+ const __m512i zmm_mask_DST1 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
+ while (width >= 32) {
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_or_si256(
+ _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8))));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ vu_buf += 32;
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
+ __m512i zmm1 = _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
+ __m512i zmm2 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
+ _mm512_castsi512_si256(zmm_dst1));
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm3 = PermutexvarEpi16(
+ zmm_quad,
+ _mm512_castsi256_si512(_mm256_or_si256(
+ _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
+ _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8))));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6);
+ __m512i zmm1 = _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6);
+ __m512i zmm2 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ int dst_bytes = width * 3;
+ __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
+ __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
+ _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
+ _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
+ _mm512_castsi512_si256(zmm_dst1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_NV21TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_I422ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
+ ptrdiff_t offset = v_buf - u_buf;
+ while (width >= 32) {
+ __m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
+ __m128i xv = _mm_loadu_si128((const __m128i*)(u_buf + offset));
+ u_buf += 16;
+ __m256i yuv = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ __m512i zmm5 = _mm512_packus_epi16(
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf)),
+ _mm512_setzero_si512());
+ y_buf += 32;
+ a_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64),
+ _mm512_unpackhi_epi16(bg, ra));
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
+ __m128i xv = _mm_maskz_loadu_epi8(k_uv, u_buf + offset);
+ __m256i yuv = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ __m512i zmm5 = _mm512_packus_epi16(
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, a_buf)),
+ _mm512_setzero_si512());
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb),
+ zmm_kYBiasToRgb);
+ __m512i zmm0 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6),
+ zmm5);
+ __m512i zmm1 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6),
+ zmm5);
+ __m512i zmm2 = _mm512_packus_epi16(
+ _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
+ zmm5);
+ __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
+ __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
+ _mm512_unpacklo_epi16(bg, ra));
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
+ _mm512_unpackhi_epi16(bg, ra));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422ALPHATOARGBROW_AVX512BW
+
+// ===========================================================================
+// Module 5: I400ToARGB & ARGB Shaders (Attenuate, Multiply, Gray, Blend)
+// ===========================================================================
+
+#ifdef HAS_I400TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7};
+ static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+ const __m512i zmm_yg =
+ _mm512_set1_epi16(*(const int16_t*)yuvconstants->kYToRgb);
+ const __m512i zmm_ygb =
+ _mm512_set1_epi16(*(const int16_t*)yuvconstants->kYBiasToRgb);
+ const __m512i zmm_lin = _mm512_loadu_si512((const __m512i*)kLinearizePackQ);
+ const __m512i zmm_bcast =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kBcastGrayByte));
+ const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
+ while (width >= 32) {
+ __m512i y16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ y_buf += 32;
+ y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
+ y16 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
+ __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
+ __m256i g256 = _mm512_castsi512_si256(g);
+ __m512i d0 = _mm512_or_si512(
+ _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
+ zmm_bcast),
+ zmm_alpha);
+ __m512i d1 = _mm512_or_si512(
+ _mm512_shuffle_epi8(
+ _mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast),
+ zmm_alpha);
+ _mm512_storeu_si512((__m512i*)dst_argb, d0);
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64), d1);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __m512i y16 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
+ y16 = _mm512_srai_epi16(
+ _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
+ __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
+ __m256i g256 = _mm512_castsi512_si256(g);
+ __m512i d0 = _mm512_or_si512(
+ _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
+ zmm_bcast),
+ zmm_alpha);
+ __m512i d1 = _mm512_or_si512(
+ _mm512_shuffle_epi8(
+ _mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast),
+ zmm_alpha);
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, d0);
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), d1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I400TOARGBROW_AVX512BW
+
+#ifdef HAS_ARGBATTENUATEROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const int8_t kAttenShuf16[16] = {6, -128, 6, -128, 6, -128,
+ -128, -128, 14, -128, 14, -128,
+ 14, -128, -128, -128};
+ const __m512i zmm_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kAttenShuf16));
+ const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
+ const __m512i zmm_bias = _mm512_set1_epi16(0x00ff);
+ while (width >= 16) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ __m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha);
+ __m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha);
+ lo = _mm512_srli_epi16(
+ _mm512_add_epi16(
+ _mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)),
+ zmm_bias),
+ 8);
+ hi = _mm512_srli_epi16(
+ _mm512_add_epi16(
+ _mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)),
+ zmm_bias),
+ 8);
+ __m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi),
+ _mm512_and_si512(s, zmm_alpha));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha);
+ __m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha);
+ lo = _mm512_srli_epi16(
+ _mm512_add_epi16(
+ _mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)),
+ zmm_bias),
+ 8);
+ hi = _mm512_srli_epi16(
+ _mm512_add_epi16(
+ _mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)),
+ zmm_bias),
+ 8);
+ __m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi),
+ _mm512_and_si512(s, zmm_alpha));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBATTENUATEROW_AVX512BW
+
+#ifdef HAS_ARGBMULTIPLYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ const __m512i zero = _mm512_setzero_si512();
+ while (width >= 16) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1);
+ src_argb += 64;
+ src_argb1 += 64;
+ __m512i lo = _mm512_mulhrs_epi16(
+ _mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1),
+ _mm512_unpacklo_epi8(s1, zero));
+ __m512i hi = _mm512_mulhrs_epi16(
+ _mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1),
+ _mm512_unpackhi_epi8(s1, zero));
+ _mm512_storeu_si512((__m512i*)dst_argb, _mm512_packus_epi16(lo, hi));
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1);
+ __m512i lo = _mm512_mulhrs_epi16(
+ _mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1),
+ _mm512_unpacklo_epi8(s1, zero));
+ __m512i hi = _mm512_mulhrs_epi16(
+ _mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1),
+ _mm512_unpackhi_epi8(s1, zero));
+ _mm512_mask_storeu_epi32(dst_argb, k, _mm512_packus_epi16(lo, hi));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBMULTIPLYROW_AVX512BW
+
+#ifdef HAS_ARGBGRAYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBGrayRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+ const __m512i c_coeff = _mm512_set1_epi32(0x004d961d);
+ const __m512i c_sub128 = _mm512_set1_epi8((char)0x80);
+ const __m512i c_one16 = _mm512_set1_epi16(1);
+ const __m512i c_bias = _mm512_set1_epi32(32896);
+ const __m512i c_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword));
+ const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u);
+ while (width >= 16) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ __m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128));
+ __m512i y32 = _mm512_srli_epi32(
+ _mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8);
+ __m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha),
+ _mm512_shuffle_epi8(y32, c_shuf));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128));
+ __m512i y32 = _mm512_srli_epi32(
+ _mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8);
+ __m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha),
+ _mm512_shuffle_epi8(y32, c_shuf));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBGRAYROW_AVX512BW
+
+#ifdef HAS_ARGBGRAYROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+ const __m512i c_coeff = _mm512_set1_epi32(0x004d961d);
+ const __m512i c_sub128 = _mm512_set1_epi8((char)0x80);
+ const __m512i c_bias = _mm512_set1_epi32(32896);
+ const __m512i c_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword));
+ const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u);
+ // Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan on RGBX inputs;
+ // see GEMINI.md).
+ while (width >= 16) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ __m512i y32 = _mm512_srli_epi32(
+ _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
+ __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
+ _mm512_and_si512(s, c_alpha));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i y32 = _mm512_srli_epi32(
+ _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
+ __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
+ _mm512_and_si512(s, c_alpha));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBGRAYROW_AVX512VNNI
+
+#ifdef HAS_ARGBBLENDROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBBlendRow_AVX512BW(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShuffleAlpha16[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80,
+ 7u, 0x80, 11u, 0x80, 11u, 0x80,
+ 15u, 0x80, 15u, 0x80};
+ const __m512i zmm_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShuffleAlpha16));
+ const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
+ const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00u);
+ const __m512i zmm_00ff = _mm512_set1_epi16(0x00ff);
+ const __m512i zmm_0001 = _mm512_set1_epi16(0x0001);
+ while (width >= 16) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1);
+ src_argb += 64;
+ src_argb1 += 64;
+ __m512i inv_a = _mm512_add_epi16(
+ _mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf),
+ zmm_0001);
+ __m512i even = _mm512_srli_epi16(
+ _mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8);
+ __m512i odd = _mm512_and_si512(
+ _mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00);
+ __m512i out = _mm512_adds_epu8(
+ _mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd);
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1);
+ __m512i inv_a = _mm512_add_epi16(
+ _mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf),
+ zmm_0001);
+ __m512i even = _mm512_srli_epi16(
+ _mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8);
+ __m512i odd = _mm512_and_si512(
+ _mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00);
+ __m512i out = _mm512_adds_epu8(
+ _mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd);
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBBLENDROW_AVX512BW
+
+#ifdef HAS_ARGBORROW_SSE2
+void ARGBOrRow_SSE2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ const __m128i val = _mm_set1_epi32((int)value);
+ while (width >= 4) {
+ __m128i s = _mm_loadu_si128((const __m128i*)src_argb);
+ src_argb += 16;
+ _mm_storeu_si128((__m128i*)dst_argb, _mm_or_si128(s, val));
+ dst_argb += 16;
+ width -= 4;
+ }
+}
+#endif // HAS_ARGBORROW_SSE2
+
+#ifdef HAS_ARGBORROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBOrRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ const __m256i val = _mm256_set1_epi32((int)value);
+ while (width >= 8) {
+ __m256i s = _mm256_loadu_si256((const __m256i*)src_argb);
+ src_argb += 32;
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_or_si256(s, val));
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBORROW_AVX2
+
+#ifdef HAS_ARGBORROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBOrRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ uint32_t value,
+ int width) {
+ const __m512i val = _mm512_set1_epi32((int)value);
+ while (width >= 16) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ _mm512_storeu_si512((__m512i*)dst_argb, _mm512_or_si512(s, val));
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
+ _mm512_mask_storeu_epi32(dst_argb, k, _mm512_or_si512(s, val));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBORROW_AVX512BW
+
+#ifdef HAS_ARGBALPHABLENDROW_SSSE3
+LIBYUV_TARGET_SSSE3
+void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ const __m128i zero = _mm_setzero_si128();
+ const __m128i k0001 = _mm_set1_epi16(0x0001);
+ const __m128i k0101 = _mm_set1_epi16(0x0101);
+ const __m128i k00ff = _mm_set1_epi16(0x00ff);
+ const __m128i k8080 = _mm_set1_epi8((char)0x80);
+ const __m128i k7f81 = _mm_set1_epi16(0x7f81);
+ const __m128i k00ffffff = _mm_set1_epi32(0x00ffffff);
+ while (width >= 4) {
+ __m128i s = _mm_loadu_si128((const __m128i*)src_argb0);
+ __m128i d = _mm_loadu_si128((const __m128i*)src_argb1);
+ src_argb0 += 16;
+ src_argb1 += 16;
+ __m128i sa = _mm_srli_epi32(s, 24);
+ __m128i da = _mm_srli_epi32(d, 24);
+ __m128i da_sa = _mm_mulhi_epu16(
+ _mm_add_epi16(_mm_mullo_epi16(da, sa), k0001), k0101);
+ __m128i da2 = _mm_add_epi32(da, _mm_sub_epi32(sa, da_sa));
+ __m128i t = _mm_srli_epi32(_mm_cmpeq_epi32(da, zero), 24);
+ __m128i sa_eff = _mm_or_si128(sa, t);
+ __m128i den_eff = _mm_or_si128(da2, t);
+ __m128i num = _mm_sub_epi32(_mm_slli_epi32(sa_eff, 8), sa_eff);
+ __m128i ratio = _mm_cvttps_epi32(
+ _mm_div_ps(_mm_cvtepi32_ps(num), _mm_cvtepi32_ps(den_eff)));
+ __m128i r_rep = _mm_shufflelo_epi16(_mm_shufflehi_epi16(ratio, 0xa0), 0xa0);
+ __m128i w =
+ _mm_or_si128(r_rep, _mm_slli_epi16(_mm_sub_epi16(k00ff, r_rep), 8));
+ __m128i w_lo = _mm_unpacklo_epi16(w, w);
+ __m128i w_hi = _mm_unpackhi_epi16(w, w);
+ __m128i sb = _mm_sub_epi8(s, k8080);
+ __m128i db = _mm_sub_epi8(d, k8080);
+ __m128i lo = _mm_mulhi_epu16(
+ _mm_add_epi16(_mm_maddubs_epi16(w_lo, _mm_unpacklo_epi8(sb, db)),
+ k7f81),
+ k0101);
+ __m128i hi = _mm_mulhi_epu16(
+ _mm_add_epi16(_mm_maddubs_epi16(w_hi, _mm_unpackhi_epi8(sb, db)),
+ k7f81),
+ k0101);
+ __m128i rgb = _mm_packus_epi16(lo, hi);
+ __m128i out =
+ _mm_or_si128(_mm_and_si128(rgb, k00ffffff), _mm_slli_epi32(da2, 24));
+ _mm_storeu_si128((__m128i*)dst_argb, out);
+ dst_argb += 16;
+ width -= 4;
+ }
+}
+#endif // HAS_ARGBALPHABLENDROW_SSSE3
+
+#ifdef HAS_ARGBALPHABLENDROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ const __m256i zero = _mm256_setzero_si256();
+ const __m256i k0001 = _mm256_set1_epi16(0x0001);
+ const __m256i k0101 = _mm256_set1_epi16(0x0101);
+ const __m256i k00ff = _mm256_set1_epi16(0x00ff);
+ const __m256i k8080 = _mm256_set1_epi8((char)0x80);
+ const __m256i k7f81 = _mm256_set1_epi16(0x7f81);
+ const __m256i k00ffffff = _mm256_set1_epi32(0x00ffffff);
+ while (width >= 8) {
+ __m256i s = _mm256_loadu_si256((const __m256i*)src_argb0);
+ __m256i d = _mm256_loadu_si256((const __m256i*)src_argb1);
+ src_argb0 += 32;
+ src_argb1 += 32;
+ __m256i sa = _mm256_srli_epi32(s, 24);
+ __m256i da = _mm256_srli_epi32(d, 24);
+ __m256i da_sa = _mm256_mulhi_epu16(
+ _mm256_add_epi16(_mm256_mullo_epi16(da, sa), k0001), k0101);
+ __m256i da2 = _mm256_add_epi32(da, _mm256_sub_epi32(sa, da_sa));
+ __m256i t = _mm256_srli_epi32(_mm256_cmpeq_epi32(da, zero), 24);
+ __m256i sa_eff = _mm256_or_si256(sa, t);
+ __m256i den_eff = _mm256_or_si256(da2, t);
+ __m256i num = _mm256_sub_epi32(_mm256_slli_epi32(sa_eff, 8), sa_eff);
+ __m256i ratio = _mm256_cvttps_epi32(
+ _mm256_div_ps(_mm256_cvtepi32_ps(num), _mm256_cvtepi32_ps(den_eff)));
+ __m256i r_rep =
+ _mm256_shufflelo_epi16(_mm256_shufflehi_epi16(ratio, 0xa0), 0xa0);
+ __m256i w = _mm256_or_si256(
+ r_rep, _mm256_slli_epi16(_mm256_sub_epi16(k00ff, r_rep), 8));
+ __m256i w_lo = _mm256_unpacklo_epi16(w, w);
+ __m256i w_hi = _mm256_unpackhi_epi16(w, w);
+ __m256i sb = _mm256_sub_epi8(s, k8080);
+ __m256i db = _mm256_sub_epi8(d, k8080);
+ __m256i lo = _mm256_mulhi_epu16(
+ _mm256_add_epi16(
+ _mm256_maddubs_epi16(w_lo, _mm256_unpacklo_epi8(sb, db)), k7f81),
+ k0101);
+ __m256i hi = _mm256_mulhi_epu16(
+ _mm256_add_epi16(
+ _mm256_maddubs_epi16(w_hi, _mm256_unpackhi_epi8(sb, db)), k7f81),
+ k0101);
+ __m256i rgb = _mm256_packus_epi16(lo, hi);
+ __m256i out = _mm256_or_si256(_mm256_and_si256(rgb, k00ffffff),
+ _mm256_slli_epi32(da2, 24));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
}
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBALPHABLENDROW_AVX2
+
+#ifdef HAS_ARGBALPHABLENDROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ const __m512i zero = _mm512_setzero_si512();
+ const __m512i k0001 = _mm512_set1_epi16(0x0001);
+ const __m512i k0101 = _mm512_set1_epi16(0x0101);
+ const __m512i k00ff = _mm512_set1_epi16(0x00ff);
+ const __m512i k8080 = _mm512_set1_epi8((char)0x80);
+ const __m512i k7f81 = _mm512_set1_epi16(0x7f81);
+ const __m512i k00ffffff = _mm512_set1_epi32(0x00ffffff);
+ while (width >= 16) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)src_argb0);
+ __m512i d = _mm512_loadu_si512((const __m512i*)src_argb1);
+ src_argb0 += 64;
+ src_argb1 += 64;
+ __m512i sa = _mm512_srli_epi32(s, 24);
+ __m512i da = _mm512_srli_epi32(d, 24);
+ __m512i da_sa = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101);
+ __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa));
+ __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero);
+ __m512i t = _mm512_maskz_set1_epi32(k_da0, 255);
+ __m512i sa_eff = _mm512_or_si512(sa, t);
+ __m512i den_eff = _mm512_or_si512(da2, t);
+ __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
+ __m512i ratio = CvttpsEpi32(
+ _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
+ __m512i r_rep =
+ _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
+ __m512i w = _mm512_or_si512(
+ r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8));
+ __m512i w_lo = _mm512_unpacklo_epi16(w, w);
+ __m512i w_hi = _mm512_unpackhi_epi16(w, w);
+ __m512i sb = _mm512_sub_epi8(s, k8080);
+ __m512i db = _mm512_sub_epi8(d, k8080);
+ __m512i lo = _mm512_mulhi_epu16(
+ _mm512_add_epi16(
+ _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81),
+ k0101);
+ __m512i hi = _mm512_mulhi_epu16(
+ _mm512_add_epi16(
+ _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81),
+ k0101);
+ __m512i rgb = _mm512_packus_epi16(lo, hi);
+ __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff),
+ _mm512_slli_epi32(da2, 24));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s = _mm512_maskz_loadu_epi32(k, src_argb0);
+ __m512i d = _mm512_maskz_loadu_epi32(k, src_argb1);
+ __m512i sa = _mm512_srli_epi32(s, 24);
+ __m512i da = _mm512_srli_epi32(d, 24);
+ __m512i da_sa = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101);
+ __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa));
+ __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero);
+ __m512i t = _mm512_maskz_set1_epi32(k_da0, 255);
+ __m512i sa_eff = _mm512_or_si512(sa, t);
+ __m512i den_eff = _mm512_or_si512(da2, t);
+ __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
+ __m512i ratio = CvttpsEpi32(
+ _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
+ __m512i r_rep =
+ _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
+ __m512i w = _mm512_or_si512(
+ r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8));
+ __m512i w_lo = _mm512_unpacklo_epi16(w, w);
+ __m512i w_hi = _mm512_unpackhi_epi16(w, w);
+ __m512i sb = _mm512_sub_epi8(s, k8080);
+ __m512i db = _mm512_sub_epi8(d, k8080);
+ __m512i lo = _mm512_mulhi_epu16(
+ _mm512_add_epi16(
+ _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81),
+ k0101);
+ __m512i hi = _mm512_mulhi_epu16(
+ _mm512_add_epi16(
+ _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81),
+ k0101);
+ __m512i rgb = _mm512_packus_epi16(lo, hi);
+ __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff),
+ _mm512_slli_epi32(da2, 24));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBALPHABLENDROW_AVX512BW
+
+// ===========================================================================
+// Module 6: AVX2 Row Functions (Minimum AVX2 Coverage)
+// ===========================================================================
+
+#ifdef HAS_ARGBSEPIAROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) {
+ static const int8_t kSepiaB[16] = {17, 68, 35, 0, 17, 68, 35, 0,
+ 17, 68, 35, 0, 17, 68, 35, 0};
+ static const int8_t kSepiaG[16] = {22, 88, 45, 0, 22, 88, 45, 0,
+ 22, 88, 45, 0, 22, 88, 45, 0};
+ static const int8_t kSepiaR[16] = {24, 98, 50, 0, 24, 98, 50, 0,
+ 24, 98, 50, 0, 24, 98, 50, 0};
+ if (width >= 8) {
+ const __m256i vb =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaB));
+ const __m256i vg =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaG));
+ const __m256i vr =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaR));
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
+ __m256i b = _mm256_maddubs_epi16(v0, vb);
+ __m256i g = _mm256_maddubs_epi16(v0, vg);
+ __m256i r = _mm256_maddubs_epi16(v0, vr);
+ __m256i bg = _mm256_srli_epi16(_mm256_hadd_epi16(b, g), 7);
+ __m256i rr = _mm256_srli_epi16(_mm256_hadd_epi16(r, r), 7);
+ bg = _mm256_packus_epi16(bg, bg);
+ rr = _mm256_packus_epi16(rr, rr);
+ bg = _mm256_unpacklo_epi8(bg, _mm256_shuffle_epi32(bg, 0x01));
+ __m256i a = _mm256_srli_epi32(v0, 24);
+ a = _mm256_packus_epi16(a, a);
+ a = _mm256_packus_epi16(a, a);
+ __m256i ra = _mm256_unpacklo_epi8(rr, a);
+ __m256i out = _mm256_unpacklo_epi16(bg, ra);
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBSepiaRow_C(dst_argb, width);
+ }
+}
+#endif // HAS_ARGBSEPIAROW_AVX2
+
+#ifdef HAS_ARGBCOLORMATRIXROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ static const int8_t kShufTranspose4x4[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+ if (width >= 8) {
+ const __m256i m = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)matrix_argb));
+ const __m256i shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufTranspose4x4));
+ const __m256i m0 = _mm256_shuffle_epi32(m, 0x00);
+ const __m256i m1 = _mm256_shuffle_epi32(m, 0x55);
+ const __m256i m2 = _mm256_shuffle_epi32(m, 0xaa);
+ const __m256i m3 = _mm256_shuffle_epi32(m, 0xff);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ src_argb += 32;
+ __m256i p01 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m0),
+ _mm256_maddubs_epi16(v0, m1));
+ __m256i p23 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m2),
+ _mm256_maddubs_epi16(v0, m3));
+ p01 = _mm256_srai_epi16(p01, 6);
+ p23 = _mm256_srai_epi16(p23, 6);
+ __m256i out = _mm256_shuffle_epi8(_mm256_packus_epi16(p01, p23), shuf);
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBColorMatrixRow_C(src_argb, dst_argb, matrix_argb, width);
+ }
+}
+#endif // HAS_ARGBCOLORMATRIXROW_AVX2
+
+#ifdef HAS_ARGBQUANTIZEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBQuantizeRow_AVX2(uint8_t* dst_argb,
+ int scale,
+ int interval_size,
+ int interval_offset,
+ int width) {
+ if (width >= 8) {
+ const __m256i v_scale = _mm256_set1_epi16((short)scale);
+ const __m256i v_size = _mm256_set1_epi16((short)interval_size);
+ const __m256i v_off = _mm256_set1_epi16((short)interval_offset);
+ const __m256i zero = _mm256_setzero_si256();
+ const __m256i alpha_mask = _mm256_set1_epi32((int)0xff000000u);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
+ __m256i a = _mm256_and_si256(v0, alpha_mask);
+ __m256i lo = _mm256_unpacklo_epi8(v0, zero);
+ __m256i hi = _mm256_unpackhi_epi8(v0, zero);
+ lo = _mm256_add_epi16(
+ _mm256_mullo_epi16(_mm256_mulhi_epu16(lo, v_scale), v_size), v_off);
+ hi = _mm256_add_epi16(
+ _mm256_mullo_epi16(_mm256_mulhi_epu16(hi, v_scale), v_size), v_off);
+ __m256i out = _mm256_or_si256(
+ _mm256_andnot_si256(alpha_mask, _mm256_packus_epi16(lo, hi)), a);
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBQuantizeRow_C(dst_argb, scale, interval_size, interval_offset, width);
+ }
+}
+#endif // HAS_ARGBQUANTIZEROW_AVX2
+
+#ifdef HAS_ARGBSHADEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBShadeRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ uint32_t value) {
+ if (width >= 8) {
+ __m256i v_val = _mm256_set1_epi32((int)value);
+ v_val = _mm256_unpacklo_epi8(v_val, v_val);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ src_argb += 32;
+ __m256i lo = _mm256_unpacklo_epi8(v0, v0);
+ __m256i hi = _mm256_unpackhi_epi8(v0, v0);
+ lo = _mm256_srli_epi16(_mm256_mulhi_epu16(lo, v_val), 8);
+ hi = _mm256_srli_epi16(_mm256_mulhi_epu16(hi, v_val), 8);
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(lo, hi));
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBShadeRow_C(src_argb, dst_argb, width, value);
+ }
+}
+#endif // HAS_ARGBSHADEROW_AVX2
+
+#ifdef HAS_SOBELXROW_AVX2
+LIBYUV_TARGET_AVX2
+void SobelXRow_AVX2(const uint8_t* src_y0,
+ const uint8_t* src_y1,
+ const uint8_t* src_y2,
+ uint8_t* dst_sobelx,
+ int width) {
+ while (width > 0) {
+ __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0));
+ __m256i b0 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2)));
+ __m256i a1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1));
+ __m256i b1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2)));
+ __m256i a2 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y2));
+ __m256i b2 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y2 + 2)));
+ __m256i d0 = _mm256_sub_epi16(a0, b0);
+ __m256i d1 = _mm256_sub_epi16(a1, b1);
+ __m256i d2 = _mm256_sub_epi16(a2, b2);
+ __m256i sum =
+ _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1));
+ sum = _mm256_abs_epi16(sum);
+ sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_sobelx, _mm256_castsi256_si128(sum));
+ src_y0 += 16;
+ src_y1 += 16;
+ src_y2 += 16;
+ dst_sobelx += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SOBELXROW_AVX2
+
+#ifdef HAS_SOBELYROW_AVX2
+LIBYUV_TARGET_AVX2
+void SobelYRow_AVX2(const uint8_t* src_y0,
+ const uint8_t* src_y1,
+ uint8_t* dst_sobely,
+ int width) {
+ while (width > 0) {
+ __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0));
+ __m256i b0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1));
+ __m256i a1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 1)));
+ __m256i b1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 1)));
+ __m256i a2 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2)));
+ __m256i b2 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2)));
+ __m256i d0 = _mm256_sub_epi16(a0, b0);
+ __m256i d1 = _mm256_sub_epi16(a1, b1);
+ __m256i d2 = _mm256_sub_epi16(a2, b2);
+ __m256i sum =
+ _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1));
+ sum = _mm256_abs_epi16(sum);
+ sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_sobely, _mm256_castsi256_si128(sum));
+ src_y0 += 16;
+ src_y1 += 16;
+ dst_sobely += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SOBELYROW_AVX2
+
+#ifdef HAS_SOBELROW_AVX2
+LIBYUV_TARGET_AVX2
+void SobelRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ const __m256i v_alpha = _mm256_set1_epi32((int)0xff000000u);
+ while (width > 0) {
+ __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx);
+ __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely);
+ __m256i s16 = _mm256_cvtepu8_epi16(_mm_adds_epu8(vx, vy));
+ s16 = _mm256_or_si256(s16, _mm256_slli_epi16(s16, 8));
+ s16 = _mm256_permute4x64_epi64(s16, 0xd8);
+ __m256i p0 = _mm256_or_si256(_mm256_unpacklo_epi16(s16, s16), v_alpha);
+ __m256i p1 = _mm256_or_si256(_mm256_unpackhi_epi16(s16, s16), v_alpha);
+ _mm256_storeu_si256((__m256i*)dst_argb, p0);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), p1);
+ src_sobelx += 16;
+ src_sobely += 16;
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SOBELROW_AVX2
+
+#ifdef HAS_SOBELTOPLANEROW_AVX2
+LIBYUV_TARGET_AVX2
+void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_y,
+ int width) {
+ while (width > 0) {
+ __m256i vx = _mm256_loadu_si256((const __m256i*)src_sobelx);
+ __m256i vy = _mm256_loadu_si256((const __m256i*)src_sobely);
+ _mm256_storeu_si256((__m256i*)dst_y, _mm256_adds_epu8(vx, vy));
+ src_sobelx += 32;
+ src_sobely += 32;
+ dst_y += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SOBELTOPLANEROW_AVX2
+
+#ifdef HAS_SOBELXYROW_AVX2
+LIBYUV_TARGET_AVX2
+void SobelXYRow_AVX2(const uint8_t* src_sobelx,
+ const uint8_t* src_sobely,
+ uint8_t* dst_argb,
+ int width) {
+ const __m256i v_ff00 = _mm256_set1_epi16((short)0xff00u);
+ while (width > 0) {
+ __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx);
+ __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely);
+ __m128i vs = _mm_adds_epu8(vx, vy);
+ __m256i bg =
+ _mm256_or_si256(_mm256_cvtepu8_epi16(vy),
+ _mm256_slli_epi16(_mm256_cvtepu8_epi16(vs), 8));
+ __m256i ra = _mm256_or_si256(_mm256_cvtepu8_epi16(vx), v_ff00);
+ bg = _mm256_permute4x64_epi64(bg, 0xd8);
+ ra = _mm256_permute4x64_epi64(ra, 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(bg, ra));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_unpackhi_epi16(bg, ra));
+ src_sobelx += 16;
+ src_sobely += 16;
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SOBELXYROW_AVX2
+
+#ifdef HAS_ARGBAFFINEROW_AVX2
+LIBYUV_API
+LIBYUV_TARGET_AVX2
+void ARGBAffineRow_AVX2(const uint8_t* src_argb,
+ int src_argb_stride,
+ uint8_t* dst_argb,
+ const float* src_dudv,
+ int width) {
+ static const float kRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f,
+ 4.0f, 5.0f, 6.0f, 7.0f};
+ float uv_tail[4] = {src_dudv[0], src_dudv[1], src_dudv[2], src_dudv[3]};
+ if (width >= 8) {
+ const __m256 ramp = _mm256_loadu_ps(kRamp);
+ const __m256 step8 = _mm256_set1_ps(8.0f);
+ __m256 du = _mm256_set1_ps(uv_tail[2]);
+ __m256 dv = _mm256_set1_ps(uv_tail[3]);
+ __m256 u =
+ _mm256_add_ps(_mm256_set1_ps(uv_tail[0]), _mm256_mul_ps(ramp, du));
+ __m256 v =
+ _mm256_add_ps(_mm256_set1_ps(uv_tail[1]), _mm256_mul_ps(ramp, dv));
+ du = _mm256_mul_ps(du, step8);
+ dv = _mm256_mul_ps(dv, step8);
+ const __m256i v_stride = _mm256_set1_epi32(src_argb_stride);
+ while (width >= 8) {
+ __m256i iu = _mm256_cvttps_epi32(u);
+ __m256i iv = _mm256_cvttps_epi32(v);
+ __m256i idx = _mm256_add_epi32(_mm256_slli_epi32(iu, 2),
+ _mm256_mullo_epi32(iv, v_stride));
+ __m256i pix = _mm256_i32gather_epi32((const int*)src_argb, idx, 1);
+ _mm256_storeu_si256((__m256i*)dst_argb, pix);
+ u = _mm256_add_ps(u, du);
+ v = _mm256_add_ps(v, dv);
+ dst_argb += 32;
+ width -= 8;
+ }
+ uv_tail[0] = _mm256_cvtss_f32(u);
+ uv_tail[1] = _mm256_cvtss_f32(v);
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBAffineRow_C(src_argb, src_argb_stride, dst_argb, uv_tail, width);
+ }
+}
+#endif // HAS_ARGBAFFINEROW_AVX2
+
+#ifdef HAS_ARGBCOLORTABLEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ if (width >= 8) {
+ const __m256i m0 = _mm256_set1_epi32(0x000000ff);
+ const __m256i m1 = _mm256_set1_epi32(0x0000ff00);
+ const __m256i m2 = _mm256_set1_epi32(0x00ff0000);
+ const __m256i m3 = _mm256_set1_epi32((int)0xff000000u);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
+ __m256i c0 =
+ _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
+ _mm256_and_si256(v0, m0), 4),
+ m0);
+ __m256i c1 = _mm256_and_si256(
+ _mm256_i32gather_epi32((const int*)table_argb,
+ _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0),
+ 4),
+ m1);
+ __m256i c2 = _mm256_and_si256(
+ _mm256_i32gather_epi32(
+ (const int*)table_argb,
+ _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4),
+ m2);
+ __m256i c3 =
+ _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
+ _mm256_srli_epi32(v0, 24), 4),
+ m3);
+ __m256i out =
+ _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBColorTableRow_C(dst_argb, table_argb, width);
+ }
+}
+#endif // HAS_ARGBCOLORTABLEROW_AVX2
+
+#ifdef HAS_RGBCOLORTABLEROW_AVX2
+LIBYUV_TARGET_AVX2
+void RGBColorTableRow_AVX2(uint8_t* dst_argb,
+ const uint8_t* table_argb,
+ int width) {
+ if (width >= 8) {
+ const __m256i m0 = _mm256_set1_epi32(0x000000ff);
+ const __m256i m1 = _mm256_set1_epi32(0x0000ff00);
+ const __m256i m2 = _mm256_set1_epi32(0x00ff0000);
+ const __m256i m3 = _mm256_set1_epi32((int)0xff000000u);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb);
+ __m256i c0 =
+ _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb,
+ _mm256_and_si256(v0, m0), 4),
+ m0);
+ __m256i c1 = _mm256_and_si256(
+ _mm256_i32gather_epi32((const int*)table_argb,
+ _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0),
+ 4),
+ m1);
+ __m256i c2 = _mm256_and_si256(
+ _mm256_i32gather_epi32(
+ (const int*)table_argb,
+ _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4),
+ m2);
+ __m256i c3 = _mm256_and_si256(v0, m3);
+ __m256i out =
+ _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ RGBColorTableRow_C(dst_argb, table_argb, width);
+ }
+}
+#endif // HAS_RGBCOLORTABLEROW_AVX2
+
+#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width,
+ const uint8_t* luma,
+ uint32_t lumacoeff) {
+ if (width >= 8) {
+ const __m256i c_br = _mm256_set1_epi32((int)(lumacoeff & 0x00ff00ffu));
+ const __m256i c_ga =
+ _mm256_set1_epi32((int)((lumacoeff >> 8) & 0x000000ffu));
+ const __m256i m_00ff = _mm256_set1_epi16(0x00ff);
+ const __m256i m_ff = _mm256_set1_epi32(0x000000ff);
+ const __m256i m_7f00 = _mm256_set1_epi32(0x00007f00);
+ const __m256i m_18 = _mm256_set1_epi32(0x00000018);
+ const __m256i m_alpha = _mm256_set1_epi32((int)0xff000000u);
+ while (width >= 8) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ src_argb += 32;
+ __m256i y = _mm256_add_epi32(
+ _mm256_madd_epi16(_mm256_and_si256(v0, m_00ff), c_br),
+ _mm256_madd_epi16(_mm256_srli_epi16(v0, 8), c_ga));
+ y = _mm256_and_si256(y, m_7f00);
+
+ __m256i b_idx =
+ _mm256_srli_epi32(_mm256_or_si256(_mm256_and_si256(v0, m_ff), y), 2);
+ __m256i b_val = _mm256_i32gather_epi32((const int*)luma, b_idx, 4);
+ __m256i b_sh = _mm256_and_si256(_mm256_slli_epi32(v0, 3), m_18);
+ b_val = _mm256_and_si256(_mm256_srlv_epi32(b_val, b_sh), m_ff);
+
+ __m256i g_idx = _mm256_srli_epi32(
+ _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 8), m_ff), y),
+ 2);
+ __m256i g_val = _mm256_i32gather_epi32((const int*)luma, g_idx, 4);
+ __m256i g_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 5), m_18);
+ g_val = _mm256_slli_epi32(
+ _mm256_and_si256(_mm256_srlv_epi32(g_val, g_sh), m_ff), 8);
+
+ __m256i r_idx = _mm256_srli_epi32(
+ _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 16), m_ff), y),
+ 2);
+ __m256i r_val = _mm256_i32gather_epi32((const int*)luma, r_idx, 4);
+ __m256i r_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 13), m_18);
+ r_val = _mm256_slli_epi32(
+ _mm256_and_si256(_mm256_srlv_epi32(r_val, r_sh), m_ff), 16);
+
+ __m256i a_val = _mm256_and_si256(v0, m_alpha);
+ __m256i out = _mm256_or_si256(_mm256_or_si256(b_val, g_val),
+ _mm256_or_si256(r_val, a_val));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBLumaColorTableRow_C(src_argb, dst_argb, width, luma, lumacoeff);
+ }
+}
+#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2
+
+#ifdef HAS_BYTETOFLOATROW_AVX2
+LIBYUV_TARGET_AVX2
+void ByteToFloatRow_AVX2(const uint8_t* src,
+ float* dst,
+ float scale,
+ int width) {
+ const __m256 v_scale = _mm256_set1_ps(scale);
+ while (width > 0) {
+ __m256i i0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)src));
+ __m256i i1 =
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(src + 8)));
+ _mm256_storeu_ps(dst, _mm256_mul_ps(_mm256_cvtepi32_ps(i0), v_scale));
+ _mm256_storeu_ps(dst + 8, _mm256_mul_ps(_mm256_cvtepi32_ps(i1), v_scale));
+ src += 16;
+ dst += 16;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_BYTETOFLOATROW_AVX2
+
+#ifdef HAS_GAUSSROW_AVX2
+LIBYUV_TARGET_AVX2
+void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) {
+ static const int8_t kShuf32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13,
+ -1, -1, -1, -1, -1, -1, -1, -1};
+ if (width >= 8) {
+ const __m256i v_128 = _mm256_set1_epi32(128);
+ const __m256i v_shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShuf32To16));
+ while (width >= 8) {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 1));
+ __m256i s2 = _mm256_loadu_si256((const __m256i*)(src + 2));
+ __m256i s3 = _mm256_loadu_si256((const __m256i*)(src + 3));
+ __m256i s4 = _mm256_loadu_si256((const __m256i*)(src + 4));
+ src += 8;
+ __m256i sum04 = _mm256_add_epi32(s0, s4);
+ __m256i sum13 = _mm256_slli_epi32(_mm256_add_epi32(s1, s3), 2);
+ __m256i sum2 =
+ _mm256_add_epi32(_mm256_slli_epi32(s2, 2), _mm256_slli_epi32(s2, 1));
+ __m256i acc =
+ _mm256_srli_epi32(_mm256_add_epi32(_mm256_add_epi32(sum04, sum13),
+ _mm256_add_epi32(sum2, v_128)),
+ 8);
+ acc = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(acc, v_shuf), 0xd8);
+ _mm_storeu_si128((__m128i*)dst, _mm256_castsi256_si128(acc));
+ dst += 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ GaussRow_C(src, dst, width);
+ }
+}
+#endif // HAS_GAUSSROW_AVX2
+
+#ifdef HAS_GAUSSROW_F32_AVX2
+LIBYUV_TARGET_AVX2
+void GaussRow_F32_AVX2(const float* src, float* dst, int width) {
+ if (width >= 8) {
+ const __m256 v4 = _mm256_set1_ps(4.0f);
+ const __m256 v6 = _mm256_set1_ps(6.0f);
+ const __m256 v_scale = _mm256_set1_ps(1.0f / 256.0f);
+ while (width >= 8) {
+ __m256 acc = _mm256_loadu_ps(src);
+ acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 1), v4));
+ acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 2), v6));
+ acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 3), v4));
+ acc = _mm256_add_ps(acc, _mm256_loadu_ps(src + 4));
+ acc = _mm256_mul_ps(acc, v_scale);
+ _mm256_storeu_ps(dst, acc);
+ src += 8;
+ dst += 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ }
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ GaussRow_F32_C(src, dst, width);
+ }
+}
+#endif // HAS_GAUSSROW_F32_AVX2
+
+#ifdef HAS_SETROW_AVX2
+LIBYUV_TARGET_AVX2
+void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) {
+ const __m256i v = _mm256_set1_epi8((char)v8);
+ while (width >= 64) {
+ _mm256_storeu_si256((__m256i*)dst, v);
+ _mm256_storeu_si256((__m256i*)(dst + 32), v);
+ dst += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ _mm256_storeu_si256((__m256i*)dst, v);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SETROW_AVX2
+
+#ifdef HAS_ARGBSETROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) {
+ const __m256i v = _mm256_set1_epi32((int)v32);
+ while (width >= 16) {
+ _mm256_storeu_si256((__m256i*)dst_argb, v);
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32), v);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ _mm256_storeu_si256((__m256i*)dst_argb, v);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBSETROW_AVX2
+
+#ifdef HAS_COPYROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) {
+ while (width > 0) {
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i v1 = _mm256_loadu_si256((const __m256i*)(src + 16));
+ _mm256_storeu_si256((__m256i*)dst, v0);
+ _mm256_storeu_si256((__m256i*)(dst + 16), v1);
+ src += 32;
+ dst += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_COPYROW_16_AVX2
+
+#ifdef HAS_DETILEROW_AVX2
+LIBYUV_TARGET_AVX2
+void DetileRow_AVX2(const uint8_t* src,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst,
+ int width) {
+ while (width >= 32) {
+ __m128i t0 = _mm_loadu_si128((const __m128i*)src);
+ __m128i t1 = _mm_loadu_si128((const __m128i*)(src + src_tile_stride));
+ src += 2 * src_tile_stride;
+ _mm256_storeu_si256((__m256i*)dst, _mm256_set_m128i(t1, t0));
+ dst += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_DETILEROW_AVX2
+
+#ifdef HAS_DETILEROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void DetileRow_16_AVX2(const uint16_t* src,
+ ptrdiff_t src_tile_stride,
+ uint16_t* dst,
+ int width) {
+ while (width >= 32) {
+ __m256i t0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i t1 = _mm256_loadu_si256((const __m256i*)(src + src_tile_stride));
+ src += 2 * src_tile_stride;
+ _mm256_storeu_si256((__m256i*)dst, t0);
+ _mm256_storeu_si256((__m256i*)(dst + 16), t1);
+ dst += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_DETILEROW_16_AVX2
+
+#ifdef HAS_DETILESPLITUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void DetileSplitUVRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_tile_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ static const uint8_t kDeinterlaceUV_Win[16] = {0, 2, 4, 6, 8, 10, 12, 14,
+ 1, 3, 5, 7, 9, 11, 13, 15};
+ const __m128i shuf128 = _mm_loadu_si128((const __m128i*)kDeinterlaceUV_Win);
+ const __m256i shuf256 = _mm256_broadcastsi128_si256(shuf128);
+ while (width >= 32) {
+ __m128i t0 = _mm_loadu_si128((const __m128i*)src_uv);
+ __m128i t1 = _mm_loadu_si128((const __m128i*)(src_uv + src_tile_stride));
+ src_uv += 2 * src_tile_stride;
+ __m256i v = _mm256_permute4x64_epi64(
+ _mm256_shuffle_epi8(_mm256_set_m128i(t1, t0), shuf256), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(v));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(v, 1));
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ __m128i v =
+ _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), shuf128);
+ _mm_storel_epi64((__m128i*)dst_u, v);
+ _mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(v, v));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_DETILESPLITUVROW_AVX2
+
+#if defined(__x86_64__) || defined(_M_X64)
+
+#ifdef HAS_ARGB1555TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ const __m512i zmm_f800 = _mm512_set1_epi16((short)0xf800);
+ const __m512i zmm_03e0 = _mm512_set1_epi16(0x03e0);
+ const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00);
+ const __m512i zmm_0108 = _mm512_set1_epi16(0x0108);
+ const __m512i zmm_4200 = _mm512_set1_epi16(0x4200);
+ while (width >= 32) {
+ __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb1555);
+ src_argb1555 += 64;
+ __m512i r = _mm512_mulhi_epu16(
+ _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108);
+ __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108);
+ __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b);
+ __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200);
+ __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00);
+ __m512i ga = _mm512_or_si512(g, a);
+ __m512i lo = _mm512_unpacklo_epi8(rb, ga);
+ __m512i hi = _mm512_unpackhi_epi8(rb, ga);
+ __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44);
+ __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee);
+ out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8);
+ out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8);
+ _mm512_storeu_si512((__m512i*)dst_argb, out0);
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64), out1);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1ULL << width) - 1ULL);
+ __m512i v0 = _mm512_maskz_loadu_epi16(k, src_argb1555);
+ __m512i r = _mm512_mulhi_epu16(
+ _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108);
+ __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108);
+ __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b);
+ __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200);
+ __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00);
+ __m512i ga = _mm512_or_si512(g, a);
+ __m512i lo = _mm512_unpacklo_epi8(rb, ga);
+ __m512i hi = _mm512_unpackhi_epi8(rb, ga);
+ __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44);
+ __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee);
+ out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8);
+ out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8);
+ if (width > 16) {
+ _mm512_storeu_si512((__m512i*)dst_argb, out0);
+ _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(k >> 16), out1);
+ } else {
+ _mm512_mask_storeu_epi32(dst_argb, (__mmask16)k, out0);
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGB1555TOARGBROW_AVX512BW
+
+#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ __m128i d128 = _mm_cvtsi32_si128((int)dither4);
+ d128 = _mm_unpacklo_epi8(d128, d128);
+ d128 = _mm_unpacklo_epi16(d128, d128);
+ const __m512i zmm_dither = _mm512_broadcast_i32x4(d128);
+ const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f);
+ const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0);
+ const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800);
+ while (width >= 16) {
+ __m512i v0 = _mm512_adds_epu8(_mm512_loadu_si512((const __m512i*)src_argb),
+ zmm_dither);
+ src_argb += 64;
+ __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
+ __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
+ __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
+ __m256i rgb16 =
+ _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
+ _mm256_storeu_si256((__m256i*)dst_rgb, rgb16);
+ dst_rgb += 32;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v0 =
+ _mm512_adds_epu8(_mm512_maskz_loadu_epi32(k, src_argb), zmm_dither);
+ __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
+ __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
+ __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
+ __m256i rgb16 =
+ _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
+ _mm256_mask_storeu_epi16(dst_rgb, k, rgb16);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW
+
+#ifdef HAS_ARGBTORGB565ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f);
+ const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0);
+ const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800);
+ while (width >= 16) {
+ __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
+ __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
+ __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
+ __m256i rgb16 =
+ _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
+ _mm256_storeu_si256((__m256i*)dst_rgb, rgb16);
+ dst_rgb += 32;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f);
+ __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0);
+ __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800);
+ __m256i rgb16 =
+ _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r));
+ _mm256_mask_storeu_epi16(dst_rgb, k, rgb16);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTORGB565ROW_AVX512BW
+
+#ifdef HAS_MERGEUVROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MergeUVRow_16_AVX512BW(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ const __m128i sh_u = _mm_cvtsi32_si128(16 - depth);
+ const __m128i sh_v = _mm_cvtsi32_si128(32 - depth);
+ while (width >= 16) {
+ __m512i u32 =
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_u));
+ __m512i v32 =
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_v));
+ src_u += 16;
+ src_v += 16;
+ u32 = _mm512_sll_epi16(u32, sh_u);
+ v32 = _mm512_sll_epi32(v32, sh_v);
+ _mm512_storeu_si512((__m512i*)dst_uv, _mm512_or_si512(u32, v32));
+ dst_uv += 32;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i u32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_u));
+ __m512i v32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_v));
+ u32 = _mm512_sll_epi16(u32, sh_u);
+ v32 = _mm512_sll_epi32(v32, sh_v);
+ _mm512_mask_storeu_epi32(dst_uv, k, _mm512_or_si512(u32, v32));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEUVROW_16_AVX512BW
+
+#ifdef HAS_DIVIDEROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void DivideRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
+ while (width >= 32) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_y);
+ src_y += 32;
+ _mm512_storeu_si512((__m512i*)dst_y, _mm512_mulhi_epu16(v, zmm_scale));
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi16(k, src_y);
+ _mm512_mask_storeu_epi16(dst_y, k, _mm512_mulhi_epu16(v, zmm_scale));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_DIVIDEROW_16_AVX512BW
+
+#ifdef HAS_YUY2TOYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ while (width >= 32) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_yuy2);
+ src_yuy2 += 64;
+ _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(v));
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi16(k, src_yuy2);
+ _mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(v));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TOYROW_AVX512BW
+
+#ifdef HAS_YUY2TOUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const ptrdiff_t stride = (ptrdiff_t)stride_yuy2;
+ while (width >= 32) {
+ __m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2);
+ __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_yuy2 + stride));
+ src_yuy2 += 64;
+ __m512i avg = _mm512_avg_epu8(r0, r1);
+ _mm_storeu_si128((__m128i*)dst_u,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8)));
+ _mm_storeu_si128((__m128i*)dst_v,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24)));
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2);
+ __m512i r1 = _mm512_maskz_loadu_epi32(k, src_yuy2 + stride);
+ __m512i avg = _mm512_avg_epu8(r0, r1);
+ _mm_mask_storeu_epi8(dst_u, k,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8)));
+ _mm_mask_storeu_epi8(dst_v, k,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TOUVROW_AVX512BW
+
+#ifdef HAS_UYVYTOYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ while (width >= 32) {
+ __m512i v = _mm512_loadu_si512((const __m512i*)src_uyvy);
+ src_uyvy += 64;
+ _mm256_storeu_si256((__m256i*)dst_y,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i v = _mm512_maskz_loadu_epi16(k, src_uyvy);
+ _mm256_mask_storeu_epi8(dst_y, k,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_UYVYTOYROW_AVX512BW
+
+#ifdef HAS_UYVYTOUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
+ int stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const ptrdiff_t stride = (ptrdiff_t)stride_uyvy;
+ while (width >= 32) {
+ __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy);
+ __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uyvy + stride));
+ src_uyvy += 64;
+ __m512i avg = _mm512_avg_epu8(r0, r1);
+ _mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(avg));
+ _mm_storeu_si128((__m128i*)dst_v,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16)));
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy);
+ __m512i r1 = _mm512_maskz_loadu_epi32(k, src_uyvy + stride);
+ __m512i avg = _mm512_avg_epu8(r0, r1);
+ _mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(avg));
+ _mm_mask_storeu_epi8(dst_v, k,
+ _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_UYVYTOUVROW_AVX512BW
+
+#ifdef HAS_ARGBSEPIAROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) {
+ const __m512i zero = _mm512_setzero_si512();
+ const __m512i c_b = _mm512_set1_epi32(0x00234411);
+ const __m512i c_g = _mm512_set1_epi32(0x002d5816);
+ const __m512i c_r = _mm512_set1_epi32(0x00326218);
+ const __m512i max255 = _mm512_set1_epi32(255);
+ const __m512i a_mask = _mm512_set1_epi32((int)0xff000000u);
+ while (width >= 16) {
+ __m512i v0 = _mm512_loadu_si512((const __m512i*)dst_argb);
+ __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7);
+ __m512i g = _mm512_min_epu32(
+ _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255);
+ __m512i r = _mm512_min_epu32(
+ _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255);
+ __m512i a = _mm512_and_si512(v0, a_mask);
+ __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
+ _mm512_or_si512(_mm512_slli_epi32(r, 16), a));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v0 = _mm512_maskz_loadu_epi32(k, dst_argb);
+ __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7);
+ __m512i g = _mm512_min_epu32(
+ _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255);
+ __m512i r = _mm512_min_epu32(
+ _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255);
+ __m512i a = _mm512_and_si512(v0, a_mask);
+ __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
+ _mm512_or_si512(_mm512_slli_epi32(r, 16), a));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBSEPIAROW_AVX512VNNI
+
+#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI
+LIBYUV_TARGET_AVX512VNNI
+void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ const __m512i zero = _mm512_setzero_si512();
+ const __m512i max255 = _mm512_set1_epi32(255);
+ const __m128i m128 = _mm_loadu_si128((const __m128i*)matrix_argb);
+ const __m512i m0 = _mm512_set1_epi32(_mm_cvtsi128_si32(m128));
+ const __m512i m1 =
+ _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 4)));
+ const __m512i m2 =
+ _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 8)));
+ const __m512i m3 =
+ _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 12)));
+ while (width >= 16) {
+ __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ src_argb += 64;
+ __m512i b = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero),
+ max255);
+ __m512i g = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero),
+ max255);
+ __m512i r = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero),
+ max255);
+ __m512i a = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero),
+ max255);
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
+ _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24)));
+ _mm512_storeu_si512((__m512i*)dst_argb, out);
+ dst_argb += 64;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb);
+ __m512i b = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero),
+ max255);
+ __m512i g = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero),
+ max255);
+ __m512i r = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero),
+ max255);
+ __m512i a = _mm512_min_epu32(
+ _mm512_max_epi32(
+ _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero),
+ max255);
+ __m512i out = _mm512_or_si512(
+ _mm512_or_si512(b, _mm512_slli_epi32(g, 8)),
+ _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24)));
+ _mm512_mask_storeu_epi32(dst_argb, k, out);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI
+
+#ifdef HAS_HALFFLOATROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfFloatRow_AVX512BW(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ const __m512 vscale = _mm512_set1_ps(scale);
+ while (width >= 16) {
+ __m512i s32 =
+ _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src));
+ src += 16;
+ __m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale);
+ __m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
+ _mm256_storeu_si256((__m256i*)dst, h16);
+ dst += 16;
+ width -= 16;
+ }
+ if (width > 0) {
+ __mmask16 k = (__mmask16)((1u << width) - 1u);
+ __m512i s32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src));
+ __m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale);
+ __m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
+ _mm256_mask_storeu_epi16(dst, k, h16);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFFLOATROW_AVX512BW
+
+#ifdef HAS_CONVERT8TO8ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert8To8Row_AVX512BW(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ const __m512i zmm_scale =
+ _mm512_slli_epi16(_mm512_set1_epi16((short)scale), 8);
+ const __m512i zmm_bias = _mm512_set1_epi8((char)bias);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ while (width >= 64) {
+ __m512i y8 = _mm512_loadu_si512((const __m512i*)src_y);
+ src_y += 64;
+ __m512i y_lo =
+ _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale);
+ __m512i y_hi =
+ _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale);
+ __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias);
+ _mm512_storeu_si512((__m512i*)dst_y, out8);
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 k = (__mmask64)((1ULL << width) - 1ULL);
+ __m512i y8 = _mm512_maskz_loadu_epi8(k, src_y);
+ __m512i y_lo =
+ _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale);
+ __m512i y_hi =
+ _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale);
+ __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias);
+ _mm512_mask_storeu_epi8(dst_y, k, out8);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT8TO8ROW_AVX512BW
+
+// ===========================================================================
+// Extended YUV (I422, I444, YUY2, UYVY, I210, I212, I410, P210, P410)
+// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI)
+// ===========================================================================
+
+static const uint16_t kSplitQuadWords_ExtWin[32] = {
+ 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
+ 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+static const uint64_t kUnpermuteAVX512_ExtWin[8] = {0, 4, 1, 5, 2, 6, 3, 7};
+static const uint16_t kPermAR30_0_Win[32] = {
+ 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
+ 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
+static const uint16_t kPermAR30_1_Win[32] = {
+ 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
+ 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
+static const uint8_t kShuffleYUY2Y_Win[16] = {0, 0, 2, 2, 4, 4, 6, 6,
+ 8, 8, 10, 10, 12, 12, 14, 14};
+static const uint8_t kShuffleYUY2UV_Win[16] = {1, 3, 1, 3, 5, 7, 5, 7,
+ 9, 11, 9, 11, 13, 15, 13, 15};
+static const uint8_t kShuffleUYVYY_Win[16] = {1, 1, 3, 3, 5, 5, 7, 7,
+ 9, 9, 11, 11, 13, 13, 15, 15};
+static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6,
+ 8, 10, 8, 10, 12, 14, 12, 14};
+
+#define YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants) \
+ const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( \
+ _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToB)); \
+ const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( \
+ _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToG)); \
+ const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( \
+ _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToR)); \
+ const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( \
+ _mm_loadu_si128((const __m128i*)(yuvconstants)->kYToRgb)); \
+ const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( \
+ _mm_loadu_si128((const __m128i*)(yuvconstants)->kYBiasToRgb)); \
+ const __m512i zmm_128 = _mm512_set1_epi8((char)0x80)
+
+#define YUVTORGB16_WIN_AVX512BW \
+ zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \
+ zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), \
+ zmm_kYBiasToRgb); \
+ __m512i b16 = \
+ _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4); \
+ __m512i g16 = \
+ _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)); \
+ __m512i r16 = _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4)
+
+#define YUVTORGB8_WIN_AVX512BW \
+ YUVTORGB16_WIN_AVX512BW; \
+ __m512i zmm0 = _mm512_packus_epi16(_mm512_srai_epi16(b16, 6), zmm5); \
+ __m512i zmm1 = _mm512_packus_epi16(_mm512_srai_epi16(g16, 6), zmm5); \
+ __m512i zmm2 = _mm512_packus_epi16(_mm512_srai_epi16(r16, 6), zmm5)
+
+#define STOREARGB_WIN_AVX512BW(dst_ptr) \
+ do { \
+ __m512i bg = \
+ PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
+ __m512i ra = \
+ PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
+ _mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \
+ _mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \
+ _mm512_unpackhi_epi16(bg, ra)); \
+ } while (0)
+
+#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \
+ do { \
+ __m512i bg = \
+ PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
+ __m512i ra = \
+ PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
+ _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \
+ _mm512_unpacklo_epi16(bg, ra)); \
+ _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \
+ _mm512_unpackhi_epi16(bg, ra)); \
+ } while (0)
+
+#define PACKAR30_WIN_AVX512BW(dst0, dst1) \
+ b16 = _mm512_max_epi16(_mm512_min_epi16(b16, zmm_3ff0), zmm_zero); \
+ g16 = _mm512_max_epi16(_mm512_min_epi16(g16, zmm_3ff0), zmm_zero); \
+ r16 = _mm512_max_epi16(_mm512_min_epi16(r16, zmm_3ff0), zmm_zero); \
+ __m512i lo16 = \
+ _mm512_or_si512(_mm512_srli_epi16(b16, 4), \
+ _mm512_and_si512(_mm512_slli_epi16(g16, 6), zmm_fc00)); \
+ __m512i hi16 = \
+ _mm512_or_si512(_mm512_or_si512(_mm512_and_si512(r16, zmm_3ff0), \
+ _mm512_srli_epi16(g16, 10)), \
+ zmm_c000); \
+ __m512i dst0 = _mm512_permutex2var_epi16(lo16, zmm_perm0, hi16); \
+ __m512i dst1 = _mm512_permutex2var_epi16(lo16, zmm_perm1, hi16)
+
+#ifdef HAS_I422TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
+ __m128i xv = _mm_loadu_si128((const __m128i*)v_buf);
+ u_buf += 16;
+ v_buf += 16;
+ __m256i uv256 = _mm256_inserti128_si256(
+ _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
+ __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf);
+ __m256i uv256 = _mm256_inserti128_si256(
+ _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TOARGBROW_AVX512BW
+
+#ifdef HAS_I422TORGBAROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m128i xu = _mm_loadu_si128((const __m128i*)u_buf);
+ __m128i xv = _mm_loadu_si128((const __m128i*)v_buf);
+ u_buf += 16;
+ v_buf += 16;
+ __m256i uv256 = _mm256_inserti128_si256(
+ _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ y_buf += 32;
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
+ __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
+ _mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr));
+ _mm512_storeu_si512((__m512i*)(dst_rgba + 64),
+ _mm512_unpackhi_epi16(ab, gr));
+ dst_rgba += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf);
+ __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf);
+ __m256i uv256 = _mm256_inserti128_si256(
+ _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
+ _mm_unpackhi_epi8(xu, xv), 1);
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
+ __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
+ _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32,
+ _mm512_unpacklo_epi16(ab, gr));
+ _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16),
+ _mm512_unpackhi_epi16(ab, gr));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TORGBAROW_AVX512BW
+
+#ifdef HAS_I444TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I444ToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i u16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
+ __m512i v16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask32 k = (__mmask32)m32;
+ __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
+ __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
+ __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I444TOARGBROW_AVX512BW
+
+#ifdef HAS_I444ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ const uint8_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ while (width >= 32) {
+ __m512i u16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
+ __m512i v16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ __m512i a16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf));
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ a_buf += 32;
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask32 k = (__mmask32)m32;
+ __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
+ __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
+ __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
+ __m512i a16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, a_buf));
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I444ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_I444TORGB24ROW_AVX512VBMI
+static const uint8_t kMaskBG_I444[64] = {
+ 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05,
+ 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52,
+ 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20,
+ 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65,
+ 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33,
+ 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77};
+static const uint8_t kMaskDST0_I444[64] = {
+ 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07,
+ 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e,
+ 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52,
+ 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d,
+ 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24,
+ 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a};
+static const uint8_t kMaskDST1_I444[64] = {
+ 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70,
+ 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39,
+ 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+LIBYUV_TARGET_AVX512VBMI
+void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_I444);
+ const __m512i zmm_mask_DST0 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST0_I444);
+ const __m512i zmm_mask_DST1 =
+ _mm512_loadu_si512((const __m512i*)kMaskDST1_I444);
+ while (width >= 32) {
+ __m512i u16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf));
+ __m512i v16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf));
+ __m512i zmm4 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB16_WIN_AVX512BW;
+ __m512i zmm0 = _mm512_srai_epi16(b16, 6);
+ __m512i zmm1 = _mm512_srai_epi16(g16, 6);
+ __m512i zmm2 = _mm512_srai_epi16(r16, 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
+ _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64),
+ _mm512_castsi512_si256(zmm_dst1));
+ dst_rgb24 += 96;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf));
+ __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf));
+ __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf));
+ __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
+ YUVTORGB16_WIN_AVX512BW;
+ __m512i zmm0 = _mm512_srai_epi16(b16, 6);
+ __m512i zmm1 = _mm512_srai_epi16(g16, 6);
+ __m512i zmm2 = _mm512_srai_epi16(r16, 6);
+ zmm0 = _mm512_packus_epi16(zmm0, zmm0);
+ zmm1 = _mm512_packus_epi16(zmm1, zmm1);
+ zmm2 = _mm512_packus_epi16(zmm2, zmm2);
+ __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
+ __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
+ __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
+ int dst_bytes = width * 3;
+ __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull;
+ __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u;
+ _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0);
+ _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1,
+ _mm512_castsi512_si256(zmm_dst1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I444TORGB24ROW_AVX512VBMI
+
+#ifdef HAS_YUY2TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm_shuf_y = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleYUY2Y_Win));
+ const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i in = _mm512_loadu_si512((const __m512i*)yuy2_buf);
+ yuy2_buf += 64;
+ __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
+ __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i in = _mm512_maskz_loadu_epi32(k2, yuy2_buf);
+ __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
+ __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TOARGBROW_AVX512BW
+
+#ifdef HAS_UYVYTOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm_shuf_y = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleUYVYY_Win));
+ const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleUYVYUV_Win));
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i in = _mm512_loadu_si512((const __m512i*)uyvy_buf);
+ uyvy_buf += 64;
+ __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
+ __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i in = _mm512_maskz_loadu_epi32(k2, uyvy_buf);
+ __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y);
+ __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_UYVYTOARGBROW_AVX512BW
+
+#ifdef HAS_I210TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m256i u_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
+ __m256i v_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 32;
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
+ __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I210TOARGBROW_AVX512BW
+
+#ifdef HAS_I210TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
+ const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
+ const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ while (width >= 32) {
+ __m256i u_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
+ __m256i v_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 32;
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_storeu_si512((__m512i*)dst_ar30, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
+ __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I210TOAR30ROW_AVX512BW
+
+#ifdef HAS_I210ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ while (width >= 32) {
+ __m256i u_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2);
+ __m256i v_s =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i a16 =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 32;
+ a_buf += 32;
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2);
+ __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m512i a16 =
+ _mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2);
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I210ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_I212TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m256i u_s =
+ _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4);
+ __m256i v_s =
+ _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 32;
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4);
+ __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4);
+ __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I212TOARGBROW_AVX512BW
+
+#ifdef HAS_I212TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_quad =
+ _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin);
+ const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
+ const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
+ const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ while (width >= 32) {
+ __m256i u_s =
+ _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4);
+ __m256i v_s =
+ _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 32;
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_storeu_si512((__m512i*)dst_ar30, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4);
+ __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4);
+ __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
+ _mm256_packus_epi16(v_s, v_s));
+ __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I212TOAR30ROW_AVX512BW
+
+#ifdef HAS_I410TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i u_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
+ __m512i v_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask32 k = (__mmask32)m32;
+ __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
+ __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I410TOARGBROW_AVX512BW
+
+#ifdef HAS_I410TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
+ const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
+ const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ while (width >= 32) {
+ __m512i u_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
+ __m512i v_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_storeu_si512((__m512i*)dst_ar30, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask32 k = (__mmask32)m32;
+ __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
+ __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I410TOAR30ROW_AVX512BW
+
+#ifdef HAS_I410ALPHATOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ while (width >= 32) {
+ __m512i u_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2);
+ __m512i v_s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2);
+ __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i a16 =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2);
+ u_buf += 32;
+ v_buf += 32;
+ y_buf += 32;
+ a_buf += 32;
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask32 k = (__mmask32)m32;
+ __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2);
+ __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2);
+ __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf);
+ __m512i a16 = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, a_buf), 2);
+ __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s),
+ _mm512_packus_epi16(v_s, v_s));
+ __m512i zmm4 =
+ _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
+ __m512i zmm5 = _mm512_packus_epi16(a16, a16);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_I410ALPHATOARGBROW_AVX512BW
+
+#ifdef HAS_P210TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void P210ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i zmm3 = _mm512_shuffle_epi8(
+ _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv);
+ y_buf += 32;
+ uv_buf += 32;
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf),
+ zmm_shuf_uv);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_P210TOARGBROW_AVX512BW
+
+#ifdef HAS_P210TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void P210ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
+ const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
+ const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win));
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
+ const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ while (width >= 32) {
+ __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i zmm3 = _mm512_shuffle_epi8(
+ _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv);
+ y_buf += 32;
+ uv_buf += 32;
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_storeu_si512((__m512i*)dst_ar30, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
+ __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf),
+ zmm_shuf_uv);
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_P210TOAR30ROW_AVX512BW
+
+#ifdef HAS_P410TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void P410ToARGBRow_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_unperm =
+ _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin);
+ const __m512i zmm5 = _mm512_set1_epi32(-1);
+ while (width >= 32) {
+ __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf);
+ __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32));
+ y_buf += 32;
+ uv_buf += 64;
+ __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
+ __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
+ __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_WIN_AVX512BW(dst_argb);
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf);
+ __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32);
+ __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
+ __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
+ __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
+ YUVTORGB8_WIN_AVX512BW;
+ STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_P410TOARGBROW_AVX512BW
+
+#ifdef HAS_P410TOAR30ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void P410ToAR30Row_AVX512BW(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants);
+ const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win);
+ const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win);
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0);
+ const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
+ const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
+ while (width >= 32) {
+ __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf);
+ __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf);
+ __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32));
+ y_buf += 32;
+ uv_buf += 64;
+ __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
+ __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
+ __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_storeu_si512((__m512i*)dst_ar30, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1);
+ dst_ar30 += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t m32 = (1u << width) - 1u;
+ __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
+ __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf);
+ __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32);
+ __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8));
+ __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8));
+ __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1);
+ YUVTORGB16_WIN_AVX512BW;
+ PACKAR30_WIN_AVX512BW(dst0, dst1);
+ _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0);
+ _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_P410TOAR30ROW_AVX512BW
+
+#if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW)
+LIBYUV_TARGET_AVX512BW
+static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src,
+ uint8_t* dst,
+ int width,
+ const uint8_t* shuffler) {
+ const __m512i zmm_shuf =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler));
+ const __m512i zmm_p0 =
+ _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
+ const __m512i zmm_p1 =
+ _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
+ const __m512i zmm_p2 =
+ _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
+
+ while (width >= 64) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src + 64));
+ __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src + 128));
+ __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src + 192));
+ src += 256;
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1);
+ __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2);
+ __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3);
+
+ _mm512_storeu_si512((__m512i*)dst, out0);
+ _mm512_storeu_si512((__m512i*)(dst + 64), out1);
+ _mm512_storeu_si512((__m512i*)(dst + 128), out2);
+ dst += 192;
+ width -= 64;
+ }
+
+ if (width > 0) {
+ uint64_t m_in = (1ULL << width) - 1ULL;
+ __m512i zmm0 = _mm512_maskz_loadu_epi32((__mmask16)m_in, src);
+ __m512i zmm1 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 16), src + 64);
+ __m512i zmm2 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 32), src + 128);
+ __m512i zmm3 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 48), src + 192);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1);
+ __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2);
+ __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3);
+
+ int bytes = width * 3;
+ __mmask64 k_out0 =
+ (bytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << bytes) - 1ULL);
+ __mmask64 k_out1 =
+ (bytes <= 64)
+ ? 0
+ : ((bytes >= 128) ? ~(__mmask64)0
+ : (__mmask64)((1ULL << (bytes - 64)) - 1ULL));
+ __mmask64 k_out2 =
+ (bytes <= 128) ? 0 : (__mmask64)((1ULL << (bytes - 128)) - 1ULL);
+
+ _mm512_mask_storeu_epi8(dst, k_out0, out0);
+ _mm512_mask_storeu_epi8(dst + 64, k_out1, out1);
+ _mm512_mask_storeu_epi8(dst + 128, k_out2, out2);
+ }
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_ARGBTORGB24ROW_AVX512BW) ||
+ // defined(HAS_ARGBTORAWROW_AVX512BW)
+
+#ifdef HAS_ARGBTORGB24ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]);
+}
+#endif // HAS_ARGBTORGB24ROW_AVX512BW
+
+#ifdef HAS_ARGBTORAWROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ static const uint8_t kShuffleMaskARGBToRAW_AVX512[16] = {
+ 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u,
+ 128u};
+ ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW_AVX512);
+}
+#endif // HAS_ARGBTORAWROW_AVX512BW
+
+#ifdef HAS_CONVERT16TO8ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert16To8Row_AVX512BW(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
+ const __m512i zmm_255 = _mm512_set1_epi16(255);
+ while (width >= 64) {
+ __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y);
+ __m512i y1 = _mm512_loadu_si512((const __m512i*)(src_y + 32));
+ y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
+ y1 = _mm512_min_epu16(_mm512_mulhi_epu16(y1, zmm_scale), zmm_255);
+ _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0));
+ _mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_cvtepi16_epi8(y1));
+ src_y += 64;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width >= 32) {
+ __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y);
+ y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
+ _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0));
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i y0 = _mm512_maskz_loadu_epi16(k, src_y);
+ y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255);
+ _mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(y0));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT16TO8ROW_AVX512BW
+
+#ifdef HAS_COPYROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ while (width >= 128) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src);
+ __m512i z1 = _mm512_loadu_si512((const __m512i*)(src + 64));
+ _mm512_storeu_si512((__m512i*)dst, z0);
+ _mm512_storeu_si512((__m512i*)(dst + 64), z1);
+ src += 128;
+ dst += 128;
+ width -= 128;
+ }
+ if (width >= 64) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src);
+ _mm512_storeu_si512((__m512i*)dst, z0);
+ src += 64;
+ dst += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 k = (__mmask64)((1ULL << width) - 1ULL);
+ __m512i z0 = _mm512_maskz_loadu_epi8(k, src);
+ _mm512_mask_storeu_epi8(dst, k, z0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_COPYROW_AVX512BW
+
+#ifdef HAS_HALFROW_16TO8_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
+ const __m512i zmm_255 = _mm512_set1_epi16(255);
+ while (width >= 64) {
+ __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uv);
+ __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride));
+ __m512i s1 =
+ _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32));
+ r0 = _mm512_avg_epu16(r0, s0);
+ r1 = _mm512_avg_epu16(r1, s1);
+ r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255);
+ r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(r0));
+ _mm256_storeu_si256((__m256i*)(dst_uv + 32), _mm512_cvtepi16_epi8(r1));
+ src_uv += 64;
+ dst_uv += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask32 k1 = (__mmask32)mask;
+ __mmask32 k2 = (__mmask32)(mask >> 32);
+ __m512i r0 = _mm512_maskz_loadu_epi16(k1, src_uv);
+ __m512i r1 = _mm512_maskz_loadu_epi16(k2, src_uv + 32);
+ __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_uv + src_uv_stride);
+ __m512i s1 = _mm512_maskz_loadu_epi16(k2, src_uv + src_uv_stride + 32);
+ r0 = _mm512_avg_epu16(r0, s0);
+ r1 = _mm512_avg_epu16(r1, s1);
+ r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255);
+ r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255);
+ _mm256_mask_storeu_epi8(dst_uv, k1, _mm512_cvtepi16_epi8(r0));
+ _mm256_mask_storeu_epi8(dst_uv + 32, k2, _mm512_cvtepi16_epi8(r1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFROW_16TO8_AVX512BW
+
+#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const __m512i zmm_mask16 = _mm512_set1_epi32(0x0000ffff);
+ const __m512i zmm_2 = _mm512_set1_epi32(2);
+ const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
+ const __m512i zmm_255 = _mm512_set1_epi16(255);
+ while (width >= 32) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
+ __m512i t0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride));
+ __m512i t1 =
+ _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32));
+ __m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16),
+ _mm512_srli_epi32(s0, 16));
+ sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16));
+ sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16));
+ sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2);
+
+ __m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16),
+ _mm512_srli_epi32(s1, 16));
+ sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16));
+ sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16));
+ sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2);
+
+ __m512i avg16 =
+ _mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)),
+ _mm512_cvtepi32_epi16(sum1), 1);
+ avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(avg16));
+ src_uv += 64;
+ dst_uv += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)mask;
+ __mmask16 k2 = (__mmask16)(mask >> 16);
+ __mmask32 k3 = (__mmask32)mask;
+ __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv);
+ __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32);
+ __m512i t0 = _mm512_maskz_loadu_epi32(k1, src_uv + src_uv_stride);
+ __m512i t1 = _mm512_maskz_loadu_epi32(k2, src_uv + src_uv_stride + 32);
+ __m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16),
+ _mm512_srli_epi32(s0, 16));
+ sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16));
+ sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16));
+ sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2);
+
+ __m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16),
+ _mm512_srli_epi32(s1, 16));
+ sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16));
+ sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16));
+ sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2);
+
+ __m512i avg16 =
+ _mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)),
+ _mm512_cvtepi32_epi16(sum1), 1);
+ avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255);
+ _mm256_mask_storeu_epi8(dst_uv, k3, _mm512_cvtepi16_epi8(avg16));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW
+
+#ifdef HAS_SPLITUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void SplitUVRow_AVX512BW(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ while (width >= 64) {
+ __m512i uv0 = _mm512_loadu_si512((const __m512i*)src_uv);
+ __m512i uv1 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
+ _mm256_storeu_si256((__m256i*)dst_u, _mm512_cvtepi16_epi8(uv0));
+ _mm256_storeu_si256((__m256i*)(dst_u + 32), _mm512_cvtepi16_epi8(uv1));
+ _mm256_storeu_si256((__m256i*)dst_v,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)));
+ _mm256_storeu_si256((__m256i*)(dst_v + 32),
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)));
+ src_uv += 128;
+ dst_u += 64;
+ dst_v += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask32 k0 = (__mmask32)mask;
+ __mmask32 k1 = (__mmask32)(mask >> 32);
+ __m512i uv0 = _mm512_maskz_loadu_epi16(k0, src_uv);
+ _mm256_mask_storeu_epi8(dst_u, k0, _mm512_cvtepi16_epi8(uv0));
+ _mm256_mask_storeu_epi8(dst_v, k0,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)));
+ if (k1) {
+ __m512i uv1 = _mm512_maskz_loadu_epi16(k1, src_uv + 64);
+ _mm256_mask_storeu_epi8(dst_u + 32, k1, _mm512_cvtepi16_epi8(uv1));
+ _mm256_mask_storeu_epi8(dst_v + 32, k1,
+ _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)));
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITUVROW_AVX512BW
+
+#ifdef HAS_CONVERT8TO16ROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void Convert8To16Row_AVX512BW(const uint8_t* src_y,
+ uint16_t* dst_y,
+ int bits,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(16 - bits);
+ while (width >= 64) {
+ __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i y1_8 = _mm256_loadu_si256((const __m256i*)(src_y + 32));
+ __m512i y0 = _mm512_cvtepu8_epi16(y0_8);
+ __m512i y1 = _mm512_cvtepu8_epi16(y1_8);
+ y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
+ shift128);
+ y1 = _mm512_srl_epi16(_mm512_or_si512(y1, _mm512_slli_epi16(y1, 8)),
+ shift128);
+ _mm512_storeu_si512((__m512i*)dst_y, y0);
+ _mm512_storeu_si512((__m512i*)(dst_y + 32), y1);
+ src_y += 64;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width >= 32) {
+ __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y);
+ __m512i y0 = _mm512_cvtepu8_epi16(y0_8);
+ y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
+ shift128);
+ _mm512_storeu_si512((__m512i*)dst_y, y0);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i y0 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_y));
+ y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)),
+ shift128);
+ _mm512_mask_storeu_epi16(dst_y, k, y0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT8TO16ROW_AVX512BW
+
+#ifdef HAS_J400TOARGBROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShuffleMaskJ400ToARGB_AVX512[64] = {
+ 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u,
+ 3u, 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u,
+ 6u, 128u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u,
+ 128u, 10u, 10u, 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u,
+ 13u, 13u, 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
+ const __m512i zmm_mask =
+ _mm512_loadu_si512((const __m512i*)kShuffleMaskJ400ToARGB_AVX512);
+ const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u);
+ while (width >= 32) {
+ __m512i z0 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)src_y));
+ __m512i z1 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)(src_y + 16)));
+ z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha);
+ z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha);
+ _mm512_storeu_si512((__m512i*)dst_argb, z0);
+ _mm512_storeu_si512((__m512i*)(dst_argb + 64), z1);
+ src_y += 32;
+ dst_argb += 128;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __m512i z0 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k0, src_y));
+ z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha);
+ _mm512_mask_storeu_epi32(dst_argb, k0, z0);
+ if (k1) {
+ __m512i z1 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k1, src_y + 16));
+ z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha);
+ _mm512_mask_storeu_epi32(dst_argb + 64, k1, z1);
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_J400TOARGBROW_AVX512BW
+
+#ifdef HAS_MERGEUVROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MergeUVRow_AVX512BW(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ while (width >= 32) {
+ __m512i u16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u));
+ __m512i v16 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v));
+ __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ _mm512_storeu_si512((__m512i*)dst_uv, uv);
+ src_u += 32;
+ src_v += 32;
+ dst_uv += 64;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_u));
+ __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_v));
+ __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8));
+ _mm512_mask_storeu_epi16(dst_uv, k, uv);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEUVROW_AVX512BW
+
+#ifdef HAS_MULTIPLYROW_16_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ const __m512i zmm_scale = _mm512_set1_epi16((short)scale);
+ while (width >= 64) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y);
+ __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_y + 32));
+ z0 = _mm512_mullo_epi16(z0, zmm_scale);
+ z1 = _mm512_mullo_epi16(z1, zmm_scale);
+ _mm512_storeu_si512((__m512i*)dst_y, z0);
+ _mm512_storeu_si512((__m512i*)(dst_y + 32), z1);
+ src_y += 64;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width >= 32) {
+ __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y);
+ z0 = _mm512_mullo_epi16(z0, zmm_scale);
+ _mm512_storeu_si512((__m512i*)dst_y, z0);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ }
+ if (width > 0) {
+ __mmask32 k = (__mmask32)((1u << width) - 1u);
+ __m512i z0 = _mm512_maskz_loadu_epi16(k, src_y);
+ z0 = _mm512_mullo_epi16(z0, zmm_scale);
+ _mm512_mask_storeu_epi16(dst_y, k, z0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MULTIPLYROW_16_AVX512BW
+
+#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct ArgbConstants* c) {
+ const __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+ const __m512i zmm_80 = _mm512_set1_epi8((char)0x80);
+ const __m512i zmm_one = _mm512_set1_epi16(1);
+ const __m512i zmm_coeff =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY));
+ __m512i zmm_add =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY));
+
+ __m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
+ zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
+ zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
+ zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
+
+ while (width >= 64) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
+ __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
+
+ zmm0 = _mm512_xor_si512(zmm0, zmm_80);
+ zmm1 = _mm512_xor_si512(zmm1, zmm_80);
+ zmm2 = _mm512_xor_si512(zmm2, zmm_80);
+ zmm3 = _mm512_xor_si512(zmm3, zmm_80);
+
+ zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
+ zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+
+ zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
+ zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+
+ zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
+ zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_storeu_si512((__m512i*)dst_y, zmm0);
+
+ src_argb += 256;
+ dst_y += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i zmm0 =
+ _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80);
+ __m512i zmm1 =
+ _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm2 =
+ _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512(),
+ zmm_80);
+ __m512i zmm3 =
+ _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512(),
+ zmm_80);
+
+ zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one);
+ zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm0, zmm1);
+
+ zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one);
+ zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one);
+ zmm2 = _mm512_packs_epi32(zmm2, zmm3);
+
+ zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
+ zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
+ _mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0);
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOYMATRIXROW_AVX512BW
+
+#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ const __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ const __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ const __m512i zmm_one = _mm512_set1_epi16(1);
+ const __m512i zmm_8000 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ const __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+
+ while (width >= 64) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128));
+ __m512i s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192));
+
+ __m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one);
+ __m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one);
+ __m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one);
+ __m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one);
+ __m512i u01 =
+ _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)),
+ 8);
+ __m512i u23 =
+ _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)),
+ 8);
+ _mm512_storeu_si512(
+ (__m512i*)dst_u,
+ PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
+
+ __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one);
+ __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one);
+ __m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one);
+ __m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one);
+ __m512i v01 =
+ _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)),
+ 8);
+ __m512i v23 =
+ _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)),
+ 8);
+ _mm512_storeu_si512(
+ (__m512i*)dst_v,
+ PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
+
+ src_argb += 256;
+ dst_u += 64;
+ dst_v += 64;
+ width -= 64;
+ }
+ if (width > 0) {
+ uint64_t mask = (1ULL << width) - 1ULL;
+ __mmask16 k0 = (__mmask16)mask;
+ __mmask16 k1 = (__mmask16)(mask >> 16);
+ __mmask16 k2 = (__mmask16)(mask >> 32);
+ __mmask16 k3 = (__mmask16)(mask >> 48);
+ __m512i s0 = _mm512_maskz_loadu_epi32(k0, src_argb);
+ __m512i s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64)
+ : _mm512_setzero_si512();
+ __m512i s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128)
+ : _mm512_setzero_si512();
+ __m512i s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192)
+ : _mm512_setzero_si512();
+
+ __m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one);
+ __m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one);
+ __m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one);
+ __m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one);
+ __m512i u01 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)), 8);
+ __m512i u23 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)), 8);
+ _mm512_mask_storeu_epi8(
+ dst_u, (__mmask64)mask,
+ PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
+
+ __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one);
+ __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one);
+ __m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one);
+ __m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one);
+ __m512i v01 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)), 8);
+ __m512i v23 = _mm512_srli_epi16(
+ _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)), 8);
+ _mm512_mask_storeu_epi8(
+ dst_v, (__mmask64)mask,
+ PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW
+
+#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct ArgbConstants* c) {
+ static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
+ 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
+ static const uint8_t kShuffleAARRGGBB_AVX512BW[16] = {
+ 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
+ const __m512i zmm_u =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
+ const __m512i zmm_v =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
+ const __m512i zmm_0101 = _mm512_set1_epi16(0x0101);
+ const __m512i zmm_one = _mm512_set1_epi16(1);
+ const __m512i zmm_8000 =
+ _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV));
+ const __m512i zmm_zero = _mm512_setzero_si512();
+ const __m512i zmm_shuf = _mm512_broadcast_i32x4(
+ _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512BW));
+ const __m512i zmm_perm_y =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
+ const __m512i zmm_perm_uv =
+ _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW);
+
+ while (width >= 32) {
+ __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb);
+ __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
+ __m512i zmm2 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb));
+ __m512i zmm3 =
+ _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64));
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
+ zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
+
+ zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
+ zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm1, zmm2);
+ zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
+
+ _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
+ _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
+
+ src_argb += 128;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ }
+ if (width > 0) {
+ uint32_t mask = (1u << width) - 1u;
+ __mmask16 k1 = (__mmask16)mask;
+ __mmask16 k2 = (__mmask16)(mask >> 16);
+ __mmask16 k3 = ~k1;
+ __mmask16 k4 = ~k2;
+ __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb);
+ __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64);
+ __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb);
+ __m512i zmm3 =
+ _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64);
+
+ zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0);
+ zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0);
+ zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0);
+ zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0);
+
+ zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
+ zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
+ zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
+ zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
+
+ zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101);
+ zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101);
+ zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101);
+ zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101);
+
+ zmm0 = _mm512_add_epi16(zmm0, zmm2);
+ zmm1 = _mm512_add_epi16(zmm1, zmm3);
+
+ zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
+ zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
+
+ zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
+
+ zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
+ zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
+ zmm0 = _mm512_packs_epi32(zmm1, zmm2);
+ zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
+ zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
+
+ int halfwidth = (width + 1) >> 1;
+ __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
+ _mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0));
+ _mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1));
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW
+
+#endif // defined(__x86_64__) || defined(_M_X64)
+
+#ifdef HAS_COPYROW_AVX
+LIBYUV_TARGET_AVX
+void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
+ do {
+ __m256i y0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i y1 = _mm256_loadu_si256((const __m256i*)(src + 32));
+ _mm256_storeu_si256((__m256i*)dst, y0);
+ _mm256_storeu_si256((__m256i*)(dst + 32), y1);
+ src += 64;
+ dst += 64;
+ width -= 64;
+ } while (width > 0);
+ _mm256_zeroupper();
}
-#endif // HAS_BLENDPLANEROW_SSSE3
+#endif // HAS_COPYROW_AVX
-#ifdef HAS_BLENDPLANEROW_AVX2
-// Blend 32 pixels at a time.
-// unsigned version of math
-// =((A2*C2)+(B2*(255-C2))+255)/256
-// signed version of math
-// =(((A2-128)*C2)+((B2-128)*(255-C2))+32768+127)/256
-__declspec(naked) void BlendPlaneRow_AVX2(const uint8_t* src0,
- const uint8_t* src1,
- const uint8_t* alpha,
- uint8_t* dst,
- int width) {
- __asm {
- push esi
- push edi
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xff00ff00
- vpsllw ymm5, ymm5, 8
- mov eax, 0x80808080 // 128 for biasing image to signed.
- vmovd xmm6, eax
- vbroadcastss ymm6, xmm6
- mov eax, 0x807f807f // 32768 + 127 for unbias and round.
- vmovd xmm7, eax
- vbroadcastss ymm7, xmm7
- mov eax, [esp + 8 + 4] // src0
- mov edx, [esp + 8 + 8] // src1
- mov esi, [esp + 8 + 12] // alpha
- mov edi, [esp + 8 + 16] // dst
- mov ecx, [esp + 8 + 20] // width
- sub eax, esi
- sub edx, esi
- sub edi, esi
-
- // 32 pixel loop.
- convertloop32:
- vmovdqu ymm0, [esi] // alpha
- vpunpckhbw ymm3, ymm0, ymm0 // 8..15, 24..31
- vpunpcklbw ymm0, ymm0, ymm0 // 0..7, 16..23
- vpxor ymm3, ymm3, ymm5 // a, 255-a
- vpxor ymm0, ymm0, ymm5 // a, 255-a
- vmovdqu ymm1, [eax + esi] // src0
- vmovdqu ymm2, [edx + esi] // src1
- vpunpckhbw ymm4, ymm1, ymm2
- vpunpcklbw ymm1, ymm1, ymm2
- vpsubb ymm4, ymm4, ymm6 // bias src0/1 - 128
- vpsubb ymm1, ymm1, ymm6 // bias src0/1 - 128
- vpmaddubsw ymm3, ymm3, ymm4
- vpmaddubsw ymm0, ymm0, ymm1
- vpaddw ymm3, ymm3, ymm7 // unbias result - 32768 and round.
- vpaddw ymm0, ymm0, ymm7 // unbias result - 32768 and round.
- vpsrlw ymm3, ymm3, 8
- vpsrlw ymm0, ymm0, 8
- vpackuswb ymm0, ymm0, ymm3
- vmovdqu [edi + esi], ymm0
- lea esi, [esi + 32]
- sub ecx, 32
- jg convertloop32
-
- pop edi
- pop esi
- vzeroupper
- ret
+#ifdef HAS_DETILETOYUY2_AVX2
+LIBYUV_TARGET_AVX2
+void DetileToYUY2_AVX2(const uint8_t* src_y,
+ ptrdiff_t src_y_tile_stride,
+ const uint8_t* src_uv,
+ ptrdiff_t src_uv_tile_stride,
+ uint8_t* dst_yuy2,
+ int width) {
+ do {
+ __m128i y0 = _mm_loadu_si128((const __m128i*)src_y);
+ __m128i y1 = _mm_loadu_si128((const __m128i*)(src_y + src_y_tile_stride));
+ __m128i uv0 = _mm_loadu_si128((const __m128i*)src_uv);
+ __m128i uv1 =
+ _mm_loadu_si128((const __m128i*)(src_uv + src_uv_tile_stride));
+ __m256i y = _mm256_inserti128_si256(_mm256_castsi128_si256(y0), y1, 1);
+ __m256i uv = _mm256_inserti128_si256(_mm256_castsi128_si256(uv0), uv1, 1);
+ __m256i lo = _mm256_unpacklo_epi8(y, uv);
+ __m256i hi = _mm256_unpackhi_epi8(y, uv);
+ _mm256_storeu_si256((__m256i*)dst_yuy2,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_y += 2 * src_y_tile_stride;
+ src_uv += 2 * src_uv_tile_stride;
+ dst_yuy2 += 64;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_DETILETOYUY2_AVX2
+
+#ifdef HAS_UNPACKMT2T_AVX2
+LIBYUV_TARGET_AVX2
+void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) {
+ const __m256i mask_00c0 = _mm256_set1_epi16(0x00c0);
+ while (size >= 80) {
+ __m256i low16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
+ __m256i l0 = _mm256_and_si256(_mm256_slli_epi16(low16, 6), mask_00c0);
+ __m256i l1 = _mm256_and_si256(_mm256_slli_epi16(low16, 4), mask_00c0);
+ __m256i l2 = _mm256_and_si256(_mm256_slli_epi16(low16, 2), mask_00c0);
+ __m256i l3 = _mm256_and_si256(low16, mask_00c0);
+
+ __m256i u0 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
+ __m256i u1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32)));
+ __m256i u2 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48)));
+ __m256i u3 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 64)));
+
+ u0 = _mm256_or_si256(
+ _mm256_or_si256(_mm256_slli_epi16(u0, 8), _mm256_srli_epi16(u0, 2)),
+ l0);
+ u1 = _mm256_or_si256(
+ _mm256_or_si256(_mm256_slli_epi16(u1, 8), _mm256_srli_epi16(u1, 2)),
+ l1);
+ u2 = _mm256_or_si256(
+ _mm256_or_si256(_mm256_slli_epi16(u2, 8), _mm256_srli_epi16(u2, 2)),
+ l2);
+ u3 = _mm256_or_si256(
+ _mm256_or_si256(_mm256_slli_epi16(u3, 8), _mm256_srli_epi16(u3, 2)),
+ l3);
+
+ _mm256_storeu_si256((__m256i*)dst, u0);
+ _mm256_storeu_si256((__m256i*)(dst + 16), u1);
+ _mm256_storeu_si256((__m256i*)(dst + 32), u2);
+ _mm256_storeu_si256((__m256i*)(dst + 48), u3);
+
+ src += 80;
+ dst += 64;
+ size -= 80;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_UNPACKMT2T_AVX2
+
+#ifdef HAS_ARGBGRAYROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
+ static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80,
+ 8, 8, 8, 0x80, 12, 12, 12, 0x80};
+ const __m256i c_coeff = _mm256_set1_epi32(0x004d961d);
+ const __m256i c_sub128 = _mm256_set1_epi8((char)0x80);
+ const __m256i c_one16 = _mm256_set1_epi16(1);
+ const __m256i c_bias = _mm256_set1_epi32(32896);
+ const __m256i c_shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufGrayDword));
+ const __m256i c_alpha = _mm256_set1_epi32((int)0xff000000u);
+
+ while (width >= 8) {
+ __m256i s = _mm256_loadu_si256((const __m256i*)src_argb);
+ src_argb += 32;
+ __m256i y16 = _mm256_maddubs_epi16(c_coeff, _mm256_sub_epi8(s, c_sub128));
+ __m256i y32 = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_madd_epi16(y16, c_one16), c_bias), 8);
+ __m256i out = _mm256_or_si256(_mm256_and_si256(s, c_alpha),
+ _mm256_shuffle_epi8(y32, c_shuf));
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ dst_argb += 32;
+ width -= 8;
}
-}
-#endif // HAS_BLENDPLANEROW_AVX2
+ _mm256_zeroupper();
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ARGBGrayRow_C(src_argb, dst_argb, width);
+ }
+}
+#endif // HAS_ARGBGRAYROW_AVX2
+
+#ifdef HAS_MERGERGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ static const uint8_t kMergeRGBShuf[5][16] = {
+ {0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u, 5u},
+ {5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u},
+ {10u, 5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u},
+ {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u,
+ 0u},
+ {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u,
+ 0u}};
+ const __m256i shuf_r = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kMergeRGBShuf[0]));
+ const __m256i shuf_g = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kMergeRGBShuf[1]));
+ const __m256i shuf_b = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kMergeRGBShuf[2]));
+ const __m256i m1 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kMergeRGBShuf[3]));
+ const __m256i m2 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kMergeRGBShuf[4]));
+
+ do {
+ __m256i r =
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_r), shuf_r);
+ __m256i g =
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_g), shuf_g);
+ __m256i b =
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_b), shuf_b);
+
+ __m256i c0 = _mm256_blendv_epi8(_mm256_blendv_epi8(r, g, m1), b, m2);
+ __m256i c1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g, b, m1), r, m2);
+ __m256i c2 = _mm256_blendv_epi8(_mm256_blendv_epi8(b, r, m1), g, m2);
+
+ _mm256_storeu_si256((__m256i*)dst_rgb,
+ _mm256_permute2x128_si256(c0, c1, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_rgb + 32),
+ _mm256_permute2x128_si256(c2, c0, 0x30));
+ _mm256_storeu_si256((__m256i*)(dst_rgb + 64),
+ _mm256_permute2x128_si256(c1, c2, 0x31));
+
+ src_r += 32;
+ src_g += 32;
+ src_b += 32;
+ dst_rgb += 96;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGERGBROW_AVX2
+
+#ifdef HAS_RAWTORGBAROW_AVX2
+LIBYUV_TARGET_AVX2
+void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ static const uint8_t kShufRAWToRGBA_Lo[16] = {0x80, 2u, 1u, 0u, 0x80, 5u,
+ 4u, 3u, 0x80, 8u, 7u, 6u,
+ 0x80, 11u, 10u, 9u};
+ static const uint8_t kShufRAWToRGBA_Hi[16] = {0x80, 6u, 5u, 4u, 0x80, 9u,
+ 8u, 7u, 0x80, 12u, 11u, 10u,
+ 0x80, 15u, 14u, 13u};
+ const __m256i shuf_lo = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Lo));
+ const __m256i shuf_hi = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Hi));
+ const __m256i alpha = _mm256_set1_epi32(0x000000ff);
+
+ do {
+ __m128i s0 = _mm_loadu_si128((const __m128i*)src_raw);
+ __m128i s1 = _mm_loadu_si128((const __m128i*)(src_raw + 12));
+ __m128i s2 = _mm_loadu_si128((const __m128i*)(src_raw + 20));
+ __m128i s3 = _mm_loadu_si128((const __m128i*)(src_raw + 32));
+ __m256i y0 = _mm256_inserti128_si256(_mm256_castsi128_si256(s0), s1, 1);
+ __m256i y1 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
+ y0 = _mm256_or_si256(_mm256_shuffle_epi8(y0, shuf_lo), alpha);
+ y1 = _mm256_or_si256(_mm256_shuffle_epi8(y1, shuf_hi), alpha);
+ _mm256_storeu_si256((__m256i*)dst_rgba, y0);
+ _mm256_storeu_si256((__m256i*)(dst_rgba + 32), y1);
+ src_raw += 48;
+ dst_rgba += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_RAWTORGBAROW_AVX2
+
+#ifdef HAS_HALFROW_AVX2
+LIBYUV_TARGET_AVX2
+void HalfRow_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int width) {
+ do {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu8(r0, r1));
+ src_uv += 32;
+ dst_uv += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFROW_AVX2
+
+#ifdef HAS_HALFROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void HalfRow_16_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint16_t* dst_uv,
+ int width) {
+ do {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu16(r0, r1));
+ src_uv += 16;
+ dst_uv += 16;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFROW_16_AVX2
+
+#ifdef HAS_HALFMERGEUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void HalfMergeUVRow_AVX2(const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_uv,
+ int width) {
+ const __m256i ymm_0101 = _mm256_set1_epi16(0x0101);
+ const __m256i ymm_zero = _mm256_setzero_si256();
+ do {
+ __m256i u0 = _mm256_loadu_si256((const __m256i*)src_u);
+ __m256i v0 = _mm256_loadu_si256((const __m256i*)src_v);
+ __m256i u1 = _mm256_loadu_si256((const __m256i*)(src_u + src_stride_u));
+ __m256i v1 = _mm256_loadu_si256((const __m256i*)(src_v + src_stride_v));
+ u0 = _mm256_add_epi16(_mm256_maddubs_epi16(u0, ymm_0101),
+ _mm256_maddubs_epi16(u1, ymm_0101));
+ v0 = _mm256_add_epi16(_mm256_maddubs_epi16(v0, ymm_0101),
+ _mm256_maddubs_epi16(v1, ymm_0101));
+ u0 = _mm256_avg_epu16(_mm256_srli_epi16(u0, 1), ymm_zero);
+ v0 = _mm256_avg_epu16(_mm256_srli_epi16(v0, 1), ymm_zero);
+ u0 = _mm256_packus_epi16(u0, u0);
+ v0 = _mm256_packus_epi16(v0, v0);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm256_unpacklo_epi8(u0, v0));
+ src_u += 32;
+ src_v += 32;
+ dst_uv += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFMERGEUVROW_AVX2
+
+#ifdef HAS_CONVERT8TO8ROW_AVX2
+LIBYUV_TARGET_AVX2
+void Convert8To8Row_AVX2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ const __m256i ymm_scale =
+ _mm256_slli_epi16(_mm256_set1_epi16((short)scale), 8);
+ const __m256i ymm_bias = _mm256_set1_epi8((char)bias);
+ const __m256i ymm_zero = _mm256_setzero_si256();
+ do {
+ __m256i y = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i lo =
+ _mm256_mulhi_epu16(_mm256_unpacklo_epi8(y, ymm_zero), ymm_scale);
+ __m256i hi =
+ _mm256_mulhi_epu16(_mm256_unpackhi_epi8(y, ymm_zero), ymm_scale);
+ __m256i out = _mm256_add_epi8(_mm256_packus_epi16(lo, hi), ymm_bias);
+ _mm256_storeu_si256((__m256i*)dst_y, out);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT8TO8ROW_AVX2
+
+#ifdef HAS_DIVIDEROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void DivideRow_16_AVX2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ if (scale == 65536) { // Divide by 1, so do memcpy.
+ if (src_y != dst_y) {
+ memcpy(dst_y, src_y, (size_t)width * 2);
+ }
+ return;
+ }
+ const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
+ while (width > 0) {
+ __m256i v0 = _mm256_mulhi_epu16(_mm256_loadu_si256((const __m256i*)src_y),
+ ymm_scale);
+ __m256i v1 = _mm256_mulhi_epu16(
+ _mm256_loadu_si256((const __m256i*)(src_y + 16)), ymm_scale);
+ _mm256_storeu_si256((__m256i*)dst_y, v0);
+ _mm256_storeu_si256((__m256i*)(dst_y + 16), v1);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_DIVIDEROW_16_AVX2
+
+#ifdef HAS_CONVERT16TO8ROW_AVX2
+LIBYUV_TARGET_AVX2
+void Convert16To8Row_AVX2(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
+ const __m256i ymm_255 = _mm256_set1_epi16(255);
+ do {
+ __m256i y0 = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
+ y0 = _mm256_min_epu16(_mm256_mulhi_epu16(y0, ymm_scale), ymm_255);
+ y1 = _mm256_min_epu16(_mm256_mulhi_epu16(y1, ymm_scale), ymm_255);
+ __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(y0, y1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_y, out);
+ src_y += 32;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_CONVERT16TO8ROW_AVX2
+
+#ifdef HAS_HALFROW_16TO8_AVX2
+LIBYUV_TARGET_AVX2
+void HalfRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
+ const __m256i ymm_255 = _mm256_set1_epi16(255);
+ do {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
+ __m256i s1 =
+ _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16));
+ r0 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r0, s0), ymm_scale),
+ ymm_255);
+ r1 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r1, s1), ymm_scale),
+ ymm_255);
+ __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(r0, r1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_uv, out);
+ src_uv += 32;
+ dst_uv += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFROW_16TO8_AVX2
+
+#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
+LIBYUV_TARGET_AVX2
+void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ const __m256i ymm_mask16 = _mm256_set1_epi32(0x0000ffff);
+ const __m256i ymm_2 = _mm256_set1_epi32(2);
+ const __m256i ymm_scale = _mm256_set1_epi16((short)scale);
+ const __m256i ymm_255 = _mm256_set1_epi16(255);
+ do {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
+ __m256i t0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride));
+ __m256i t1 =
+ _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16));
+
+ __m256i sum0 = _mm256_add_epi32(_mm256_and_si256(s0, ymm_mask16),
+ _mm256_srli_epi32(s0, 16));
+ sum0 = _mm256_add_epi32(sum0, _mm256_and_si256(t0, ymm_mask16));
+ sum0 = _mm256_add_epi32(sum0, _mm256_srli_epi32(t0, 16));
+ sum0 = _mm256_srli_epi32(_mm256_add_epi32(sum0, ymm_2), 2);
+
+ __m256i sum1 = _mm256_add_epi32(_mm256_and_si256(s1, ymm_mask16),
+ _mm256_srli_epi32(s1, 16));
+ sum1 = _mm256_add_epi32(sum1, _mm256_and_si256(t1, ymm_mask16));
+ sum1 = _mm256_add_epi32(sum1, _mm256_srli_epi32(t1, 16));
+ sum1 = _mm256_srli_epi32(_mm256_add_epi32(sum1, ymm_2), 2);
+
+ __m256i avg16 =
+ _mm256_permute4x64_epi64(_mm256_packus_epi32(sum0, sum1), 0xd8);
+ avg16 = _mm256_min_epu16(_mm256_mulhi_epu16(avg16, ymm_scale), ymm_255);
+ __m256i out =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(avg16, avg16), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(out));
+ src_uv += 32;
+ dst_uv += 16;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_HALFWIDTHROW_16TO8_AVX2
+
+#ifdef HAS_HALFWIDTHROW_16TO8_ODD_AVX2
+LIBYUV_TARGET_AVX2
+void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv,
+ ptrdiff_t src_uv_stride,
+ uint8_t* dst_uv,
+ int scale,
+ int width) {
+ int n = (width - 1) & ~15;
+ int r = (width - 1) & 15;
+ SIMD_ALIGNED(uint16_t pair[64]);
+ SIMD_ALIGNED(uint8_t out[16]);
+ if (n > 0) {
+ HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n);
+ src_uv += n * 2;
+ dst_uv += n;
+ }
+ if (r > 0) {
+ HalfWidthRow_16To8_Any_AVX2(src_uv, src_uv_stride, dst_uv, scale, r);
+ src_uv += r * 2;
+ dst_uv += r;
+ }
+ memset(pair, 0, sizeof(pair));
+ pair[0] = src_uv[0];
+ pair[1] = src_uv[0];
+ pair[32] = src_uv[src_uv_stride];
+ pair[33] = src_uv[src_uv_stride];
+ HalfWidthRow_16To8_AVX2(pair, 32, out, scale, 16);
+ dst_uv[0] = out[0];
+}
+#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2
+
+#ifdef HAS_GAUSSCOL_AVX2
+LIBYUV_TARGET_AVX2
+void GaussCol_AVX2(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width) {
+ while (width >= 8) {
+ __m256i v0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src0));
+ __m256i v1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src1));
+ __m256i v2 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src2));
+ __m256i v3 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src3));
+ __m256i v4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src4));
+ __m256i v04 = _mm256_add_epi32(v0, v4);
+ __m256i v13 = _mm256_slli_epi32(_mm256_add_epi32(v1, v3), 2);
+ __m256i v2_6 =
+ _mm256_add_epi32(_mm256_slli_epi32(v2, 2), _mm256_slli_epi32(v2, 1));
+ _mm256_storeu_si256((__m256i*)dst,
+ _mm256_add_epi32(_mm256_add_epi32(v04, v13), v2_6));
+ src0 += 8;
+ src1 += 8;
+ src2 += 8;
+ src3 += 8;
+ src4 += 8;
+ dst += 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ GaussCol_C(src0, src1, src2, src3, src4, dst, width);
+ }
+}
+#endif // HAS_GAUSSCOL_AVX2
+
+#ifdef HAS_GAUSSCOL_F32_AVX2
+LIBYUV_TARGET_AVX2
+void GaussCol_F32_AVX2(const float* src0,
+ const float* src1,
+ const float* src2,
+ const float* src3,
+ const float* src4,
+ float* dst,
+ int width) {
+ const __m256 c4 = _mm256_set1_ps(4.0f);
+ const __m256 c6 = _mm256_set1_ps(6.0f);
+ while (width >= 8) {
+ __m256 s0 = _mm256_loadu_ps(src0);
+ __m256 s1 = _mm256_loadu_ps(src1);
+ __m256 s2 = _mm256_loadu_ps(src2);
+ __m256 s3 = _mm256_loadu_ps(src3);
+ __m256 s4 = _mm256_loadu_ps(src4);
+ __m256 sum = _mm256_add_ps(s0, _mm256_mul_ps(s1, c4));
+ sum = _mm256_add_ps(sum, _mm256_mul_ps(s2, c6));
+ sum = _mm256_add_ps(sum, _mm256_mul_ps(s3, c4));
+ sum = _mm256_add_ps(sum, s4);
+ _mm256_storeu_ps(dst, sum);
+ src0 += 8;
+ src1 += 8;
+ src2 += 8;
+ src3 += 8;
+ src4 += 8;
+ dst += 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ GaussCol_F32_C(src0, src1, src2, src3, src4, dst, width);
+ }
+}
+#endif // HAS_GAUSSCOL_F32_AVX2
+
+#ifdef HAS_SCALEMAXSAMPLES_AVX2
+LIBYUV_TARGET_AVX2
+float ScaleMaxSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width) {
+ __m256 vmax = _mm256_setzero_ps();
+ const __m256 vscale = _mm256_set1_ps(scale);
+ while (width >= 8) {
+ __m256 v = _mm256_loadu_ps(src);
+ src += 8;
+ vmax = _mm256_max_ps(v, vmax);
+ _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
+ dst += 8;
+ width -= 8;
+ }
+ __m128 m128 =
+ _mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax));
+ m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128);
+ m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128);
+ float fmax = _mm_cvtss_f32(m128);
+ _mm256_zeroupper();
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ for (; width > 0; --width) {
+ float v = *src++;
+ float vs = v * scale;
+ fmax = (v > fmax) ? v : fmax;
+ *dst++ = vs;
+ }
+ return fmax;
+}
+#endif // HAS_SCALEMAXSAMPLES_AVX2
+
+#ifdef HAS_SCALESUMSAMPLES_AVX2
+LIBYUV_TARGET_AVX2
+float ScaleSumSamples_AVX2(const float* src,
+ float* dst,
+ float scale,
+ int width) {
+ __m256 vsum = _mm256_setzero_ps();
+ const __m256 vscale = _mm256_set1_ps(scale);
+ while (width >= 8) {
+ __m256 v = _mm256_loadu_ps(src);
+ src += 8;
+ vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v));
+ _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
+ dst += 8;
+ width -= 8;
+ }
+ __m128 s128 =
+ _mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1));
+ s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128));
+ s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1));
+ float fsum = _mm_cvtss_f32(s128);
+ _mm256_zeroupper();
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ for (; width > 0; --width) {
+ float v = *src++;
+ fsum += v * v;
+ *dst++ = v * scale;
+ }
+ return fsum;
+}
+#endif // HAS_SCALESUMSAMPLES_AVX2
+
+#ifdef HAS_SCALESAMPLES_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) {
+ const __m256 vscale = _mm256_set1_ps(scale);
+ while (width >= 8) {
+ __m256 v = _mm256_loadu_ps(src);
+ src += 8;
+ _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale));
+ dst += 8;
+ width -= 8;
+ }
+ _mm256_zeroupper();
+ if (width > 0) {
+ // TODO(fbarchard): Handle remainder with masking or SIMD Any function.
+ ScaleSamples_C(src, dst, scale, width);
+ }
+}
+#endif // HAS_SCALESAMPLES_AVX2
+
+#if defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2)
+LIBYUV_TARGET_AVX2
+static inline void ARGBToRGBRow_AVX2(const uint8_t* src,
+ uint8_t* dst,
+ int width,
+ const uint8_t* shuffler) {
+ static const uint32_t kPermdRGB24_AVX[8] = {0, 1, 2, 4, 5, 6, 3, 7};
+ const __m256i ymm_shuf =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler));
+ const __m256i ymm_perm =
+ _mm256_loadu_si256((const __m256i*)kPermdRGB24_AVX);
+ do {
+ __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src + 32));
+ __m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src + 64));
+ __m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src + 96));
+ src += 128;
+
+ ymm0 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm0, ymm_shuf),
+ ymm_perm);
+ ymm1 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm1, ymm_shuf),
+ ymm_perm);
+ ymm2 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm2, ymm_shuf),
+ ymm_perm);
+ ymm3 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm3, ymm_shuf),
+ ymm_perm);
+
+ _mm256_storeu_si256(
+ (__m256i*)dst,
+ _mm256_or_si256(ymm0, _mm256_permute4x64_epi64(ymm1, 0x3f)));
+ _mm256_storeu_si256(
+ (__m256i*)(dst + 32),
+ _mm256_or_si256(_mm256_permute4x64_epi64(ymm1, 0xf9),
+ _mm256_permute4x64_epi64(ymm2, 0x4f)));
+ _mm256_storeu_si256(
+ (__m256i*)(dst + 64),
+ _mm256_or_si256(_mm256_permute4x64_epi64(ymm2, 0xfe),
+ _mm256_permute4x64_epi64(ymm3, 0x93)));
+ dst += 96;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2)
+
+#ifdef HAS_ARGBTORGB24ROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRGB24[0]);
+}
+#endif // HAS_ARGBTORGB24ROW_AVX2
+
+#ifdef HAS_ARGBTORAWROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ static const uint8_t kShuffleMaskARGBToRAW_AVX2[16] = {
+ 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u,
+ 128u};
+ ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRAW_AVX2);
+}
+#endif // HAS_ARGBTORAWROW_AVX2
-#ifdef HAS_ARGBBLENDROW_SSSE3
-// Shuffle table for isolating alpha.
-static const uvec8 kShuffleAlpha = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80,
- 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80};
-
-// Blend 8 pixels at a time.
-__declspec(naked) void ARGBBlendRow_SSSE3(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- pcmpeqb xmm7, xmm7 // generate constant 0x0001
- psrlw xmm7, 15
- pcmpeqb xmm6, xmm6 // generate mask 0x00ff00ff
- psrlw xmm6, 8
- pcmpeqb xmm5, xmm5 // generate mask 0xff00ff00
- psllw xmm5, 8
- pcmpeqb xmm4, xmm4 // generate mask 0xff000000
- pslld xmm4, 24
- sub ecx, 4
- jl convertloop4b // less than 4 pixels?
-
- // 4 pixel loop.
- convertloop4:
- movdqu xmm3, [eax] // src argb
- lea eax, [eax + 16]
- movdqa xmm0, xmm3 // src argb
- pxor xmm3, xmm4 // ~alpha
- movdqu xmm2, [esi] // _r_b
- pshufb xmm3, xmmword ptr kShuffleAlpha // alpha
- pand xmm2, xmm6 // _r_b
- paddw xmm3, xmm7 // 256 - alpha
- pmullw xmm2, xmm3 // _r_b * alpha
- movdqu xmm1, [esi] // _a_g
- lea esi, [esi + 16]
- psrlw xmm1, 8 // _a_g
- por xmm0, xmm4 // set alpha to 255
- pmullw xmm1, xmm3 // _a_g * alpha
- psrlw xmm2, 8 // _r_b convert to 8 bits again
- paddusb xmm0, xmm2 // + src argb
- pand xmm1, xmm5 // a_g_ convert to 8 bits again
- paddusb xmm0, xmm1 // + src argb
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jge convertloop4
-
- convertloop4b:
- add ecx, 4 - 1
- jl convertloop1b
-
- // 1 pixel loop.
- convertloop1:
- movd xmm3, [eax] // src argb
- lea eax, [eax + 4]
- movdqa xmm0, xmm3 // src argb
- pxor xmm3, xmm4 // ~alpha
- movd xmm2, [esi] // _r_b
- pshufb xmm3, xmmword ptr kShuffleAlpha // alpha
- pand xmm2, xmm6 // _r_b
- paddw xmm3, xmm7 // 256 - alpha
- pmullw xmm2, xmm3 // _r_b * alpha
- movd xmm1, [esi] // _a_g
- lea esi, [esi + 4]
- psrlw xmm1, 8 // _a_g
- por xmm0, xmm4 // set alpha to 255
- pmullw xmm1, xmm3 // _a_g * alpha
- psrlw xmm2, 8 // _r_b convert to 8 bits again
- paddusb xmm0, xmm2 // + src argb
- pand xmm1, xmm5 // a_g_ convert to 8 bits again
- paddusb xmm0, xmm1 // + src argb
- movd [edx], xmm0
- lea edx, [edx + 4]
- sub ecx, 1
- jge convertloop1
-
- convertloop1b:
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBBLENDROW_SSSE3
-
-#ifdef HAS_ARGBATTENUATEROW_SSSE3
-// Shuffle table duplicating alpha.
-static const uvec8 kShuffleAlpha0 = {
- 3u, 3u, 3u, 3u, 3u, 3u, 128u, 128u, 7u, 7u, 7u, 7u, 7u, 7u, 128u, 128u,
-};
-static const uvec8 kShuffleAlpha1 = {
- 11u, 11u, 11u, 11u, 11u, 11u, 128u, 128u,
- 15u, 15u, 15u, 15u, 15u, 15u, 128u, 128u,
-};
-__declspec(naked) void ARGBAttenuateRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb0
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- pcmpeqb xmm3, xmm3 // generate mask 0xff000000
- pslld xmm3, 24
- movdqa xmm4, xmmword ptr kShuffleAlpha0
- movdqa xmm5, xmmword ptr kShuffleAlpha1
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels
- pshufb xmm0, xmm4 // isolate first 2 alphas
- movdqu xmm1, [eax] // read 4 pixels
- punpcklbw xmm1, xmm1 // first 2 pixel rgbs
- pmulhuw xmm0, xmm1 // rgb * a
- movdqu xmm1, [eax] // read 4 pixels
- pshufb xmm1, xmm5 // isolate next 2 alphas
- movdqu xmm2, [eax] // read 4 pixels
- punpckhbw xmm2, xmm2 // next 2 pixel rgbs
- pmulhuw xmm1, xmm2 // rgb * a
- movdqu xmm2, [eax] // mask original alpha
- lea eax, [eax + 16]
- pand xmm2, xmm3
- psrlw xmm0, 8
- psrlw xmm1, 8
- packuswb xmm0, xmm1
- por xmm0, xmm2 // copy original alpha
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
-
- ret
- }
-}
-#endif // HAS_ARGBATTENUATEROW_SSSE3
-
-#ifdef HAS_ARGBATTENUATEROW_AVX2
-// Shuffle table duplicating alpha.
-static const uvec8 kShuffleAlpha_AVX2 = {6u, 7u, 6u, 7u, 6u, 7u,
- 128u, 128u, 14u, 15u, 14u, 15u,
- 14u, 15u, 128u, 128u};
-__declspec(naked) void ARGBAttenuateRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb0
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- vbroadcastf128 ymm4, xmmword ptr kShuffleAlpha_AVX2
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xff000000
- vpslld ymm5, ymm5, 24
-
- convertloop:
- vmovdqu ymm6, [eax] // read 8 pixels.
- vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated.
- vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated.
- vpshufb ymm2, ymm0, ymm4 // low 4 alphas
- vpshufb ymm3, ymm1, ymm4 // high 4 alphas
- vpmulhuw ymm0, ymm0, ymm2 // rgb * a
- vpmulhuw ymm1, ymm1, ymm3 // rgb * a
- vpand ymm6, ymm6, ymm5 // isolate alpha
- vpsrlw ymm0, ymm0, 8
- vpsrlw ymm1, ymm1, 8
- vpackuswb ymm0, ymm0, ymm1 // unmutated.
- vpor ymm0, ymm0, ymm6 // copy original alpha
- vmovdqu [eax + edx], ymm0
- lea eax, [eax + 32]
- sub ecx, 8
- jg convertloop
-
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBATTENUATEROW_AVX2
-
-#ifdef HAS_ARGBUNATTENUATEROW_SSE2
-// Unattenuate 4 pixels at a time.
-__declspec(naked) void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push ebx
- push esi
- push edi
- mov eax, [esp + 12 + 4] // src_argb
- mov edx, [esp + 12 + 8] // dst_argb
- mov ecx, [esp + 12 + 12] // width
- lea ebx, fixed_invtbl8
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels
- movzx esi, byte ptr [eax + 3] // first alpha
- movzx edi, byte ptr [eax + 7] // second alpha
- punpcklbw xmm0, xmm0 // first 2
- movd xmm2, dword ptr [ebx + esi * 4]
- movd xmm3, dword ptr [ebx + edi * 4]
- pshuflw xmm2, xmm2, 040h // first 4 inv_alpha words. 1, a, a, a
- pshuflw xmm3, xmm3, 040h // next 4 inv_alpha words
- movlhps xmm2, xmm3
- pmulhuw xmm0, xmm2 // rgb * a
-
- movdqu xmm1, [eax] // read 4 pixels
- movzx esi, byte ptr [eax + 11] // third alpha
- movzx edi, byte ptr [eax + 15] // forth alpha
- punpckhbw xmm1, xmm1 // next 2
- movd xmm2, dword ptr [ebx + esi * 4]
- movd xmm3, dword ptr [ebx + edi * 4]
- pshuflw xmm2, xmm2, 040h // first 4 inv_alpha words
- pshuflw xmm3, xmm3, 040h // next 4 inv_alpha words
- movlhps xmm2, xmm3
- pmulhuw xmm1, xmm2 // rgb * a
- lea eax, [eax + 16]
- packuswb xmm0, xmm1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
-
- pop edi
- pop esi
- pop ebx
- ret
- }
-}
-#endif // HAS_ARGBUNATTENUATEROW_SSE2
+#ifdef HAS_SPLITUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void SplitUVRow_AVX2(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
+ __m256i u = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_and_si256(uv0, mask_00ff),
+ _mm256_and_si256(uv1, mask_00ff)),
+ 0xd8);
+ __m256i v = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_srli_epi16(uv0, 8),
+ _mm256_srli_epi16(uv1, 8)),
+ 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_u, u);
+ _mm256_storeu_si256((__m256i*)dst_v, v);
+ src_uv += 64;
+ dst_u += 32;
+ dst_v += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITUVROW_AVX2
-#ifdef HAS_ARGBUNATTENUATEROW_AVX2
-// Shuffle table duplicating alpha.
-static const uvec8 kUnattenShuffleAlpha_AVX2 = {
- 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u};
-// TODO(fbarchard): Enable USE_GATHER for future hardware if faster.
-// USE_GATHER is not on by default, due to being a slow instruction.
-#ifdef USE_GATHER
-__declspec(naked) void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb0
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- sub edx, eax
- vbroadcastf128 ymm4, xmmword ptr kUnattenShuffleAlpha_AVX2
-
- convertloop:
- vmovdqu ymm6, [eax] // read 8 pixels.
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0xffffffff for gather.
- vpsrld ymm2, ymm6, 24 // alpha in low 8 bits.
- vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated.
- vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated.
- vpgatherdd ymm3, [ymm2 * 4 + fixed_invtbl8], ymm5 // ymm5 cleared. 1, a
- vpunpcklwd ymm2, ymm3, ymm3 // low 4 inverted alphas. mutated. 1, 1, a, a
- vpunpckhwd ymm3, ymm3, ymm3 // high 4 inverted alphas. mutated.
- vpshufb ymm2, ymm2, ymm4 // replicate low 4 alphas. 1, a, a, a
- vpshufb ymm3, ymm3, ymm4 // replicate high 4 alphas
- vpmulhuw ymm0, ymm0, ymm2 // rgb * ia
- vpmulhuw ymm1, ymm1, ymm3 // rgb * ia
- vpackuswb ymm0, ymm0, ymm1 // unmutated.
- vmovdqu [eax + edx], ymm0
- lea eax, [eax + 32]
- sub ecx, 8
- jg convertloop
-
- vzeroupper
- ret
- }
-}
-#else // USE_GATHER
-__declspec(naked) void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
-
- push ebx
- push esi
- push edi
- mov eax, [esp + 12 + 4] // src_argb
- mov edx, [esp + 12 + 8] // dst_argb
- mov ecx, [esp + 12 + 12] // width
- sub edx, eax
- lea ebx, fixed_invtbl8
- vbroadcastf128 ymm5, xmmword ptr kUnattenShuffleAlpha_AVX2
-
- convertloop:
- // replace VPGATHER
- movzx esi, byte ptr [eax + 3] // alpha0
- movzx edi, byte ptr [eax + 7] // alpha1
- vmovd xmm0, dword ptr [ebx + esi * 4] // [1,a0]
- vmovd xmm1, dword ptr [ebx + edi * 4] // [1,a1]
- movzx esi, byte ptr [eax + 11] // alpha2
- movzx edi, byte ptr [eax + 15] // alpha3
- vpunpckldq xmm6, xmm0, xmm1 // [1,a1,1,a0]
- vmovd xmm2, dword ptr [ebx + esi * 4] // [1,a2]
- vmovd xmm3, dword ptr [ebx + edi * 4] // [1,a3]
- movzx esi, byte ptr [eax + 19] // alpha4
- movzx edi, byte ptr [eax + 23] // alpha5
- vpunpckldq xmm7, xmm2, xmm3 // [1,a3,1,a2]
- vmovd xmm0, dword ptr [ebx + esi * 4] // [1,a4]
- vmovd xmm1, dword ptr [ebx + edi * 4] // [1,a5]
- movzx esi, byte ptr [eax + 27] // alpha6
- movzx edi, byte ptr [eax + 31] // alpha7
- vpunpckldq xmm0, xmm0, xmm1 // [1,a5,1,a4]
- vmovd xmm2, dword ptr [ebx + esi * 4] // [1,a6]
- vmovd xmm3, dword ptr [ebx + edi * 4] // [1,a7]
- vpunpckldq xmm2, xmm2, xmm3 // [1,a7,1,a6]
- vpunpcklqdq xmm3, xmm6, xmm7 // [1,a3,1,a2,1,a1,1,a0]
- vpunpcklqdq xmm0, xmm0, xmm2 // [1,a7,1,a6,1,a5,1,a4]
- vinserti128 ymm3, ymm3, xmm0, 1 // [1,a7,1,a6,1,a5,1,a4,1,a3,1,a2,1,a1,1,a0]
- // end of VPGATHER
-
- vmovdqu ymm6, [eax] // read 8 pixels.
- vpunpcklbw ymm0, ymm6, ymm6 // low 4 pixels. mutated.
- vpunpckhbw ymm1, ymm6, ymm6 // high 4 pixels. mutated.
- vpunpcklwd ymm2, ymm3, ymm3 // low 4 inverted alphas. mutated. 1, 1, a, a
- vpunpckhwd ymm3, ymm3, ymm3 // high 4 inverted alphas. mutated.
- vpshufb ymm2, ymm2, ymm5 // replicate low 4 alphas. 1, a, a, a
- vpshufb ymm3, ymm3, ymm5 // replicate high 4 alphas
- vpmulhuw ymm0, ymm0, ymm2 // rgb * ia
- vpmulhuw ymm1, ymm1, ymm3 // rgb * ia
- vpackuswb ymm0, ymm0, ymm1 // unmutated.
- vmovdqu [eax + edx], ymm0
- lea eax, [eax + 32]
- sub ecx, 8
- jg convertloop
-
- pop edi
- pop esi
- pop ebx
- vzeroupper
- ret
- }
-}
-#endif // USE_GATHER
-#endif // HAS_ARGBATTENUATEROW_AVX2
-
-#ifdef HAS_ARGBGRAYROW_SSSE3
-// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels.
-__declspec(naked) void ARGBGrayRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_argb */
- mov ecx, [esp + 12] /* width */
- movdqa xmm4, xmmword ptr kARGBToYJ
- movdqa xmm5, xmmword ptr kAddYJ64
-
- convertloop:
- movdqu xmm0, [eax] // G
- movdqu xmm1, [eax + 16]
- pmaddubsw xmm0, xmm4
- pmaddubsw xmm1, xmm4
- phaddw xmm0, xmm1
- paddw xmm0, xmm5 // Add .5 for rounding.
- psrlw xmm0, 7
- packuswb xmm0, xmm0 // 8 G bytes
- movdqu xmm2, [eax] // A
- movdqu xmm3, [eax + 16]
- lea eax, [eax + 32]
- psrld xmm2, 24
- psrld xmm3, 24
- packuswb xmm2, xmm3
- packuswb xmm2, xmm2 // 8 A bytes
- movdqa xmm3, xmm0 // Weave into GG, GA, then GGGA
- punpcklbw xmm0, xmm0 // 8 GG words
- punpcklbw xmm3, xmm2 // 8 GA words
- movdqa xmm1, xmm0
- punpcklwd xmm0, xmm3 // GGGA first 4
- punpckhwd xmm1, xmm3 // GGGA next 4
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-#endif // HAS_ARGBGRAYROW_SSSE3
-
-#ifdef HAS_ARGBSEPIAROW_SSSE3
-// b = (r * 35 + g * 68 + b * 17) >> 7
-// g = (r * 45 + g * 88 + b * 22) >> 7
-// r = (r * 50 + g * 98 + b * 24) >> 7
-// Constant for ARGB color to sepia tone.
-static const vec8 kARGBToSepiaB = {17, 68, 35, 0, 17, 68, 35, 0,
- 17, 68, 35, 0, 17, 68, 35, 0};
-
-static const vec8 kARGBToSepiaG = {22, 88, 45, 0, 22, 88, 45, 0,
- 22, 88, 45, 0, 22, 88, 45, 0};
-
-static const vec8 kARGBToSepiaR = {24, 98, 50, 0, 24, 98, 50, 0,
- 24, 98, 50, 0, 24, 98, 50, 0};
-
-// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels.
-__declspec(naked) void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width) {
- __asm {
- mov eax, [esp + 4] /* dst_argb */
- mov ecx, [esp + 8] /* width */
- movdqa xmm2, xmmword ptr kARGBToSepiaB
- movdqa xmm3, xmmword ptr kARGBToSepiaG
- movdqa xmm4, xmmword ptr kARGBToSepiaR
-
- convertloop:
- movdqu xmm0, [eax] // B
- movdqu xmm6, [eax + 16]
- pmaddubsw xmm0, xmm2
- pmaddubsw xmm6, xmm2
- phaddw xmm0, xmm6
- psrlw xmm0, 7
- packuswb xmm0, xmm0 // 8 B values
- movdqu xmm5, [eax] // G
- movdqu xmm1, [eax + 16]
- pmaddubsw xmm5, xmm3
- pmaddubsw xmm1, xmm3
- phaddw xmm5, xmm1
- psrlw xmm5, 7
- packuswb xmm5, xmm5 // 8 G values
- punpcklbw xmm0, xmm5 // 8 BG values
- movdqu xmm5, [eax] // R
- movdqu xmm1, [eax + 16]
- pmaddubsw xmm5, xmm4
- pmaddubsw xmm1, xmm4
- phaddw xmm5, xmm1
- psrlw xmm5, 7
- packuswb xmm5, xmm5 // 8 R values
- movdqu xmm6, [eax] // A
- movdqu xmm1, [eax + 16]
- psrld xmm6, 24
- psrld xmm1, 24
- packuswb xmm6, xmm1
- packuswb xmm6, xmm6 // 8 A values
- punpcklbw xmm5, xmm6 // 8 RA values
- movdqa xmm1, xmm0 // Weave BG, RA together
- punpcklwd xmm0, xmm5 // BGRA first 4
- punpckhwd xmm1, xmm5 // BGRA next 4
- movdqu [eax], xmm0
- movdqu [eax + 16], xmm1
- lea eax, [eax + 32]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-#endif // HAS_ARGBSEPIAROW_SSSE3
-
-#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3
-// Tranform 8 ARGB pixels (32 bytes) with color matrix.
-// Same as Sepia except matrix is provided.
-// TODO(fbarchard): packuswbs only use half of the reg. To make RGBA, combine R
-// and B into a high and low, then G/A, unpackl/hbw and then unpckl/hwd.
-__declspec(naked) void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const int8_t* matrix_argb,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_argb */
- mov ecx, [esp + 12] /* matrix_argb */
- movdqu xmm5, [ecx]
- pshufd xmm2, xmm5, 0x00
- pshufd xmm3, xmm5, 0x55
- pshufd xmm4, xmm5, 0xaa
- pshufd xmm5, xmm5, 0xff
- mov ecx, [esp + 16] /* width */
-
- convertloop:
- movdqu xmm0, [eax] // B
- movdqu xmm7, [eax + 16]
- pmaddubsw xmm0, xmm2
- pmaddubsw xmm7, xmm2
- movdqu xmm6, [eax] // G
- movdqu xmm1, [eax + 16]
- pmaddubsw xmm6, xmm3
- pmaddubsw xmm1, xmm3
- phaddsw xmm0, xmm7 // B
- phaddsw xmm6, xmm1 // G
- psraw xmm0, 6 // B
- psraw xmm6, 6 // G
- packuswb xmm0, xmm0 // 8 B values
- packuswb xmm6, xmm6 // 8 G values
- punpcklbw xmm0, xmm6 // 8 BG values
- movdqu xmm1, [eax] // R
- movdqu xmm7, [eax + 16]
- pmaddubsw xmm1, xmm4
- pmaddubsw xmm7, xmm4
- phaddsw xmm1, xmm7 // R
- movdqu xmm6, [eax] // A
- movdqu xmm7, [eax + 16]
- pmaddubsw xmm6, xmm5
- pmaddubsw xmm7, xmm5
- phaddsw xmm6, xmm7 // A
- psraw xmm1, 6 // R
- psraw xmm6, 6 // A
- packuswb xmm1, xmm1 // 8 R values
- packuswb xmm6, xmm6 // 8 A values
- punpcklbw xmm1, xmm6 // 8 RA values
- movdqa xmm6, xmm0 // Weave BG, RA together
- punpcklwd xmm0, xmm1 // BGRA first 4
- punpckhwd xmm6, xmm1 // BGRA next 4
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm6
- lea eax, [eax + 32]
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-#endif // HAS_ARGBCOLORMATRIXROW_SSSE3
-
-#ifdef HAS_ARGBQUANTIZEROW_SSE2
-// Quantize 4 ARGB pixels (16 bytes).
-__declspec(naked) void ARGBQuantizeRow_SSE2(uint8_t* dst_argb,
- int scale,
- int interval_size,
- int interval_offset,
- int width) {
- __asm {
- mov eax, [esp + 4] /* dst_argb */
- movd xmm2, [esp + 8] /* scale */
- movd xmm3, [esp + 12] /* interval_size */
- movd xmm4, [esp + 16] /* interval_offset */
- mov ecx, [esp + 20] /* width */
- pshuflw xmm2, xmm2, 040h
- pshufd xmm2, xmm2, 044h
- pshuflw xmm3, xmm3, 040h
- pshufd xmm3, xmm3, 044h
- pshuflw xmm4, xmm4, 040h
- pshufd xmm4, xmm4, 044h
- pxor xmm5, xmm5 // constant 0
- pcmpeqb xmm6, xmm6 // generate mask 0xff000000
- pslld xmm6, 24
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels
- punpcklbw xmm0, xmm5 // first 2 pixels
- pmulhuw xmm0, xmm2 // pixel * scale >> 16
- movdqu xmm1, [eax] // read 4 pixels
- punpckhbw xmm1, xmm5 // next 2 pixels
- pmulhuw xmm1, xmm2
- pmullw xmm0, xmm3 // * interval_size
- movdqu xmm7, [eax] // read 4 pixels
- pmullw xmm1, xmm3
- pand xmm7, xmm6 // mask alpha
- paddw xmm0, xmm4 // + interval_size / 2
- paddw xmm1, xmm4
- packuswb xmm0, xmm1
- por xmm0, xmm7
- movdqu [eax], xmm0
- lea eax, [eax + 16]
- sub ecx, 4
- jg convertloop
- ret
- }
-}
-#endif // HAS_ARGBQUANTIZEROW_SSE2
-
-#ifdef HAS_ARGBSHADEROW_SSE2
-// Shade 4 pixels at a time by specified value.
-__declspec(naked) void ARGBShadeRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- uint32_t value) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // width
- movd xmm2, [esp + 16] // value
- punpcklbw xmm2, xmm2
- punpcklqdq xmm2, xmm2
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels
- lea eax, [eax + 16]
- movdqa xmm1, xmm0
- punpcklbw xmm0, xmm0 // first 2
- punpckhbw xmm1, xmm1 // next 2
- pmulhuw xmm0, xmm2 // argb * value
- pmulhuw xmm1, xmm2 // argb * value
- psrlw xmm0, 8
- psrlw xmm1, 8
- packuswb xmm0, xmm1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
-
- ret
- }
-}
-#endif // HAS_ARGBSHADEROW_SSE2
-
-#ifdef HAS_ARGBMULTIPLYROW_SSE2
-// Multiply 2 rows of ARGB pixels together, 4 pixels at a time.
-__declspec(naked) void ARGBMultiplyRow_SSE2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- pxor xmm5, xmm5 // constant 0
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels from src_argb0
- movdqu xmm2, [esi] // read 4 pixels from src_argb1
- movdqu xmm1, xmm0
- movdqu xmm3, xmm2
- punpcklbw xmm0, xmm0 // first 2
- punpckhbw xmm1, xmm1 // next 2
- punpcklbw xmm2, xmm5 // first 2
- punpckhbw xmm3, xmm5 // next 2
- pmulhuw xmm0, xmm2 // src_argb0 * src_argb1 first 2
- pmulhuw xmm1, xmm3 // src_argb0 * src_argb1 next 2
- lea eax, [eax + 16]
- lea esi, [esi + 16]
- packuswb xmm0, xmm1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBMULTIPLYROW_SSE2
-
-#ifdef HAS_ARGBADDROW_SSE2
-// Add 2 rows of ARGB pixels together, 4 pixels at a time.
-// TODO(fbarchard): Port this to posix, neon and other math functions.
-__declspec(naked) void ARGBAddRow_SSE2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
-
- sub ecx, 4
- jl convertloop49
-
- convertloop4:
- movdqu xmm0, [eax] // read 4 pixels from src_argb0
- lea eax, [eax + 16]
- movdqu xmm1, [esi] // read 4 pixels from src_argb1
- lea esi, [esi + 16]
- paddusb xmm0, xmm1 // src_argb0 + src_argb1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jge convertloop4
-
- convertloop49:
- add ecx, 4 - 1
- jl convertloop19
-
- convertloop1:
- movd xmm0, [eax] // read 1 pixels from src_argb0
- lea eax, [eax + 4]
- movd xmm1, [esi] // read 1 pixels from src_argb1
- lea esi, [esi + 4]
- paddusb xmm0, xmm1 // src_argb0 + src_argb1
- movd [edx], xmm0
- lea edx, [edx + 4]
- sub ecx, 1
- jge convertloop1
-
- convertloop19:
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBADDROW_SSE2
-
-#ifdef HAS_ARGBSUBTRACTROW_SSE2
-// Subtract 2 rows of ARGB pixels together, 4 pixels at a time.
-__declspec(naked) void ARGBSubtractRow_SSE2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
-
- convertloop:
- movdqu xmm0, [eax] // read 4 pixels from src_argb0
- lea eax, [eax + 16]
- movdqu xmm1, [esi] // read 4 pixels from src_argb1
- lea esi, [esi + 16]
- psubusb xmm0, xmm1 // src_argb0 - src_argb1
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBSUBTRACTROW_SSE2
-
-#ifdef HAS_ARGBMULTIPLYROW_AVX2
-// Multiply 2 rows of ARGB pixels together, 8 pixels at a time.
-__declspec(naked) void ARGBMultiplyRow_AVX2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- vpxor ymm5, ymm5, ymm5 // constant 0
-
- convertloop:
- vmovdqu ymm1, [eax] // read 8 pixels from src_argb0
- lea eax, [eax + 32]
- vmovdqu ymm3, [esi] // read 8 pixels from src_argb1
- lea esi, [esi + 32]
- vpunpcklbw ymm0, ymm1, ymm1 // low 4
- vpunpckhbw ymm1, ymm1, ymm1 // high 4
- vpunpcklbw ymm2, ymm3, ymm5 // low 4
- vpunpckhbw ymm3, ymm3, ymm5 // high 4
- vpmulhuw ymm0, ymm0, ymm2 // src_argb0 * src_argb1 low 4
- vpmulhuw ymm1, ymm1, ymm3 // src_argb0 * src_argb1 high 4
- vpackuswb ymm0, ymm0, ymm1
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
-
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_ARGBMULTIPLYROW_AVX2
+#ifdef HAS_AR64TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void AR64ToARGBRow_AVX2(const uint16_t* src_ar64,
+ uint8_t* dst_argb,
+ int width) {
+ do {
+ __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ar64),
+ 8);
+ __m256i s1 = _mm256_srli_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ar64 + 16)), 8);
+ __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ src_ar64 += 32;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_AR64TOARGBROW_AVX2
+
+#ifdef HAS_AB64TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u,
+ 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
+ const __m256i shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufSwapRB));
+ do {
+ __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ab64),
+ 8);
+ __m256i s1 = _mm256_srli_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ab64 + 16)), 8);
+ __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
+ out = _mm256_shuffle_epi8(out, shuf);
+ _mm256_storeu_si256((__m256i*)dst_argb, out);
+ src_ab64 += 32;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_AB64TOARGBROW_AVX2
+
+#ifdef HAS_ARGBTOAR64ROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToAR64Row_AVX2(const uint8_t* src_argb,
+ uint16_t* dst_ar64,
+ int width) {
+ do {
+ __m256i s = _mm256_permute4x64_epi64(
+ _mm256_loadu_si256((const __m256i*)src_argb), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ar64, _mm256_unpacklo_epi8(s, s));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), _mm256_unpackhi_epi8(s, s));
+ src_argb += 32;
+ dst_ar64 += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOAR64ROW_AVX2
+
+#ifdef HAS_ARGBTOAB64ROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBToAB64Row_AVX2(const uint8_t* src_argb,
+ uint16_t* dst_ab64,
+ int width) {
+ static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u,
+ 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u};
+ const __m256i shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufSwapRB));
+ do {
+ __m256i s = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)src_argb), shuf);
+ s = _mm256_permute4x64_epi64(s, 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ab64, _mm256_unpacklo_epi8(s, s));
+ _mm256_storeu_si256((__m256i*)(dst_ab64 + 16), _mm256_unpackhi_epi8(s, s));
+ src_argb += 32;
+ dst_ab64 += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBTOAB64ROW_AVX2
#ifdef HAS_ARGBADDROW_AVX2
-// Add 2 rows of ARGB pixels together, 8 pixels at a time.
-__declspec(naked) void ARGBAddRow_AVX2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
-
- convertloop:
- vmovdqu ymm0, [eax] // read 8 pixels from src_argb0
- lea eax, [eax + 32]
- vpaddusb ymm0, ymm0, [esi] // add 8 pixels from src_argb1
- lea esi, [esi + 32]
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
-
- pop esi
- vzeroupper
- ret
- }
+LIBYUV_TARGET_AVX2
+void ARGBAddRow_AVX2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ do {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_adds_epu8(s0, s1));
+ src_argb += 32;
+ src_argb1 += 32;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
}
#endif // HAS_ARGBADDROW_AVX2
#ifdef HAS_ARGBSUBTRACTROW_AVX2
-// Subtract 2 rows of ARGB pixels together, 8 pixels at a time.
-__declspec(naked) void ARGBSubtractRow_AVX2(const uint8_t* src_argb0,
- const uint8_t* src_argb1,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb0
- mov esi, [esp + 4 + 8] // src_argb1
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
-
- convertloop:
- vmovdqu ymm0, [eax] // read 8 pixels from src_argb0
- lea eax, [eax + 32]
- vpsubusb ymm0, ymm0, [esi] // src_argb0 - src_argb1
- lea esi, [esi + 32]
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 8
- jg convertloop
-
- pop esi
- vzeroupper
- ret
- }
+LIBYUV_TARGET_AVX2
+void ARGBSubtractRow_AVX2(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ do {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1);
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_subs_epu8(s0, s1));
+ src_argb += 32;
+ src_argb1 += 32;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
}
#endif // HAS_ARGBSUBTRACTROW_AVX2
-#ifdef HAS_SOBELXROW_SSE2
-// SobelX as a matrix is
-// -1 0 1
-// -2 0 2
-// -1 0 1
-__declspec(naked) void SobelXRow_SSE2(const uint8_t* src_y0,
- const uint8_t* src_y1,
- const uint8_t* src_y2,
- uint8_t* dst_sobelx,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_y0
- mov esi, [esp + 8 + 8] // src_y1
- mov edi, [esp + 8 + 12] // src_y2
- mov edx, [esp + 8 + 16] // dst_sobelx
- mov ecx, [esp + 8 + 20] // width
- sub esi, eax
- sub edi, eax
- sub edx, eax
- pxor xmm5, xmm5 // constant 0
-
- convertloop:
- movq xmm0, qword ptr [eax] // read 8 pixels from src_y0[0]
- movq xmm1, qword ptr [eax + 2] // read 8 pixels from src_y0[2]
- punpcklbw xmm0, xmm5
- punpcklbw xmm1, xmm5
- psubw xmm0, xmm1
- movq xmm1, qword ptr [eax + esi] // read 8 pixels from src_y1[0]
- movq xmm2, qword ptr [eax + esi + 2] // read 8 pixels from src_y1[2]
- punpcklbw xmm1, xmm5
- punpcklbw xmm2, xmm5
- psubw xmm1, xmm2
- movq xmm2, qword ptr [eax + edi] // read 8 pixels from src_y2[0]
- movq xmm3, qword ptr [eax + edi + 2] // read 8 pixels from src_y2[2]
- punpcklbw xmm2, xmm5
- punpcklbw xmm3, xmm5
- psubw xmm2, xmm3
- paddw xmm0, xmm2
- paddw xmm0, xmm1
- paddw xmm0, xmm1
- pxor xmm1, xmm1 // abs = max(xmm0, -xmm0). SSSE3 could use pabsw
- psubw xmm1, xmm0
- pmaxsw xmm0, xmm1
- packuswb xmm0, xmm0
- movq qword ptr [eax + edx], xmm0
- lea eax, [eax + 8]
- sub ecx, 8
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-#endif // HAS_SOBELXROW_SSE2
-
-#ifdef HAS_SOBELYROW_SSE2
-// SobelY as a matrix is
-// -1 -2 -1
-// 0 0 0
-// 1 2 1
-__declspec(naked) void SobelYRow_SSE2(const uint8_t* src_y0,
- const uint8_t* src_y1,
- uint8_t* dst_sobely,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_y0
- mov esi, [esp + 4 + 8] // src_y1
- mov edx, [esp + 4 + 12] // dst_sobely
- mov ecx, [esp + 4 + 16] // width
- sub esi, eax
- sub edx, eax
- pxor xmm5, xmm5 // constant 0
-
- convertloop:
- movq xmm0, qword ptr [eax] // read 8 pixels from src_y0[0]
- movq xmm1, qword ptr [eax + esi] // read 8 pixels from src_y1[0]
- punpcklbw xmm0, xmm5
- punpcklbw xmm1, xmm5
- psubw xmm0, xmm1
- movq xmm1, qword ptr [eax + 1] // read 8 pixels from src_y0[1]
- movq xmm2, qword ptr [eax + esi + 1] // read 8 pixels from src_y1[1]
- punpcklbw xmm1, xmm5
- punpcklbw xmm2, xmm5
- psubw xmm1, xmm2
- movq xmm2, qword ptr [eax + 2] // read 8 pixels from src_y0[2]
- movq xmm3, qword ptr [eax + esi + 2] // read 8 pixels from src_y1[2]
- punpcklbw xmm2, xmm5
- punpcklbw xmm3, xmm5
- psubw xmm2, xmm3
- paddw xmm0, xmm2
- paddw xmm0, xmm1
- paddw xmm0, xmm1
- pxor xmm1, xmm1 // abs = max(xmm0, -xmm0). SSSE3 could use pabsw
- psubw xmm1, xmm0
- pmaxsw xmm0, xmm1
- packuswb xmm0, xmm0
- movq qword ptr [eax + edx], xmm0
- lea eax, [eax + 8]
- sub ecx, 8
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_SOBELYROW_SSE2
-
-#ifdef HAS_SOBELROW_SSE2
-// Adds Sobel X and Sobel Y and stores Sobel into ARGB.
-// A = 255
-// R = Sobel
-// G = Sobel
-// B = Sobel
-__declspec(naked) void SobelRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_sobelx
- mov esi, [esp + 4 + 8] // src_sobely
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- sub esi, eax
- pcmpeqb xmm5, xmm5 // alpha 255
- pslld xmm5, 24 // 0xff000000
-
- convertloop:
- movdqu xmm0, [eax] // read 16 pixels src_sobelx
- movdqu xmm1, [eax + esi] // read 16 pixels src_sobely
- lea eax, [eax + 16]
- paddusb xmm0, xmm1 // sobel = sobelx + sobely
- movdqa xmm2, xmm0 // GG
- punpcklbw xmm2, xmm0 // First 8
- punpckhbw xmm0, xmm0 // Next 8
- movdqa xmm1, xmm2 // GGGG
- punpcklwd xmm1, xmm2 // First 4
- punpckhwd xmm2, xmm2 // Next 4
- por xmm1, xmm5 // GGGA
- por xmm2, xmm5
- movdqa xmm3, xmm0 // GGGG
- punpcklwd xmm3, xmm0 // Next 4
- punpckhwd xmm0, xmm0 // Last 4
- por xmm3, xmm5 // GGGA
- por xmm0, xmm5
- movdqu [edx], xmm1
- movdqu [edx + 16], xmm2
- movdqu [edx + 32], xmm3
- movdqu [edx + 48], xmm0
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_SOBELROW_SSE2
-
-#ifdef HAS_SOBELTOPLANEROW_SSE2
-// Adds Sobel X and Sobel Y and stores Sobel into a plane.
-__declspec(naked) void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_y,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_sobelx
- mov esi, [esp + 4 + 8] // src_sobely
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- sub esi, eax
-
- convertloop:
- movdqu xmm0, [eax] // read 16 pixels src_sobelx
- movdqu xmm1, [eax + esi] // read 16 pixels src_sobely
- lea eax, [eax + 16]
- paddusb xmm0, xmm1 // sobel = sobelx + sobely
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_SOBELTOPLANEROW_SSE2
-
-#ifdef HAS_SOBELXYROW_SSE2
-// Mixes Sobel X, Sobel Y and Sobel into ARGB.
-// A = 255
-// R = Sobel X
-// G = Sobel
-// B = Sobel Y
-__declspec(naked) void SobelXYRow_SSE2(const uint8_t* src_sobelx,
- const uint8_t* src_sobely,
- uint8_t* dst_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_sobelx
- mov esi, [esp + 4 + 8] // src_sobely
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // width
- sub esi, eax
- pcmpeqb xmm5, xmm5 // alpha 255
-
- convertloop:
- movdqu xmm0, [eax] // read 16 pixels src_sobelx
- movdqu xmm1, [eax + esi] // read 16 pixels src_sobely
- lea eax, [eax + 16]
- movdqa xmm2, xmm0
- paddusb xmm2, xmm1 // sobel = sobelx + sobely
- movdqa xmm3, xmm0 // XA
- punpcklbw xmm3, xmm5
- punpckhbw xmm0, xmm5
- movdqa xmm4, xmm1 // YS
- punpcklbw xmm4, xmm2
- punpckhbw xmm1, xmm2
- movdqa xmm6, xmm4 // YSXA
- punpcklwd xmm6, xmm3 // First 4
- punpckhwd xmm4, xmm3 // Next 4
- movdqa xmm7, xmm1 // YSXA
- punpcklwd xmm7, xmm0 // Next 4
- punpckhwd xmm1, xmm0 // Last 4
- movdqu [edx], xmm6
- movdqu [edx + 16], xmm4
- movdqu [edx + 32], xmm7
- movdqu [edx + 48], xmm1
- lea edx, [edx + 64]
- sub ecx, 16
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_SOBELXYROW_SSE2
-
-#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2
-// Consider float CumulativeSum.
-// Consider calling CumulativeSum one row at time as needed.
-// Consider circular CumulativeSum buffer of radius * 2 + 1 height.
-// Convert cumulative sum for an area to an average for 1 pixel.
-// topleft is pointer to top left of CumulativeSum buffer for area.
-// botleft is pointer to bottom left of CumulativeSum buffer.
-// width is offset from left to right of area in CumulativeSum buffer measured
-// in number of ints.
-// area is the number of pixels in the area being averaged.
-// dst points to pixel to store result to.
-// count is number of averaged pixels to produce.
-// Does 4 pixels at a time.
-// This function requires alignment on accumulation buffer pointers.
-void CumulativeSumToAverageRow_SSE2(const int32_t* topleft,
+#ifdef HAS_ARGBCOPYALPHAROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
+ const __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000u);
+ do {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 32));
+ __m256i d0 = _mm256_loadu_si256((const __m256i*)dst);
+ __m256i d1 = _mm256_loadu_si256((const __m256i*)(dst + 32));
+ _mm256_storeu_si256((__m256i*)dst, _mm256_blendv_epi8(d0, s0, mask_alpha));
+ _mm256_storeu_si256((__m256i*)(dst + 32),
+ _mm256_blendv_epi8(d1, s1, mask_alpha));
+ src += 64;
+ dst += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBCOPYALPHAROW_AVX2
+
+#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const float* poly,
+ int width) {
+ const __m256 c0 = _mm256_broadcast_ps((const __m128*)(poly + 0));
+ const __m256 c1 = _mm256_broadcast_ps((const __m128*)(poly + 4));
+ const __m256 c2 = _mm256_broadcast_ps((const __m128*)(poly + 8));
+ const __m256 c3 = _mm256_broadcast_ps((const __m128*)(poly + 12));
+ do {
+ __m128i s8 = _mm_loadl_epi64((const __m128i*)src_argb);
+ src_argb += 8;
+ __m256 x = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(s8));
+ __m256 x2 = _mm256_mul_ps(x, x);
+ __m256 res = _mm256_add_ps(c0, _mm256_mul_ps(c1, x));
+ res = _mm256_add_ps(res, _mm256_mul_ps(c2, x2));
+ res = _mm256_add_ps(res, _mm256_mul_ps(_mm256_mul_ps(c3, x), x2));
+ __m256i i32 = _mm256_cvttps_epi32(res);
+ __m256i u16 = _mm256_permute4x64_epi64(_mm256_packs_epi32(i32, i32), 0xd8);
+ __m128i u8 = _mm_packus_epi16(_mm256_castsi256_si128(u16),
+ _mm256_castsi256_si128(u16));
+ _mm_storel_epi64((__m128i*)dst_argb, u8);
+ dst_argb += 8;
+ width -= 2;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBPOLYNOMIALROW_AVX2
+
+#ifdef HAS_ARGBUNATTENUATEROW_AVX2
+LIBYUV_TARGET_AVX2
+void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ static const uint8_t kUnattenShuffleAlpha[16] = {
+ 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u};
+ const __m256i ymm_shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kUnattenShuffleAlpha));
+ do {
+ __m256i ymm3 = _mm256_setr_epi32(
+ (int)fixed_invtbl8[src_argb[3]], (int)fixed_invtbl8[src_argb[7]],
+ (int)fixed_invtbl8[src_argb[11]], (int)fixed_invtbl8[src_argb[15]],
+ (int)fixed_invtbl8[src_argb[19]], (int)fixed_invtbl8[src_argb[23]],
+ (int)fixed_invtbl8[src_argb[27]], (int)fixed_invtbl8[src_argb[31]]);
+ __m256i ymm6 = _mm256_loadu_si256((const __m256i*)src_argb);
+ __m256i ymm0 = _mm256_unpacklo_epi8(ymm6, ymm6);
+ __m256i ymm1 = _mm256_unpackhi_epi8(ymm6, ymm6);
+ __m256i ymm2 =
+ _mm256_shuffle_epi8(_mm256_unpacklo_epi16(ymm3, ymm3), ymm_shuf);
+ ymm3 = _mm256_shuffle_epi8(_mm256_unpackhi_epi16(ymm3, ymm3), ymm_shuf);
+ ymm0 = _mm256_mulhi_epu16(ymm0, ymm2);
+ ymm1 = _mm256_mulhi_epu16(ymm1, ymm3);
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(ymm0, ymm1));
+ src_argb += 32;
+ dst_argb += 32;
+ width -= 8;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_ARGBUNATTENUATEROW_AVX2
+
+#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2
+LIBYUV_TARGET_AVX2
+void ComputeCumulativeSumRow_AVX2(const uint8_t* row,
+ int32_t* cumsum,
+ const int32_t* previous_cumsum,
+ int width) {
+ static const uint8_t kCumSumTranspose[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+ static const int8_t kCumSumMaddLo[32] = {
+ 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
+ 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0};
+ static const int8_t kCumSumMaddHi[32] = {
+ 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+ __m256i ymm0 = _mm256_setzero_si256();
+ const __m256i ymm5 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kCumSumTranspose));
+ const __m256i ymm6 = _mm256_loadu_si256((const __m256i*)kCumSumMaddLo);
+ const __m256i ymm7 = _mm256_loadu_si256((const __m256i*)kCumSumMaddHi);
+ const __m256i ymm8 = _mm256_set1_epi16(1);
+
+ while (width >= 4) {
+ __m256i ymm2 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)row));
+ row += 16;
+ ymm2 = _mm256_shuffle_epi8(ymm2, ymm5);
+ __m256i ymm3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm6), ymm8);
+ __m256i ymm4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm7), ymm8);
+ __m256i sum4 = _mm256_permute2x128_si256(ymm4, ymm4, 0x11);
+ ymm3 = _mm256_add_epi32(
+ _mm256_add_epi32(ymm3,
+ _mm256_loadu_si256((const __m256i*)previous_cumsum)),
+ ymm0);
+ ymm4 = _mm256_add_epi32(
+ _mm256_add_epi32(
+ ymm4, _mm256_loadu_si256((const __m256i*)(previous_cumsum + 8))),
+ ymm0);
+ previous_cumsum += 16;
+ ymm0 = _mm256_add_epi32(ymm0, sum4);
+ _mm256_storeu_si256((__m256i*)cumsum, ymm3);
+ _mm256_storeu_si256((__m256i*)(cumsum + 8), ymm4);
+ cumsum += 16;
+ width -= 4;
+ }
+
+ __m128i xmm0 = _mm256_castsi256_si128(ymm0);
+ while (width > 0) {
+ int32_t px;
+ memcpy(&px, row, sizeof(px));
+ row += 4;
+ __m128i xmm2 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(px));
+ xmm0 = _mm_add_epi32(xmm0, xmm2);
+ xmm2 = _mm_add_epi32(xmm0,
+ _mm_loadu_si128((const __m128i*)previous_cumsum));
+ previous_cumsum += 4;
+ _mm_storeu_si128((__m128i*)cumsum, xmm2);
+ cumsum += 4;
+ --width;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2
+
+#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+LIBYUV_TARGET_AVX2
+void CumulativeSumToAverageRow_AVX2(const int32_t* topleft,
const int32_t* botleft,
int width,
int area,
uint8_t* dst,
int count) {
- __asm {
- mov eax, topleft // eax topleft
- mov esi, botleft // esi botleft
- mov edx, width
- movd xmm5, area
- mov edi, dst
- mov ecx, count
- cvtdq2ps xmm5, xmm5
- rcpss xmm4, xmm5 // 1.0f / area
- pshufd xmm4, xmm4, 0
- sub ecx, 4
- jl l4b
-
- cmp area, 128 // 128 pixels will not overflow 15 bits.
- ja l4
-
- pshufd xmm5, xmm5, 0 // area
- pcmpeqb xmm6, xmm6 // constant of 65536.0 - 1 = 65535.0
- psrld xmm6, 16
- cvtdq2ps xmm6, xmm6
- addps xmm5, xmm6 // (65536.0 + area - 1)
- mulps xmm5, xmm4 // (65536.0 + area - 1) * 1 / area
- cvtps2dq xmm5, xmm5 // 0.16 fixed point
- packssdw xmm5, xmm5 // 16 bit shorts
-
- // 4 pixel loop small blocks.
- s4:
- // top left
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
-
- // - top right
- psubd xmm0, [eax + edx * 4]
- psubd xmm1, [eax + edx * 4 + 16]
- psubd xmm2, [eax + edx * 4 + 32]
- psubd xmm3, [eax + edx * 4 + 48]
- lea eax, [eax + 64]
-
- // - bottom left
- psubd xmm0, [esi]
- psubd xmm1, [esi + 16]
- psubd xmm2, [esi + 32]
- psubd xmm3, [esi + 48]
-
- // + bottom right
- paddd xmm0, [esi + edx * 4]
- paddd xmm1, [esi + edx * 4 + 16]
- paddd xmm2, [esi + edx * 4 + 32]
- paddd xmm3, [esi + edx * 4 + 48]
- lea esi, [esi + 64]
-
- packssdw xmm0, xmm1 // pack 4 pixels into 2 registers
- packssdw xmm2, xmm3
-
- pmulhuw xmm0, xmm5
- pmulhuw xmm2, xmm5
-
- packuswb xmm0, xmm2
- movdqu [edi], xmm0
- lea edi, [edi + 16]
- sub ecx, 4
- jge s4
-
- jmp l4b
-
- // 4 pixel loop
- l4:
- // top left
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + 32]
- movdqu xmm3, [eax + 48]
-
- // - top right
- psubd xmm0, [eax + edx * 4]
- psubd xmm1, [eax + edx * 4 + 16]
- psubd xmm2, [eax + edx * 4 + 32]
- psubd xmm3, [eax + edx * 4 + 48]
- lea eax, [eax + 64]
-
- // - bottom left
- psubd xmm0, [esi]
- psubd xmm1, [esi + 16]
- psubd xmm2, [esi + 32]
- psubd xmm3, [esi + 48]
-
- // + bottom right
- paddd xmm0, [esi + edx * 4]
- paddd xmm1, [esi + edx * 4 + 16]
- paddd xmm2, [esi + edx * 4 + 32]
- paddd xmm3, [esi + edx * 4 + 48]
- lea esi, [esi + 64]
-
- cvtdq2ps xmm0, xmm0 // Average = Sum * 1 / Area
- cvtdq2ps xmm1, xmm1
- mulps xmm0, xmm4
- mulps xmm1, xmm4
- cvtdq2ps xmm2, xmm2
- cvtdq2ps xmm3, xmm3
- mulps xmm2, xmm4
- mulps xmm3, xmm4
- cvtps2dq xmm0, xmm0
- cvtps2dq xmm1, xmm1
- cvtps2dq xmm2, xmm2
- cvtps2dq xmm3, xmm3
- packssdw xmm0, xmm1
- packssdw xmm2, xmm3
- packuswb xmm0, xmm2
- movdqu [edi], xmm0
- lea edi, [edi + 16]
- sub ecx, 4
- jge l4
-
- l4b:
- add ecx, 4 - 1
- jl l1b
-
- // 1 pixel loop
- l1:
- movdqu xmm0, [eax]
- psubd xmm0, [eax + edx * 4]
- lea eax, [eax + 16]
- psubd xmm0, [esi]
- paddd xmm0, [esi + edx * 4]
- lea esi, [esi + 16]
- cvtdq2ps xmm0, xmm0
- mulps xmm0, xmm4
- cvtps2dq xmm0, xmm0
- packssdw xmm0, xmm0
- packuswb xmm0, xmm0
- movd dword ptr [edi], xmm0
- lea edi, [edi + 4]
- sub ecx, 1
- jge l1
- l1b:
- }
-}
-#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2
-
-#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2
-// Creates a table of cumulative sums where each value is a sum of all values
-// above and to the left of the value.
-void ComputeCumulativeSumRow_SSE2(const uint8_t* row,
- int32_t* cumsum,
- const int32_t* previous_cumsum,
- int width) {
- __asm {
- mov eax, row
- mov edx, cumsum
- mov esi, previous_cumsum
- mov ecx, width
- pxor xmm0, xmm0
- pxor xmm1, xmm1
-
- sub ecx, 4
- jl l4b
- test edx, 15
- jne l4b
-
- // 4 pixel loop
- l4:
- movdqu xmm2, [eax] // 4 argb pixels 16 bytes.
- lea eax, [eax + 16]
- movdqa xmm4, xmm2
-
- punpcklbw xmm2, xmm1
- movdqa xmm3, xmm2
- punpcklwd xmm2, xmm1
- punpckhwd xmm3, xmm1
-
- punpckhbw xmm4, xmm1
- movdqa xmm5, xmm4
- punpcklwd xmm4, xmm1
- punpckhwd xmm5, xmm1
-
- paddd xmm0, xmm2
- movdqu xmm2, [esi] // previous row above.
- paddd xmm2, xmm0
-
- paddd xmm0, xmm3
- movdqu xmm3, [esi + 16]
- paddd xmm3, xmm0
-
- paddd xmm0, xmm4
- movdqu xmm4, [esi + 32]
- paddd xmm4, xmm0
-
- paddd xmm0, xmm5
- movdqu xmm5, [esi + 48]
- lea esi, [esi + 64]
- paddd xmm5, xmm0
-
- movdqu [edx], xmm2
- movdqu [edx + 16], xmm3
- movdqu [edx + 32], xmm4
- movdqu [edx + 48], xmm5
-
- lea edx, [edx + 64]
- sub ecx, 4
- jge l4
-
- l4b:
- add ecx, 4 - 1
- jl l1b
-
- // 1 pixel loop
- l1:
- movd xmm2, dword ptr [eax] // 1 argb pixel
- lea eax, [eax + 4]
- punpcklbw xmm2, xmm1
- punpcklwd xmm2, xmm1
- paddd xmm0, xmm2
- movdqu xmm2, [esi]
- lea esi, [esi + 16]
- paddd xmm2, xmm0
- movdqu [edx], xmm2
- lea edx, [edx + 16]
- sub ecx, 1
- jge l1
-
- l1b:
- }
-}
-#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2
-
-#ifdef HAS_ARGBAFFINEROW_SSE2
-// Copy ARGB pixels from source image with slope to a row of destination.
-__declspec(naked) LIBYUV_API void ARGBAffineRow_SSE2(const uint8_t* src_argb,
- int src_argb_stride,
- uint8_t* dst_argb,
- const float* uv_dudv,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 12] // src_argb
- mov esi, [esp + 16] // stride
- mov edx, [esp + 20] // dst_argb
- mov ecx, [esp + 24] // pointer to uv_dudv
- movq xmm2, qword ptr [ecx] // uv
- movq xmm7, qword ptr [ecx + 8] // dudv
- mov ecx, [esp + 28] // width
- shl esi, 16 // 4, stride
- add esi, 4
- movd xmm5, esi
- sub ecx, 4
- jl l4b
-
- // setup for 4 pixel loop
- pshufd xmm7, xmm7, 0x44 // dup dudv
- pshufd xmm5, xmm5, 0 // dup 4, stride
- movdqa xmm0, xmm2 // x0, y0, x1, y1
- addps xmm0, xmm7
- movlhps xmm2, xmm0
- movdqa xmm4, xmm7
- addps xmm4, xmm4 // dudv *= 2
- movdqa xmm3, xmm2 // x2, y2, x3, y3
- addps xmm3, xmm4
- addps xmm4, xmm4 // dudv *= 4
-
- // 4 pixel loop
- l4:
- cvttps2dq xmm0, xmm2 // x, y float to int first 2
- cvttps2dq xmm1, xmm3 // x, y float to int next 2
- packssdw xmm0, xmm1 // x, y as 8 shorts
- pmaddwd xmm0, xmm5 // offsets = x * 4 + y * stride.
- movd esi, xmm0
- pshufd xmm0, xmm0, 0x39 // shift right
- movd edi, xmm0
- pshufd xmm0, xmm0, 0x39 // shift right
- movd xmm1, [eax + esi] // read pixel 0
- movd xmm6, [eax + edi] // read pixel 1
- punpckldq xmm1, xmm6 // combine pixel 0 and 1
- addps xmm2, xmm4 // x, y += dx, dy first 2
- movq qword ptr [edx], xmm1
- movd esi, xmm0
- pshufd xmm0, xmm0, 0x39 // shift right
- movd edi, xmm0
- movd xmm6, [eax + esi] // read pixel 2
- movd xmm0, [eax + edi] // read pixel 3
- punpckldq xmm6, xmm0 // combine pixel 2 and 3
- addps xmm3, xmm4 // x, y += dx, dy next 2
- movq qword ptr 8[edx], xmm6
- lea edx, [edx + 16]
- sub ecx, 4
- jge l4
-
- l4b:
- add ecx, 4 - 1
- jl l1b
-
- // 1 pixel loop
- l1:
- cvttps2dq xmm0, xmm2 // x, y float to int
- packssdw xmm0, xmm0 // x, y as shorts
- pmaddwd xmm0, xmm5 // offset = x * 4 + y * stride
- addps xmm2, xmm7 // x, y += dx, dy
- movd esi, xmm0
- movd xmm0, [eax + esi] // copy a pixel
- movd [edx], xmm0
- lea edx, [edx + 4]
- sub ecx, 1
- jge l1
- l1b:
- pop edi
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBAFFINEROW_SSE2
+ __m128 xmm5 = _mm_cvtsi32_ss(_mm_setzero_ps(), area);
+ __m128 xmm4 = _mm_div_ss(_mm_set_ss(1.0f), xmm5);
+ xmm4 = _mm_shuffle_ps(xmm4, xmm4, 0);
+
+ if (count >= 4) {
+ __m256 ymm4 = _mm256_insertf128_ps(_mm256_castps128_ps256(xmm4), xmm4, 1);
+ if (area > 1 && area <= 0x80) {
+ __m128 xmm6 = _mm_cvtepi32_ps(_mm_set1_epi32(0x0000ffff));
+ xmm6 = _mm_mul_ss(_mm_add_ss(xmm5, xmm6), xmm4);
+ __m128i i6 = _mm_cvtps_epi32(xmm6);
+ i6 = _mm_packus_epi32(i6, i6);
+ __m256i ymm6 = _mm256_broadcastw_epi16(i6);
+ while (count >= 4) {
+ __m256i ymm0 = _mm256_sub_epi32(
+ _mm256_loadu_si256((const __m256i*)topleft),
+ _mm256_loadu_si256((const __m256i*)(topleft + width)));
+ __m256i ymm1 = _mm256_sub_epi32(
+ _mm256_loadu_si256((const __m256i*)(topleft + 8)),
+ _mm256_loadu_si256((const __m256i*)(topleft + width + 8)));
+ topleft += 16;
+ ymm0 = _mm256_sub_epi32(ymm0,
+ _mm256_loadu_si256((const __m256i*)botleft));
+ ymm1 = _mm256_sub_epi32(
+ ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8)));
+ ymm0 = _mm256_add_epi32(
+ ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width)));
+ ymm1 = _mm256_add_epi32(
+ ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8)));
+ botleft += 16;
+ ymm0 = _mm256_mulhi_epu16(_mm256_packus_epi32(ymm0, ymm1), ymm6);
+ __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
+ _mm256_extracti128_si256(ymm0, 1));
+ _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8));
+ dst += 16;
+ count -= 4;
+ }
+ } else {
+ while (count >= 4) {
+ __m256i ymm0 = _mm256_sub_epi32(
+ _mm256_loadu_si256((const __m256i*)topleft),
+ _mm256_loadu_si256((const __m256i*)(topleft + width)));
+ __m256i ymm1 = _mm256_sub_epi32(
+ _mm256_loadu_si256((const __m256i*)(topleft + 8)),
+ _mm256_loadu_si256((const __m256i*)(topleft + width + 8)));
+ topleft += 16;
+ ymm0 = _mm256_sub_epi32(ymm0,
+ _mm256_loadu_si256((const __m256i*)botleft));
+ ymm1 = _mm256_sub_epi32(
+ ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8)));
+ ymm0 = _mm256_add_epi32(
+ ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width)));
+ ymm1 = _mm256_add_epi32(
+ ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8)));
+ botleft += 16;
+ ymm0 =
+ _mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4));
+ ymm1 =
+ _mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4));
+ ymm0 = _mm256_packus_epi32(ymm0, ymm1);
+ __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0),
+ _mm256_extracti128_si256(ymm0, 1));
+ _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8));
+ dst += 16;
+ count -= 4;
+ }
+ }
+ _mm256_zeroupper();
+ }
+
+ while (count > 0) {
+ __m128i d0 = _mm_sub_epi32(
+ _mm_loadu_si128((const __m128i*)topleft),
+ _mm_loadu_si128((const __m128i*)(topleft + width)));
+ topleft += 4;
+ __m128i d1 = _mm_sub_epi32(
+ _mm_loadu_si128((const __m128i*)(botleft + width)),
+ _mm_loadu_si128((const __m128i*)botleft));
+ botleft += 4;
+ d0 = _mm_add_epi32(d0, d1);
+ d0 = _mm_cvttps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4));
+ d0 = _mm_packus_epi32(d0, d0);
+ d0 = _mm_packus_epi16(d0, d0);
+ int32_t out32 = _mm_cvtsi128_si32(d0);
+ memcpy(dst, &out32, sizeof(out32));
+ dst += 4;
+ --count;
+ }
+}
+#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
+
+#ifdef HAS_I422TOYUY2ROW_AVX2
+LIBYUV_TARGET_AVX2
+void I422ToYUY2Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_yuy2,
+ int width) {
+ do {
+ __m128i u = _mm_loadu_si128((const __m128i*)src_u);
+ __m128i v = _mm_loadu_si128((const __m128i*)src_v);
+ __m256i uv = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)),
+ _mm_unpackhi_epi8(u, v), 1);
+ __m256i y = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i lo = _mm256_unpacklo_epi8(y, uv);
+ __m256i hi = _mm256_unpackhi_epi8(y, uv);
+ _mm256_storeu_si256((__m256i*)dst_yuy2,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_y += 32;
+ src_u += 16;
+ src_v += 16;
+ dst_yuy2 += 64;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TOYUY2ROW_AVX2
+
+#ifdef HAS_I422TOUYVYROW_AVX2
+LIBYUV_TARGET_AVX2
+void I422ToUYVYRow_AVX2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uyvy,
+ int width) {
+ do {
+ __m128i u = _mm_loadu_si128((const __m128i*)src_u);
+ __m128i v = _mm_loadu_si128((const __m128i*)src_v);
+ __m256i uv = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)),
+ _mm_unpackhi_epi8(u, v), 1);
+ __m256i y = _mm256_loadu_si256((const __m256i*)src_y);
+ __m256i lo = _mm256_unpacklo_epi8(uv, y);
+ __m256i hi = _mm256_unpackhi_epi8(uv, y);
+ _mm256_storeu_si256((__m256i*)dst_uyvy,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_uyvy + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_y += 32;
+ src_u += 16;
+ src_v += 16;
+ dst_uyvy += 64;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_I422TOUYVYROW_AVX2
+
+#ifdef HAS_MERGEARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeARGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ do {
+ __m128i r = _mm_loadu_si128((const __m128i*)src_r);
+ __m128i g = _mm_loadu_si128((const __m128i*)src_g);
+ __m128i b = _mm_loadu_si128((const __m128i*)src_b);
+ __m128i a = _mm_loadu_si128((const __m128i*)src_a);
+ __m256i bg = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
+ _mm_unpackhi_epi8(b, g), 1);
+ __m256i ra = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
+ _mm_unpackhi_epi8(r, a), 1);
+ __m256i lo = _mm256_unpacklo_epi16(bg, ra);
+ __m256i hi = _mm256_unpackhi_epi16(bg, ra);
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ src_a += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEARGBROW_AVX2
+
+#ifdef HAS_MERGEXRGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeXRGBRow_AVX2(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ const __m128i a = _mm_set1_epi8((char)0xff);
+ do {
+ __m128i r = _mm_loadu_si128((const __m128i*)src_r);
+ __m128i g = _mm_loadu_si128((const __m128i*)src_g);
+ __m128i b = _mm_loadu_si128((const __m128i*)src_b);
+ __m256i bg = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
+ _mm_unpackhi_epi8(b, g), 1);
+ __m256i ra = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
+ _mm_unpackhi_epi8(r, a), 1);
+ __m256i lo = _mm256_unpacklo_epi16(bg, ra);
+ __m256i hi = _mm256_unpackhi_epi16(bg, ra);
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEXRGBROW_AVX2
+
+#ifdef HAS_MERGEARGB16TO8ROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeARGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(depth - 8);
+ const __m256i ymm_255 = _mm256_set1_epi16(255);
+ do {
+ __m256i r16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128),
+ ymm_255);
+ __m256i g16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128),
+ ymm_255);
+ __m256i b16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128),
+ ymm_255);
+ __m256i a16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_a), shift128),
+ ymm_255);
+ __m256i br8 =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8);
+ __m256i ga8 =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, a16), 0xd8);
+ __m128i b = _mm256_castsi256_si128(br8);
+ __m128i r = _mm256_extracti128_si256(br8, 1);
+ __m128i g = _mm256_castsi256_si128(ga8);
+ __m128i a = _mm256_extracti128_si256(ga8, 1);
+ __m256i bg = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
+ _mm_unpackhi_epi8(b, g), 1);
+ __m256i ra = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
+ _mm_unpackhi_epi8(r, a), 1);
+ __m256i lo = _mm256_unpacklo_epi16(bg, ra);
+ __m256i hi = _mm256_unpackhi_epi16(bg, ra);
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ src_a += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEARGB16TO8ROW_AVX2
+
+#ifdef HAS_MERGEXRGB16TO8ROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_argb,
+ int depth,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(depth - 8);
+ const __m256i ymm_255 = _mm256_set1_epi16(255);
+ const __m128i a = _mm_set1_epi8((char)0xff);
+ do {
+ __m256i r16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128),
+ ymm_255);
+ __m256i g16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128),
+ ymm_255);
+ __m256i b16 = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128),
+ ymm_255);
+ __m256i br8 =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8);
+ __m256i g8 =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, g16), 0xd8);
+ __m128i b = _mm256_castsi256_si128(br8);
+ __m128i r = _mm256_extracti128_si256(br8, 1);
+ __m128i g = _mm256_castsi256_si128(g8);
+ __m256i bg = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)),
+ _mm_unpackhi_epi8(b, g), 1);
+ __m256i ra = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)),
+ _mm_unpackhi_epi8(r, a), 1);
+ __m256i lo = _mm256_unpacklo_epi16(bg, ra);
+ __m256i hi = _mm256_unpackhi_epi16(bg, ra);
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_permute2x128_si256(lo, hi, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_permute2x128_si256(lo, hi, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ dst_argb += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEXRGB16TO8ROW_AVX2
+
+#ifdef HAS_MERGEAR64ROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeAR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ const uint16_t* src_a,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
+ const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
+ const __m256i ymm_max = _mm256_set1_epi16((short)max_val);
+ do {
+ __m256i r = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max),
+ shift128);
+ __m256i g = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max),
+ shift128);
+ __m256i b = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max),
+ shift128);
+ __m256i a = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_a), ymm_max),
+ shift128);
+ __m256i bg_lo = _mm256_unpacklo_epi16(b, g);
+ __m256i bg_hi = _mm256_unpackhi_epi16(b, g);
+ __m256i ra_lo = _mm256_unpacklo_epi16(r, a);
+ __m256i ra_hi = _mm256_unpackhi_epi16(r, a);
+ __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo);
+ __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo);
+ __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi);
+ __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi);
+ _mm256_storeu_si256((__m256i*)dst_ar64,
+ _mm256_permute2x128_si256(p0_8, p2_10, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 16),
+ _mm256_permute2x128_si256(p4_12, p6_14, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 32),
+ _mm256_permute2x128_si256(p0_8, p2_10, 0x31));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 48),
+ _mm256_permute2x128_si256(p4_12, p6_14, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ src_a += 16;
+ dst_ar64 += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEAR64ROW_AVX2
+
+#ifdef HAS_MERGEXR64ROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeXR64Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint16_t* dst_ar64,
+ int depth,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
+ const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u);
+ const __m256i ymm_max = _mm256_set1_epi16((short)max_val);
+ const __m256i a = _mm256_set1_epi16((short)0xffff);
+ do {
+ __m256i r = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max),
+ shift128);
+ __m256i g = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max),
+ shift128);
+ __m256i b = _mm256_sll_epi16(
+ _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max),
+ shift128);
+ __m256i bg_lo = _mm256_unpacklo_epi16(b, g);
+ __m256i bg_hi = _mm256_unpackhi_epi16(b, g);
+ __m256i ra_lo = _mm256_unpacklo_epi16(r, a);
+ __m256i ra_hi = _mm256_unpackhi_epi16(r, a);
+ __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo);
+ __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo);
+ __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi);
+ __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi);
+ _mm256_storeu_si256((__m256i*)dst_ar64,
+ _mm256_permute2x128_si256(p0_8, p2_10, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 16),
+ _mm256_permute2x128_si256(p4_12, p6_14, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 32),
+ _mm256_permute2x128_si256(p0_8, p2_10, 0x31));
+ _mm256_storeu_si256((__m256i*)(dst_ar64 + 48),
+ _mm256_permute2x128_si256(p4_12, p6_14, 0x31));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ dst_ar64 += 64;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEXR64ROW_AVX2
+
+#ifdef HAS_NV21TOYUV24ROW_AVX2
+LIBYUV_TARGET_AVX2
+void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ static const uint8_t kYUV24Shuffle[3][16] = {
+ {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12},
+ {9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15},
+ {2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}};
+ const __m256i ymm4 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kYUV24Shuffle[0]));
+ const __m256i ymm5 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kYUV24Shuffle[1]));
+ const __m256i ymm6 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kYUV24Shuffle[2]));
+
+ do {
+ __m256 ymm2 = _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_y));
+ __m256 ymm3 =
+ _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_vu));
+ __m256i c0 = _mm256_shuffle_epi8(
+ _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x44)), ymm4);
+ __m256i c1 = _mm256_shuffle_epi8(
+ _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x99)), ymm5);
+ __m256i c2 = _mm256_shuffle_epi8(
+ _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0xee)), ymm6);
+ _mm256_storeu_si256((__m256i*)dst_yuv24,
+ _mm256_permute2x128_si256(c0, c1, 0x20));
+ _mm256_storeu_si256((__m256i*)(dst_yuv24 + 32),
+ _mm256_permute2x128_si256(c2, c0, 0x30));
+ _mm256_storeu_si256((__m256i*)(dst_yuv24 + 64),
+ _mm256_permute2x128_si256(c1, c2, 0x31));
+ src_y += 32;
+ src_vu += 32;
+ dst_yuv24 += 96;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_NV21TOYUV24ROW_AVX2
+
+#ifdef HAS_NV21TOYUV24ROW_AVX512VBMI
+LIBYUV_TARGET_AVX512VBMI
+void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_yuv24,
+ int width) {
+ static const uint8_t kNV21ToYUV24Perm_AVX512VBMI[3][64] = {
+ {64, 65, 0, 64, 65, 1, 66, 67, 2, 66, 67, 3, 68, 69, 4, 68,
+ 69, 5, 70, 71, 6, 70, 71, 7, 72, 73, 8, 72, 73, 9, 74, 75,
+ 10, 74, 75, 11, 76, 77, 12, 76, 77, 13, 78, 79, 14, 78, 79, 15,
+ 80, 81, 16, 80, 81, 17, 82, 83, 18, 82, 83, 19, 84, 85, 20, 84},
+ {85, 21, 86, 87, 22, 86, 87, 23, 88, 89, 24, 88, 89,
+ 25, 90, 91, 26, 90, 91, 27, 92, 93, 28, 92, 93, 29,
+ 94, 95, 30, 94, 95, 31, 96, 97, 32, 96, 97, 33, 98,
+ 99, 34, 98, 99, 35, 100, 101, 36, 100, 101, 37, 102, 103,
+ 38, 102, 103, 39, 104, 105, 40, 104, 105, 41, 106, 107},
+ {42, 106, 107, 43, 108, 109, 44, 108, 109, 45, 110, 111, 46,
+ 110, 111, 47, 112, 113, 48, 112, 113, 49, 114, 115, 50, 114,
+ 115, 51, 116, 117, 52, 116, 117, 53, 118, 119, 54, 118, 119,
+ 55, 120, 121, 56, 120, 121, 57, 122, 123, 58, 122, 123, 59,
+ 124, 125, 60, 124, 125, 61, 126, 127, 62, 126, 127, 63}};
+ __m512i zmm_p0 =
+ _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[0]);
+ __m512i zmm_p1 =
+ _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[1]);
+ __m512i zmm_p2 =
+ _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[2]);
+ LIBYUV_HIDE_CONSTANT(zmm_p0);
+ LIBYUV_HIDE_CONSTANT(zmm_p1);
+ LIBYUV_HIDE_CONSTANT(zmm_p2);
+
+ while (width >= 64) {
+ __m512i zy = _mm512_loadu_si512((const __m512i*)src_y);
+ __m512i zvu = _mm512_loadu_si512((const __m512i*)src_vu);
+ __m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu);
+ __m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu);
+ __m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu);
+ _mm512_storeu_si512((__m512i*)dst_yuv24, dst0);
+ _mm512_storeu_si512((__m512i*)(dst_yuv24 + 64), dst1);
+ _mm512_storeu_si512((__m512i*)(dst_yuv24 + 128), dst2);
+ src_y += 64;
+ src_vu += 64;
+ dst_yuv24 += 192;
+ width -= 64;
+ }
+ if (width > 0) {
+ __mmask64 ky = (__mmask64)((1ULL << width) - 1ULL);
+ int vu_bytes = (width + 1) & ~1;
+ __mmask64 kvu = (vu_bytes >= 64) ? ~(__mmask64)0
+ : (__mmask64)((1ULL << vu_bytes) - 1ULL);
+ __m512i zy = _mm512_maskz_loadu_epi8(ky, src_y);
+ __m512i zvu = _mm512_maskz_loadu_epi8(kvu, src_vu);
+ __m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu);
+ __m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu);
+ __m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu);
+ int nbytes = width * 3;
+ __mmask64 k0 =
+ (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL);
+ __mmask64 k1 = (nbytes <= 64) ? 0
+ : (nbytes >= 128)
+ ? ~(__mmask64)0
+ : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL);
+ __mmask64 k2 =
+ (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL);
+ _mm512_mask_storeu_epi8(dst_yuv24, k0, dst0);
+ if (k1) {
+ _mm512_mask_storeu_epi8(dst_yuv24 + 64, k1, dst1);
+ }
+ if (k2) {
+ _mm512_mask_storeu_epi8(dst_yuv24 + 128, k2, dst2);
+ }
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_NV21TOYUV24ROW_AVX512VBMI
+
+#ifdef HAS_SPLITARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void SplitARGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+ const __m256i shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufSplitARGB));
+ do {
+ __m256i t0 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)src_argb), shuf);
+ __m256i t1 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf);
+ __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20);
+ __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31);
+ __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8);
+ __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg));
+ _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1));
+ _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra));
+ _mm_storeu_si128((__m128i*)dst_a, _mm256_extracti128_si256(ra, 1));
+ src_argb += 64;
+ dst_r += 16;
+ dst_g += 16;
+ dst_b += 16;
+ dst_a += 16;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITARGBROW_AVX2
+
+#ifdef HAS_SPLITXRGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void SplitXRGBRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13,
+ 2, 6, 10, 14, 3, 7, 11, 15};
+ const __m256i shuf = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kShufSplitARGB));
+ do {
+ __m256i t0 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)src_argb), shuf);
+ __m256i t1 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf);
+ __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20);
+ __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31);
+ __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8);
+ __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg));
+ _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1));
+ _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra));
+ src_argb += 64;
+ dst_r += 16;
+ dst_g += 16;
+ dst_b += 16;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITXRGBROW_AVX2
+
+#ifdef HAS_SPLITRGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void SplitRGBRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ static const uint8_t kSplitRGBShuffle[5][16] = {
+ {0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u},
+ {1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u},
+ {2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u},
+ {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u,
+ 0u},
+ {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u,
+ 0u}};
+ const __m256i s0 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[0]));
+ const __m256i s1 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[1]));
+ const __m256i s2 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[2]));
+ const __m256i m0 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[3]));
+ const __m256i m1 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[4]));
+
+ do {
+ __m256i ymm4 = _mm256_loadu_si256((const __m256i*)src_rgb);
+ __m256i ymm5 = _mm256_loadu_si256((const __m256i*)(src_rgb + 32));
+ __m256i ymm6 = _mm256_loadu_si256((const __m256i*)(src_rgb + 64));
+ src_rgb += 96;
+ __m256i ymm1 = _mm256_blend_epi32(ymm4, ymm5, 0xf0);
+ __m256i ymm2 = _mm256_permute2x128_si256(ymm4, ymm6, 0x21);
+ __m256i ymm3 = _mm256_blend_epi32(ymm5, ymm6, 0xf0);
+ ymm4 = _mm256_blendv_epi8(ymm1, ymm3, m0);
+ ymm5 = _mm256_blendv_epi8(ymm2, ymm1, m0);
+ ymm6 = _mm256_blendv_epi8(ymm3, ymm2, m0);
+ ymm1 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm4, ymm5, m1), s0);
+ ymm2 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm5, ymm6, m1), s1);
+ ymm3 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm6, ymm4, m1), s2);
+ _mm256_storeu_si256((__m256i*)dst_r, ymm1);
+ _mm256_storeu_si256((__m256i*)dst_g, ymm2);
+ _mm256_storeu_si256((__m256i*)dst_b, ymm3);
+ dst_r += 32;
+ dst_g += 32;
+ dst_b += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITRGBROW_AVX2
+
+#ifdef HAS_SPLITUVROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void SplitUVRow_16_AVX2(const uint16_t* src_uv,
+ uint16_t* dst_u,
+ uint16_t* dst_v,
+ int depth,
+ int width) {
+ const __m128i shift128 = _mm_cvtsi32_si128(16 - depth);
+ const __m256i mask16 = _mm256_set1_epi32(0x0000ffff);
+ do {
+ __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16));
+ __m256i u = _mm256_permute4x64_epi64(
+ _mm256_packus_epi32(_mm256_and_si256(uv0, mask16),
+ _mm256_and_si256(uv1, mask16)),
+ 0xd8);
+ __m256i v = _mm256_permute4x64_epi64(
+ _mm256_packus_epi32(_mm256_srli_epi32(uv0, 16),
+ _mm256_srli_epi32(uv1, 16)),
+ 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_u, _mm256_srl_epi16(u, shift128));
+ _mm256_storeu_si256((__m256i*)dst_v, _mm256_srl_epi16(v, shift128));
+ src_uv += 32;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 16;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_SPLITUVROW_16_AVX2
-#ifdef HAS_INTERPOLATEROW_AVX2
-// Bilinear filter 32x2 -> 32x1
-__declspec(naked) void InterpolateRow_AVX2(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- __asm {
- push esi
- push edi
- mov edi, [esp + 8 + 4] // dst_ptr
- mov esi, [esp + 8 + 8] // src_ptr
- mov edx, [esp + 8 + 12] // src_stride
- mov ecx, [esp + 8 + 16] // dst_width
- mov eax, [esp + 8 + 20] // source_y_fraction (0..255)
- // Dispatch to specialized filters if applicable.
- cmp eax, 0
- je xloop100 // 0 / 256. Blend 100 / 0.
- sub edi, esi
- cmp eax, 128
- je xloop50 // 128 /256 is 0.50. Blend 50 / 50.
-
- vmovd xmm0, eax // high fraction 0..255
- neg eax
- add eax, 256
- vmovd xmm5, eax // low fraction 256..1
- vpunpcklbw xmm5, xmm5, xmm0
- vpunpcklwd xmm5, xmm5, xmm5
- vbroadcastss ymm5, xmm5
-
- mov eax, 0x80808080 // 128b for bias and rounding.
- vmovd xmm4, eax
- vbroadcastss ymm4, xmm4
-
- xloop:
- vmovdqu ymm0, [esi]
- vmovdqu ymm2, [esi + edx]
- vpunpckhbw ymm1, ymm0, ymm2 // mutates
- vpunpcklbw ymm0, ymm0, ymm2
- vpsubb ymm1, ymm1, ymm4 // bias to signed image
- vpsubb ymm0, ymm0, ymm4
- vpmaddubsw ymm1, ymm5, ymm1
- vpmaddubsw ymm0, ymm5, ymm0
- vpaddw ymm1, ymm1, ymm4 // unbias and round
- vpaddw ymm0, ymm0, ymm4
- vpsrlw ymm1, ymm1, 8
- vpsrlw ymm0, ymm0, 8
- vpackuswb ymm0, ymm0, ymm1 // unmutates
- vmovdqu [esi + edi], ymm0
- lea esi, [esi + 32]
- sub ecx, 32
- jg xloop
- jmp xloop99
-
- // Blend 50 / 50.
- xloop50:
- vmovdqu ymm0, [esi]
- vpavgb ymm0, ymm0, [esi + edx]
- vmovdqu [esi + edi], ymm0
- lea esi, [esi + 32]
- sub ecx, 32
- jg xloop50
- jmp xloop99
-
- // Blend 100 / 0 - Copy row unchanged.
- xloop100:
- rep movsb
-
- xloop99:
- pop edi
- pop esi
- vzeroupper
- ret
- }
+#ifdef HAS_YUY2TOYROW_AVX2
+LIBYUV_TARGET_AVX2
+void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i s0 =
+ _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_yuy2),
+ mask_00ff);
+ __m256i s1 =
+ _mm256_and_si256(_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
+ mask_00ff);
+ _mm256_storeu_si256(
+ (__m256i*)dst_y,
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8));
+ src_yuy2 += 64;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
}
-#endif // HAS_INTERPOLATEROW_AVX2
+#endif // HAS_YUY2TOYROW_AVX2
-// Bilinear filter 16x2 -> 16x1
-// TODO(fbarchard): Consider allowing 256 using memcpy.
-__declspec(naked) void InterpolateRow_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- __asm {
- push esi
- push edi
-
- mov edi, [esp + 8 + 4] // dst_ptr
- mov esi, [esp + 8 + 8] // src_ptr
- mov edx, [esp + 8 + 12] // src_stride
- mov ecx, [esp + 8 + 16] // dst_width
- mov eax, [esp + 8 + 20] // source_y_fraction (0..255)
- sub edi, esi
- // Dispatch to specialized filters if applicable.
- cmp eax, 0
- je xloop100 // 0 /256. Blend 100 / 0.
- cmp eax, 128
- je xloop50 // 128 / 256 is 0.50. Blend 50 / 50.
-
- movd xmm0, eax // high fraction 0..255
- neg eax
- add eax, 256
- movd xmm5, eax // low fraction 255..1
- punpcklbw xmm5, xmm0
- punpcklwd xmm5, xmm5
- pshufd xmm5, xmm5, 0
- mov eax, 0x80808080 // 128 for biasing image to signed.
- movd xmm4, eax
- pshufd xmm4, xmm4, 0x00
-
- xloop:
- movdqu xmm0, [esi]
- movdqu xmm2, [esi + edx]
- movdqu xmm1, xmm0
- punpcklbw xmm0, xmm2
- punpckhbw xmm1, xmm2
- psubb xmm0, xmm4 // bias image by -128
- psubb xmm1, xmm4
- movdqa xmm2, xmm5
- movdqa xmm3, xmm5
- pmaddubsw xmm2, xmm0
- pmaddubsw xmm3, xmm1
- paddw xmm2, xmm4
- paddw xmm3, xmm4
- psrlw xmm2, 8
- psrlw xmm3, 8
- packuswb xmm2, xmm3
- movdqu [esi + edi], xmm2
- lea esi, [esi + 16]
- sub ecx, 16
- jg xloop
- jmp xloop99
-
- // Blend 50 / 50.
- xloop50:
- movdqu xmm0, [esi]
- movdqu xmm1, [esi + edx]
- pavgb xmm0, xmm1
- movdqu [esi + edi], xmm0
- lea esi, [esi + 16]
- sub ecx, 16
- jg xloop50
- jmp xloop99
-
- // Blend 100 / 0 - Copy row unchanged.
- xloop100:
- movdqu xmm0, [esi]
- movdqu [esi + edi], xmm0
- lea esi, [esi + 16]
- sub ecx, 16
- jg xloop100
-
- xloop99:
- pop edi
- pop esi
- ret
- }
-}
-
-// For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA.
-__declspec(naked) void ARGBShuffleRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // shuffler
- movdqu xmm5, [ecx]
- mov ecx, [esp + 16] // width
-
- wloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- pshufb xmm0, xmm5
- pshufb xmm1, xmm5
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 8
- jg wloop
- ret
- }
-}
+#ifdef HAS_UYVYTOYROW_AVX2
+LIBYUV_TARGET_AVX2
+void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
+ do {
+ __m256i s0 =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_uyvy), 8);
+ __m256i s1 =
+ _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)(src_uyvy + 32)),
+ 8);
+ _mm256_storeu_si256(
+ (__m256i*)dst_y,
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8));
+ src_uyvy += 64;
+ dst_y += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_UYVYTOYROW_AVX2
+
+#ifdef HAS_YUY2TOUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i y0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_yuy2),
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2)));
+ __m256i y1 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32)));
+ __m256i uv = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
+ 0xd8);
+ __m256i u = _mm256_and_si256(uv, mask_00ff);
+ __m256i v = _mm256_srli_epi16(uv, 8);
+ u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
+ v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
+ src_yuy2 += 64;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TOUVROW_AVX2
+
+#ifdef HAS_YUY2TOUV422ROW_AVX2
+LIBYUV_TARGET_AVX2
+void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i y0 = _mm256_loadu_si256((const __m256i*)src_yuy2);
+ __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32));
+ __m256i uv = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
+ 0xd8);
+ __m256i u = _mm256_and_si256(uv, mask_00ff);
+ __m256i v = _mm256_srli_epi16(uv, 8);
+ u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
+ v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
+ src_yuy2 += 64;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TOUV422ROW_AVX2
+
+#ifdef HAS_YUY2TONVUVROW_AVX2
+LIBYUV_TARGET_AVX2
+void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2,
+ int stride_yuy2,
+ uint8_t* dst_uv,
+ int width) {
+ do {
+ __m256i y0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_yuy2),
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2)));
+ __m256i y1 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32)));
+ __m256i uv = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)),
+ 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_uv, uv);
+ src_yuy2 += 64;
+ dst_uv += 32;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // HAS_YUY2TONVUVROW_AVX2
+
+#if defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
+LIBYUV_TARGET_AVX2
+void UYVYToUVRow_AVX2(const uint8_t* src_uyvy,
+ int stride_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i y0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_uyvy),
+ _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy)));
+ __m256i y1 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)(src_uyvy + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy + 32)));
+ __m256i uv = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff),
+ _mm256_and_si256(y1, mask_00ff)),
+ 0xd8);
+ __m256i u = _mm256_and_si256(uv, mask_00ff);
+ __m256i v = _mm256_srli_epi16(uv, 8);
+ u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
+ v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
+ src_uyvy += 64;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
+
+#if defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
+LIBYUV_TARGET_AVX2
+void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ const __m256i mask_00ff = _mm256_set1_epi16(0x00ff);
+ do {
+ __m256i y0 = _mm256_loadu_si256((const __m256i*)src_uyvy);
+ __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_uyvy + 32));
+ __m256i uv = _mm256_permute4x64_epi64(
+ _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff),
+ _mm256_and_si256(y1, mask_00ff)),
+ 0xd8);
+ __m256i u = _mm256_and_si256(uv, mask_00ff);
+ __m256i v = _mm256_srli_epi16(uv, 8);
+ u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8);
+ v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u));
+ _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v));
+ src_uyvy += 64;
+ dst_u += 16;
+ dst_v += 16;
+ width -= 32;
+ } while (width > 0);
+ _mm256_zeroupper();
+}
+#endif // defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2)
-#ifdef HAS_ARGBSHUFFLEROW_AVX2
-__declspec(naked) void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const uint8_t* shuffler,
- int width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- mov edx, [esp + 8] // dst_argb
- mov ecx, [esp + 12] // shuffler
- vbroadcastf128 ymm5, [ecx] // same shuffle in high as low.
- mov ecx, [esp + 16] // width
-
- wloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpshufb ymm0, ymm0, ymm5
- vpshufb ymm1, ymm1, ymm5
- vmovdqu [edx], ymm0
- vmovdqu [edx + 32], ymm1
- lea edx, [edx + 64]
- sub ecx, 16
- jg wloop
-
- vzeroupper
- ret
- }
+#ifdef HAS_I444TOARGBROW_AVX2
+LIBYUV_TARGET_AVX2
+void I444ToARGBRow_AVX2(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const __m256i ymm_kUVToB =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
+ const __m256i ymm_kUVToG =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
+ const __m256i ymm_kUVToR =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
+ const __m256i ymm_kYToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
+ const __m256i ymm_kYBiasToRgb =
+ _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
+ const __m256i ymm_128 = _mm256_set1_epi8((char)0x80);
+ const __m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
+
+ while (width >= 16) {
+ __m128i xmm_u = _mm_loadu_si128((const __m128i*)u_buf);
+ __m128i xmm_v = _mm_loadu_si128((const __m128i*)v_buf);
+ __m128i xmm_y = _mm_loadu_si128((const __m128i*)y_buf);
+ u_buf += 16;
+ v_buf += 16;
+ y_buf += 16;
+
+ __m256i ymm3 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_u, xmm_v)),
+ _mm_unpackhi_epi8(xmm_u, xmm_v), 1);
+ __m256i ymm4 = _mm256_inserti128_si256(
+ _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_y, xmm_y)),
+ _mm_unpackhi_epi8(xmm_y, xmm_y), 1);
+
+ ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
+ ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
+
+ __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
+ __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
+ __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
+
+ ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
+
+ ymm0 = _mm256_srai_epi16(_mm256_adds_epi16(ymm0, ymm4), 6);
+ ymm1 = _mm256_srai_epi16(_mm256_subs_epi16(ymm4, ymm1), 6);
+ ymm2 = _mm256_srai_epi16(_mm256_adds_epi16(ymm2, ymm4), 6);
+
+ ymm0 = _mm256_packus_epi16(ymm0, ymm0);
+ ymm1 = _mm256_packus_epi16(ymm1, ymm1);
+ ymm2 = _mm256_packus_epi16(ymm2, ymm2);
+
+ ymm0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm0, ymm1), 0xd8);
+ ymm2 =
+ _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm2, ymm_alpha), 0xd8);
+
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(ymm0, ymm2));
+ _mm256_storeu_si256((__m256i*)(dst_argb + 32),
+ _mm256_unpackhi_epi16(ymm0, ymm2));
+
+ dst_argb += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
}
-#endif // HAS_ARGBSHUFFLEROW_AVX2
-
-// YUY2 - Macro-pixel = 2 image pixels
-// Y0U0Y1V0....Y2U2Y3V2...Y4U4Y5V4....
-
-// UYVY - Macro-pixel = 2 image pixels
-// U0Y0V0Y1
-
-__declspec(naked) void I422ToYUY2Row_SSE2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_frame,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_y
- mov esi, [esp + 8 + 8] // src_u
- mov edx, [esp + 8 + 12] // src_v
- mov edi, [esp + 8 + 16] // dst_frame
- mov ecx, [esp + 8 + 20] // width
- sub edx, esi
-
- convertloop:
- movq xmm2, qword ptr [esi] // U
- movq xmm3, qword ptr [esi + edx] // V
- lea esi, [esi + 8]
- punpcklbw xmm2, xmm3 // UV
- movdqu xmm0, [eax] // Y
- lea eax, [eax + 16]
- movdqa xmm1, xmm0
- punpcklbw xmm0, xmm2 // YUYV
- punpckhbw xmm1, xmm2
- movdqu [edi], xmm0
- movdqu [edi + 16], xmm1
- lea edi, [edi + 32]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-__declspec(naked) void I422ToUYVYRow_SSE2(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_frame,
- int width) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_y
- mov esi, [esp + 8 + 8] // src_u
- mov edx, [esp + 8 + 12] // src_v
- mov edi, [esp + 8 + 16] // dst_frame
- mov ecx, [esp + 8 + 20] // width
- sub edx, esi
-
- convertloop:
- movq xmm2, qword ptr [esi] // U
- movq xmm3, qword ptr [esi + edx] // V
- lea esi, [esi + 8]
- punpcklbw xmm2, xmm3 // UV
- movdqu xmm0, [eax] // Y
- movdqa xmm1, xmm2
- lea eax, [eax + 16]
- punpcklbw xmm1, xmm0 // UYVY
- punpckhbw xmm2, xmm0
- movdqu [edi], xmm1
- movdqu [edi + 16], xmm2
- lea edi, [edi + 32]
- sub ecx, 16
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-
-#ifdef HAS_ARGBPOLYNOMIALROW_SSE2
-__declspec(naked) void ARGBPolynomialRow_SSE2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const float* poly,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] /* src_argb */
- mov edx, [esp + 4 + 8] /* dst_argb */
- mov esi, [esp + 4 + 12] /* poly */
- mov ecx, [esp + 4 + 16] /* width */
- pxor xmm3, xmm3 // 0 constant for zero extending bytes to ints.
-
- // 2 pixel loop.
- convertloop:
- // pmovzxbd xmm0, dword ptr [eax] // BGRA pixel
- // pmovzxbd xmm4, dword ptr [eax + 4] // BGRA pixel
- movq xmm0, qword ptr [eax] // BGRABGRA
- lea eax, [eax + 8]
- punpcklbw xmm0, xmm3
- movdqa xmm4, xmm0
- punpcklwd xmm0, xmm3 // pixel 0
- punpckhwd xmm4, xmm3 // pixel 1
- cvtdq2ps xmm0, xmm0 // 4 floats
- cvtdq2ps xmm4, xmm4
- movdqa xmm1, xmm0 // X
- movdqa xmm5, xmm4
- mulps xmm0, [esi + 16] // C1 * X
- mulps xmm4, [esi + 16]
- addps xmm0, [esi] // result = C0 + C1 * X
- addps xmm4, [esi]
- movdqa xmm2, xmm1
- movdqa xmm6, xmm5
- mulps xmm2, xmm1 // X * X
- mulps xmm6, xmm5
- mulps xmm1, xmm2 // X * X * X
- mulps xmm5, xmm6
- mulps xmm2, [esi + 32] // C2 * X * X
- mulps xmm6, [esi + 32]
- mulps xmm1, [esi + 48] // C3 * X * X * X
- mulps xmm5, [esi + 48]
- addps xmm0, xmm2 // result += C2 * X * X
- addps xmm4, xmm6
- addps xmm0, xmm1 // result += C3 * X * X * X
- addps xmm4, xmm5
- cvttps2dq xmm0, xmm0
- cvttps2dq xmm4, xmm4
- packuswb xmm0, xmm4
- packuswb xmm0, xmm0
- movq qword ptr [edx], xmm0
- lea edx, [edx + 8]
- sub ecx, 2
- jg convertloop
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBPOLYNOMIALROW_SSE2
+#endif // HAS_I444TOARGBROW_AVX2
-#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
-__declspec(naked) void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
- uint8_t* dst_argb,
- const float* poly,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src_argb */
- mov edx, [esp + 8] /* dst_argb */
- mov ecx, [esp + 12] /* poly */
- vbroadcastf128 ymm4, [ecx] // C0
- vbroadcastf128 ymm5, [ecx + 16] // C1
- vbroadcastf128 ymm6, [ecx + 32] // C2
- vbroadcastf128 ymm7, [ecx + 48] // C3
- mov ecx, [esp + 16] /* width */
-
- // 2 pixel loop.
- convertloop:
- vpmovzxbd ymm0, qword ptr [eax] // 2 BGRA pixels
- lea eax, [eax + 8]
- vcvtdq2ps ymm0, ymm0 // X 8 floats
- vmulps ymm2, ymm0, ymm0 // X * X
- vmulps ymm3, ymm0, ymm7 // C3 * X
- vfmadd132ps ymm0, ymm4, ymm5 // result = C0 + C1 * X
- vfmadd231ps ymm0, ymm2, ymm6 // result += C2 * X * X
- vfmadd231ps ymm0, ymm2, ymm3 // result += C3 * X * X * X
- vcvttps2dq ymm0, ymm0
- vpackusdw ymm0, ymm0, ymm0 // b0g0r0a0_00000000_b0g0r0a0_00000000
- vpermq ymm0, ymm0, 0xd8 // b0g0r0a0_b0g0r0a0_00000000_00000000
- vpackuswb xmm0, xmm0, xmm0 // bgrabgra_00000000_00000000_00000000
- vmovq qword ptr [edx], xmm0
- lea edx, [edx + 8]
- sub ecx, 2
- jg convertloop
- vzeroupper
- ret
+#ifdef HAS_MERGEXR30ROW_AVX2
+LIBYUV_TARGET_AVX2
+void MergeXR30Row_AVX2(const uint16_t* src_r,
+ const uint16_t* src_g,
+ const uint16_t* src_b,
+ uint8_t* dst_ar30,
+ int depth,
+ int width) {
+ int shift = depth - 10;
+ __m128i v_shift = _mm_cvtsi32_si128(shift);
+ __m256i v_alpha = _mm256_set1_epi16(0x0030);
+ __m256i v_max = _mm256_set1_epi16(1023);
+ while (width > 0) {
+ __m256i r = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), v_shift),
+ v_max);
+ __m256i g = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), v_shift),
+ v_max);
+ __m256i b = _mm256_min_epu16(
+ _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), v_shift),
+ v_max);
+ r = _mm256_slli_epi16(_mm256_permute4x64_epi64(r, 0xd8), 4);
+ g = _mm256_permute4x64_epi64(g, 0xd8);
+ b = _mm256_permute4x64_epi64(b, 0xd8);
+ __m256i rb_lo = _mm256_unpacklo_epi16(b, r);
+ __m256i rb_hi = _mm256_unpackhi_epi16(b, r);
+ __m256i ag_lo = _mm256_slli_epi32(_mm256_unpacklo_epi16(g, v_alpha), 10);
+ __m256i ag_hi = _mm256_slli_epi32(_mm256_unpackhi_epi16(g, v_alpha), 10);
+ _mm256_storeu_si256((__m256i*)dst_ar30, _mm256_or_si256(rb_lo, ag_lo));
+ _mm256_storeu_si256((__m256i*)(dst_ar30 + 32),
+ _mm256_or_si256(rb_hi, ag_hi));
+ src_r += 16;
+ src_g += 16;
+ src_b += 16;
+ dst_ar30 += 64;
+ width -= 16;
+ }
+ _mm256_zeroupper();
+}
+#endif // HAS_MERGEXR30ROW_AVX2
+
+#ifdef HAS_MERGEUVROW_16_AVX2
+LIBYUV_TARGET_AVX2
+void MergeUVRow_16_AVX2(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ const __m128i sh_u = _mm_cvtsi32_si128(16 - depth);
+ const __m128i sh_v = _mm_cvtsi32_si128(32 - depth);
+ while (width > 0) {
+ __m256i u32 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_u));
+ __m256i v32 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_v));
+ src_u += 8;
+ src_v += 8;
+ u32 = _mm256_sll_epi16(u32, sh_u);
+ v32 = _mm256_sll_epi32(v32, sh_v);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm256_or_si256(u32, v32));
+ dst_uv += 16;
+ width -= 8;
}
+ _mm256_zeroupper();
}
-#endif // HAS_ARGBPOLYNOMIALROW_AVX2
-
-#ifdef HAS_HALFFLOATROW_SSE2
-static float kExpBias = 1.9259299444e-34f;
-__declspec(naked) void HalfFloatRow_SSE2(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src */
- mov edx, [esp + 8] /* dst */
- movd xmm4, dword ptr [esp + 12] /* scale */
- mov ecx, [esp + 16] /* width */
- mulss xmm4, kExpBias
- pshufd xmm4, xmm4, 0
- pxor xmm5, xmm5
- sub edx, eax
-
- // 8 pixel loop.
- convertloop:
- movdqu xmm2, xmmword ptr [eax] // 8 shorts
- add eax, 16
- movdqa xmm3, xmm2
- punpcklwd xmm2, xmm5
- cvtdq2ps xmm2, xmm2 // convert 8 ints to floats
- punpckhwd xmm3, xmm5
- cvtdq2ps xmm3, xmm3
- mulps xmm2, xmm4
- mulps xmm3, xmm4
- psrld xmm2, 13
- psrld xmm3, 13
- packssdw xmm2, xmm3
- movdqu [eax + edx - 16], xmm2
- sub ecx, 8
- jg convertloop
- ret
- }
-}
-#endif // HAS_HALFFLOATROW_SSE2
-
-#ifdef HAS_HALFFLOATROW_AVX2
-__declspec(naked) void HalfFloatRow_AVX2(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src */
- mov edx, [esp + 8] /* dst */
- movd xmm4, dword ptr [esp + 12] /* scale */
- mov ecx, [esp + 16] /* width */
-
- vmulss xmm4, xmm4, kExpBias
- vbroadcastss ymm4, xmm4
- vpxor ymm5, ymm5, ymm5
- sub edx, eax
-
- // 16 pixel loop.
- convertloop:
- vmovdqu ymm2, [eax] // 16 shorts
- add eax, 32
- vpunpckhwd ymm3, ymm2, ymm5 // convert 16 shorts to 16 ints
- vpunpcklwd ymm2, ymm2, ymm5
- vcvtdq2ps ymm3, ymm3 // convert 16 ints to floats
- vcvtdq2ps ymm2, ymm2
- vmulps ymm3, ymm3, ymm4 // scale to adjust exponent for 5 bit range.
- vmulps ymm2, ymm2, ymm4
- vpsrld ymm3, ymm3, 13 // float convert to 8 half floats truncate
- vpsrld ymm2, ymm2, 13
- vpackssdw ymm2, ymm2, ymm3
- vmovdqu [eax + edx - 32], ymm2
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_HALFFLOATROW_AVX2
-
-#ifdef HAS_HALFFLOATROW_F16C
-__declspec(naked) void HalfFloatRow_F16C(const uint16_t* src,
- uint16_t* dst,
- float scale,
- int width) {
- __asm {
- mov eax, [esp + 4] /* src */
- mov edx, [esp + 8] /* dst */
- vbroadcastss ymm4, [esp + 12] /* scale */
- mov ecx, [esp + 16] /* width */
- sub edx, eax
-
- // 16 pixel loop.
- convertloop:
- vpmovzxwd ymm2, xmmword ptr [eax] // 8 shorts -> 8 ints
- vpmovzxwd ymm3, xmmword ptr [eax + 16] // 8 more shorts
- add eax, 32
- vcvtdq2ps ymm2, ymm2 // convert 8 ints to floats
- vcvtdq2ps ymm3, ymm3
- vmulps ymm2, ymm2, ymm4 // scale to normalized range 0 to 1
- vmulps ymm3, ymm3, ymm4
- vcvtps2ph xmm2, ymm2, 3 // float convert to 8 half floats truncate
- vcvtps2ph xmm3, ymm3, 3
- vmovdqu [eax + edx + 32], xmm2
- vmovdqu [eax + edx + 32 + 16], xmm3
- sub ecx, 16
- jg convertloop
- vzeroupper
- ret
- }
-}
-#endif // HAS_HALFFLOATROW_F16C
-
-#ifdef HAS_ARGBCOLORTABLEROW_X86
-// Tranform ARGB pixels with color table.
-__declspec(naked) void ARGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] /* dst_argb */
- mov esi, [esp + 4 + 8] /* table_argb */
- mov ecx, [esp + 4 + 12] /* width */
-
- // 1 pixel loop.
- convertloop:
- movzx edx, byte ptr [eax]
- lea eax, [eax + 4]
- movzx edx, byte ptr [esi + edx * 4]
- mov byte ptr [eax - 4], dl
- movzx edx, byte ptr [eax - 4 + 1]
- movzx edx, byte ptr [esi + edx * 4 + 1]
- mov byte ptr [eax - 4 + 1], dl
- movzx edx, byte ptr [eax - 4 + 2]
- movzx edx, byte ptr [esi + edx * 4 + 2]
- mov byte ptr [eax - 4 + 2], dl
- movzx edx, byte ptr [eax - 4 + 3]
- movzx edx, byte ptr [esi + edx * 4 + 3]
- mov byte ptr [eax - 4 + 3], dl
- dec ecx
- jg convertloop
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBCOLORTABLEROW_X86
-
-#ifdef HAS_RGBCOLORTABLEROW_X86
-// Tranform RGB pixels with color table.
-__declspec(naked) void RGBColorTableRow_X86(uint8_t* dst_argb,
- const uint8_t* table_argb,
- int width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] /* dst_argb */
- mov esi, [esp + 4 + 8] /* table_argb */
- mov ecx, [esp + 4 + 12] /* width */
-
- // 1 pixel loop.
- convertloop:
- movzx edx, byte ptr [eax]
- lea eax, [eax + 4]
- movzx edx, byte ptr [esi + edx * 4]
- mov byte ptr [eax - 4], dl
- movzx edx, byte ptr [eax - 4 + 1]
- movzx edx, byte ptr [esi + edx * 4 + 1]
- mov byte ptr [eax - 4 + 1], dl
- movzx edx, byte ptr [eax - 4 + 2]
- movzx edx, byte ptr [esi + edx * 4 + 2]
- mov byte ptr [eax - 4 + 2], dl
- dec ecx
- jg convertloop
-
- pop esi
- ret
- }
-}
-#endif // HAS_RGBCOLORTABLEROW_X86
-
-#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3
-// Tranform RGB pixels with luma table.
-__declspec(naked) void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_argb,
- int width,
- const uint8_t* luma,
- uint32_t lumacoeff) {
- __asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] /* src_argb */
- mov edi, [esp + 8 + 8] /* dst_argb */
- mov ecx, [esp + 8 + 12] /* width */
- movd xmm2, dword ptr [esp + 8 + 16] // luma table
- movd xmm3, dword ptr [esp + 8 + 20] // lumacoeff
- pshufd xmm2, xmm2, 0
- pshufd xmm3, xmm3, 0
- pcmpeqb xmm4, xmm4 // generate mask 0xff00ff00
- psllw xmm4, 8
- pxor xmm5, xmm5
-
- // 4 pixel loop.
- convertloop:
- movdqu xmm0, xmmword ptr [eax] // generate luma ptr
- pmaddubsw xmm0, xmm3
- phaddw xmm0, xmm0
- pand xmm0, xmm4 // mask out low bits
- punpcklwd xmm0, xmm5
- paddd xmm0, xmm2 // add table base
- movd esi, xmm0
- pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32
-
- movzx edx, byte ptr [eax]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi], dl
- movzx edx, byte ptr [eax + 1]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 1], dl
- movzx edx, byte ptr [eax + 2]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 2], dl
- movzx edx, byte ptr [eax + 3] // copy alpha.
- mov byte ptr [edi + 3], dl
-
- movd esi, xmm0
- pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32
-
- movzx edx, byte ptr [eax + 4]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 4], dl
- movzx edx, byte ptr [eax + 5]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 5], dl
- movzx edx, byte ptr [eax + 6]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 6], dl
- movzx edx, byte ptr [eax + 7] // copy alpha.
- mov byte ptr [edi + 7], dl
-
- movd esi, xmm0
- pshufd xmm0, xmm0, 0x39 // 00111001 to rotate right 32
-
- movzx edx, byte ptr [eax + 8]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 8], dl
- movzx edx, byte ptr [eax + 9]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 9], dl
- movzx edx, byte ptr [eax + 10]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 10], dl
- movzx edx, byte ptr [eax + 11] // copy alpha.
- mov byte ptr [edi + 11], dl
-
- movd esi, xmm0
-
- movzx edx, byte ptr [eax + 12]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 12], dl
- movzx edx, byte ptr [eax + 13]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 13], dl
- movzx edx, byte ptr [eax + 14]
- movzx edx, byte ptr [esi + edx]
- mov byte ptr [edi + 14], dl
- movzx edx, byte ptr [eax + 15] // copy alpha.
- mov byte ptr [edi + 15], dl
-
- lea eax, [eax + 16]
- lea edi, [edi + 16]
- sub ecx, 4
- jg convertloop
-
- pop edi
- pop esi
- ret
- }
-}
-#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3
-
-#endif // defined(_M_X64)
+#endif // HAS_MERGEUVROW_16_AVX2
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
-#endif // !defined(LIBYUV_DISABLE_X86) && (defined(_M_IX86) || defined(_M_X64))
+#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
+ // defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) &&
+ // ((defined(_MSC_VER) && !defined(__clang__)) ||
+ // defined(LIBYUV_ENABLE_ROWWIN))
diff --git a/libs/libyuv/source/scale.cc b/libs/libyuv/source/scale.cc
index 30066d5b34..f2157409cb 100644
--- a/libs/libyuv/source/scale.cc
+++ b/libs/libyuv/source/scale.cc
@@ -11,12 +11,15 @@
#include "libyuv/scale.h"
#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
#include <string.h>
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h" // For CopyPlane
#include "libyuv/row.h"
#include "libyuv/scale_row.h"
+#include "libyuv/scale_uv.h" // For UVScale
#ifdef __cplusplus
namespace libyuv {
@@ -28,6 +31,7 @@ static __inline int Abs(int v) {
}
#define SUBSAMPLE(v, a, s) (v < 0) ? (-((-v + a) >> s)) : ((v + a) >> s)
+#define CENTERSTART(dx, s) (dx < 0) ? -((-dx >> 1) + s) : ((dx >> 1) + s)
// Scale plane, 1/2
// This is an optimized version for scaling down a plane to 1/2 of
@@ -37,8 +41,8 @@ static void ScalePlaneDown2(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr,
enum FilterMode filtering) {
@@ -49,7 +53,7 @@ static void ScalePlaneDown2(int src_width,
? ScaleRowDown2_C
: (filtering == kFilterLinear ? ScaleRowDown2Linear_C
: ScaleRowDown2Box_C);
- int row_stride = src_stride << 1;
+ ptrdiff_t row_stride = src_stride * 2;
(void)src_width;
(void)src_height;
if (!filtering) {
@@ -72,6 +76,13 @@ static void ScalePlaneDown2(int src_width,
}
}
#endif
+#if defined(HAS_SCALEROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_SME
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_SME
+ : ScaleRowDown2Box_SME;
+ }
+#endif
#if defined(HAS_SCALEROWDOWN2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleRowDown2 =
@@ -103,36 +114,62 @@ static void ScalePlaneDown2(int src_width,
}
}
#endif
-#if defined(HAS_SCALEROWDOWN2_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
+#if defined(HAS_SCALEROWDOWN2_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleRowDown2 =
filtering == kFilterNone
- ? ScaleRowDown2_Any_MSA
- : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_MSA
- : ScaleRowDown2Box_Any_MSA);
- if (IS_ALIGNED(dst_width, 32)) {
- ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_MSA
+ ? ScaleRowDown2_AVX512BW
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_AVX512BW
+ : ScaleRowDown2Box_AVX512BW);
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN2_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleRowDown2 =
+ filtering == kFilterNone
+ ? ScaleRowDown2_Any_LSX
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_LSX
+ : ScaleRowDown2Box_Any_LSX);
+ if (IS_ALIGNED(dst_width, 16)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_LSX
: (filtering == kFilterLinear
- ? ScaleRowDown2Linear_MSA
- : ScaleRowDown2Box_MSA);
+ ? ScaleRowDown2Linear_LSX
+ : ScaleRowDown2Box_LSX);
}
}
#endif
-#if defined(HAS_SCALEROWDOWN2_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
+#if defined(HAS_SCALEROWDOWN2_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowDown2 = filtering == kFilterNone
+ ? ScaleRowDown2_RVV
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_RVV
+ : ScaleRowDown2Box_RVV);
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
ScaleRowDown2 =
filtering == kFilterNone
- ? ScaleRowDown2_Any_MMI
- : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_MMI
- : ScaleRowDown2Box_Any_MMI);
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_MMI
- : (filtering == kFilterLinear
- ? ScaleRowDown2Linear_MMI
- : ScaleRowDown2Box_MMI);
+ ? ScaleRowDown2_Any_WASMSIMD
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_WASMSIMD
+ : ScaleRowDown2Box_Any_WASMSIMD);
+ if (IS_ALIGNED(dst_width, 16)) {
+ ScaleRowDown2 =
+ filtering == kFilterNone
+ ? ScaleRowDown2_WASMSIMD
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_WASMSIMD
+ : ScaleRowDown2Box_WASMSIMD);
}
}
#endif
+#if defined(HAS_SCALEROWDOWN2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleRowDown2 = filtering == kFilterNone
+ ? ScaleRowDown2_HVX
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_HVX
+ : ScaleRowDown2Box_HVX);
+ }
+#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@@ -149,8 +186,8 @@ static void ScalePlaneDown2_16(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr,
enum FilterMode filtering) {
@@ -161,7 +198,7 @@ static void ScalePlaneDown2_16(int src_width,
? ScaleRowDown2_16_C
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16_C
: ScaleRowDown2Box_16_C);
- int row_stride = src_stride << 1;
+ ptrdiff_t row_stride = src_stride * 2;
(void)src_width;
(void)src_height;
if (!filtering) {
@@ -169,27 +206,40 @@ static void ScalePlaneDown2_16(int src_width,
src_stride = 0;
}
+#if defined(HAS_SCALEROWDOWN2_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowDown2 =
+ filtering == kFilterNone
+ ? ScaleRowDown2_16_Any_AVX2
+ : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_Any_AVX2
+ : ScaleRowDown2Box_16_Any_AVX2);
+ if (IS_ALIGNED(dst_width, 16)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_AVX2
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_16_AVX2
+ : ScaleRowDown2Box_16_AVX2;
+ }
+ }
+#endif
#if defined(HAS_SCALEROWDOWN2_16_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) {
- ScaleRowDown2 =
- filtering ? ScaleRowDown2Box_16_NEON : ScaleRowDown2_16_NEON;
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_NEON
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_16_NEON
+ : ScaleRowDown2Box_16_NEON;
}
#endif
-#if defined(HAS_SCALEROWDOWN2_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) {
- ScaleRowDown2 =
- filtering == kFilterNone
- ? ScaleRowDown2_16_SSE2
- : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2
- : ScaleRowDown2Box_16_SSE2);
+#if defined(HAS_SCALEROWDOWN2_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_SME
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_16_SME
+ : ScaleRowDown2Box_16_SME;
}
#endif
-#if defined(HAS_SCALEROWDOWN2_16_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 4)) {
- ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_MMI
+#if defined(HAS_SCALEROWDOWN2_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_HVX
: (filtering == kFilterLinear
- ? ScaleRowDown2Linear_16_MMI
- : ScaleRowDown2Box_16_MMI);
+ ? ScaleRowDown2Linear_16_HVX
+ : ScaleRowDown2Box_16_HVX);
}
#endif
@@ -212,8 +262,8 @@ static void ScalePlaneDown4(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr,
enum FilterMode filtering) {
@@ -221,7 +271,7 @@ static void ScalePlaneDown4(int src_width,
void (*ScaleRowDown4)(const uint8_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, int dst_width) =
filtering ? ScaleRowDown4Box_C : ScaleRowDown4_C;
- int row_stride = src_stride << 2;
+ ptrdiff_t row_stride = src_stride * 4;
(void)src_width;
(void)src_height;
if (!filtering) {
@@ -232,7 +282,7 @@ static void ScalePlaneDown4(int src_width,
if (TestCpuFlag(kCpuHasNEON)) {
ScaleRowDown4 =
filtering ? ScaleRowDown4Box_Any_NEON : ScaleRowDown4_Any_NEON;
- if (IS_ALIGNED(dst_width, 8)) {
+ if (IS_ALIGNED(dst_width, 16)) {
ScaleRowDown4 = filtering ? ScaleRowDown4Box_NEON : ScaleRowDown4_NEON;
}
}
@@ -255,24 +305,35 @@ static void ScalePlaneDown4(int src_width,
}
}
#endif
-#if defined(HAS_SCALEROWDOWN4_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
+#if defined(HAS_SCALEROWDOWN4_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
ScaleRowDown4 =
- filtering ? ScaleRowDown4Box_Any_MSA : ScaleRowDown4_Any_MSA;
- if (IS_ALIGNED(dst_width, 16)) {
- ScaleRowDown4 = filtering ? ScaleRowDown4Box_MSA : ScaleRowDown4_MSA;
+ filtering ? ScaleRowDown4Box_Any_LSX : ScaleRowDown4_Any_LSX;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleRowDown4 = filtering ? ScaleRowDown4Box_LSX : ScaleRowDown4_LSX;
}
}
#endif
-#if defined(HAS_SCALEROWDOWN4_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
+#if defined(HAS_SCALEROWDOWN4_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowDown4 = filtering ? ScaleRowDown4Box_RVV : ScaleRowDown4_RVV;
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN4_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
ScaleRowDown4 =
- filtering ? ScaleRowDown4Box_Any_MMI : ScaleRowDown4_Any_MMI;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleRowDown4 = filtering ? ScaleRowDown4Box_MMI : ScaleRowDown4_MMI;
+ filtering ? ScaleRowDown4Box_Any_WASMSIMD : ScaleRowDown4_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 16)) {
+ ScaleRowDown4 =
+ filtering ? ScaleRowDown4Box_WASMSIMD : ScaleRowDown4_WASMSIMD;
}
}
#endif
+#if defined(HAS_SCALEROWDOWN4_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleRowDown4 = filtering ? ScaleRowDown4Box_HVX : ScaleRowDown4_HVX;
+ }
+#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@@ -288,8 +349,8 @@ static void ScalePlaneDown4_16(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr,
enum FilterMode filtering) {
@@ -297,30 +358,13 @@ static void ScalePlaneDown4_16(int src_width,
void (*ScaleRowDown4)(const uint16_t* src_ptr, ptrdiff_t src_stride,
uint16_t* dst_ptr, int dst_width) =
filtering ? ScaleRowDown4Box_16_C : ScaleRowDown4_16_C;
- int row_stride = src_stride << 2;
+ ptrdiff_t row_stride = src_stride * 4;
(void)src_width;
(void)src_height;
if (!filtering) {
src_ptr += src_stride * 2; // Point to row 2.
src_stride = 0;
}
-#if defined(HAS_SCALEROWDOWN4_16_NEON)
- if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) {
- ScaleRowDown4 =
- filtering ? ScaleRowDown4Box_16_NEON : ScaleRowDown4_16_NEON;
- }
-#endif
-#if defined(HAS_SCALEROWDOWN4_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleRowDown4 =
- filtering ? ScaleRowDown4Box_16_SSE2 : ScaleRowDown4_16_SSE2;
- }
-#endif
-#if defined(HAS_SCALEROWDOWN4_16_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 8)) {
- ScaleRowDown4 = filtering ? ScaleRowDown4Box_16_MMI : ScaleRowDown4_16_MMI;
- }
-#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@@ -337,8 +381,8 @@ static void ScalePlaneDown34(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr,
enum FilterMode filtering) {
@@ -347,7 +391,7 @@ static void ScalePlaneDown34(int src_width,
uint8_t* dst_ptr, int dst_width);
void (*ScaleRowDown34_1)(const uint8_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, int dst_width);
- const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
+ const ptrdiff_t filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
(void)src_width;
(void)src_height;
assert(dst_width % 3 == 0);
@@ -360,14 +404,11 @@ static void ScalePlaneDown34(int src_width,
}
#if defined(HAS_SCALEROWDOWN34_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_Any_NEON;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_NEON;
- }
+#if defined(__aarch64__)
+ if (dst_width % 48 == 0) {
+#else
if (dst_width % 24 == 0) {
+#endif
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_NEON;
ScaleRowDown34_1 = ScaleRowDown34_NEON;
@@ -375,45 +416,87 @@ static void ScalePlaneDown34(int src_width,
ScaleRowDown34_0 = ScaleRowDown34_0_Box_NEON;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_NEON;
}
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_NEON;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_NEON;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_NEON;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_NEON;
+ }
}
}
#endif
-#if defined(HAS_SCALEROWDOWN34_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_Any_MSA;
+#if defined(HAS_SCALEROWDOWN34_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ if (dst_width % 12 == 0) {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_LSX;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_LSX;
+ }
} else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_MSA;
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_LSX;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_LSX;
+ }
}
- if (dst_width % 48 == 0) {
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN34_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ if (dst_width % 24 == 0) {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_AVX2;
+ ScaleRowDown34_1 = ScaleRowDown34_AVX2;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_AVX2;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_AVX2;
+ }
+ } else {
if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_MSA;
+ ScaleRowDown34_0 = ScaleRowDown34_Any_AVX2;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_AVX2;
} else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_MSA;
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_AVX2;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_AVX2;
}
}
}
#endif
-#if defined(HAS_SCALEROWDOWN34_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
+#if defined(HAS_SCALEROWDOWN34_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3;
+ ScaleRowDown34_0 = ScaleRowDown34_RVV;
+ ScaleRowDown34_1 = ScaleRowDown34_RVV;
} else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3;
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_RVV;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_RVV;
}
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN34_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
if (dst_width % 24 == 0) {
if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_SSSE3;
+ ScaleRowDown34_0 = ScaleRowDown34_WASMSIMD;
+ ScaleRowDown34_1 = ScaleRowDown34_WASMSIMD;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_WASMSIMD;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_WASMSIMD;
+ }
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_WASMSIMD;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_WASMSIMD;
} else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3;
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_WASMSIMD;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_WASMSIMD;
}
}
}
@@ -446,8 +529,8 @@ static void ScalePlaneDown34_16(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr,
enum FilterMode filtering) {
@@ -456,7 +539,7 @@ static void ScalePlaneDown34_16(int src_width,
uint16_t* dst_ptr, int dst_width);
void (*ScaleRowDown34_1)(const uint16_t* src_ptr, ptrdiff_t src_stride,
uint16_t* dst_ptr, int dst_width);
- const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
+ const ptrdiff_t filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
(void)src_width;
(void)src_height;
assert(dst_width % 3 == 0);
@@ -467,28 +550,6 @@ static void ScalePlaneDown34_16(int src_width,
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C;
}
-#if defined(HAS_SCALEROWDOWN34_16_NEON)
- if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_16_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_16_NEON;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON;
- }
- }
-#endif
-#if defined(HAS_SCALEROWDOWN34_16_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3;
- }
- }
-#endif
for (y = 0; y < dst_height - 2; y += 3) {
ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width);
@@ -532,8 +593,8 @@ static void ScalePlaneDown38(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr,
enum FilterMode filtering) {
@@ -542,7 +603,7 @@ static void ScalePlaneDown38(int src_width,
uint8_t* dst_ptr, int dst_width);
void (*ScaleRowDown38_2)(const uint8_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, int dst_width);
- const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
+ const ptrdiff_t filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
assert(dst_width % 3 == 0);
(void)src_width;
(void)src_height;
@@ -574,41 +635,73 @@ static void ScalePlaneDown38(int src_width,
}
}
#endif
-#if defined(HAS_SCALEROWDOWN38_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
+#if defined(HAS_SCALEROWDOWN38_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ if (!filtering) {
+ ScaleRowDown38_3 = ScaleRowDown38_Any_AVX2;
+ ScaleRowDown38_2 = ScaleRowDown38_Any_AVX2;
+ } else {
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_AVX2;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_AVX2;
+ }
+ if (dst_width % 12 == 0) {
+ if (!filtering) {
+ ScaleRowDown38_3 = ScaleRowDown38_AVX2;
+ ScaleRowDown38_2 = ScaleRowDown38_AVX2;
+ } else {
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_AVX2;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_AVX2;
+ }
+ }
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN38_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
if (!filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_Any_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_Any_SSSE3;
+ ScaleRowDown38_3 = ScaleRowDown38_Any_LSX;
+ ScaleRowDown38_2 = ScaleRowDown38_Any_LSX;
} else {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_SSSE3;
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_LSX;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_LSX;
}
- if (dst_width % 12 == 0 && !filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_SSSE3;
+ if (dst_width % 12 == 0) {
+ if (!filtering) {
+ ScaleRowDown38_3 = ScaleRowDown38_LSX;
+ ScaleRowDown38_2 = ScaleRowDown38_LSX;
+ } else {
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_LSX;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_LSX;
+ }
}
- if (dst_width % 6 == 0 && filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_SSSE3;
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN38_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ if (!filtering) {
+ ScaleRowDown38_3 = ScaleRowDown38_RVV;
+ ScaleRowDown38_2 = ScaleRowDown38_RVV;
+ } else {
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_RVV;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_RVV;
}
}
#endif
-#if defined(HAS_SCALEROWDOWN38_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
+#if defined(HAS_SCALEROWDOWN38_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
if (!filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_Any_MSA;
- ScaleRowDown38_2 = ScaleRowDown38_Any_MSA;
+ ScaleRowDown38_3 = ScaleRowDown38_Any_WASMSIMD;
+ ScaleRowDown38_2 = ScaleRowDown38_Any_WASMSIMD;
} else {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_MSA;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_MSA;
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_WASMSIMD;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_WASMSIMD;
}
if (dst_width % 12 == 0) {
if (!filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_MSA;
- ScaleRowDown38_2 = ScaleRowDown38_MSA;
+ ScaleRowDown38_3 = ScaleRowDown38_WASMSIMD;
+ ScaleRowDown38_2 = ScaleRowDown38_WASMSIMD;
} else {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_MSA;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_MSA;
+ ScaleRowDown38_3 = ScaleRowDown38_3_Box_WASMSIMD;
+ ScaleRowDown38_2 = ScaleRowDown38_2_Box_WASMSIMD;
}
}
}
@@ -641,8 +734,8 @@ static void ScalePlaneDown38_16(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr,
enum FilterMode filtering) {
@@ -651,7 +744,7 @@ static void ScalePlaneDown38_16(int src_width,
uint16_t* dst_ptr, int dst_width);
void (*ScaleRowDown38_2)(const uint16_t* src_ptr, ptrdiff_t src_stride,
uint16_t* dst_ptr, int dst_width);
- const int filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
+ const ptrdiff_t filter_stride = (filtering == kFilterLinear) ? 0 : src_stride;
(void)src_width;
(void)src_height;
assert(dst_width % 3 == 0);
@@ -662,28 +755,6 @@ static void ScalePlaneDown38_16(int src_width,
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C;
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C;
}
-#if defined(HAS_SCALEROWDOWN38_16_NEON)
- if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) {
- if (!filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_16_NEON;
- ScaleRowDown38_2 = ScaleRowDown38_16_NEON;
- } else {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON;
- }
- }
-#endif
-#if defined(HAS_SCALEROWDOWN38_16_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
- if (!filtering) {
- ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3;
- } else {
- ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3;
- ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3;
- }
- }
-#endif
for (y = 0; y < dst_height - 2; y += 3) {
ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width);
@@ -736,9 +807,6 @@ static void ScaleAddCols2_C(int dst_width,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr) {
-#ifdef __clang_analyzer__
- *dst_ptr = 0;
-#else
int i;
int scaletbl[2];
int minboxwidth = dx >> 16;
@@ -749,11 +817,12 @@ static void ScaleAddCols2_C(int dst_width,
int ix = x >> 16;
x += dx;
boxwidth = MIN1((x >> 16) - ix);
- *dst_ptr++ =
- SumPixels(boxwidth, src_ptr + ix) * scaletbl[boxwidth - minboxwidth] >>
- 16;
+ int scaletbl_index = boxwidth - minboxwidth;
+ assert((scaletbl_index == 0) || (scaletbl_index == 1));
+ *dst_ptr++ = (uint8_t)(SumPixels(boxwidth, src_ptr + ix) *
+ scaletbl[scaletbl_index] >>
+ 16);
}
-#endif
}
static void ScaleAddCols2_16_C(int dst_width,
@@ -762,9 +831,6 @@ static void ScaleAddCols2_16_C(int dst_width,
int dx,
const uint32_t* src_ptr,
uint16_t* dst_ptr) {
-#ifdef __clang_analyzer__
- * dst_ptr = 0;
-#else
int i;
int scaletbl[2];
int minboxwidth = dx >> 16;
@@ -775,11 +841,11 @@ static void ScaleAddCols2_16_C(int dst_width,
int ix = x >> 16;
x += dx;
boxwidth = MIN1((x >> 16) - ix);
- *dst_ptr++ = SumPixels_16(boxwidth, src_ptr + ix) *
- scaletbl[boxwidth - minboxwidth] >>
- 16;
+ int scaletbl_index = boxwidth - minboxwidth;
+ assert((scaletbl_index == 0) || (scaletbl_index == 1));
+ *dst_ptr++ =
+ SumPixels_16(boxwidth, src_ptr + ix) * scaletbl[scaletbl_index] >> 16;
}
-#endif
}
static void ScaleAddCols0_C(int dst_width,
@@ -793,7 +859,7 @@ static void ScaleAddCols0_C(int dst_width,
(void)dx;
src_ptr += (x >> 16);
for (i = 0; i < dst_width; ++i) {
- *dst_ptr++ = src_ptr[i] * scaleval >> 16;
+ *dst_ptr++ = (uint8_t)(src_ptr[i] * scaleval >> 16);
}
}
@@ -808,7 +874,7 @@ static void ScaleAddCols1_C(int dst_width,
int i;
x >>= 16;
for (i = 0; i < dst_width; ++i) {
- *dst_ptr++ = SumPixels(boxwidth, src_ptr + x) * scaleval >> 16;
+ *dst_ptr++ = (uint8_t)(SumPixels(boxwidth, src_ptr + x) * scaleval >> 16);
x += boxwidth;
}
}
@@ -822,6 +888,7 @@ static void ScaleAddCols1_16_C(int dst_width,
int boxwidth = MIN1(dx >> 16);
int scaleval = 65536 / (boxwidth * boxheight);
int i;
+ x >>= 16;
for (i = 0; i < dst_width; ++i) {
*dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16;
x += boxwidth;
@@ -835,27 +902,29 @@ static void ScaleAddCols1_16_C(int dst_width,
// one pixel of destination using fixed point (16.16) to step
// through source, sampling a box of pixel with simple
// averaging.
-static void ScalePlaneBox(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_ptr,
- uint8_t* dst_ptr) {
+static int ScalePlaneBox(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr) {
int j, k;
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
int dx = 0;
int dy = 0;
- const int max_y = (src_height << 16);
+ const uint32_t max_y = (uint32_t)src_height << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
{
// Allocate a row buffer of uint16_t.
align_buffer_64(row16, src_width * 2);
+ if (!row16)
+ return 1;
void (*ScaleAddCols)(int dst_width, int boxheight, int x, int dx,
const uint16_t* src_ptr, uint8_t* dst_ptr) =
(dx & 0xffff) ? ScaleAddCols2_C
@@ -878,6 +947,22 @@ static void ScalePlaneBox(int src_width,
}
}
#endif
+#if defined(HAS_SCALEADDROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ScaleAddRow = ScaleAddRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SCALEADDCOLS1_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && !(dx & 0xffff) &&
+ dx != 0x10000) {
+ ScaleAddCols = ScaleAddCols1_AVX512BW;
+ }
+#endif
+#if defined(HAS_SCALEADDCOLS2_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && (dx & 0xffff)) {
+ ScaleAddCols = ScaleAddCols2_AVX512BW;
+ }
+#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;
@@ -886,32 +971,43 @@ static void ScalePlaneBox(int src_width,
}
}
#endif
-#if defined(HAS_SCALEADDROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ScaleAddRow = ScaleAddRow_Any_MSA;
+#if defined(HAS_SCALEADDROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleAddRow = ScaleAddRow_Any_LSX;
if (IS_ALIGNED(src_width, 16)) {
- ScaleAddRow = ScaleAddRow_MSA;
+ ScaleAddRow = ScaleAddRow_LSX;
}
}
#endif
-#if defined(HAS_SCALEADDROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ScaleAddRow = ScaleAddRow_Any_MMI;
- if (IS_ALIGNED(src_width, 8)) {
- ScaleAddRow = ScaleAddRow_MMI;
+#if defined(HAS_SCALEADDROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleAddRow = ScaleAddRow_RVV;
+ }
+#endif
+#if defined(HAS_SCALEADDROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleAddRow = ScaleAddRow_HVX;
+ }
+#endif
+#if defined(HAS_SCALEADDROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASM)) {
+ ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
+ if (IS_ALIGNED(src_width, 16)) {
+ ScaleAddRow = ScaleAddRow_WASMSIMD;
}
}
#endif
+ uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
- int iy = y >> 16;
+ int iy = uy >> 16;
const uint8_t* src = src_ptr + iy * src_stride;
- y += dy;
- if (y > max_y) {
- y = max_y;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
}
- boxheight = MIN1((y >> 16) - iy);
+ boxheight = MIN1((int)(uy >> 16) - iy);
memset(row16, 0, src_width * 2);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint16_t*)(row16), src_width);
@@ -922,55 +1018,48 @@ static void ScalePlaneBox(int src_width,
}
free_aligned_buffer_64(row16);
}
+ return 0;
}
-static void ScalePlaneBox_16(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint16_t* src_ptr,
- uint16_t* dst_ptr) {
+static int ScalePlaneBox_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
int j, k;
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
int dx = 0;
int dy = 0;
- const int max_y = (src_height << 16);
+ const uint32_t max_y = (uint32_t)src_height << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
{
// Allocate a row buffer of uint32_t.
align_buffer_64(row32, src_width * 4);
+ if (!row32)
+ return 1;
void (*ScaleAddCols)(int dst_width, int boxheight, int x, int dx,
const uint32_t* src_ptr, uint16_t* dst_ptr) =
(dx & 0xffff) ? ScaleAddCols2_16_C : ScaleAddCols1_16_C;
void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr,
int src_width) = ScaleAddRow_16_C;
-#if defined(HAS_SCALEADDROW_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) {
- ScaleAddRow = ScaleAddRow_16_SSE2;
- }
-#endif
-
-#if defined(HAS_SCALEADDROW_16_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(src_width, 4)) {
- ScaleAddRow = ScaleAddRow_16_MMI;
- }
-#endif
+ uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
- int iy = y >> 16;
+ int iy = uy >> 16;
const uint16_t* src = src_ptr + iy * src_stride;
- y += dy;
- if (y > max_y) {
- y = max_y;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
}
- boxheight = MIN1((y >> 16) - iy);
+ boxheight = MIN1((int)(uy >> 16) - iy);
memset(row32, 0, src_width * 4);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint32_t*)(row32), src_width);
@@ -981,18 +1070,19 @@ static void ScalePlaneBox_16(int src_width,
}
free_aligned_buffer_64(row32);
}
+ return 0;
}
// Scale plane down with bilinear interpolation.
-void ScalePlaneBilinearDown(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- enum FilterMode filtering) {
+static int ScalePlaneBilinearDown(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ enum FilterMode filtering) {
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
@@ -1001,27 +1091,21 @@ void ScalePlaneBilinearDown(int src_width,
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row buffer.
align_buffer_64(row, src_width);
+ if (!row)
+ return 1;
- const int max_y = (src_height - 1) << 16;
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
int j;
- void (*ScaleFilterCols)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ void (*ScaleFilterCols)(uint8_t* dst_ptr, const uint8_t* src_ptr,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleFilterCols64_C : ScaleFilterCols_C;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(src_width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -1030,6 +1114,11 @@ void ScalePlaneBilinearDown(int src_width,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -1038,26 +1127,54 @@ void ScalePlaneBilinearDown(int src_width,
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(src_width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(src_width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+ InterpolateRow = InterpolateRow_LASX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(src_width, 16)) {
- InterpolateRow = InterpolateRow_MMI;
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_SCALEFILTERCOLS_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_SSSE3;
+#if defined(HAS_SCALEFILTERCOLS_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_AVX2;
}
#endif
#if defined(HAS_SCALEFILTERCOLS_NEON)
@@ -1068,46 +1185,53 @@ void ScalePlaneBilinearDown(int src_width,
}
}
#endif
-#if defined(HAS_SCALEFILTERCOLS_MSA)
- if (TestCpuFlag(kCpuHasMSA) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 16)) {
- ScaleFilterCols = ScaleFilterCols_MSA;
+#if defined(HAS_SCALEFILTERCOLS_LSX)
+ if (TestCpuFlag(kCpuHasLSX) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_Any_LSX;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleFilterCols = ScaleFilterCols_LSX;
}
}
#endif
- if (y > max_y) {
- y = max_y;
+#if defined(HAS_SCALEFILTERCOLS_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleFilterCols = ScaleFilterCols_WASMSIMD;
+ }
}
+#endif
+ uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
- int yi = y >> 16;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ int yi = uy >> 16;
const uint8_t* src = src_ptr + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleFilterCols(dst_ptr, src, dst_width, x, dx);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (uy >> 8) & 255;
InterpolateRow(row, src, src_stride, src_width, yf);
ScaleFilterCols(dst_ptr, row, dst_width, x, dx);
}
dst_ptr += dst_stride;
- y += dy;
- if (y > max_y) {
- y = max_y;
- }
+ uy += dy;
}
free_aligned_buffer_64(row);
+ return 0;
}
-void ScalePlaneBilinearDown_16(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint16_t* src_ptr,
- uint16_t* dst_ptr,
- enum FilterMode filtering) {
+static int ScalePlaneBilinearDown_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ enum FilterMode filtering) {
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
@@ -1116,115 +1240,115 @@ void ScalePlaneBilinearDown_16(int src_width,
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row buffer.
align_buffer_64(row, src_width * 2);
+ if (!row)
+ return 1;
- const int max_y = (src_height - 1) << 16;
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
int j;
- void (*ScaleFilterCols)(uint16_t * dst_ptr, const uint16_t* src_ptr,
+ void (*ScaleFilterCols)(uint16_t* dst_ptr, const uint16_t* src_ptr,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleFilterCols64_16_C : ScaleFilterCols_16_C;
- void (*InterpolateRow)(uint16_t * dst_ptr, const uint16_t* src_ptr,
+ void (*InterpolateRow)(uint16_t* dst_ptr, const uint16_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_16_C;
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
-#if defined(HAS_INTERPOLATEROW_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- InterpolateRow = InterpolateRow_Any_16_SSE2;
- if (IS_ALIGNED(src_width, 16)) {
- InterpolateRow = InterpolateRow_16_SSE2;
- }
- }
-#endif
-#if defined(HAS_INTERPOLATEROW_16_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_16_SSSE3;
- if (IS_ALIGNED(src_width, 16)) {
- InterpolateRow = InterpolateRow_16_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_16_AVX2;
+ InterpolateRow = InterpolateRow_16_Any_AVX2;
if (IS_ALIGNED(src_width, 32)) {
InterpolateRow = InterpolateRow_16_AVX2;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_16_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_16_NEON;
+ InterpolateRow = InterpolateRow_16_Any_NEON;
if (IS_ALIGNED(src_width, 16)) {
InterpolateRow = InterpolateRow_16_NEON;
}
}
#endif
-
-#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_16_SSSE3;
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_16_HVX;
}
#endif
- if (y > max_y) {
- y = max_y;
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(src_width, 8)) {
+ InterpolateRow = InterpolateRow_16_WASMSIMD;
+ }
}
+#endif
+ uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
- int yi = y >> 16;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ int yi = uy >> 16;
const uint16_t* src = src_ptr + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleFilterCols(dst_ptr, src, dst_width, x, dx);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (uy >> 8) & 255;
InterpolateRow((uint16_t*)row, src, src_stride, src_width, yf);
ScaleFilterCols(dst_ptr, (uint16_t*)row, dst_width, x, dx);
}
dst_ptr += dst_stride;
- y += dy;
- if (y > max_y) {
- y = max_y;
- }
+ uy += dy;
}
free_aligned_buffer_64(row);
+ return 0;
}
// Scale up down with bilinear interpolation.
-void ScalePlaneBilinearUp(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_ptr,
- uint8_t* dst_ptr,
- enum FilterMode filtering) {
+static int ScalePlaneBilinearUp(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ enum FilterMode filtering) {
+ assert(src_width > 0);
+ assert(src_height > 0);
+ assert(dst_width > 0);
+ assert(dst_height > 0);
+
int j;
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
int dx = 0;
int dy = 0;
- const int max_y = (src_height - 1) << 16;
- void (*InterpolateRow)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- void (*ScaleFilterCols)(uint8_t * dst_ptr, const uint8_t* src_ptr,
+ void (*ScaleFilterCols)(uint8_t* dst_ptr, const uint8_t* src_ptr,
int dst_width, int x, int dx) =
filtering ? ScaleFilterCols_C : ScaleCols_C;
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
&dx, &dy);
+ assert(dy <= 65536);
src_width = Abs(src_width);
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(dst_width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -1233,6 +1357,11 @@ void ScalePlaneBilinearUp(int src_width,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -1241,13 +1370,41 @@ void ScalePlaneBilinearUp(int src_width,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
+ }
+ }
+#endif
if (filtering && src_width >= 32768) {
ScaleFilterCols = ScaleFilterCols64_C;
}
-#if defined(HAS_SCALEFILTERCOLS_SSSE3)
- if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_SSSE3;
+#if defined(HAS_SCALEFILTERCOLS_AVX2)
+ if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_AVX2;
}
#endif
#if defined(HAS_SCALEFILTERCOLS_NEON)
@@ -1258,41 +1415,47 @@ void ScalePlaneBilinearUp(int src_width,
}
}
#endif
-#if defined(HAS_SCALEFILTERCOLS_MSA)
- if (filtering && TestCpuFlag(kCpuHasMSA) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 16)) {
- ScaleFilterCols = ScaleFilterCols_MSA;
+#if defined(HAS_SCALEFILTERCOLS_LSX)
+ if (filtering && TestCpuFlag(kCpuHasLSX) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_Any_LSX;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleFilterCols = ScaleFilterCols_LSX;
}
}
#endif
- if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
- ScaleFilterCols = ScaleColsUp2_C;
-#if defined(HAS_SCALECOLS_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleFilterCols = ScaleColsUp2_SSE2;
+#if defined(HAS_SCALEFILTERCOLS_WASMSIMD)
+ if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleFilterCols = ScaleFilterCols_WASMSIMD;
}
+ }
#endif
-#if defined(HAS_SCALECOLS_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 8)) {
- ScaleFilterCols = ScaleColsUp2_MMI;
+ if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
+ ScaleFilterCols = ScaleColsUp2_C;
+#if defined(HAS_SCALECOLSUP2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
+ ScaleFilterCols = ScaleColsUp2_AVX2;
}
#endif
}
- if (y > max_y) {
- y = max_y;
+ uint32_t uy = y;
+ if (uy > max_y) {
+ uy = max_y;
}
{
- int yi = y >> 16;
+ int yi = uy >> 16;
const uint8_t* src = src_ptr + yi * src_stride;
// Allocate 2 row buffers.
- const int kRowSize = (dst_width + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
+ const int row_size = (dst_width + 31) & ~31;
+ align_buffer_64(row, row_size * 2);
+ if (!row)
+ return 1;
uint8_t* rowptr = row;
- int rowstride = kRowSize;
+ ptrdiff_t rowstride = row_size;
int lasty = yi;
ScaleFilterCols(rowptr, src, dst_width, x, dx);
@@ -1300,167 +1463,523 @@ void ScalePlaneBilinearUp(int src_width,
src += src_stride;
}
ScaleFilterCols(rowptr + rowstride, src, dst_width, x, dx);
- src += src_stride;
+ if (src_height > 2) {
+ src += src_stride;
+ }
+ // 2-row rolling buffer:
+ // rowptr and (rowptr + rowstride) hold the scaled rows for yi and yi + 1.
+ // Because dy <= 65536 (dy <= 1.0 in 16.16), yi advances in unit steps.
+ // When yi != lasty:
+ // 1. Scale the next source row into the older buffer (rowptr).
+ // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
+ // so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
+ // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
+ // otherwise clamp src at (src_height - 1) to avoid reading out of
+ // bounds.
for (j = 0; j < dst_height; ++j) {
- yi = y >> 16;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ yi = uy >> 16;
if (yi != lasty) {
- if (y > max_y) {
- y = max_y;
- yi = y >> 16;
- src = src_ptr + yi * src_stride;
- }
- if (yi != lasty) {
- ScaleFilterCols(rowptr, src, dst_width, x, dx);
- rowptr += rowstride;
- rowstride = -rowstride;
- lasty = yi;
+ ScaleFilterCols(rowptr, src, dst_width, x, dx);
+ rowptr += rowstride;
+ rowstride = -rowstride;
+ lasty = yi;
+ if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (uy >> 8) & 255;
InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf);
}
dst_ptr += dst_stride;
- y += dy;
+ uy += dy;
}
free_aligned_buffer_64(row);
}
+ return 0;
}
-void ScalePlaneBilinearUp_16(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint16_t* src_ptr,
- uint16_t* dst_ptr,
- enum FilterMode filtering) {
- int j;
- // Initial source x/y coordinate and step values as 16.16 fixed point.
- int x = 0;
- int y = 0;
- int dx = 0;
- int dy = 0;
- const int max_y = (src_height - 1) << 16;
- void (*InterpolateRow)(uint16_t * dst_ptr, const uint16_t* src_ptr,
- ptrdiff_t src_stride, int dst_width,
- int source_y_fraction) = InterpolateRow_16_C;
- void (*ScaleFilterCols)(uint16_t * dst_ptr, const uint16_t* src_ptr,
- int dst_width, int x, int dx) =
- filtering ? ScaleFilterCols_16_C : ScaleCols_16_C;
- ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
- &dx, &dy);
- src_width = Abs(src_width);
+// Scale plane, horizontally up by 2 times.
+// Uses linear filter horizontally, nearest vertically.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original width, using linear interpolation.
+// This is used to scale U and V planes of I422 to I444.
+static void ScalePlaneUp2_Linear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr) {
+ void (*ScaleRowUp)(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) =
+ ScaleRowUp2_Linear_Any_C;
+ int i;
+
+ (void)src_width;
+ // This function can only scale up by 2 times horizontally.
+ assert(src_width == ((dst_width + 1) / 2));
-#if defined(HAS_INTERPOLATEROW_16_SSE2)
+#ifdef HAS_SCALEROWUP2_LINEAR_SSE2
if (TestCpuFlag(kCpuHasSSE2)) {
- InterpolateRow = InterpolateRow_Any_16_SSE2;
- if (IS_ALIGNED(dst_width, 16)) {
- InterpolateRow = InterpolateRow_16_SSE2;
+ ScaleRowUp = ScaleRowUp2_Linear_Any_SSE2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_SSSE3
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp = ScaleRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp = ScaleRowUp2_Linear_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp = ScaleRowUp2_Linear_Any_NEON;
+ }
+#endif
+#ifdef HAS_SCALEROWUP2_LINEAR_RVV
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowUp = ScaleRowUp2_Linear_RVV;
+ }
+#endif
+
+ if (dst_height == 1) {
+ ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
+ dst_width);
+ } else {
+ uint32_t uy = (1 << 15) - 1;
+ int dy = FixedDiv(src_height - 1, dst_height - 1);
+ for (i = 0; i < dst_height; ++i) {
+ int yi = uy >> 16;
+ ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
+ dst_ptr += dst_stride;
+ uy += dy;
}
}
+}
+
+// Scale plane, up by 2 times.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original size, using bilinear interpolation.
+// This is used to scale U and V planes of I420 to I444.
+static void ScalePlaneUp2_Bilinear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr) {
+ void (*Scale2RowUp)(const uint8_t* src_ptr, ptrdiff_t src_stride,
+ uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
+ ScaleRowUp2_Bilinear_Any_C;
+ int x;
+
+ (void)src_width;
+ // This function can only scale up by 2 times.
+ assert(src_width == ((dst_width + 1) / 2));
+ assert(src_height == ((dst_height + 1) / 2));
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSE2
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_Any_SSE2;
+ }
#endif
-#if defined(HAS_INTERPOLATEROW_16_SSSE3)
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSSE3
if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_16_SSSE3;
- if (IS_ALIGNED(dst_width, 16)) {
- InterpolateRow = InterpolateRow_16_SSSE3;
+ Scale2RowUp = ScaleRowUp2_Bilinear_Any_SSSE3;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_Any_NEON;
+ }
+#endif
+#ifdef HAS_SCALEROWUP2_BILINEAR_RVV
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_RVV;
+ }
+#endif
+
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ dst_ptr += dst_stride;
+ for (x = 0; x < src_height - 1; ++x) {
+ Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
+ src_ptr += src_stride;
+ // TODO(fbarchard): Test performance of writing one row of destination at a
+ // time.
+ dst_ptr += 2 * dst_stride;
+ }
+ if (!(dst_height & 1)) {
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ }
+}
+
+// Scale at most 14 bit plane, horizontally up by 2 times.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original width, using linear interpolation.
+// stride is in count of uint16_t.
+// This is used to scale U and V planes of I210 to I410 and I212 to I412.
+static void ScalePlaneUp2_12_Linear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
+ void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ int i;
+
+ (void)src_width;
+ // This function can only scale up by 2 times horizontally.
+ assert(src_width == ((dst_width + 1) / 2));
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_SSSE3
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp = ScaleRowUp2_Linear_12_Any_SSSE3;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp = ScaleRowUp2_Linear_12_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp = ScaleRowUp2_Linear_12_Any_NEON;
+ }
+#endif
+
+ if (dst_height == 1) {
+ ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
+ dst_width);
+ } else {
+ uint32_t uy = (1 << 15) - 1;
+ int dy = FixedDiv(src_height - 1, dst_height - 1);
+ for (i = 0; i < dst_height; ++i) {
+ int yi = uy >> 16;
+ ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
+ dst_ptr += dst_stride;
+ uy += dy;
}
}
+}
+
+// Scale at most 12 bit plane, up by 2 times.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original size, using bilinear interpolation.
+// stride is in count of uint16_t.
+// This is used to scale U and V planes of I010 to I410 and I012 to I412.
+static void ScalePlaneUp2_12_Bilinear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
+ void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride,
+ uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
+ ScaleRowUp2_Bilinear_16_Any_C;
+ int x;
+
+ (void)src_width;
+ // This function can only scale up by 2 times.
+ assert(src_width == ((dst_width + 1) / 2));
+ assert(src_height == ((dst_height + 1) / 2));
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_SSSE3
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_12_Any_SSSE3;
+ }
#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_12_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_12_Any_NEON;
+ }
+#endif
+
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ dst_ptr += dst_stride;
+ for (x = 0; x < src_height - 1; ++x) {
+ Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
+ src_ptr += src_stride;
+ dst_ptr += 2 * dst_stride;
+ }
+ if (!(dst_height & 1)) {
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ }
+}
+
+static void ScalePlaneUp2_16_Linear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
+ void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr,
+ int dst_width) = ScaleRowUp2_Linear_16_Any_C;
+ int i;
+
+ (void)src_width;
+ // This function can only scale up by 2 times horizontally.
+ assert(src_width == ((dst_width + 1) / 2));
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_SSE2
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ScaleRowUp = ScaleRowUp2_Linear_16_Any_SSE2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp = ScaleRowUp2_Linear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp = ScaleRowUp2_Linear_16_Any_NEON;
+ }
+#endif
+
+ if (dst_height == 1) {
+ ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
+ dst_width);
+ } else {
+ uint32_t uy = (1 << 15) - 1;
+ int dy = FixedDiv(src_height - 1, dst_height - 1);
+ for (i = 0; i < dst_height; ++i) {
+ int yi = uy >> 16;
+ ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
+ dst_ptr += dst_stride;
+ uy += dy;
+ }
+ }
+}
+
+static void ScalePlaneUp2_16_Bilinear(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
+ void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride,
+ uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
+ ScaleRowUp2_Bilinear_16_Any_C;
+ int x;
+
+ (void)src_width;
+ // This function can only scale up by 2 times.
+ assert(src_width == ((dst_width + 1) / 2));
+ assert(src_height == ((dst_height + 1) / 2));
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_SSE2
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_16_Any_SSE2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp = ScaleRowUp2_Bilinear_16_Any_NEON;
+ }
+#endif
+
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ dst_ptr += dst_stride;
+ for (x = 0; x < src_height - 1; ++x) {
+ Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
+ src_ptr += src_stride;
+ dst_ptr += 2 * dst_stride;
+ }
+ if (!(dst_height & 1)) {
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ }
+}
+
+static int ScalePlaneBilinearUp_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ enum FilterMode filtering) {
+ assert(src_width > 0);
+ assert(src_height > 0);
+ assert(dst_width > 0);
+ assert(dst_height > 0);
+
+ int j;
+ // Initial source x/y coordinate and step values as 16.16 fixed point.
+ int x = 0;
+ int y = 0;
+ int dx = 0;
+ int dy = 0;
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ void (*InterpolateRow)(uint16_t* dst_ptr, const uint16_t* src_ptr,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_16_C;
+ void (*ScaleFilterCols)(uint16_t* dst_ptr, const uint16_t* src_ptr,
+ int dst_width, int x, int dx) =
+ filtering ? ScaleFilterCols_16_C : ScaleCols_16_C;
+ ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
+ &dx, &dy);
+ assert(dy <= 65536);
+ src_width = Abs(src_width);
+
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_16_AVX2;
+ InterpolateRow = InterpolateRow_16_Any_AVX2;
if (IS_ALIGNED(dst_width, 32)) {
InterpolateRow = InterpolateRow_16_AVX2;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_16_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_16_NEON;
+ InterpolateRow = InterpolateRow_16_Any_NEON;
if (IS_ALIGNED(dst_width, 16)) {
InterpolateRow = InterpolateRow_16_NEON;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_16_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 8)) {
+ InterpolateRow = InterpolateRow_16_WASMSIMD;
+ }
+ }
+#endif
if (filtering && src_width >= 32768) {
ScaleFilterCols = ScaleFilterCols64_16_C;
}
-#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
- if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleFilterCols = ScaleFilterCols_16_SSSE3;
- }
-#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleFilterCols = ScaleColsUp2_16_C;
-#if defined(HAS_SCALECOLS_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleFilterCols = ScaleColsUp2_16_SSE2;
- }
-#endif
-#if defined(HAS_SCALECOLS_16_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 8)) {
- ScaleFilterCols = ScaleColsUp2_16_MMI;
- }
-#endif
}
- if (y > max_y) {
- y = max_y;
+ uint32_t uy = y;
+ if (uy > max_y) {
+ uy = max_y;
}
{
- int yi = y >> 16;
+ int yi = uy >> 16;
const uint16_t* src = src_ptr + yi * src_stride;
// Allocate 2 row buffers.
- const int kRowSize = (dst_width + 31) & ~31;
- align_buffer_64(row, kRowSize * 4);
-
- uint16_t* rowptr = (uint16_t*)row;
- int rowstride = kRowSize;
+ const int row_size = (dst_width + 31) & ~31;
+ align_buffer_64(row, row_size * 4);
+ ptrdiff_t rowstride = row_size;
int lasty = yi;
+ uint16_t* rowptr = (uint16_t*)row;
+ if (!row)
+ return 1;
ScaleFilterCols(rowptr, src, dst_width, x, dx);
if (src_height > 1) {
src += src_stride;
}
ScaleFilterCols(rowptr + rowstride, src, dst_width, x, dx);
- src += src_stride;
+ if (src_height > 2) {
+ src += src_stride;
+ }
+ // 2-row rolling buffer:
+ // rowptr and (rowptr + rowstride) hold the scaled rows for yi and yi + 1.
+ // Because dy <= 65536 (dy <= 1.0 in 16.16), yi advances in unit steps.
+ // When yi != lasty:
+ // 1. Scale the next source row into the older buffer (rowptr).
+ // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
+ // so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
+ // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
+ // otherwise clamp src at (src_height - 1) to avoid reading out of
+ // bounds.
for (j = 0; j < dst_height; ++j) {
- yi = y >> 16;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ yi = uy >> 16;
if (yi != lasty) {
- if (y > max_y) {
- y = max_y;
- yi = y >> 16;
- src = src_ptr + yi * src_stride;
- }
- if (yi != lasty) {
- ScaleFilterCols(rowptr, src, dst_width, x, dx);
- rowptr += rowstride;
- rowstride = -rowstride;
- lasty = yi;
+ ScaleFilterCols(rowptr, src, dst_width, x, dx);
+ rowptr += rowstride;
+ rowstride = -rowstride;
+ lasty = yi;
+ if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (uy >> 8) & 255;
InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf);
}
dst_ptr += dst_stride;
- y += dy;
+ uy += dy;
}
free_aligned_buffer_64(row);
}
+ return 0;
}
// Scale Plane to/from any dimensions, without interpolation.
@@ -1472,12 +1991,12 @@ static void ScalePlaneSimple(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr) {
int i;
- void (*ScaleCols)(uint8_t * dst_ptr, const uint8_t* src_ptr, int dst_width,
+ void (*ScaleCols)(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width,
int x, int dx) = ScaleCols_C;
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
@@ -1490,22 +2009,19 @@ static void ScalePlaneSimple(int src_width,
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleCols = ScaleColsUp2_C;
-#if defined(HAS_SCALECOLS_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleCols = ScaleColsUp2_SSE2;
- }
-#endif
-#if defined(HAS_SCALECOLS_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 8)) {
- ScaleCols = ScaleColsUp2_MMI;
+#if defined(HAS_SCALECOLSUP2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
+ ScaleCols = ScaleColsUp2_AVX2;
}
#endif
}
+ uint32_t uy = y;
for (i = 0; i < dst_height; ++i) {
- ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx);
+ int yi = uy >> 16;
+ ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx);
dst_ptr += dst_stride;
- y += dy;
+ uy += dy;
}
}
@@ -1513,12 +2029,12 @@ static void ScalePlaneSimple_16(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr) {
int i;
- void (*ScaleCols)(uint16_t * dst_ptr, const uint16_t* src_ptr, int dst_width,
+ void (*ScaleCols)(uint16_t* dst_ptr, const uint16_t* src_ptr, int dst_width,
int x, int dx) = ScaleCols_16_C;
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
@@ -1531,38 +2047,37 @@ static void ScalePlaneSimple_16(int src_width,
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleCols = ScaleColsUp2_16_C;
-#if defined(HAS_SCALECOLS_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleCols = ScaleColsUp2_16_SSE2;
- }
-#endif
-#if defined(HAS_SCALECOLS_16_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 8)) {
- ScaleCols = ScaleColsUp2_16_MMI;
- }
-#endif
}
+ uint32_t uy = y;
for (i = 0; i < dst_height; ++i) {
- ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx);
+ int yi = uy >> 16;
+ ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx);
dst_ptr += dst_stride;
- y += dy;
+ uy += dy;
}
}
// Scale a plane.
// This function dispatches to a specialized scaler based on scale factor.
-
LIBYUV_API
-void ScalePlane(const uint8_t* src,
- int src_stride,
- int src_width,
- int src_height,
- uint8_t* dst,
- int dst_stride,
- int dst_width,
- int dst_height,
- enum FilterMode filtering) {
+int ScalePlane(const uint8_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint8_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ // Reject dimensions larger than 32768 (or smaller than -32768 for height).
+ // This prevents FixedDiv signed integer overflows that can lead to division
+ // by zero/overflow crashes (SIGFPE on x86) or incorrect step calculations.
+ if (!src || src_width <= 0 || src_height == 0 || src_width > 32768 ||
+ src_height < -32768 || src_height > 32768 || !dst || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
// Simplify filtering when possible.
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
filtering);
@@ -1570,23 +2085,44 @@ void ScalePlane(const uint8_t* src,
// Negative height means invert the image.
if (src_height < 0) {
src_height = -src_height;
- src = src + (src_height - 1) * src_stride;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
src_stride = -src_stride;
}
-
// Use specialized scales to improve performance for common resolutions.
// For example, all the 1/2 scalings will use ScalePlaneDown2()
if (dst_width == src_width && dst_height == src_height) {
// Straight copy.
CopyPlane(src, src_stride, dst, dst_stride, dst_width, dst_height);
- return;
+ return 0;
}
if (dst_width == src_width && filtering != kFilterBox) {
- int dy = FixedDiv(src_height, dst_height);
+ int dy = 0;
+ int y = 0;
+ // Check for 1 pixel and avoid FixedDiv overflow.
+ // TODO(b/573815252): Fix center row calculation when dst_height == 1 and
+ // src_height == 32768.
+ if (dst_height == 1 && src_height >= 32768) {
+ dy = FixedDiv(src_height, src_height);
+ y = CENTERSTART(dy, -32768);
+ } else if (dst_height <= src_height) {
+ // When scaling down, use the center 2 rows to filter.
+ dy = FixedDiv(src_height, dst_height);
+ y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter.
+ } else if (src_height > 1 && dst_height > 1) {
+ // Point sampling uses centered source/destination stepping, matching
+ // ScaleSlope. Bilinear scaling maps the source endpoints.
+ if (filtering == kFilterNone) {
+ dy = FixedDiv(src_height, dst_height);
+ y = CENTERSTART(dy, 0);
+ } else {
+ // When scaling up, last row of destination uses the last 2 source rows.
+ dy = FixedDiv1(src_height, dst_height);
+ }
+ }
// Arbitrary scale vertically, but unscaled horizontally.
ScalePlaneVertical(src_height, dst_width, dst_height, src_stride,
- dst_stride, src, dst, 0, 0, dy, 1, filtering);
- return;
+ dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering);
+ return 0;
}
if (dst_width <= Abs(src_width) && dst_height <= src_height) {
// Scale down.
@@ -1594,58 +2130,75 @@ void ScalePlane(const uint8_t* src,
// optimized, 3/4
ScalePlaneDown34(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst, filtering);
- return;
+ return 0;
}
if (2 * dst_width == src_width && 2 * dst_height == src_height) {
// optimized, 1/2
ScalePlaneDown2(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst, filtering);
- return;
+ return 0;
}
// 3/8 rounded up for odd sized chroma height.
if (8 * dst_width == 3 * src_width && 8 * dst_height == 3 * src_height) {
// optimized, 3/8
ScalePlaneDown38(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst, filtering);
- return;
+ return 0;
}
if (4 * dst_width == src_width && 4 * dst_height == src_height &&
(filtering == kFilterBox || filtering == kFilterNone)) {
// optimized, 1/4
ScalePlaneDown4(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst, filtering);
- return;
+ return 0;
}
}
if (filtering == kFilterBox && dst_height * 2 < src_height) {
- ScalePlaneBox(src_width, src_height, dst_width, dst_height, src_stride,
- dst_stride, src, dst);
- return;
+ return ScalePlaneBox(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ }
+ if ((dst_width + 1) / 2 == src_width && filtering == kFilterLinear) {
+ ScalePlaneUp2_Linear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width &&
+ (filtering == kFilterBilinear || filtering == kFilterBox)) {
+ ScalePlaneUp2_Bilinear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
}
if (filtering && dst_height > src_height) {
- ScalePlaneBilinearUp(src_width, src_height, dst_width, dst_height,
- src_stride, dst_stride, src, dst, filtering);
- return;
+ return ScalePlaneBilinearUp(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst, filtering);
}
if (filtering) {
- ScalePlaneBilinearDown(src_width, src_height, dst_width, dst_height,
- src_stride, dst_stride, src, dst, filtering);
- return;
+ return ScalePlaneBilinearDown(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst, filtering);
}
ScalePlaneSimple(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst);
+ return 0;
}
LIBYUV_API
-void ScalePlane_16(const uint16_t* src,
- int src_stride,
- int src_width,
- int src_height,
- uint16_t* dst,
- int dst_stride,
- int dst_width,
- int dst_height,
- enum FilterMode filtering) {
+int ScalePlane_16(const uint16_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint16_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ // Reject dimensions larger than 32768 (or smaller than -32768 for height).
+ // This prevents FixedDiv signed integer overflows that can lead to division
+ // by zero/overflow crashes (SIGFPE on x86) or incorrect step calculations.
+ if (!src || src_width <= 0 || src_height == 0 || src_width > 32768 ||
+ src_height < -32768 || src_height > 32768 || !dst || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
// Simplify filtering when possible.
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
filtering);
@@ -1653,23 +2206,44 @@ void ScalePlane_16(const uint16_t* src,
// Negative height means invert the image.
if (src_height < 0) {
src_height = -src_height;
- src = src + (src_height - 1) * src_stride;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
src_stride = -src_stride;
}
-
// Use specialized scales to improve performance for common resolutions.
// For example, all the 1/2 scalings will use ScalePlaneDown2()
if (dst_width == src_width && dst_height == src_height) {
// Straight copy.
CopyPlane_16(src, src_stride, dst, dst_stride, dst_width, dst_height);
- return;
+ return 0;
}
if (dst_width == src_width && filtering != kFilterBox) {
- int dy = FixedDiv(src_height, dst_height);
- // Arbitrary scale vertically, but unscaled vertically.
+ int dy = 0;
+ int y = 0;
+ // Check for 1 pixel and avoid FixedDiv overflow.
+ // TODO(b/573815252): Fix center row calculation when dst_height == 1 and
+ // src_height == 32768.
+ if (dst_height == 1 && src_height >= 32768) {
+ dy = FixedDiv(src_height, src_height);
+ y = CENTERSTART(dy, -32768);
+ } else if (dst_height <= src_height) {
+ // When scaling down, use the center 2 rows to filter.
+ dy = FixedDiv(src_height, dst_height);
+ y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter.
+ } else if (src_height > 1 && dst_height > 1) {
+ // Point sampling uses centered source/destination stepping, matching
+ // ScaleSlope. Bilinear scaling maps the source endpoints.
+ if (filtering == kFilterNone) {
+ dy = FixedDiv(src_height, dst_height);
+ y = CENTERSTART(dy, 0);
+ } else {
+ // When scaling up, last row of destination uses the last 2 source rows.
+ dy = FixedDiv1(src_height, dst_height);
+ }
+ }
+ // Arbitrary scale vertically, but unscaled horizontally.
ScalePlaneVertical_16(src_height, dst_width, dst_height, src_stride,
- dst_stride, src, dst, 0, 0, dy, 1, filtering);
- return;
+ dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering);
+ return 0;
}
if (dst_width <= Abs(src_width) && dst_height <= src_height) {
// Scale down.
@@ -1677,46 +2251,101 @@ void ScalePlane_16(const uint16_t* src,
// optimized, 3/4
ScalePlaneDown34_16(src_width, src_height, dst_width, dst_height,
src_stride, dst_stride, src, dst, filtering);
- return;
+ return 0;
}
if (2 * dst_width == src_width && 2 * dst_height == src_height) {
// optimized, 1/2
ScalePlaneDown2_16(src_width, src_height, dst_width, dst_height,
src_stride, dst_stride, src, dst, filtering);
- return;
+ return 0;
}
// 3/8 rounded up for odd sized chroma height.
if (8 * dst_width == 3 * src_width && 8 * dst_height == 3 * src_height) {
// optimized, 3/8
ScalePlaneDown38_16(src_width, src_height, dst_width, dst_height,
src_stride, dst_stride, src, dst, filtering);
- return;
+ return 0;
}
if (4 * dst_width == src_width && 4 * dst_height == src_height &&
(filtering == kFilterBox || filtering == kFilterNone)) {
// optimized, 1/4
ScalePlaneDown4_16(src_width, src_height, dst_width, dst_height,
src_stride, dst_stride, src, dst, filtering);
- return;
+ return 0;
}
}
if (filtering == kFilterBox && dst_height * 2 < src_height) {
- ScalePlaneBox_16(src_width, src_height, dst_width, dst_height, src_stride,
- dst_stride, src, dst);
- return;
+ return ScalePlaneBox_16(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ }
+ if ((dst_width + 1) / 2 == src_width && filtering == kFilterLinear) {
+ ScalePlaneUp2_16_Linear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width &&
+ (filtering == kFilterBilinear || filtering == kFilterBox)) {
+ ScalePlaneUp2_16_Bilinear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
}
if (filtering && dst_height > src_height) {
- ScalePlaneBilinearUp_16(src_width, src_height, dst_width, dst_height,
- src_stride, dst_stride, src, dst, filtering);
- return;
+ return ScalePlaneBilinearUp_16(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst, filtering);
}
if (filtering) {
- ScalePlaneBilinearDown_16(src_width, src_height, dst_width, dst_height,
- src_stride, dst_stride, src, dst, filtering);
- return;
+ return ScalePlaneBilinearDown_16(src_width, src_height, dst_width,
+ dst_height, src_stride, dst_stride, src,
+ dst, filtering);
}
ScalePlaneSimple_16(src_width, src_height, dst_width, dst_height, src_stride,
dst_stride, src, dst);
+ return 0;
+}
+
+LIBYUV_API
+int ScalePlane_12(const uint16_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint16_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ // Reject dimensions larger than 32768 (or smaller than -32768 for height).
+ // This prevents FixedDiv signed integer overflows that can lead to division
+ // by zero/overflow crashes (SIGFPE on x86) or incorrect step calculations.
+ if (!src || src_width <= 0 || src_height == 0 || src_width > 32768 ||
+ src_height < -32768 || src_height > 32768 || !dst || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ // Simplify filtering when possible.
+ filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
+ filtering);
+
+ // Negative height means invert the image.
+ if (src_height < 0) {
+ src_height = -src_height;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
+ src_stride = -src_stride;
+ }
+
+ if ((dst_width + 1) / 2 == src_width && filtering == kFilterLinear) {
+ ScalePlaneUp2_12_Linear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width &&
+ (filtering == kFilterBilinear || filtering == kFilterBox)) {
+ ScalePlaneUp2_12_Bilinear(src_width, src_height, dst_width, dst_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+
+ return ScalePlane_16(src, src_stride, src_width, src_height, dst, dst_stride,
+ dst_width, dst_height, filtering);
}
// Scale an I420 image.
@@ -1740,23 +2369,31 @@ int I420Scale(const uint8_t* src_y,
int dst_width,
int dst_height,
enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
int src_halfheight = SUBSAMPLE(src_height, 1, 1);
int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
int dst_halfheight = SUBSAMPLE(dst_height, 1, 1);
- if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 ||
- src_width > 32768 || src_height > 32768 || !dst_y || !dst_u || !dst_v ||
- dst_width <= 0 || dst_height <= 0) {
- return -1;
- }
- ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y, dst_stride_y,
- dst_width, dst_height, filtering);
- ScalePlane(src_u, src_stride_u, src_halfwidth, src_halfheight, dst_u,
- dst_stride_u, dst_halfwidth, dst_halfheight, filtering);
- ScalePlane(src_v, src_stride_v, src_halfwidth, src_halfheight, dst_v,
- dst_stride_v, dst_halfwidth, dst_halfheight, filtering);
- return 0;
+ r = ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_u, src_stride_u, src_halfwidth, src_halfheight, dst_u,
+ dst_stride_u, dst_halfwidth, dst_halfheight, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, src_halfwidth, src_halfheight, dst_v,
+ dst_stride_v, dst_halfwidth, dst_halfheight, filtering);
+ return r;
}
LIBYUV_API
@@ -1777,23 +2414,76 @@ int I420Scale_16(const uint16_t* src_y,
int dst_width,
int dst_height,
enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
int src_halfheight = SUBSAMPLE(src_height, 1, 1);
int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
int dst_halfheight = SUBSAMPLE(dst_height, 1, 1);
- if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 ||
- src_width > 32768 || src_height > 32768 || !dst_y || !dst_u || !dst_v ||
- dst_width <= 0 || dst_height <= 0) {
+
+ r = ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_u, src_stride_u, src_halfwidth, src_halfheight, dst_u,
+ dst_stride_u, dst_halfwidth, dst_halfheight, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_v, src_stride_v, src_halfwidth, src_halfheight, dst_v,
+ dst_stride_v, dst_halfwidth, dst_halfheight, filtering);
+ return r;
+}
+
+LIBYUV_API
+int I420Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
return -1;
}
+ int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
+ int src_halfheight = SUBSAMPLE(src_height, 1, 1);
+ int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
+ int dst_halfheight = SUBSAMPLE(dst_height, 1, 1);
- ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y, dst_stride_y,
- dst_width, dst_height, filtering);
- ScalePlane_16(src_u, src_stride_u, src_halfwidth, src_halfheight, dst_u,
- dst_stride_u, dst_halfwidth, dst_halfheight, filtering);
- ScalePlane_16(src_v, src_stride_v, src_halfwidth, src_halfheight, dst_v,
- dst_stride_v, dst_halfwidth, dst_halfheight, filtering);
- return 0;
+ r = ScalePlane_12(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_u, src_stride_u, src_halfwidth, src_halfheight, dst_u,
+ dst_stride_u, dst_halfwidth, dst_halfheight, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, src_halfwidth, src_halfheight, dst_v,
+ dst_stride_v, dst_halfwidth, dst_halfheight, filtering);
+ return r;
}
// Scale an I444 image.
@@ -1817,19 +2507,27 @@ int I444Scale(const uint8_t* src_y,
int dst_width,
int dst_height,
enum FilterMode filtering) {
- if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 ||
- src_width > 32768 || src_height > 32768 || !dst_y || !dst_u || !dst_v ||
- dst_width <= 0 || dst_height <= 0) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
return -1;
}
- ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y, dst_stride_y,
- dst_width, dst_height, filtering);
- ScalePlane(src_u, src_stride_u, src_width, src_height, dst_u, dst_stride_u,
- dst_width, dst_height, filtering);
- ScalePlane(src_v, src_stride_v, src_width, src_height, dst_v, dst_stride_v,
- dst_width, dst_height, filtering);
- return 0;
+ r = ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_u, src_stride_u, src_width, src_height, dst_u,
+ dst_stride_u, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, src_width, src_height, dst_v,
+ dst_stride_v, dst_width, dst_height, filtering);
+ return r;
}
LIBYUV_API
@@ -1850,19 +2548,496 @@ int I444Scale_16(const uint16_t* src_y,
int dst_width,
int dst_height,
enum FilterMode filtering) {
- if (!src_y || !src_u || !src_v || src_width == 0 || src_height == 0 ||
- src_width > 32768 || src_height > 32768 || !dst_y || !dst_u || !dst_v ||
- dst_width <= 0 || dst_height <= 0) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
return -1;
}
- ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y, dst_stride_y,
- dst_width, dst_height, filtering);
- ScalePlane_16(src_u, src_stride_u, src_width, src_height, dst_u, dst_stride_u,
- dst_width, dst_height, filtering);
- ScalePlane_16(src_v, src_stride_v, src_width, src_height, dst_v, dst_stride_v,
- dst_width, dst_height, filtering);
- return 0;
+ r = ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_u, src_stride_u, src_width, src_height, dst_u,
+ dst_stride_u, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_v, src_stride_v, src_width, src_height, dst_v,
+ dst_stride_v, dst_width, dst_height, filtering);
+ return r;
+}
+
+LIBYUV_API
+int I444Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+
+ r = ScalePlane_12(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_u, src_stride_u, src_width, src_height, dst_u,
+ dst_stride_u, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, src_width, src_height, dst_v,
+ dst_stride_v, dst_width, dst_height, filtering);
+ return r;
+}
+
+// Scale an I422 image.
+// This function in turn calls a scaling function for each plane.
+
+LIBYUV_API
+int I422Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
+ int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
+
+ r = ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_u, src_stride_u, src_halfwidth, src_height, dst_u,
+ dst_stride_u, dst_halfwidth, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane(src_v, src_stride_v, src_halfwidth, src_height, dst_v,
+ dst_stride_v, dst_halfwidth, dst_height, filtering);
+ return r;
+}
+
+LIBYUV_API
+int I422Scale_16(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
+ int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
+
+ r = ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_u, src_stride_u, src_halfwidth, src_height, dst_u,
+ dst_stride_u, dst_halfwidth, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_16(src_v, src_stride_v, src_halfwidth, src_height, dst_v,
+ dst_stride_v, dst_halfwidth, dst_height, filtering);
+ return r;
+}
+
+LIBYUV_API
+int I422Scale_12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_u,
+ int dst_stride_u,
+ uint16_t* dst_v,
+ int dst_stride_v,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_u || !src_v || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_u || !dst_v || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ int src_halfwidth = SUBSAMPLE(src_width, 1, 1);
+ int dst_halfwidth = SUBSAMPLE(dst_width, 1, 1);
+
+ r = ScalePlane_12(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_u, src_stride_u, src_halfwidth, src_height, dst_u,
+ dst_stride_u, dst_halfwidth, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = ScalePlane_12(src_v, src_stride_v, src_halfwidth, src_height, dst_v,
+ dst_stride_v, dst_halfwidth, dst_height, filtering);
+ return r;
+}
+
+// Scale NV12 / NV16 / NV24. subsample_x/y are 1 to half that axis, else 0.
+
+static int NVScale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering,
+ int subsample_x,
+ int subsample_y) {
+ int r;
+ int src_uv_width;
+ int src_uv_height;
+ int dst_uv_width;
+ int dst_uv_height;
+
+ if (!src_y || !src_uv || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_uv || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ src_uv_width = SUBSAMPLE(src_width, subsample_x, subsample_x);
+ src_uv_height = SUBSAMPLE(src_height, subsample_y, subsample_y);
+ dst_uv_width = SUBSAMPLE(dst_width, subsample_x, subsample_x);
+ dst_uv_height = SUBSAMPLE(dst_height, subsample_y, subsample_y);
+
+ r = ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = UVScale(src_uv, src_stride_uv, src_uv_width, src_uv_height, dst_uv,
+ dst_stride_uv, dst_uv_width, dst_uv_height, filtering);
+ return r;
+}
+
+// Scale P010 / P210 / P410. subsample_x/y are 1 to half that axis, else 0.
+// Samples are 10-bit in the MSBs. Convert to 10-bit LSB, ScalePlane_16, then
+// back to MSB. UVScale_16 cannot general-scale, so UV is split then merged.
+
+static int PxxxScale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering,
+ int subsample_x,
+ int subsample_y) {
+ int r;
+ int src_uv_width;
+ int src_uv_height;
+ int dst_uv_width;
+ int dst_uv_height;
+ size_t src_uv_plane;
+ size_t dst_uv_plane;
+ size_t src_uv_bytes;
+ size_t dst_uv_bytes;
+
+ if (!src_y || !src_uv || src_width <= 0 || src_height == 0 ||
+ src_height == INT_MIN || !dst_y || !dst_uv || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ // ConvertToLSBPlane_16 / SplitUVPlane_16 invert dst, not src. Point both
+ // planes at the last row so ScalePlane_16 and the UV split see a top-down
+ // image.
+ if (src_height < 0) {
+ const int abs_height = -src_height;
+ const int src_uv_h = SUBSAMPLE(abs_height, subsample_y, subsample_y);
+ src_y = src_y + (ptrdiff_t)(abs_height - 1) * src_stride_y;
+ src_stride_y = -src_stride_y;
+ src_uv = src_uv + (ptrdiff_t)(src_uv_h - 1) * src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ src_height = abs_height;
+ }
+
+ src_uv_width = SUBSAMPLE(src_width, subsample_x, subsample_x);
+ src_uv_height = SUBSAMPLE(src_height, subsample_y, subsample_y);
+ dst_uv_width = SUBSAMPLE(dst_width, subsample_x, subsample_x);
+ dst_uv_height = SUBSAMPLE(dst_height, subsample_y, subsample_y);
+
+ if (src_width == dst_width && src_height == dst_height) {
+ r = ScalePlane_16(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ } else {
+ size_t src_y_plane;
+ size_t src_y_bytes;
+ if ((size_t)src_width > SIZE_MAX / (size_t)src_height) {
+ return -1;
+ }
+ src_y_plane = (size_t)src_width * (size_t)src_height;
+ if (src_y_plane > SIZE_MAX / sizeof(uint16_t)) {
+ return 1;
+ }
+ src_y_bytes = src_y_plane * sizeof(uint16_t);
+ align_buffer_64(src_y_lsb, src_y_bytes);
+ if (!src_y_lsb) {
+ return 1;
+ }
+ ConvertToLSBPlane_16(src_y, src_stride_y, (uint16_t*)src_y_lsb, src_width,
+ src_width, src_height, 10);
+ r = ScalePlane_16((uint16_t*)src_y_lsb, src_width, src_width, src_height,
+ dst_y, dst_stride_y, dst_width, dst_height, filtering);
+ if (r == 0) {
+ ConvertToMSBPlane_16(dst_y, dst_stride_y, dst_y, dst_stride_y, dst_width,
+ dst_height, 10);
+ }
+ free_aligned_buffer_64(src_y_lsb);
+ }
+ if (r != 0) {
+ return r;
+ }
+
+ if (src_uv_width == dst_uv_width && src_uv_height == dst_uv_height &&
+ src_uv_width <= INT_MAX / 2) {
+ CopyPlane_16(src_uv, src_stride_uv, dst_uv, dst_stride_uv, src_uv_width * 2,
+ src_uv_height);
+ return 0;
+ }
+
+ if (src_uv_width <= 0 || src_uv_height <= 0 || dst_uv_width <= 0 ||
+ dst_uv_height <= 0 ||
+ (size_t)src_uv_width > SIZE_MAX / (size_t)src_uv_height ||
+ (size_t)dst_uv_width > SIZE_MAX / (size_t)dst_uv_height) {
+ return -1;
+ }
+ src_uv_plane = (size_t)src_uv_width * (size_t)src_uv_height;
+ dst_uv_plane = (size_t)dst_uv_width * (size_t)dst_uv_height;
+ if (src_uv_plane > SIZE_MAX / (2u * sizeof(uint16_t)) ||
+ dst_uv_plane > SIZE_MAX / (2u * sizeof(uint16_t))) {
+ return 1;
+ }
+ src_uv_bytes = src_uv_plane * sizeof(uint16_t) * 2u;
+ dst_uv_bytes = dst_uv_plane * sizeof(uint16_t) * 2u;
+
+ align_buffer_64(src_uv_planes, src_uv_bytes);
+ align_buffer_64(dst_uv_planes, dst_uv_bytes);
+ if (!src_uv_planes || !dst_uv_planes) {
+ free_aligned_buffer_64(src_uv_planes);
+ free_aligned_buffer_64(dst_uv_planes);
+ return 1;
+ }
+
+ {
+ uint16_t* src_u = (uint16_t*)src_uv_planes;
+ uint16_t* src_v = src_u + src_uv_plane;
+ uint16_t* dst_u = (uint16_t*)dst_uv_planes;
+ uint16_t* dst_v = dst_u + dst_uv_plane;
+
+ SplitUVPlane_16(src_uv, src_stride_uv, src_u, src_uv_width, src_v,
+ src_uv_width, src_uv_width, src_uv_height, 10);
+ r = ScalePlane_16(src_u, src_uv_width, src_uv_width, src_uv_height, dst_u,
+ dst_uv_width, dst_uv_width, dst_uv_height, filtering);
+ if (r == 0) {
+ r = ScalePlane_16(src_v, src_uv_width, src_uv_width, src_uv_height, dst_v,
+ dst_uv_width, dst_uv_width, dst_uv_height, filtering);
+ }
+ if (r == 0) {
+ MergeUVPlane_16(dst_u, dst_uv_width, dst_v, dst_uv_width, dst_uv,
+ dst_stride_uv, dst_uv_width, dst_uv_height, 10);
+ }
+ }
+
+ free_aligned_buffer_64(src_uv_planes);
+ free_aligned_buffer_64(dst_uv_planes);
+ return r;
+}
+
+LIBYUV_API
+int NV12Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return NVScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 1, 1);
+}
+
+LIBYUV_API
+int NV16Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return NVScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 1, 0);
+}
+
+LIBYUV_API
+int NV24Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return NVScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 0, 0);
+}
+
+LIBYUV_API
+int P010Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return PxxxScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 1, 1);
+}
+
+LIBYUV_API
+int P210Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return PxxxScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 1, 0);
+}
+
+LIBYUV_API
+int P410Scale(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ return PxxxScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width,
+ src_height, dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ dst_width, dst_height, filtering, 0, 0);
}
// Deprecated api
@@ -1883,7 +3058,7 @@ int Scale(const uint8_t* src_y,
int dst_stride_v,
int dst_width,
int dst_height,
- LIBYUV_BOOL interpolate) {
+ int interpolate) {
return I420Scale(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, src_width, src_height, dst_y, dst_stride_y,
dst_u, dst_stride_u, dst_v, dst_stride_v, dst_width,
diff --git a/libs/libyuv/source/scale_any.cc b/libs/libyuv/source/scale_any.cc
index 17831372cf..ec903768e1 100644
--- a/libs/libyuv/source/scale_any.cc
+++ b/libs/libyuv/source/scale_any.cc
@@ -8,61 +8,17 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <assert.h>
#include <string.h> // For memset/memcpy
#include "libyuv/scale.h"
#include "libyuv/scale_row.h"
-#include "libyuv/basic_types.h"
-
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// Definition for ScaleFilterCols, ScaleARGBCols and ScaleARGBFilterCols
-#define CANY(NAMEANY, TERP_SIMD, TERP_C, BPP, MASK) \
- void NAMEANY(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, \
- int dx) { \
- int r = dst_width & MASK; \
- int n = dst_width & ~MASK; \
- if (n > 0) { \
- TERP_SIMD(dst_ptr, src_ptr, n, x, dx); \
- } \
- TERP_C(dst_ptr + n * BPP, src_ptr, r, x + n * dx, dx); \
- }
-
-#ifdef HAS_SCALEFILTERCOLS_NEON
-CANY(ScaleFilterCols_Any_NEON, ScaleFilterCols_NEON, ScaleFilterCols_C, 1, 7)
-#endif
-#ifdef HAS_SCALEFILTERCOLS_MSA
-CANY(ScaleFilterCols_Any_MSA, ScaleFilterCols_MSA, ScaleFilterCols_C, 1, 15)
-#endif
-#ifdef HAS_SCALEARGBCOLS_NEON
-CANY(ScaleARGBCols_Any_NEON, ScaleARGBCols_NEON, ScaleARGBCols_C, 4, 7)
-#endif
-#ifdef HAS_SCALEARGBCOLS_MSA
-CANY(ScaleARGBCols_Any_MSA, ScaleARGBCols_MSA, ScaleARGBCols_C, 4, 3)
-#endif
-#ifdef HAS_SCALEARGBCOLS_MMI
-CANY(ScaleARGBCols_Any_MMI, ScaleARGBCols_MMI, ScaleARGBCols_C, 4, 0)
-#endif
-#ifdef HAS_SCALEARGBFILTERCOLS_NEON
-CANY(ScaleARGBFilterCols_Any_NEON,
- ScaleARGBFilterCols_NEON,
- ScaleARGBFilterCols_C,
- 4,
- 3)
-#endif
-#ifdef HAS_SCALEARGBFILTERCOLS_MSA
-CANY(ScaleARGBFilterCols_Any_MSA,
- ScaleARGBFilterCols_MSA,
- ScaleARGBFilterCols_C,
- 4,
- 7)
-#endif
-#undef CANY
-
// Fixed scale down.
// Mask may be non-power of 2, so use MOD
#define SDANY(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, BPP, MASK) \
@@ -73,8 +29,9 @@ CANY(ScaleARGBFilterCols_Any_MSA,
if (n > 0) { \
SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \
} \
- SCALEROWDOWN_C(src_ptr + (n * FACTOR) * BPP, src_stride, \
- dst_ptr + n * BPP, r); \
+ ptrdiff_t np = n; \
+ SCALEROWDOWN_C(src_ptr + (np * FACTOR) * BPP, src_stride, \
+ dst_ptr + np * BPP, r); \
}
// Fixed scale down for odd source width. Used by I420Blend subsampling.
@@ -88,8 +45,22 @@ CANY(ScaleARGBFilterCols_Any_MSA,
if (n > 0) { \
SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \
} \
- SCALEROWDOWN_C(src_ptr + (n * FACTOR) * BPP, src_stride, \
- dst_ptr + n * BPP, r + 1); \
+ ptrdiff_t np = n; \
+ SCALEROWDOWN_C(src_ptr + (np * FACTOR) * BPP, src_stride, \
+ dst_ptr + np * BPP, r + 1); \
+ }
+
+// Fixed scale down for 16-bit planar.
+#define SDANY16(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, MASK) \
+ void NAMEANY(const uint16_t* src_ptr, ptrdiff_t src_stride, \
+ uint16_t* dst_ptr, int dst_width) { \
+ int r = (int)((unsigned int)dst_width % (MASK + 1)); /* NOLINT */ \
+ int n = dst_width - r; \
+ if (n > 0) { \
+ SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \
+ } \
+ ptrdiff_t np = n; \
+ SCALEROWDOWN_C(src_ptr + (np * FACTOR), src_stride, dst_ptr + np, r); \
}
#ifdef HAS_SCALEROWDOWN2_SSSE3
@@ -113,6 +84,30 @@ SDODD(ScaleRowDown2Box_Odd_SSSE3,
1,
15)
#endif
+#ifdef HAS_SCALEUVROWDOWN2_AVX2
+SDANY(ScaleUVRowDown2_Any_AVX2,
+ ScaleUVRowDown2_AVX2,
+ ScaleUVRowDown2_C,
+ 2,
+ 2,
+ 7)
+#endif
+#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2
+SDANY(ScaleUVRowDown2Linear_Any_AVX2,
+ ScaleUVRowDown2Linear_AVX2,
+ ScaleUVRowDown2Linear_C,
+ 2,
+ 2,
+ 7)
+#endif
+#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2
+SDANY(ScaleUVRowDown2Box_Any_AVX2,
+ ScaleUVRowDown2Box_AVX2,
+ ScaleUVRowDown2Box_C,
+ 2,
+ 2,
+ 7)
+#endif
#ifdef HAS_SCALEROWDOWN2_AVX2
SDANY(ScaleRowDown2_Any_AVX2, ScaleRowDown2_AVX2, ScaleRowDown2_C, 2, 1, 31)
SDANY(ScaleRowDown2Linear_Any_AVX2,
@@ -134,6 +129,23 @@ SDODD(ScaleRowDown2Box_Odd_AVX2,
1,
31)
#endif
+#ifdef HAS_SCALEROWDOWN2_16_AVX2
+SDANY16(ScaleRowDown2_16_Any_AVX2,
+ ScaleRowDown2_16_AVX2,
+ ScaleRowDown2_16_C,
+ 2,
+ 15)
+SDANY16(ScaleRowDown2Linear_16_Any_AVX2,
+ ScaleRowDown2Linear_16_AVX2,
+ ScaleRowDown2Linear_16_C,
+ 2,
+ 15)
+SDANY16(ScaleRowDown2Box_16_Any_AVX2,
+ ScaleRowDown2Box_16_AVX2,
+ ScaleRowDown2Box_16_C,
+ 2,
+ 15)
+#endif
#ifdef HAS_SCALEROWDOWN2_NEON
SDANY(ScaleRowDown2_Any_NEON, ScaleRowDown2_NEON, ScaleRowDown2_C, 2, 1, 15)
SDANY(ScaleRowDown2Linear_Any_NEON,
@@ -155,41 +167,44 @@ SDODD(ScaleRowDown2Box_Odd_NEON,
1,
15)
#endif
-#ifdef HAS_SCALEROWDOWN2_MSA
-SDANY(ScaleRowDown2_Any_MSA, ScaleRowDown2_MSA, ScaleRowDown2_C, 2, 1, 31)
-SDANY(ScaleRowDown2Linear_Any_MSA,
- ScaleRowDown2Linear_MSA,
- ScaleRowDown2Linear_C,
+#ifdef HAS_SCALEUVROWDOWN2_NEON
+SDANY(ScaleUVRowDown2_Any_NEON,
+ ScaleUVRowDown2_NEON,
+ ScaleUVRowDown2_C,
2,
- 1,
- 31)
-SDANY(ScaleRowDown2Box_Any_MSA,
- ScaleRowDown2Box_MSA,
- ScaleRowDown2Box_C,
2,
- 1,
- 31)
+ 7)
#endif
-#ifdef HAS_SCALEROWDOWN2_MMI
-SDANY(ScaleRowDown2_Any_MMI, ScaleRowDown2_MMI, ScaleRowDown2_C, 2, 1, 7)
-SDANY(ScaleRowDown2Linear_Any_MMI,
- ScaleRowDown2Linear_MMI,
- ScaleRowDown2Linear_C,
+#ifdef HAS_SCALEUVROWDOWN2LINEAR_NEON
+SDANY(ScaleUVRowDown2Linear_Any_NEON,
+ ScaleUVRowDown2Linear_NEON,
+ ScaleUVRowDown2Linear_C,
+ 2,
2,
- 1,
7)
-SDANY(ScaleRowDown2Box_Any_MMI,
- ScaleRowDown2Box_MMI,
- ScaleRowDown2Box_C,
+#endif
+#ifdef HAS_SCALEUVROWDOWN2BOX_NEON
+SDANY(ScaleUVRowDown2Box_Any_NEON,
+ ScaleUVRowDown2Box_NEON,
+ ScaleUVRowDown2Box_C,
+ 2,
2,
- 1,
7)
-SDODD(ScaleRowDown2Box_Odd_MMI,
- ScaleRowDown2Box_MMI,
- ScaleRowDown2Box_Odd_C,
+#endif
+#ifdef HAS_SCALEROWDOWN2_LSX
+SDANY(ScaleRowDown2_Any_LSX, ScaleRowDown2_LSX, ScaleRowDown2_C, 2, 1, 15)
+SDANY(ScaleRowDown2Linear_Any_LSX,
+ ScaleRowDown2Linear_LSX,
+ ScaleRowDown2Linear_C,
2,
1,
- 7)
+ 15)
+SDANY(ScaleRowDown2Box_Any_LSX,
+ ScaleRowDown2Box_LSX,
+ ScaleRowDown2Box_C,
+ 2,
+ 1,
+ 15)
#endif
#ifdef HAS_SCALEROWDOWN4_SSSE3
SDANY(ScaleRowDown4_Any_SSSE3, ScaleRowDown4_SSSE3, ScaleRowDown4_C, 4, 1, 7)
@@ -210,7 +225,7 @@ SDANY(ScaleRowDown4Box_Any_AVX2,
15)
#endif
#ifdef HAS_SCALEROWDOWN4_NEON
-SDANY(ScaleRowDown4_Any_NEON, ScaleRowDown4_NEON, ScaleRowDown4_C, 4, 1, 7)
+SDANY(ScaleRowDown4_Any_NEON, ScaleRowDown4_NEON, ScaleRowDown4_C, 4, 1, 15)
SDANY(ScaleRowDown4Box_Any_NEON,
ScaleRowDown4Box_NEON,
ScaleRowDown4Box_C,
@@ -218,45 +233,56 @@ SDANY(ScaleRowDown4Box_Any_NEON,
1,
7)
#endif
-#ifdef HAS_SCALEROWDOWN4_MSA
-SDANY(ScaleRowDown4_Any_MSA, ScaleRowDown4_MSA, ScaleRowDown4_C, 4, 1, 15)
-SDANY(ScaleRowDown4Box_Any_MSA,
- ScaleRowDown4Box_MSA,
- ScaleRowDown4Box_C,
- 4,
- 1,
- 15)
-#endif
-#ifdef HAS_SCALEROWDOWN4_MMI
-SDANY(ScaleRowDown4_Any_MMI, ScaleRowDown4_MMI, ScaleRowDown4_C, 4, 1, 7)
-SDANY(ScaleRowDown4Box_Any_MMI,
- ScaleRowDown4Box_MMI,
+#ifdef HAS_SCALEROWDOWN4_LSX
+SDANY(ScaleRowDown4_Any_LSX, ScaleRowDown4_LSX, ScaleRowDown4_C, 4, 1, 7)
+SDANY(ScaleRowDown4Box_Any_LSX,
+ ScaleRowDown4Box_LSX,
ScaleRowDown4Box_C,
4,
1,
7)
#endif
-#ifdef HAS_SCALEROWDOWN34_SSSE3
-SDANY(ScaleRowDown34_Any_SSSE3,
- ScaleRowDown34_SSSE3,
+#ifdef HAS_SCALEROWDOWN34_AVX2
+SDANY(ScaleRowDown34_Any_AVX2,
+ ScaleRowDown34_AVX2,
ScaleRowDown34_C,
4 / 3,
1,
23)
-SDANY(ScaleRowDown34_0_Box_Any_SSSE3,
- ScaleRowDown34_0_Box_SSSE3,
+SDANY(ScaleRowDown34_0_Box_Any_AVX2,
+ ScaleRowDown34_0_Box_AVX2,
ScaleRowDown34_0_Box_C,
4 / 3,
1,
23)
-SDANY(ScaleRowDown34_1_Box_Any_SSSE3,
- ScaleRowDown34_1_Box_SSSE3,
+SDANY(ScaleRowDown34_1_Box_Any_AVX2,
+ ScaleRowDown34_1_Box_AVX2,
ScaleRowDown34_1_Box_C,
4 / 3,
1,
23)
#endif
#ifdef HAS_SCALEROWDOWN34_NEON
+#ifdef __aarch64__
+SDANY(ScaleRowDown34_Any_NEON,
+ ScaleRowDown34_NEON,
+ ScaleRowDown34_C,
+ 4 / 3,
+ 1,
+ 47)
+SDANY(ScaleRowDown34_0_Box_Any_NEON,
+ ScaleRowDown34_0_Box_NEON,
+ ScaleRowDown34_0_Box_C,
+ 4 / 3,
+ 1,
+ 47)
+SDANY(ScaleRowDown34_1_Box_Any_NEON,
+ ScaleRowDown34_1_Box_NEON,
+ ScaleRowDown34_1_Box_C,
+ 4 / 3,
+ 1,
+ 47)
+#else
SDANY(ScaleRowDown34_Any_NEON,
ScaleRowDown34_NEON,
ScaleRowDown34_C,
@@ -276,45 +302,46 @@ SDANY(ScaleRowDown34_1_Box_Any_NEON,
1,
23)
#endif
-#ifdef HAS_SCALEROWDOWN34_MSA
-SDANY(ScaleRowDown34_Any_MSA,
- ScaleRowDown34_MSA,
+#endif
+#ifdef HAS_SCALEROWDOWN34_LSX
+SDANY(ScaleRowDown34_Any_LSX,
+ ScaleRowDown34_LSX,
ScaleRowDown34_C,
4 / 3,
1,
- 47)
-SDANY(ScaleRowDown34_0_Box_Any_MSA,
- ScaleRowDown34_0_Box_MSA,
+ 11)
+SDANY(ScaleRowDown34_0_Box_Any_LSX,
+ ScaleRowDown34_0_Box_LSX,
ScaleRowDown34_0_Box_C,
4 / 3,
1,
- 47)
-SDANY(ScaleRowDown34_1_Box_Any_MSA,
- ScaleRowDown34_1_Box_MSA,
+ 11)
+SDANY(ScaleRowDown34_1_Box_Any_LSX,
+ ScaleRowDown34_1_Box_LSX,
ScaleRowDown34_1_Box_C,
4 / 3,
1,
- 47)
+ 11)
#endif
-#ifdef HAS_SCALEROWDOWN38_SSSE3
-SDANY(ScaleRowDown38_Any_SSSE3,
- ScaleRowDown38_SSSE3,
+#ifdef HAS_SCALEROWDOWN38_AVX2
+SDANY(ScaleRowDown38_Any_AVX2,
+ ScaleRowDown38_AVX2,
ScaleRowDown38_C,
8 / 3,
1,
11)
-SDANY(ScaleRowDown38_3_Box_Any_SSSE3,
- ScaleRowDown38_3_Box_SSSE3,
+SDANY(ScaleRowDown38_3_Box_Any_AVX2,
+ ScaleRowDown38_3_Box_AVX2,
ScaleRowDown38_3_Box_C,
8 / 3,
1,
- 5)
-SDANY(ScaleRowDown38_2_Box_Any_SSSE3,
- ScaleRowDown38_2_Box_SSSE3,
+ 11)
+SDANY(ScaleRowDown38_2_Box_Any_AVX2,
+ ScaleRowDown38_2_Box_AVX2,
ScaleRowDown38_2_Box_C,
8 / 3,
1,
- 5)
+ 11)
#endif
#ifdef HAS_SCALEROWDOWN38_NEON
SDANY(ScaleRowDown38_Any_NEON,
@@ -336,46 +363,46 @@ SDANY(ScaleRowDown38_2_Box_Any_NEON,
1,
11)
#endif
-#ifdef HAS_SCALEROWDOWN38_MSA
-SDANY(ScaleRowDown38_Any_MSA,
- ScaleRowDown38_MSA,
+#ifdef HAS_SCALEROWDOWN38_LSX
+SDANY(ScaleRowDown38_Any_LSX,
+ ScaleRowDown38_LSX,
ScaleRowDown38_C,
8 / 3,
1,
11)
-SDANY(ScaleRowDown38_3_Box_Any_MSA,
- ScaleRowDown38_3_Box_MSA,
+SDANY(ScaleRowDown38_3_Box_Any_LSX,
+ ScaleRowDown38_3_Box_LSX,
ScaleRowDown38_3_Box_C,
8 / 3,
1,
11)
-SDANY(ScaleRowDown38_2_Box_Any_MSA,
- ScaleRowDown38_2_Box_MSA,
+SDANY(ScaleRowDown38_2_Box_Any_LSX,
+ ScaleRowDown38_2_Box_LSX,
ScaleRowDown38_2_Box_C,
8 / 3,
1,
11)
#endif
-#ifdef HAS_SCALEARGBROWDOWN2_SSE2
-SDANY(ScaleARGBRowDown2_Any_SSE2,
- ScaleARGBRowDown2_SSE2,
+#ifdef HAS_SCALEARGBROWDOWN2_AVX2
+SDANY(ScaleARGBRowDown2_Any_AVX2,
+ ScaleARGBRowDown2_AVX2,
ScaleARGBRowDown2_C,
2,
4,
- 3)
-SDANY(ScaleARGBRowDown2Linear_Any_SSE2,
- ScaleARGBRowDown2Linear_SSE2,
+ 7)
+SDANY(ScaleARGBRowDown2Linear_Any_AVX2,
+ ScaleARGBRowDown2Linear_AVX2,
ScaleARGBRowDown2Linear_C,
2,
4,
- 3)
-SDANY(ScaleARGBRowDown2Box_Any_SSE2,
- ScaleARGBRowDown2Box_SSE2,
+ 7)
+SDANY(ScaleARGBRowDown2Box_Any_AVX2,
+ ScaleARGBRowDown2Box_AVX2,
ScaleARGBRowDown2Box_C,
2,
4,
- 3)
+ 7)
#endif
#ifdef HAS_SCALEARGBROWDOWN2_NEON
SDANY(ScaleARGBRowDown2_Any_NEON,
@@ -397,46 +424,128 @@ SDANY(ScaleARGBRowDown2Box_Any_NEON,
4,
7)
#endif
-#ifdef HAS_SCALEARGBROWDOWN2_MSA
-SDANY(ScaleARGBRowDown2_Any_MSA,
- ScaleARGBRowDown2_MSA,
+#ifdef HAS_SCALEARGBROWDOWN2_LSX
+SDANY(ScaleARGBRowDown2_Any_LSX,
+ ScaleARGBRowDown2_LSX,
ScaleARGBRowDown2_C,
2,
4,
3)
-SDANY(ScaleARGBRowDown2Linear_Any_MSA,
- ScaleARGBRowDown2Linear_MSA,
+SDANY(ScaleARGBRowDown2Linear_Any_LSX,
+ ScaleARGBRowDown2Linear_LSX,
ScaleARGBRowDown2Linear_C,
2,
4,
3)
-SDANY(ScaleARGBRowDown2Box_Any_MSA,
- ScaleARGBRowDown2Box_MSA,
+SDANY(ScaleARGBRowDown2Box_Any_LSX,
+ ScaleARGBRowDown2Box_LSX,
ScaleARGBRowDown2Box_C,
2,
4,
3)
#endif
-#ifdef HAS_SCALEARGBROWDOWN2_MMI
-SDANY(ScaleARGBRowDown2_Any_MMI,
- ScaleARGBRowDown2_MMI,
+#ifdef HAS_SCALEROWDOWN2_WASMSIMD
+SDANY(ScaleRowDown2_Any_WASMSIMD,
+ ScaleRowDown2_WASMSIMD,
+ ScaleRowDown2_C,
+ 2,
+ 1,
+ 15)
+SDANY(ScaleRowDown2Linear_Any_WASMSIMD,
+ ScaleRowDown2Linear_WASMSIMD,
+ ScaleRowDown2Linear_C,
+ 2,
+ 1,
+ 15)
+SDANY(ScaleRowDown2Box_Any_WASMSIMD,
+ ScaleRowDown2Box_WASMSIMD,
+ ScaleRowDown2Box_C,
+ 2,
+ 1,
+ 15)
+SDODD(ScaleRowDown2Box_Odd_WASMSIMD,
+ ScaleRowDown2Box_WASMSIMD,
+ ScaleRowDown2Box_Odd_C,
+ 2,
+ 1,
+ 15)
+#endif
+#ifdef HAS_SCALEROWDOWN4_WASMSIMD
+SDANY(ScaleRowDown4_Any_WASMSIMD,
+ ScaleRowDown4_WASMSIMD,
+ ScaleRowDown4_C,
+ 4,
+ 1,
+ 15)
+SDANY(ScaleRowDown4Box_Any_WASMSIMD,
+ ScaleRowDown4Box_WASMSIMD,
+ ScaleRowDown4Box_C,
+ 4,
+ 1,
+ 7)
+#endif
+#ifdef HAS_SCALEROWDOWN34_WASMSIMD
+SDANY(ScaleRowDown34_Any_WASMSIMD,
+ ScaleRowDown34_WASMSIMD,
+ ScaleRowDown34_C,
+ 4 / 3,
+ 1,
+ 23)
+SDANY(ScaleRowDown34_0_Box_Any_WASMSIMD,
+ ScaleRowDown34_0_Box_WASMSIMD,
+ ScaleRowDown34_0_Box_C,
+ 4 / 3,
+ 1,
+ 23)
+SDANY(ScaleRowDown34_1_Box_Any_WASMSIMD,
+ ScaleRowDown34_1_Box_WASMSIMD,
+ ScaleRowDown34_1_Box_C,
+ 4 / 3,
+ 1,
+ 23)
+#endif
+#ifdef HAS_SCALEROWDOWN38_WASMSIMD
+SDANY(ScaleRowDown38_Any_WASMSIMD,
+ ScaleRowDown38_WASMSIMD,
+ ScaleRowDown38_C,
+ 8 / 3,
+ 1,
+ 11)
+SDANY(ScaleRowDown38_3_Box_Any_WASMSIMD,
+ ScaleRowDown38_3_Box_WASMSIMD,
+ ScaleRowDown38_3_Box_C,
+ 8 / 3,
+ 1,
+ 11)
+SDANY(ScaleRowDown38_2_Box_Any_WASMSIMD,
+ ScaleRowDown38_2_Box_WASMSIMD,
+ ScaleRowDown38_2_Box_C,
+ 8 / 3,
+ 1,
+ 11)
+#endif
+#ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD
+SDANY(ScaleARGBRowDown2_Any_WASMSIMD,
+ ScaleARGBRowDown2_WASMSIMD,
ScaleARGBRowDown2_C,
2,
4,
- 1)
-SDANY(ScaleARGBRowDown2Linear_Any_MMI,
- ScaleARGBRowDown2Linear_MMI,
+ 3)
+SDANY(ScaleARGBRowDown2Linear_Any_WASMSIMD,
+ ScaleARGBRowDown2Linear_WASMSIMD,
ScaleARGBRowDown2Linear_C,
2,
4,
- 1)
-SDANY(ScaleARGBRowDown2Box_Any_MMI,
- ScaleARGBRowDown2Box_MMI,
+ 3)
+SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD,
+ ScaleARGBRowDown2Box_WASMSIMD,
ScaleARGBRowDown2Box_C,
2,
4,
- 1)
+ 3)
#endif
+#undef SDANY16
+#undef SDODD
#undef SDANY
// Scale down by even scale factor.
@@ -448,21 +557,22 @@ SDANY(ScaleARGBRowDown2Box_Any_MMI,
if (n > 0) { \
SCALEROWDOWN_SIMD(src_ptr, src_stride, src_stepx, dst_ptr, n); \
} \
- SCALEROWDOWN_C(src_ptr + (n * src_stepx) * BPP, src_stride, src_stepx, \
- dst_ptr + n * BPP, r); \
+ ptrdiff_t np = n; \
+ SCALEROWDOWN_C(src_ptr + (np * src_stepx) * BPP, src_stride, src_stepx, \
+ dst_ptr + np * BPP, r); \
}
-#ifdef HAS_SCALEARGBROWDOWNEVEN_SSE2
-SDAANY(ScaleARGBRowDownEven_Any_SSE2,
- ScaleARGBRowDownEven_SSE2,
+#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2
+SDAANY(ScaleARGBRowDownEven_Any_AVX2,
+ ScaleARGBRowDownEven_AVX2,
ScaleARGBRowDownEven_C,
4,
- 3)
-SDAANY(ScaleARGBRowDownEvenBox_Any_SSE2,
- ScaleARGBRowDownEvenBox_SSE2,
+ 7)
+SDAANY(ScaleARGBRowDownEvenBox_Any_AVX2,
+ ScaleARGBRowDownEvenBox_AVX2,
ScaleARGBRowDownEvenBox_C,
4,
- 3)
+ 7)
#endif
#ifdef HAS_SCALEARGBROWDOWNEVEN_NEON
SDAANY(ScaleARGBRowDownEven_Any_NEON,
@@ -476,49 +586,75 @@ SDAANY(ScaleARGBRowDownEvenBox_Any_NEON,
4,
3)
#endif
-#ifdef HAS_SCALEARGBROWDOWNEVEN_MSA
-SDAANY(ScaleARGBRowDownEven_Any_MSA,
- ScaleARGBRowDownEven_MSA,
+#ifdef HAS_SCALEARGBROWDOWNEVEN_LSX
+SDAANY(ScaleARGBRowDownEven_Any_LSX,
+ ScaleARGBRowDownEven_LSX,
ScaleARGBRowDownEven_C,
4,
3)
-SDAANY(ScaleARGBRowDownEvenBox_Any_MSA,
- ScaleARGBRowDownEvenBox_MSA,
+SDAANY(ScaleARGBRowDownEvenBox_Any_LSX,
+ ScaleARGBRowDownEvenBox_LSX,
ScaleARGBRowDownEvenBox_C,
4,
3)
#endif
-#ifdef HAS_SCALEARGBROWDOWNEVEN_MMI
-SDAANY(ScaleARGBRowDownEven_Any_MMI,
- ScaleARGBRowDownEven_MMI,
+#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2
+SDAANY(ScaleUVRowDownEven_Any_AVX2,
+ ScaleUVRowDownEven_AVX2,
+ ScaleUVRowDownEven_C,
+ 2,
+ 7)
+#endif
+#ifdef HAS_SCALEUVROWDOWNEVEN_NEON
+SDAANY(ScaleUVRowDownEven_Any_NEON,
+ ScaleUVRowDownEven_NEON,
+ ScaleUVRowDownEven_C,
+ 2,
+ 3)
+#endif
+#ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD
+SDAANY(ScaleARGBRowDownEven_Any_WASMSIMD,
+ ScaleARGBRowDownEven_WASMSIMD,
ScaleARGBRowDownEven_C,
4,
- 1)
-SDAANY(ScaleARGBRowDownEvenBox_Any_MMI,
- ScaleARGBRowDownEvenBox_MMI,
+ 3)
+SDAANY(ScaleARGBRowDownEvenBox_Any_WASMSIMD,
+ ScaleARGBRowDownEvenBox_WASMSIMD,
ScaleARGBRowDownEvenBox_C,
4,
- 1)
+ 3)
#endif
#ifdef SASIMDONLY
// This also works and uses memcpy and SIMD instead of C, but is slower on ARM
+#ifdef LIBYUV_MSAN
+#define MSAN_MEMSET(a, b, c) memset(a, b, c)
+#else
+#define MSAN_MEMSET(a, b, c)
+#endif
+
// Add rows box filter scale down. Using macro from row_any
#define SAROW(NAMEANY, ANY_SIMD, SBPP, BPP, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint16_t* dst_ptr, int width) { \
- SIMD_ALIGNED(uint16_t dst_temp[32]); \
SIMD_ALIGNED(uint8_t src_temp[32]); \
- memset(dst_temp, 0, 32 * 2); /* for msan */ \
+ static_assert((MASK + 1) * SBPP <= sizeof(src_temp), \
+ "src_temp buffer too small"); \
+ SIMD_ALIGNED(uint16_t dst_temp[32]); \
+ static_assert((MASK + 1) * BPP <= sizeof(dst_temp), \
+ "dst_temp buffer too small"); \
+ MSAN_MEMSET(src_temp, 0, sizeof(src_temp)); /* for msan */ \
+ MSAN_MEMSET(dst_temp, 0, sizeof(dst_temp)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, dst_ptr, n); \
} \
- memcpy(src_temp, src_ptr + n * SBPP, r * SBPP); \
- memcpy(dst_temp, dst_ptr + n * BPP, r * BPP); \
+ ptrdiff_t np = n; \
+ memcpy(src_temp, src_ptr + np * SBPP, r * SBPP); \
+ memcpy(dst_temp, dst_ptr + np * BPP, r * BPP); \
ANY_SIMD(src_temp, dst_temp, MASK + 1); \
- memcpy(dst_ptr + n * BPP, dst_temp, r * BPP); \
+ memcpy(dst_ptr + np * BPP, dst_temp, r * BPP); \
}
#ifdef HAS_SCALEADDROW_SSE2
@@ -530,11 +666,11 @@ SAROW(ScaleAddRow_Any_AVX2, ScaleAddRow_AVX2, 1, 2, 31)
#ifdef HAS_SCALEADDROW_NEON
SAROW(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, 1, 2, 15)
#endif
-#ifdef HAS_SCALEADDROW_MSA
-SAROW(ScaleAddRow_Any_MSA, ScaleAddRow_MSA, 1, 2, 15)
+#ifdef HAS_SCALEADDROW_LSX
+SAROW(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, 1, 2, 15)
#endif
-#ifdef HAS_SCALEADDROW_MMI
-SAROW(ScaleAddRow_Any_MMI, ScaleAddRow_MMI, 1, 2, 7)
+#ifdef HAS_SCALEADDROW_WASMSIMD
+SAROW(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, 1, 2, 15)
#endif
#undef SAANY
@@ -559,16 +695,523 @@ SAANY(ScaleAddRow_Any_AVX2, ScaleAddRow_AVX2, ScaleAddRow_C, 31)
#ifdef HAS_SCALEADDROW_NEON
SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15)
#endif
-#ifdef HAS_SCALEADDROW_MSA
-SAANY(ScaleAddRow_Any_MSA, ScaleAddRow_MSA, ScaleAddRow_C, 15)
+#ifdef HAS_SCALEADDROW_LSX
+SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15)
#endif
-#ifdef HAS_SCALEADDROW_MMI
-SAANY(ScaleAddRow_Any_MMI, ScaleAddRow_MMI, ScaleAddRow_C, 7)
+#ifdef HAS_SCALEADDROW_WASMSIMD
+SAANY(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, ScaleAddRow_C, 15)
#endif
#undef SAANY
#endif // SASIMDONLY
+// Definition for ScaleFilterCols, ScaleARGBCols and ScaleARGBFilterCols
+#define CANY(NAMEANY, TERP_SIMD, TERP_C, BPP, MASK) \
+ void NAMEANY(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, \
+ int dx) { \
+ int r = dst_width & MASK; \
+ int n = dst_width & ~MASK; \
+ if (n > 0) { \
+ TERP_SIMD(dst_ptr, src_ptr, n, x, dx); \
+ } \
+ ptrdiff_t np = n; \
+ TERP_C(dst_ptr + np * BPP, src_ptr, r, x + np * dx, dx); \
+ }
+
+#ifdef HAS_SCALEFILTERCOLS_NEON
+CANY(ScaleFilterCols_Any_NEON, ScaleFilterCols_NEON, ScaleFilterCols_C, 1, 7)
+#endif
+#ifdef HAS_SCALEFILTERCOLS_LSX
+CANY(ScaleFilterCols_Any_LSX, ScaleFilterCols_LSX, ScaleFilterCols_C, 1, 3)
+#endif
+#ifdef HAS_SCALEARGBCOLS_NEON
+CANY(ScaleARGBCols_Any_NEON, ScaleARGBCols_NEON, ScaleARGBCols_C, 4, 7)
+#endif
+#ifdef HAS_SCALEARGBCOLS_LSX
+CANY(ScaleARGBCols_Any_LSX, ScaleARGBCols_LSX, ScaleARGBCols_C, 4, 3)
+#endif
+#ifdef HAS_SCALEARGBFILTERCOLS_NEON
+CANY(ScaleARGBFilterCols_Any_NEON,
+ ScaleARGBFilterCols_NEON,
+ ScaleARGBFilterCols_C,
+ 4,
+ 3)
+#endif
+#ifdef HAS_SCALEARGBFILTERCOLS_LSX
+CANY(ScaleARGBFilterCols_Any_LSX,
+ ScaleARGBFilterCols_LSX,
+ ScaleARGBFilterCols_C,
+ 4,
+ 3)
+#endif
+#ifdef HAS_SCALEFILTERCOLS_WASMSIMD
+CANY(ScaleFilterCols_Any_WASMSIMD,
+ ScaleFilterCols_WASMSIMD,
+ ScaleFilterCols_C,
+ 1,
+ 7)
+#endif
+#ifdef HAS_SCALEARGBCOLS_WASMSIMD
+CANY(ScaleARGBCols_Any_WASMSIMD, ScaleARGBCols_WASMSIMD, ScaleARGBCols_C, 4, 3)
+#endif
+#ifdef HAS_SCALEARGBFILTERCOLS_WASMSIMD
+CANY(ScaleARGBFilterCols_Any_WASMSIMD,
+ ScaleARGBFilterCols_WASMSIMD,
+ ScaleARGBFilterCols_C,
+ 4,
+ 3)
+#endif
+#undef CANY
+
+// Scale up horizontally 2 times using linear filter.
+#define SUH2LANY(NAME, SIMD, C, MASK, PTYPE) \
+ void NAME(const PTYPE* src_ptr, PTYPE* dst_ptr, int dst_width) { \
+ int work_width = (dst_width - 1) & ~1; \
+ int r = work_width & MASK; \
+ int n = work_width & ~MASK; \
+ dst_ptr[0] = src_ptr[0]; \
+ if (work_width > 0) { \
+ if (n != 0) { \
+ SIMD(src_ptr, dst_ptr + 1, n); \
+ } \
+ C(src_ptr + (n / 2), dst_ptr + n + 1, r); \
+ } \
+ dst_ptr[dst_width - 1] = src_ptr[(dst_width - 1) / 2]; \
+ }
+
+// Even the C versions need to be wrapped, because boundary pixels have to
+// be handled differently
+
+SUH2LANY(ScaleRowUp2_Linear_Any_C,
+ ScaleRowUp2_Linear_C,
+ ScaleRowUp2_Linear_C,
+ 0,
+ uint8_t)
+
+SUH2LANY(ScaleRowUp2_Linear_16_Any_C,
+ ScaleRowUp2_Linear_16_C,
+ ScaleRowUp2_Linear_16_C,
+ 0,
+ uint16_t)
+
+#ifdef HAS_SCALEROWUP2_LINEAR_SSE2
+SUH2LANY(ScaleRowUp2_Linear_Any_SSE2,
+ ScaleRowUp2_Linear_SSE2,
+ ScaleRowUp2_Linear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_SSSE3
+SUH2LANY(ScaleRowUp2_Linear_Any_SSSE3,
+ ScaleRowUp2_Linear_SSSE3,
+ ScaleRowUp2_Linear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_SSSE3
+SUH2LANY(ScaleRowUp2_Linear_12_Any_SSSE3,
+ ScaleRowUp2_Linear_12_SSSE3,
+ ScaleRowUp2_Linear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_SSE2
+SUH2LANY(ScaleRowUp2_Linear_16_Any_SSE2,
+ ScaleRowUp2_Linear_16_SSE2,
+ ScaleRowUp2_Linear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_AVX2
+SUH2LANY(ScaleRowUp2_Linear_Any_AVX2,
+ ScaleRowUp2_Linear_AVX2,
+ ScaleRowUp2_Linear_C,
+ 31,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2
+SUH2LANY(ScaleRowUp2_Linear_12_Any_AVX2,
+ ScaleRowUp2_Linear_12_AVX2,
+ ScaleRowUp2_Linear_16_C,
+ 31,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2
+SUH2LANY(ScaleRowUp2_Linear_16_Any_AVX2,
+ ScaleRowUp2_Linear_16_AVX2,
+ ScaleRowUp2_Linear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_NEON
+#ifdef __aarch64__
+SUH2LANY(ScaleRowUp2_Linear_Any_NEON,
+ ScaleRowUp2_Linear_NEON,
+ ScaleRowUp2_Linear_C,
+ 31,
+ uint8_t)
+#else
+SUH2LANY(ScaleRowUp2_Linear_Any_NEON,
+ ScaleRowUp2_Linear_NEON,
+ ScaleRowUp2_Linear_C,
+ 15,
+ uint8_t)
+#endif
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_NEON
+SUH2LANY(ScaleRowUp2_Linear_12_Any_NEON,
+ ScaleRowUp2_Linear_12_NEON,
+ ScaleRowUp2_Linear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_NEON
+SUH2LANY(ScaleRowUp2_Linear_16_Any_NEON,
+ ScaleRowUp2_Linear_16_NEON,
+ ScaleRowUp2_Linear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#undef SUH2LANY
+
+// Scale up 2 times using bilinear filter.
+// This function produces 2 rows at a time.
+#define SU2BLANY(NAME, SIMD, C, MASK, PTYPE) \
+ void NAME(const PTYPE* src_ptr, ptrdiff_t src_stride, PTYPE* dst_ptr, \
+ ptrdiff_t dst_stride, int dst_width) { \
+ int work_width = (dst_width - 1) & ~1; \
+ int r = work_width & MASK; \
+ int n = work_width & ~MASK; \
+ const PTYPE* sa = src_ptr; \
+ const PTYPE* sb = src_ptr + src_stride; \
+ PTYPE* da = dst_ptr; \
+ PTYPE* db = dst_ptr + dst_stride; \
+ da[0] = (3 * sa[0] + sb[0] + 2) >> 2; \
+ db[0] = (sa[0] + 3 * sb[0] + 2) >> 2; \
+ if (work_width > 0) { \
+ if (n != 0) { \
+ SIMD(sa, sb - sa, da + 1, db - da, n); \
+ } \
+ C(sa + (n / 2), sb - sa, da + n + 1, db - da, r); \
+ } \
+ da[dst_width - 1] = \
+ (3 * sa[(dst_width - 1) / 2] + sb[(dst_width - 1) / 2] + 2) >> 2; \
+ db[dst_width - 1] = \
+ (sa[(dst_width - 1) / 2] + 3 * sb[(dst_width - 1) / 2] + 2) >> 2; \
+ }
+
+SU2BLANY(ScaleRowUp2_Bilinear_Any_C,
+ ScaleRowUp2_Bilinear_C,
+ ScaleRowUp2_Bilinear_C,
+ 0,
+ uint8_t)
+
+SU2BLANY(ScaleRowUp2_Bilinear_16_Any_C,
+ ScaleRowUp2_Bilinear_16_C,
+ ScaleRowUp2_Bilinear_16_C,
+ 0,
+ uint16_t)
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSE2
+SU2BLANY(ScaleRowUp2_Bilinear_Any_SSE2,
+ ScaleRowUp2_Bilinear_SSE2,
+ ScaleRowUp2_Bilinear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_SSSE3
+SU2BLANY(ScaleRowUp2_Bilinear_12_Any_SSSE3,
+ ScaleRowUp2_Bilinear_12_SSSE3,
+ ScaleRowUp2_Bilinear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_SSE2
+SU2BLANY(ScaleRowUp2_Bilinear_16_Any_SSE2,
+ ScaleRowUp2_Bilinear_16_SSE2,
+ ScaleRowUp2_Bilinear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSSE3
+SU2BLANY(ScaleRowUp2_Bilinear_Any_SSSE3,
+ ScaleRowUp2_Bilinear_SSSE3,
+ ScaleRowUp2_Bilinear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2
+SU2BLANY(ScaleRowUp2_Bilinear_Any_AVX2,
+ ScaleRowUp2_Bilinear_AVX2,
+ ScaleRowUp2_Bilinear_C,
+ 31,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2
+SU2BLANY(ScaleRowUp2_Bilinear_12_Any_AVX2,
+ ScaleRowUp2_Bilinear_12_AVX2,
+ ScaleRowUp2_Bilinear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2
+SU2BLANY(ScaleRowUp2_Bilinear_16_Any_AVX2,
+ ScaleRowUp2_Bilinear_16_AVX2,
+ ScaleRowUp2_Bilinear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_NEON
+SU2BLANY(ScaleRowUp2_Bilinear_Any_NEON,
+ ScaleRowUp2_Bilinear_NEON,
+ ScaleRowUp2_Bilinear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_NEON
+SU2BLANY(ScaleRowUp2_Bilinear_12_Any_NEON,
+ ScaleRowUp2_Bilinear_12_NEON,
+ ScaleRowUp2_Bilinear_16_C,
+ 15,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_NEON
+SU2BLANY(ScaleRowUp2_Bilinear_16_Any_NEON,
+ ScaleRowUp2_Bilinear_16_NEON,
+ ScaleRowUp2_Bilinear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#undef SU2BLANY
+
+// Scale bi-planar plane up horizontally 2 times using linear filter.
+#define SBUH2LANY(NAME, SIMD, C, MASK, PTYPE) \
+ void NAME(const PTYPE* src_ptr, PTYPE* dst_ptr, int dst_width) { \
+ int work_width = (dst_width - 1) & ~1; \
+ int r = work_width & MASK; \
+ int n = work_width & ~MASK; \
+ dst_ptr[0] = src_ptr[0]; \
+ dst_ptr[1] = src_ptr[1]; \
+ if (work_width > 0) { \
+ if (n != 0) { \
+ SIMD(src_ptr, dst_ptr + 2, n); \
+ } \
+ ptrdiff_t np = n; \
+ C(src_ptr + np, dst_ptr + 2 * np + 2, r); \
+ } \
+ dst_ptr[2 * dst_width - 2] = src_ptr[((dst_width + 1) & ~1) - 2]; \
+ dst_ptr[2 * dst_width - 1] = src_ptr[((dst_width + 1) & ~1) - 1]; \
+ }
+
+SBUH2LANY(ScaleUVRowUp2_Linear_Any_C,
+ ScaleUVRowUp2_Linear_C,
+ ScaleUVRowUp2_Linear_C,
+ 0,
+ uint8_t)
+
+SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_C,
+ ScaleUVRowUp2_Linear_16_C,
+ ScaleUVRowUp2_Linear_16_C,
+ 0,
+ uint16_t)
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_SSSE3
+SBUH2LANY(ScaleUVRowUp2_Linear_Any_SSSE3,
+ ScaleUVRowUp2_Linear_SSSE3,
+ ScaleUVRowUp2_Linear_C,
+ 7,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
+SBUH2LANY(ScaleUVRowUp2_Linear_Any_AVX2,
+ ScaleUVRowUp2_Linear_AVX2,
+ ScaleUVRowUp2_Linear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_SSE41,
+ ScaleUVRowUp2_Linear_16_SSE41,
+ ScaleUVRowUp2_Linear_16_C,
+ 3,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_AVX2,
+ ScaleUVRowUp2_Linear_16_AVX2,
+ ScaleUVRowUp2_Linear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_NEON
+SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON,
+ ScaleUVRowUp2_Linear_NEON,
+ ScaleUVRowUp2_Linear_C,
+ 7,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_NEON
+SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_NEON,
+ ScaleUVRowUp2_Linear_16_NEON,
+ ScaleUVRowUp2_Linear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#undef SBUH2LANY
+
+// Scale bi-planar plane up 2 times using bilinear filter.
+// This function produces 2 rows at a time.
+#define SBU2BLANY(NAME, SIMD, C, MASK, PTYPE) \
+ void NAME(const PTYPE* src_ptr, ptrdiff_t src_stride, PTYPE* dst_ptr, \
+ ptrdiff_t dst_stride, int dst_width) { \
+ int work_width = (dst_width - 1) & ~1; \
+ int r = work_width & MASK; \
+ int n = work_width & ~MASK; \
+ const PTYPE* sa = src_ptr; \
+ const PTYPE* sb = src_ptr + src_stride; \
+ PTYPE* da = dst_ptr; \
+ PTYPE* db = dst_ptr + dst_stride; \
+ da[0] = (3 * sa[0] + sb[0] + 2) >> 2; \
+ db[0] = (sa[0] + 3 * sb[0] + 2) >> 2; \
+ da[1] = (3 * sa[1] + sb[1] + 2) >> 2; \
+ db[1] = (sa[1] + 3 * sb[1] + 2) >> 2; \
+ if (work_width > 0) { \
+ if (n != 0) { \
+ SIMD(sa, sb - sa, da + 2, db - da, n); \
+ } \
+ ptrdiff_t np = n; \
+ C(sa + np, sb - sa, da + 2 * np + 2, db - da, r); \
+ } \
+ da[2 * dst_width - 2] = (3 * sa[((dst_width + 1) & ~1) - 2] + \
+ sb[((dst_width + 1) & ~1) - 2] + 2) >> \
+ 2; \
+ db[2 * dst_width - 2] = (sa[((dst_width + 1) & ~1) - 2] + \
+ 3 * sb[((dst_width + 1) & ~1) - 2] + 2) >> \
+ 2; \
+ da[2 * dst_width - 1] = (3 * sa[((dst_width + 1) & ~1) - 1] + \
+ sb[((dst_width + 1) & ~1) - 1] + 2) >> \
+ 2; \
+ db[2 * dst_width - 1] = (sa[((dst_width + 1) & ~1) - 1] + \
+ 3 * sb[((dst_width + 1) & ~1) - 1] + 2) >> \
+ 2; \
+ }
+
+SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_C,
+ ScaleUVRowUp2_Bilinear_C,
+ ScaleUVRowUp2_Bilinear_C,
+ 0,
+ uint8_t)
+
+SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_C,
+ ScaleUVRowUp2_Bilinear_16_C,
+ ScaleUVRowUp2_Bilinear_16_C,
+ 0,
+ uint16_t)
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_SSSE3
+SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_SSSE3,
+ ScaleUVRowUp2_Bilinear_SSSE3,
+ ScaleUVRowUp2_Bilinear_C,
+ 7,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
+SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_AVX2,
+ ScaleUVRowUp2_Bilinear_AVX2,
+ ScaleUVRowUp2_Bilinear_C,
+ 15,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_SSE41,
+ ScaleUVRowUp2_Bilinear_16_SSE41,
+ ScaleUVRowUp2_Bilinear_16_C,
+ 3,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_AVX2,
+ ScaleUVRowUp2_Bilinear_16_AVX2,
+ ScaleUVRowUp2_Bilinear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_NEON
+SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_NEON,
+ ScaleUVRowUp2_Bilinear_NEON,
+ ScaleUVRowUp2_Bilinear_C,
+ 7,
+ uint8_t)
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_NEON
+SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON,
+ ScaleUVRowUp2_Bilinear_16_NEON,
+ ScaleUVRowUp2_Bilinear_16_C,
+ 7,
+ uint16_t)
+#endif
+
+#undef SBU2BLANY
+
+// Scale UV columns by 3 box filter.
+#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \
+ void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \
+ uint8_t* dst_uv) { \
+ int r = dst_width & MASK; \
+ int n = dst_width & ~MASK; \
+ if (n > 0) { \
+ SIMD(n, scaleval, src_uv, dst_uv); \
+ } \
+ C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \
+ }
+#ifdef HAS_SCALEUVADDCOLS3_AVX2
+SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7)
+#endif
+#ifdef HAS_SCALEUVADDCOLS3_NEON
+SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7)
+#endif
+#ifdef HAS_SCALEUVADDCOLS3_LSX
+SUVAC3ANY(ScaleUVAddCols3_Any_LSX, ScaleUVAddCols3_LSX, ScaleUVAddCols3_C, 7)
+#endif
+#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
+SUVAC3ANY(ScaleUVAddCols3_Any_WASMSIMD,
+ ScaleUVAddCols3_WASMSIMD,
+ ScaleUVAddCols3_C,
+ 7)
+#endif
+#undef SUVAC3ANY
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/scale_argb.cc b/libs/libyuv/source/scale_argb.cc
index beef380a8e..53280ea166 100644
--- a/libs/libyuv/source/scale_argb.cc
+++ b/libs/libyuv/source/scale_argb.cc
@@ -11,11 +11,15 @@
#include "libyuv/scale.h"
#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h" // For CopyARGB
#include "libyuv/row.h"
+#include "libyuv/scale_argb.h"
#include "libyuv/scale_row.h"
#ifdef __cplusplus
@@ -34,8 +38,8 @@ static void ScaleARGBDown2(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_argb,
uint8_t* dst_argb,
int x,
@@ -44,7 +48,7 @@ static void ScaleARGBDown2(int src_width,
int dy,
enum FilterMode filtering) {
int j;
- int row_stride = src_stride * (dy >> 16);
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
void (*ScaleARGBRowDown2)(const uint8_t* src_argb, ptrdiff_t src_stride,
uint8_t* dst_argb, int dst_width) =
filtering == kFilterNone
@@ -63,19 +67,19 @@ static void ScaleARGBDown2(int src_width,
src_argb += (y >> 16) * src_stride + ((x >> 16) - 1) * 4;
}
-#if defined(HAS_SCALEARGBROWDOWN2_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
+#if defined(HAS_SCALEARGBROWDOWN2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_Any_SSE2
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_SSE2
- : ScaleARGBRowDown2Box_Any_SSE2);
- if (IS_ALIGNED(dst_width, 4)) {
+ ? ScaleARGBRowDown2_Any_AVX2
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_AVX2
+ : ScaleARGBRowDown2Box_Any_AVX2);
+ if (IS_ALIGNED(dst_width, 8)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_SSE2
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_SSE2
- : ScaleARGBRowDown2Box_SSE2);
+ ? ScaleARGBRowDown2_AVX2
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_AVX2
+ : ScaleARGBRowDown2Box_AVX2);
}
}
#endif
@@ -95,38 +99,66 @@ static void ScaleARGBDown2(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWN2_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
+#if defined(HAS_SCALEARGBROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_SME
+ : filtering == kFilterLinear
+ ? ScaleARGBRowDown2Linear_SME
+ : ScaleARGBRowDown2Box_SME;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_Any_MSA
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_MSA
- : ScaleARGBRowDown2Box_Any_MSA);
+ ? ScaleARGBRowDown2_Any_LSX
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_LSX
+ : ScaleARGBRowDown2Box_Any_LSX);
if (IS_ALIGNED(dst_width, 4)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_MSA
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_MSA
- : ScaleARGBRowDown2Box_MSA);
+ ? ScaleARGBRowDown2_LSX
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_LSX
+ : ScaleARGBRowDown2Box_LSX);
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWN2_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
+#if defined(HAS_SCALEARGBROWDOWN2_RVV) && \
+ defined(HAS_SCALEARGBROWDOWN2LINEAR_RVV) && \
+ defined(HAS_SCALEARGBROWDOWN2BOX_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBRowDown2 =
+ filtering == kFilterNone
+ ? ScaleARGBRowDown2_RVV
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_RVV
+ : ScaleARGBRowDown2Box_RVV);
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_Any_MMI
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_MMI
- : ScaleARGBRowDown2Box_Any_MMI);
- if (IS_ALIGNED(dst_width, 2)) {
+ ? ScaleARGBRowDown2_Any_WASMSIMD
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_WASMSIMD
+ : ScaleARGBRowDown2Box_Any_WASMSIMD);
+ if (IS_ALIGNED(dst_width, 4)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
- ? ScaleARGBRowDown2_MMI
- : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_MMI
- : ScaleARGBRowDown2Box_MMI);
+ ? ScaleARGBRowDown2_WASMSIMD
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_WASMSIMD
+ : ScaleARGBRowDown2Box_WASMSIMD);
}
}
#endif
+#if defined(HAS_SCALEARGBROWDOWN2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleARGBRowDown2 =
+ filtering == kFilterNone
+ ? ScaleARGBRowDown2_HVX
+ : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_HVX
+ : ScaleARGBRowDown2Box_HVX);
+ }
+#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@@ -141,23 +173,28 @@ static void ScaleARGBDown2(int src_width,
// ScaleARGB ARGB, 1/4
// This is an optimized version for scaling down a ARGB to 1/4 of
// its original size.
-static void ScaleARGBDown4Box(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_argb,
- uint8_t* dst_argb,
- int x,
- int dx,
- int y,
- int dy) {
+static int ScaleARGBDown4Box(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int x,
+ int dx,
+ int y,
+ int dy) {
int j;
// Allocate 2 rows of ARGB.
- const int kRowSize = (dst_width * 2 * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
- int row_stride = src_stride * (dy >> 16);
+ const int row_size = (dst_width * 2 * 4 + 31) & ~31;
+ // TODO(fbarchard): Remove this row buffer and implement a ScaleARGBRowDown4
+ // but implemented via a 2 pass wrapper that uses a very small array on the
+ // stack with a horizontal loop.
+ align_buffer_64(row, row_size * 2);
+ if (!row)
+ return 1;
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
void (*ScaleARGBRowDown2)(const uint8_t* src_argb, ptrdiff_t src_stride,
uint8_t* dst_argb, int dst_width) =
ScaleARGBRowDown2Box_C;
@@ -168,11 +205,11 @@ static void ScaleARGBDown4Box(int src_width,
(void)dx;
assert(dx == 65536 * 4); // Test scale factor of 4.
assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
-#if defined(HAS_SCALEARGBROWDOWN2_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_SSE2;
- if (IS_ALIGNED(dst_width, 4)) {
- ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SSE2;
+#if defined(HAS_SCALEARGBROWDOWN2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_AVX2;
}
}
#endif
@@ -184,16 +221,40 @@ static void ScaleARGBDown4Box(int src_width,
}
}
#endif
+#if defined(HAS_SCALEARGBROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SME;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2BOX_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_RVV;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_HVX;
+ }
+#endif
for (j = 0; j < dst_height; ++j) {
ScaleARGBRowDown2(src_argb, src_stride, row, dst_width * 2);
- ScaleARGBRowDown2(src_argb + src_stride * 2, src_stride, row + kRowSize,
+ ScaleARGBRowDown2(src_argb + src_stride * 2, src_stride, row + row_size,
dst_width * 2);
- ScaleARGBRowDown2(row, kRowSize, dst_argb, dst_width);
+ ScaleARGBRowDown2(row, row_size, dst_argb, dst_width);
src_argb += row_stride;
dst_argb += dst_stride;
}
free_aligned_buffer_64(row);
+ return 0;
}
// ScaleARGB ARGB Even
@@ -203,8 +264,8 @@ static void ScaleARGBDownEven(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_argb,
uint8_t* dst_argb,
int x,
@@ -214,7 +275,7 @@ static void ScaleARGBDownEven(int src_width,
enum FilterMode filtering) {
int j;
int col_step = dx >> 16;
- int row_stride = (dy >> 16) * src_stride;
+ ptrdiff_t row_stride = (dy >> 16) * src_stride;
void (*ScaleARGBRowDownEven)(const uint8_t* src_argb, ptrdiff_t src_stride,
int src_step, uint8_t* dst_argb, int dst_width) =
filtering ? ScaleARGBRowDownEvenBox_C : ScaleARGBRowDownEven_C;
@@ -223,13 +284,13 @@ static void ScaleARGBDownEven(int src_width,
assert(IS_ALIGNED(src_width, 2));
assert(IS_ALIGNED(src_height, 2));
src_argb += (y >> 16) * src_stride + (x >> 16) * 4;
-#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_SSE2
- : ScaleARGBRowDownEven_Any_SSE2;
- if (IS_ALIGNED(dst_width, 4)) {
+#if defined(HAS_SCALEARGBROWDOWNEVEN_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_AVX2
+ : ScaleARGBRowDownEven_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
ScaleARGBRowDownEven =
- filtering ? ScaleARGBRowDownEvenBox_SSE2 : ScaleARGBRowDownEven_SSE2;
+ filtering ? ScaleARGBRowDownEvenBox_AVX2 : ScaleARGBRowDownEven_AVX2;
}
}
#endif
@@ -243,23 +304,33 @@ static void ScaleARGBDownEven(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWNEVEN_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_MSA
- : ScaleARGBRowDownEven_Any_MSA;
+#if defined(HAS_SCALEARGBROWDOWNEVEN_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_LSX
+ : ScaleARGBRowDownEven_Any_LSX;
if (IS_ALIGNED(dst_width, 4)) {
ScaleARGBRowDownEven =
- filtering ? ScaleARGBRowDownEvenBox_MSA : ScaleARGBRowDownEven_MSA;
+ filtering ? ScaleARGBRowDownEvenBox_LSX : ScaleARGBRowDownEven_LSX;
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWNEVEN_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_MMI
- : ScaleARGBRowDownEven_Any_MMI;
- if (IS_ALIGNED(dst_width, 2)) {
- ScaleARGBRowDownEven =
- filtering ? ScaleARGBRowDownEvenBox_MMI : ScaleARGBRowDownEven_MMI;
+#if defined(HAS_SCALEARGBROWDOWNEVENBOX_RVV)
+ if (filtering && TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBRowDownEven = ScaleARGBRowDownEvenBox_RVV;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWNEVEN_RVV)
+ if (!filtering && TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_WASMSIMD
+ : ScaleARGBRowDownEven_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_WASMSIMD
+ : ScaleARGBRowDownEven_WASMSIMD;
}
}
#endif
@@ -275,24 +346,24 @@ static void ScaleARGBDownEven(int src_width,
}
// Scale ARGB down with bilinear interpolation.
-static void ScaleARGBBilinearDown(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_argb,
- uint8_t* dst_argb,
- int x,
- int dx,
- int y,
- int dy,
- enum FilterMode filtering) {
+static int ScaleARGBBilinearDown(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
int j;
- void (*InterpolateRow)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*InterpolateRow)(uint8_t* dst_argb, const uint8_t* src_argb,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- void (*ScaleARGBFilterCols)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*ScaleARGBFilterCols)(uint8_t* dst_argb, const uint8_t* src_argb,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleARGBFilterCols64_C : ScaleARGBFilterCols_C;
int64_t xlast = x + (int64_t)(dst_width - 1) * dx;
@@ -308,14 +379,6 @@ static void ScaleARGBBilinearDown(int src_width,
clip_src_width = (int)(xr - xl) * 4; // Width aligned to 4.
src_argb += xl * 4;
x -= (int)(xl << 16);
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(clip_src_width, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -324,6 +387,11 @@ static void ScaleARGBBilinearDown(int src_width,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -332,17 +400,53 @@ static void ScaleARGBBilinearDown(int src_width,
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(clip_src_width, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
-#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3;
+#if defined(HAS_SCALEARGBFILTERCOLS_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2;
}
#endif
#if defined(HAS_SCALEARGBFILTERCOLS_NEON)
@@ -353,73 +457,87 @@ static void ScaleARGBBilinearDown(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBFILTERCOLS_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_MSA;
+#if defined(HAS_SCALEARGBFILTERCOLS_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LSX;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_LSX;
}
}
#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
+ }
+#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_WASMSIMD;
+ }
+ }
+#endif
+
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of ARGB.
{
- align_buffer_64(row, clip_src_width * 4);
+ // clip_src_width is already in bytes (4 per ARGB pixel).
+ align_buffer_64(row, clip_src_width);
+ if (!row)
+ return 1;
- const int max_y = (src_height - 1) << 16;
- if (y > max_y) {
- y = max_y;
- }
+ const int64_t max_y = (int64_t)(src_height - 1) << 16;
+ int64_t y64 = y;
for (j = 0; j < dst_height; ++j) {
- int yi = y >> 16;
+ if (y64 > max_y) {
+ y64 = max_y;
+ }
+ int yi = (int)(y64 >> 16);
const uint8_t* src = src_argb + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleARGBFilterCols(dst_argb, src, dst_width, x, dx);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (int)((y64 >> 8) & 255);
InterpolateRow(row, src, src_stride, clip_src_width, yf);
ScaleARGBFilterCols(dst_argb, row, dst_width, x, dx);
}
dst_argb += dst_stride;
- y += dy;
- if (y > max_y) {
- y = max_y;
- }
+ y64 += dy;
}
free_aligned_buffer_64(row);
}
+ return 0;
}
// Scale ARGB up with bilinear interpolation.
-static void ScaleARGBBilinearUp(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride,
- int dst_stride,
- const uint8_t* src_argb,
- uint8_t* dst_argb,
- int x,
- int dx,
- int y,
- int dy,
- enum FilterMode filtering) {
+static int ScaleARGBBilinearUp(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ assert(src_width > 0);
+ assert(src_height > 0);
+ assert(dst_width > 0);
+ assert(dst_height > 0);
+ assert(dy <= 65536);
+
int j;
- void (*InterpolateRow)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*InterpolateRow)(uint8_t* dst_argb, const uint8_t* src_argb,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- void (*ScaleARGBFilterCols)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*ScaleARGBFilterCols)(uint8_t* dst_argb, const uint8_t* src_argb,
int dst_width, int x, int dx) =
filtering ? ScaleARGBFilterCols_C : ScaleARGBCols_C;
- const int max_y = (src_height - 1) << 16;
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(dst_width, 4)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
+ const int64_t max_y = (int64_t)(src_height - 1) << 16;
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -428,6 +546,11 @@ static void ScaleARGBBilinearUp(int src_width,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -436,19 +559,47 @@ static void ScaleARGBBilinearUp(int src_width,
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
- if (IS_ALIGNED(dst_width, 8)) {
- InterpolateRow = InterpolateRow_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(dst_width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(dst_width, 2)) {
- InterpolateRow = InterpolateRow_MMI;
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(dst_width, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
@@ -456,9 +607,9 @@ static void ScaleARGBBilinearUp(int src_width,
ScaleARGBFilterCols =
filtering ? ScaleARGBFilterCols64_C : ScaleARGBCols64_C;
}
-#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3)
- if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3;
+#if defined(HAS_SCALEARGBFILTERCOLS_AVX2)
+ if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2;
}
#endif
#if defined(HAS_SCALEARGBFILTERCOLS_NEON)
@@ -469,17 +620,22 @@ static void ScaleARGBBilinearUp(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBFILTERCOLS_MSA)
- if (filtering && TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_MSA;
+#if defined(HAS_SCALEARGBFILTERCOLS_LSX)
+ if (filtering && TestCpuFlag(kCpuHasLSX)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LSX;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_LSX;
}
}
#endif
-#if defined(HAS_SCALEARGBCOLS_SSE2)
- if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
- ScaleARGBFilterCols = ScaleARGBCols_SSE2;
+#if defined(HAS_SCALEARGBFILTERCOLS_RVV)
+ if (filtering && TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
+ }
+#endif
+#if defined(HAS_SCALEARGBCOLS_AVX2)
+ if (!filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBCols_AVX2;
}
#endif
#if defined(HAS_SCALEARGBCOLS_NEON)
@@ -490,50 +646,61 @@ static void ScaleARGBBilinearUp(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBCOLS_MSA)
- if (!filtering && TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBFilterCols = ScaleARGBCols_Any_MSA;
+#if defined(HAS_SCALEARGBCOLS_LSX)
+ if (!filtering && TestCpuFlag(kCpuHasLSX)) {
+ ScaleARGBFilterCols = ScaleARGBCols_Any_LSX;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBFilterCols = ScaleARGBCols_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_WASMSIMD)
+ if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_Any_WASMSIMD;
if (IS_ALIGNED(dst_width, 4)) {
- ScaleARGBFilterCols = ScaleARGBCols_MSA;
+ ScaleARGBFilterCols = ScaleARGBFilterCols_WASMSIMD;
}
}
#endif
-#if defined(HAS_SCALEARGBCOLS_MMI)
- if (!filtering && TestCpuFlag(kCpuHasMMI)) {
- ScaleARGBFilterCols = ScaleARGBCols_Any_MMI;
- if (IS_ALIGNED(dst_width, 1)) {
- ScaleARGBFilterCols = ScaleARGBCols_MMI;
+#if defined(HAS_SCALEARGBCOLS_WASMSIMD)
+ if (!filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleARGBFilterCols = ScaleARGBCols_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBFilterCols = ScaleARGBCols_WASMSIMD;
}
}
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleARGBFilterCols = ScaleARGBColsUp2_C;
-#if defined(HAS_SCALEARGBCOLSUP2_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2;
+#if defined(HAS_SCALEARGBCOLSUP2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
+ ScaleARGBFilterCols = ScaleARGBColsUp2_AVX2;
}
#endif
-#if defined(HAS_SCALEARGBCOLSUP2_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 4)) {
- ScaleARGBFilterCols = ScaleARGBColsUp2_MMI;
+#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(dst_width, 8)) {
+ ScaleARGBFilterCols = ScaleARGBColsUp2_WASMSIMD;
}
#endif
}
- if (y > max_y) {
- y = max_y;
+ int64_t y64 = y;
+ if (y64 > max_y) {
+ y64 = max_y;
}
{
- int yi = y >> 16;
+ int yi = (int)(y64 >> 16);
const uint8_t* src = src_argb + yi * src_stride;
// Allocate 2 rows of ARGB.
- const int kRowSize = (dst_width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
+ const int row_size = (dst_width * 4 + 31) & ~31;
+ align_buffer_64(row, row_size * 2);
+ if (!row)
+ return 1;
uint8_t* rowptr = row;
- int rowstride = kRowSize;
+ ptrdiff_t rowstride = row_size;
int lasty = yi;
ScaleARGBFilterCols(rowptr, src, dst_width, x, dx);
@@ -541,279 +708,48 @@ static void ScaleARGBBilinearUp(int src_width,
src += src_stride;
}
ScaleARGBFilterCols(rowptr + rowstride, src, dst_width, x, dx);
- src += src_stride;
+ if (src_height > 2) {
+ src += src_stride;
+ }
+ // 2-row rolling buffer:
+ // rowptr and (rowptr + rowstride) hold the scaled rows for yi and yi + 1.
+ // Because dy <= 65536 (dy <= 1.0 in 16.16), yi advances in unit steps.
+ // When yi != lasty:
+ // 1. Scale the next source row into the older buffer (rowptr).
+ // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
+ // so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
+ // 3. Advance src by 1 row if row yi + 2 exists ((y64 + 65536) < max_y),
+ // otherwise clamp src at (src_height - 1) to avoid reading out of
+ // bounds.
for (j = 0; j < dst_height; ++j) {
- yi = y >> 16;
+ if (y64 > max_y) {
+ y64 = max_y;
+ }
+ yi = (int)(y64 >> 16);
if (yi != lasty) {
- if (y > max_y) {
- y = max_y;
- yi = y >> 16;
- src = src_argb + yi * src_stride;
- }
- if (yi != lasty) {
- ScaleARGBFilterCols(rowptr, src, dst_width, x, dx);
- rowptr += rowstride;
- rowstride = -rowstride;
- lasty = yi;
+ ScaleARGBFilterCols(rowptr, src, dst_width, x, dx);
+ rowptr += rowstride;
+ rowstride = -rowstride;
+ lasty = yi;
+ if ((y64 + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_argb, rowptr, 0, dst_width * 4, 0);
} else {
- int yf = (y >> 8) & 255;
+ int yf = (int)((y64 >> 8) & 255);
InterpolateRow(dst_argb, rowptr, rowstride, dst_width * 4, yf);
}
dst_argb += dst_stride;
- y += dy;
+ y64 += dy;
}
free_aligned_buffer_64(row);
}
+ return 0;
}
-#ifdef YUVSCALEUP
-// Scale YUV to ARGB up with bilinear interpolation.
-static void ScaleYUVToARGBBilinearUp(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- int src_stride_y,
- int src_stride_u,
- int src_stride_v,
- int dst_stride_argb,
- const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
- uint8_t* dst_argb,
- int x,
- int dx,
- int y,
- int dy,
- enum FilterMode filtering) {
- int j;
- void (*I422ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf,
- const uint8_t* v_buf, uint8_t* rgb_buf, int width) =
- I422ToARGBRow_C;
-#if defined(HAS_I422TOARGBROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- I422ToARGBRow = I422ToARGBRow_Any_SSSE3;
- if (IS_ALIGNED(src_width, 8)) {
- I422ToARGBRow = I422ToARGBRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- I422ToARGBRow = I422ToARGBRow_Any_AVX2;
- if (IS_ALIGNED(src_width, 16)) {
- I422ToARGBRow = I422ToARGBRow_AVX2;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- I422ToARGBRow = I422ToARGBRow_Any_NEON;
- if (IS_ALIGNED(src_width, 8)) {
- I422ToARGBRow = I422ToARGBRow_NEON;
- }
- }
-#endif
-#if defined(HAS_I422TOARGBROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- I422ToARGBRow = I422ToARGBRow_Any_MSA;
- if (IS_ALIGNED(src_width, 8)) {
- I422ToARGBRow = I422ToARGBRow_MSA;
- }
- }
-#endif
-
- void (*InterpolateRow)(uint8_t * dst_argb, const uint8_t* src_argb,
- ptrdiff_t src_stride, int dst_width,
- int source_y_fraction) = InterpolateRow_C;
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(dst_width, 4)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
-#if defined(HAS_INTERPOLATEROW_AVX2)
- if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_AVX2;
- if (IS_ALIGNED(dst_width, 8)) {
- InterpolateRow = InterpolateRow_AVX2;
- }
- }
-#endif
-#if defined(HAS_INTERPOLATEROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_NEON;
- if (IS_ALIGNED(dst_width, 4)) {
- InterpolateRow = InterpolateRow_NEON;
- }
- }
-#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
- if (IS_ALIGNED(dst_width, 8)) {
- InterpolateRow = InterpolateRow_MSA;
- }
- }
-#endif
-
- void (*ScaleARGBFilterCols)(uint8_t * dst_argb, const uint8_t* src_argb,
- int dst_width, int x, int dx) =
- filtering ? ScaleARGBFilterCols_C : ScaleARGBCols_C;
- if (src_width >= 32768) {
- ScaleARGBFilterCols =
- filtering ? ScaleARGBFilterCols64_C : ScaleARGBCols64_C;
- }
-#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3)
- if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3;
- }
-#endif
-#if defined(HAS_SCALEARGBFILTERCOLS_NEON)
- if (filtering && TestCpuFlag(kCpuHasNEON)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_Any_NEON;
- if (IS_ALIGNED(dst_width, 4)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_NEON;
- }
- }
-#endif
-#if defined(HAS_SCALEARGBFILTERCOLS_MSA)
- if (filtering && TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBFilterCols_MSA;
- }
- }
-#endif
-#if defined(HAS_SCALEARGBCOLS_SSE2)
- if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
- ScaleARGBFilterCols = ScaleARGBCols_SSE2;
- }
-#endif
-#if defined(HAS_SCALEARGBCOLS_NEON)
- if (!filtering && TestCpuFlag(kCpuHasNEON)) {
- ScaleARGBFilterCols = ScaleARGBCols_Any_NEON;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBCols_NEON;
- }
- }
-#endif
-#if defined(HAS_SCALEARGBCOLS_MSA)
- if (!filtering && TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBFilterCols = ScaleARGBCols_Any_MSA;
- if (IS_ALIGNED(dst_width, 4)) {
- ScaleARGBFilterCols = ScaleARGBCols_MSA;
- }
- }
-#endif
-#if defined(HAS_SCALEARGBCOLS_MMI)
- if (!filtering && TestCpuFlag(kCpuHasMMI)) {
- ScaleARGBFilterCols = ScaleARGBCols_Any_MMI;
- if (IS_ALIGNED(dst_width, 1)) {
- ScaleARGBFilterCols = ScaleARGBCols_MMI;
- }
- }
-#endif
- if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
- ScaleARGBFilterCols = ScaleARGBColsUp2_C;
-#if defined(HAS_SCALEARGBCOLSUP2_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2;
- }
-#endif
-#if defined(HAS_SCALEARGBCOLSUP2_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 4)) {
- ScaleARGBFilterCols = ScaleARGBColsUp2_MMI;
- }
-#endif
- }
-
- const int max_y = (src_height - 1) << 16;
- if (y > max_y) {
- y = max_y;
- }
- const int kYShift = 1; // Shift Y by 1 to convert Y plane to UV coordinate.
- int yi = y >> 16;
- int uv_yi = yi >> kYShift;
- const uint8_t* src_row_y = src_y + yi * src_stride_y;
- const uint8_t* src_row_u = src_u + uv_yi * src_stride_u;
- const uint8_t* src_row_v = src_v + uv_yi * src_stride_v;
-
- // Allocate 2 rows of ARGB.
- const int kRowSize = (dst_width * 4 + 31) & ~31;
- align_buffer_64(row, kRowSize * 2);
-
- // Allocate 1 row of ARGB for source conversion.
- align_buffer_64(argb_row, src_width * 4);
-
- uint8_t* rowptr = row;
- int rowstride = kRowSize;
- int lasty = yi;
-
- // TODO(fbarchard): Convert first 2 rows of YUV to ARGB.
- ScaleARGBFilterCols(rowptr, src_row_y, dst_width, x, dx);
- if (src_height > 1) {
- src_row_y += src_stride_y;
- if (yi & 1) {
- src_row_u += src_stride_u;
- src_row_v += src_stride_v;
- }
- }
- ScaleARGBFilterCols(rowptr + rowstride, src_row_y, dst_width, x, dx);
- if (src_height > 2) {
- src_row_y += src_stride_y;
- if (!(yi & 1)) {
- src_row_u += src_stride_u;
- src_row_v += src_stride_v;
- }
- }
-
- for (j = 0; j < dst_height; ++j) {
- yi = y >> 16;
- if (yi != lasty) {
- if (y > max_y) {
- y = max_y;
- yi = y >> 16;
- uv_yi = yi >> kYShift;
- src_row_y = src_y + yi * src_stride_y;
- src_row_u = src_u + uv_yi * src_stride_u;
- src_row_v = src_v + uv_yi * src_stride_v;
- }
- if (yi != lasty) {
- // TODO(fbarchard): Convert the clipped region of row.
- I422ToARGBRow(src_row_y, src_row_u, src_row_v, argb_row, src_width);
- ScaleARGBFilterCols(rowptr, argb_row, dst_width, x, dx);
- rowptr += rowstride;
- rowstride = -rowstride;
- lasty = yi;
- src_row_y += src_stride_y;
- if (yi & 1) {
- src_row_u += src_stride_u;
- src_row_v += src_stride_v;
- }
- }
- }
- if (filtering == kFilterLinear) {
- InterpolateRow(dst_argb, rowptr, 0, dst_width * 4, 0);
- } else {
- int yf = (y >> 8) & 255;
- InterpolateRow(dst_argb, rowptr, rowstride, dst_width * 4, yf);
- }
- dst_argb += dst_stride_argb;
- y += dy;
- }
- free_aligned_buffer_64(row);
- free_aligned_buffer_64(row_argb);
-}
-#endif
-
// Scale ARGB to/from any dimensions, without interpolation.
// Fixed point math is used for performance: The upper 16 bits
// of x and dx is the integer part of the source position and
@@ -823,8 +759,8 @@ static void ScaleARGBSimple(int src_width,
int src_height,
int dst_width,
int dst_height,
- int src_stride,
- int dst_stride,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
const uint8_t* src_argb,
uint8_t* dst_argb,
int x,
@@ -832,13 +768,13 @@ static void ScaleARGBSimple(int src_width,
int y,
int dy) {
int j;
- void (*ScaleARGBCols)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*ScaleARGBCols)(uint8_t* dst_argb, const uint8_t* src_argb,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C;
- (void)src_height;
-#if defined(HAS_SCALEARGBCOLS_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
- ScaleARGBCols = ScaleARGBCols_SSE2;
+ const int64_t max_y = (int64_t)(src_height - 1) << 16;
+#if defined(HAS_SCALEARGBCOLS_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
+ ScaleARGBCols = ScaleARGBCols_AVX2;
}
#endif
#if defined(HAS_SCALEARGBCOLS_NEON)
@@ -849,60 +785,64 @@ static void ScaleARGBSimple(int src_width,
}
}
#endif
-#if defined(HAS_SCALEARGBCOLS_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- ScaleARGBCols = ScaleARGBCols_Any_MSA;
+#if defined(HAS_SCALEARGBCOLS_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleARGBCols = ScaleARGBCols_Any_LSX;
if (IS_ALIGNED(dst_width, 4)) {
- ScaleARGBCols = ScaleARGBCols_MSA;
+ ScaleARGBCols = ScaleARGBCols_LSX;
}
}
#endif
-#if defined(HAS_SCALEARGBCOLS_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- ScaleARGBCols = ScaleARGBCols_Any_MMI;
- if (IS_ALIGNED(dst_width, 1)) {
- ScaleARGBCols = ScaleARGBCols_MMI;
+#if defined(HAS_SCALEARGBCOLS_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) {
+ ScaleARGBCols = ScaleARGBCols_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleARGBCols = ScaleARGBCols_WASMSIMD;
}
}
#endif
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleARGBCols = ScaleARGBColsUp2_C;
-#if defined(HAS_SCALEARGBCOLSUP2_SSE2)
- if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
- ScaleARGBCols = ScaleARGBColsUp2_SSE2;
+#if defined(HAS_SCALEARGBCOLSUP2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
+ ScaleARGBCols = ScaleARGBColsUp2_AVX2;
}
#endif
-#if defined(HAS_SCALEARGBCOLSUP2_MMI)
- if (TestCpuFlag(kCpuHasMMI) && IS_ALIGNED(dst_width, 4)) {
- ScaleARGBCols = ScaleARGBColsUp2_MMI;
+#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(dst_width, 8)) {
+ ScaleARGBCols = ScaleARGBColsUp2_WASMSIMD;
}
#endif
}
+ int64_t y64 = y;
for (j = 0; j < dst_height; ++j) {
- ScaleARGBCols(dst_argb, src_argb + (y >> 16) * src_stride, dst_width, x,
+ if (y64 > max_y) {
+ y64 = max_y;
+ }
+ ScaleARGBCols(dst_argb, src_argb + (y64 >> 16) * src_stride, dst_width, x,
dx);
dst_argb += dst_stride;
- y += dy;
+ y64 += dy;
}
}
// ScaleARGB a ARGB.
// This function in turn calls a scaling function
// suitable for handling the desired resolutions.
-static void ScaleARGB(const uint8_t* src,
- int src_stride,
- int src_width,
- int src_height,
- uint8_t* dst,
- int dst_stride,
- int dst_width,
- int dst_height,
- int clip_x,
- int clip_y,
- int clip_width,
- int clip_height,
- enum FilterMode filtering) {
+static int ScaleARGB(const uint8_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint8_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ int clip_x,
+ int clip_y,
+ int clip_width,
+ int clip_height,
+ enum FilterMode filtering) {
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
@@ -916,7 +856,7 @@ static void ScaleARGB(const uint8_t* src,
// Negative src_height means invert the image.
if (src_height < 0) {
src_height = -src_height;
- src = src + (src_height - 1) * src_stride;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
src_stride = -src_stride;
}
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
@@ -931,8 +871,8 @@ static void ScaleARGB(const uint8_t* src,
if (clip_y) {
int64_t clipf = (int64_t)(clip_y)*dy;
y += (clipf & 0xffff);
- src += (clipf >> 16) * src_stride;
- dst += clip_y * dst_stride;
+ src += (clipf >> 16) * (ptrdiff_t)src_stride;
+ dst += clip_y * (ptrdiff_t)dst_stride;
}
// Special case for integer step values.
@@ -942,56 +882,55 @@ static void ScaleARGB(const uint8_t* src,
} else {
// Optimized even scale down. ie 2, 4, 6, 8, 10x.
if (!(dx & 0x10000) && !(dy & 0x10000)) {
- if (dx == 0x20000) {
+ if (dx == 0x20000 && dy == 0x20000) {
// Optimized 1/2 downsample.
ScaleARGBDown2(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
- return;
+ return 0;
}
- if (dx == 0x40000 && filtering == kFilterBox) {
+ if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
// Optimized 1/4 box downsample.
- ScaleARGBDown4Box(src_width, src_height, clip_width, clip_height,
- src_stride, dst_stride, src, dst, x, dx, y, dy);
- return;
+ return ScaleARGBDown4Box(src_width, src_height, clip_width,
+ clip_height, src_stride, dst_stride, src,
+ dst, x, dx, y, dy);
}
ScaleARGBDownEven(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
- return;
+ return 0;
}
// Optimized odd scale down. ie 3, 5, 7, 9x.
if ((dx & 0x10000) && (dy & 0x10000)) {
filtering = kFilterNone;
if (dx == 0x10000 && dy == 0x10000) {
// Straight copy.
- ARGBCopy(src + (y >> 16) * src_stride + (x >> 16) * 4, src_stride,
- dst, dst_stride, clip_width, clip_height);
- return;
+ ARGBCopy(src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 4,
+ src_stride, dst, dst_stride, clip_width, clip_height);
+ return 0;
}
}
}
}
if (dx == 0x10000 && (x & 0xffff) == 0) {
- // Arbitrary scale vertically, but unscaled vertically.
+ // Arbitrary scale vertically, but unscaled horizontally.
ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
- dst_stride, src, dst, x, y, dy, 4, filtering);
- return;
+ dst_stride, src, dst, x, y, dy, /*bpp=*/4, filtering);
+ return 0;
}
if (filtering && dy < 65536) {
- ScaleARGBBilinearUp(src_width, src_height, clip_width, clip_height,
- src_stride, dst_stride, src, dst, x, dx, y, dy,
- filtering);
- return;
+ return ScaleARGBBilinearUp(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
}
if (filtering) {
- ScaleARGBBilinearDown(src_width, src_height, clip_width, clip_height,
- src_stride, dst_stride, src, dst, x, dx, y, dy,
- filtering);
- return;
+ return ScaleARGBBilinearDown(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
}
ScaleARGBSimple(src_width, src_height, clip_width, clip_height, src_stride,
dst_stride, src, dst, x, dx, y, dy);
+ return 0;
}
LIBYUV_API
@@ -1008,17 +947,17 @@ int ARGBScaleClip(const uint8_t* src_argb,
int clip_width,
int clip_height,
enum FilterMode filtering) {
- if (!src_argb || src_width == 0 || src_height == 0 || !dst_argb ||
- dst_width <= 0 || dst_height <= 0 || clip_x < 0 || clip_y < 0 ||
- clip_width > 32768 || clip_height > 32768 ||
+ if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 ||
+ src_height == 0 || src_height > 32768 || src_height < -32768 ||
+ !dst_argb || dst_width <= 0 || dst_height <= 0 || clip_x < 0 ||
+ clip_y < 0 || clip_width > 32768 || clip_height > 32768 ||
(clip_x + clip_width) > dst_width ||
(clip_y + clip_height) > dst_height) {
return -1;
}
- ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb,
- dst_stride_argb, dst_width, dst_height, clip_x, clip_y, clip_width,
- clip_height, filtering);
- return 0;
+ return ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb,
+ dst_stride_argb, dst_width, dst_height, clip_x, clip_y,
+ clip_width, clip_height, filtering);
}
// Scale an ARGB image.
@@ -1032,14 +971,14 @@ int ARGBScale(const uint8_t* src_argb,
int dst_width,
int dst_height,
enum FilterMode filtering) {
- if (!src_argb || src_width == 0 || src_height == 0 || src_width > 32768 ||
- src_height > 32768 || !dst_argb || dst_width <= 0 || dst_height <= 0) {
+ if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 ||
+ src_height == 0 || src_height > 32768 || src_height < -32768 ||
+ !dst_argb || dst_width <= 0 || dst_height <= 0) {
return -1;
}
- ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb,
- dst_stride_argb, dst_width, dst_height, 0, 0, dst_width, dst_height,
- filtering);
- return 0;
+ return ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb,
+ dst_stride_argb, dst_width, dst_height, 0, 0, dst_width,
+ dst_height, filtering);
}
// Scale with YUV conversion to ARGB and clipping.
@@ -1063,16 +1002,32 @@ int YUVToARGBScaleClip(const uint8_t* src_y,
int clip_width,
int clip_height,
enum FilterMode filtering) {
- uint8_t* argb_buffer = (uint8_t*)malloc(src_width * src_height * 4);
int r;
(void)src_fourcc; // TODO(fbarchard): implement and/or assert.
(void)dst_fourcc;
+ if (!src_y || !src_u || !src_v || !dst_argb || src_width <= 0 ||
+ src_width > INT_MAX / 4 || src_height == 0 || src_height == INT_MIN ||
+ dst_width <= 0 || dst_height <= 0 || clip_width <= 0 ||
+ clip_height <= 0) {
+ return -1;
+ }
+ const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ const uint64_t argb_buffer_size = (uint64_t)src_width * abs_src_height * 4;
+#if UINT64_MAX > SIZE_MAX
+ if (argb_buffer_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+#endif
+ uint8_t* argb_buffer = (uint8_t*)malloc((size_t)argb_buffer_size);
+ if (!argb_buffer) {
+ return 1; // Out of memory runtime error.
+ }
I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
argb_buffer, src_width * 4, src_width, src_height);
- r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, src_height, dst_argb,
- dst_stride_argb, dst_width, dst_height, clip_x, clip_y,
- clip_width, clip_height, filtering);
+ r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, abs_src_height,
+ dst_argb, dst_stride_argb, dst_width, dst_height, clip_x,
+ clip_y, clip_width, clip_height, filtering);
free(argb_buffer);
return r;
}
diff --git a/libs/libyuv/source/scale_common.cc b/libs/libyuv/source/scale_common.cc
index 6369027175..a07a2183eb 100644
--- a/libs/libyuv/source/scale_common.cc
+++ b/libs/libyuv/source/scale_common.cc
@@ -23,6 +23,12 @@ namespace libyuv {
extern "C" {
#endif
+#ifdef __cplusplus
+#define STATIC_CAST(type, expr) static_cast<type>(expr)
+#else
+#define STATIC_CAST(type, expr) (type)(expr)
+#endif
+
static __inline int Abs(int v) {
return v >= 0 ? v : -v;
}
@@ -198,36 +204,35 @@ void ScaleRowDown4Box_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
- intptr_t stride = src_stride;
int x;
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] +
- src_ptr[stride + 0] + src_ptr[stride + 1] + src_ptr[stride + 2] +
- src_ptr[stride + 3] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2] +
- src_ptr[stride * 2 + 3] + src_ptr[stride * 3 + 0] +
- src_ptr[stride * 3 + 1] + src_ptr[stride * 3 + 2] +
- src_ptr[stride * 3 + 3] + 8) >>
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride * 2 + 0] + src_ptr[src_stride * 2 + 1] +
+ src_ptr[src_stride * 2 + 2] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 3 + 0] + src_ptr[src_stride * 3 + 1] +
+ src_ptr[src_stride * 3 + 2] + src_ptr[src_stride * 3 + 3] + 8) >>
4;
dst[1] = (src_ptr[4] + src_ptr[5] + src_ptr[6] + src_ptr[7] +
- src_ptr[stride + 4] + src_ptr[stride + 5] + src_ptr[stride + 6] +
- src_ptr[stride + 7] + src_ptr[stride * 2 + 4] +
- src_ptr[stride * 2 + 5] + src_ptr[stride * 2 + 6] +
- src_ptr[stride * 2 + 7] + src_ptr[stride * 3 + 4] +
- src_ptr[stride * 3 + 5] + src_ptr[stride * 3 + 6] +
- src_ptr[stride * 3 + 7] + 8) >>
+ src_ptr[src_stride + 4] + src_ptr[src_stride + 5] +
+ src_ptr[src_stride + 6] + src_ptr[src_stride + 7] +
+ src_ptr[src_stride * 2 + 4] + src_ptr[src_stride * 2 + 5] +
+ src_ptr[src_stride * 2 + 6] + src_ptr[src_stride * 2 + 7] +
+ src_ptr[src_stride * 3 + 4] + src_ptr[src_stride * 3 + 5] +
+ src_ptr[src_stride * 3 + 6] + src_ptr[src_stride * 3 + 7] + 8) >>
4;
dst += 2;
src_ptr += 8;
}
if (dst_width & 1) {
dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] +
- src_ptr[stride + 0] + src_ptr[stride + 1] + src_ptr[stride + 2] +
- src_ptr[stride + 3] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2] +
- src_ptr[stride * 2 + 3] + src_ptr[stride * 3 + 0] +
- src_ptr[stride * 3 + 1] + src_ptr[stride * 3 + 2] +
- src_ptr[stride * 3 + 3] + 8) >>
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride * 2 + 0] + src_ptr[src_stride * 2 + 1] +
+ src_ptr[src_stride * 2 + 2] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 3 + 0] + src_ptr[src_stride * 3 + 1] +
+ src_ptr[src_stride * 3 + 2] + src_ptr[src_stride * 3 + 3] + 8) >>
4;
}
}
@@ -236,36 +241,35 @@ void ScaleRowDown4Box_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst,
int dst_width) {
- intptr_t stride = src_stride;
int x;
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] +
- src_ptr[stride + 0] + src_ptr[stride + 1] + src_ptr[stride + 2] +
- src_ptr[stride + 3] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2] +
- src_ptr[stride * 2 + 3] + src_ptr[stride * 3 + 0] +
- src_ptr[stride * 3 + 1] + src_ptr[stride * 3 + 2] +
- src_ptr[stride * 3 + 3] + 8) >>
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride * 2 + 0] + src_ptr[src_stride * 2 + 1] +
+ src_ptr[src_stride * 2 + 2] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 3 + 0] + src_ptr[src_stride * 3 + 1] +
+ src_ptr[src_stride * 3 + 2] + src_ptr[src_stride * 3 + 3] + 8) >>
4;
dst[1] = (src_ptr[4] + src_ptr[5] + src_ptr[6] + src_ptr[7] +
- src_ptr[stride + 4] + src_ptr[stride + 5] + src_ptr[stride + 6] +
- src_ptr[stride + 7] + src_ptr[stride * 2 + 4] +
- src_ptr[stride * 2 + 5] + src_ptr[stride * 2 + 6] +
- src_ptr[stride * 2 + 7] + src_ptr[stride * 3 + 4] +
- src_ptr[stride * 3 + 5] + src_ptr[stride * 3 + 6] +
- src_ptr[stride * 3 + 7] + 8) >>
+ src_ptr[src_stride + 4] + src_ptr[src_stride + 5] +
+ src_ptr[src_stride + 6] + src_ptr[src_stride + 7] +
+ src_ptr[src_stride * 2 + 4] + src_ptr[src_stride * 2 + 5] +
+ src_ptr[src_stride * 2 + 6] + src_ptr[src_stride * 2 + 7] +
+ src_ptr[src_stride * 3 + 4] + src_ptr[src_stride * 3 + 5] +
+ src_ptr[src_stride * 3 + 6] + src_ptr[src_stride * 3 + 7] + 8) >>
4;
dst += 2;
src_ptr += 8;
}
if (dst_width & 1) {
dst[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[3] +
- src_ptr[stride + 0] + src_ptr[stride + 1] + src_ptr[stride + 2] +
- src_ptr[stride + 3] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2] +
- src_ptr[stride * 2 + 3] + src_ptr[stride * 3 + 0] +
- src_ptr[stride * 3 + 1] + src_ptr[stride * 3 + 2] +
- src_ptr[stride * 3 + 3] + 8) >>
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride * 2 + 0] + src_ptr[src_stride * 2 + 1] +
+ src_ptr[src_stride * 2 + 2] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 3 + 0] + src_ptr[src_stride * 3 + 1] +
+ src_ptr[src_stride * 3 + 2] + src_ptr[src_stride * 3 + 3] + 8) >>
4;
}
}
@@ -400,6 +404,95 @@ void ScaleRowDown34_1_Box_16_C(const uint16_t* src_ptr,
}
}
+// Sample position: (O is src sample position, X is dst sample position)
+//
+// v dst_ptr at here v stop at here
+// X O X X O X X O X X O X X O X
+// ^ src_ptr at here
+void ScaleRowUp2_Linear_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ dst_ptr[2 * x + 0] = (src_ptr[x + 0] * 3 + src_ptr[x + 1] * 1 + 2) >> 2;
+ dst_ptr[2 * x + 1] = (src_ptr[x + 0] * 1 + src_ptr[x + 1] * 3 + 2) >> 2;
+ }
+}
+
+// Sample position: (O is src sample position, X is dst sample position)
+//
+// src_ptr at here
+// X v X X X X X X X X X
+// O O O O O
+// X X X X X X X X X X
+// ^ dst_ptr at here ^ stop at here
+// X X X X X X X X X X
+// O O O O O
+// X X X X X X X X X X
+void ScaleRowUp2_Bilinear_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ uint8_t* d = dst_ptr;
+ uint8_t* e = dst_ptr + dst_stride;
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ d[2 * x + 0] =
+ (s[x + 0] * 9 + s[x + 1] * 3 + t[x + 0] * 3 + t[x + 1] * 1 + 8) >> 4;
+ d[2 * x + 1] =
+ (s[x + 0] * 3 + s[x + 1] * 9 + t[x + 0] * 1 + t[x + 1] * 3 + 8) >> 4;
+ e[2 * x + 0] =
+ (s[x + 0] * 3 + s[x + 1] * 1 + t[x + 0] * 9 + t[x + 1] * 3 + 8) >> 4;
+ e[2 * x + 1] =
+ (s[x + 0] * 1 + s[x + 1] * 3 + t[x + 0] * 3 + t[x + 1] * 9 + 8) >> 4;
+ }
+}
+
+// Only suitable for at most 14 bit range.
+void ScaleRowUp2_Linear_16_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ dst_ptr[2 * x + 0] = (src_ptr[x + 0] * 3 + src_ptr[x + 1] * 1 + 2) >> 2;
+ dst_ptr[2 * x + 1] = (src_ptr[x + 0] * 1 + src_ptr[x + 1] * 3 + 2) >> 2;
+ }
+}
+
+// Only suitable for at most 12bit range.
+void ScaleRowUp2_Bilinear_16_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* s = src_ptr;
+ const uint16_t* t = src_ptr + src_stride;
+ uint16_t* d = dst_ptr;
+ uint16_t* e = dst_ptr + dst_stride;
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ d[2 * x + 0] =
+ (s[x + 0] * 9 + s[x + 1] * 3 + t[x + 0] * 3 + t[x + 1] * 1 + 8) >> 4;
+ d[2 * x + 1] =
+ (s[x + 0] * 3 + s[x + 1] * 9 + t[x + 0] * 1 + t[x + 1] * 3 + 8) >> 4;
+ e[2 * x + 0] =
+ (s[x + 0] * 3 + s[x + 1] * 1 + t[x + 0] * 9 + t[x + 1] * 3 + 8) >> 4;
+ e[2 * x + 1] =
+ (s[x + 0] * 1 + s[x + 1] * 3 + t[x + 0] * 3 + t[x + 1] * 9 + 8) >> 4;
+ }
+}
+
// Scales a single row of pixels using point sampling.
void ScaleCols_C(uint8_t* dst_ptr,
const uint8_t* src_ptr,
@@ -541,10 +634,10 @@ void ScaleFilterCols64_C(uint8_t* dst_ptr,
#undef BLENDER
// Same as 8 bit arm blender but return is cast to uint16_t
-#define BLENDER(a, b, f) \
- (uint16_t)( \
- (int)(a) + \
- (int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> 16))
+#define BLENDER(a, b, f) \
+ (uint16_t)((int)(a) + \
+ (int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> \
+ 16))
void ScaleFilterCols_16_C(uint16_t* dst_ptr,
const uint16_t* src_ptr,
@@ -639,27 +732,26 @@ void ScaleRowDown38_3_Box_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
- intptr_t stride = src_stride;
int i;
assert((dst_width % 3 == 0) && (dst_width > 0));
for (i = 0; i < dst_width; i += 3) {
- dst_ptr[0] =
- (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[stride + 0] +
- src_ptr[stride + 1] + src_ptr[stride + 2] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2]) *
- (65536 / 9) >>
- 16;
- dst_ptr[1] =
- (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[stride + 3] +
- src_ptr[stride + 4] + src_ptr[stride + 5] + src_ptr[stride * 2 + 3] +
- src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5]) *
- (65536 / 9) >>
- 16;
- dst_ptr[2] =
- (src_ptr[6] + src_ptr[7] + src_ptr[stride + 6] + src_ptr[stride + 7] +
- src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7]) *
- (65536 / 6) >>
- 16;
+ dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] +
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride * 2 + 0] +
+ src_ptr[src_stride * 2 + 1] + src_ptr[src_stride * 2 + 2]) *
+ (65536 / 9) >>
+ 16;
+ dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] +
+ src_ptr[src_stride + 3] + src_ptr[src_stride + 4] +
+ src_ptr[src_stride + 5] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 2 + 4] + src_ptr[src_stride * 2 + 5]) *
+ (65536 / 9) >>
+ 16;
+ dst_ptr[2] = (src_ptr[6] + src_ptr[7] + src_ptr[src_stride + 6] +
+ src_ptr[src_stride + 7] + src_ptr[src_stride * 2 + 6] +
+ src_ptr[src_stride * 2 + 7]) *
+ (65536 / 6) >>
+ 16;
src_ptr += 8;
dst_ptr += 3;
}
@@ -669,27 +761,26 @@ void ScaleRowDown38_3_Box_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int dst_width) {
- intptr_t stride = src_stride;
int i;
assert((dst_width % 3 == 0) && (dst_width > 0));
for (i = 0; i < dst_width; i += 3) {
- dst_ptr[0] =
- (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[stride + 0] +
- src_ptr[stride + 1] + src_ptr[stride + 2] + src_ptr[stride * 2 + 0] +
- src_ptr[stride * 2 + 1] + src_ptr[stride * 2 + 2]) *
- (65536 / 9) >>
- 16;
- dst_ptr[1] =
- (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[stride + 3] +
- src_ptr[stride + 4] + src_ptr[stride + 5] + src_ptr[stride * 2 + 3] +
- src_ptr[stride * 2 + 4] + src_ptr[stride * 2 + 5]) *
- (65536 / 9) >>
- 16;
- dst_ptr[2] =
- (src_ptr[6] + src_ptr[7] + src_ptr[stride + 6] + src_ptr[stride + 7] +
- src_ptr[stride * 2 + 6] + src_ptr[stride * 2 + 7]) *
- (65536 / 6) >>
- 16;
+ dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] +
+ src_ptr[src_stride + 0] + src_ptr[src_stride + 1] +
+ src_ptr[src_stride + 2] + src_ptr[src_stride * 2 + 0] +
+ src_ptr[src_stride * 2 + 1] + src_ptr[src_stride * 2 + 2]) *
+ (65536u / 9u) >>
+ 16;
+ dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] +
+ src_ptr[src_stride + 3] + src_ptr[src_stride + 4] +
+ src_ptr[src_stride + 5] + src_ptr[src_stride * 2 + 3] +
+ src_ptr[src_stride * 2 + 4] + src_ptr[src_stride * 2 + 5]) *
+ (65536u / 9u) >>
+ 16;
+ dst_ptr[2] = (src_ptr[6] + src_ptr[7] + src_ptr[src_stride + 6] +
+ src_ptr[src_stride + 7] + src_ptr[src_stride * 2 + 6] +
+ src_ptr[src_stride * 2 + 7]) *
+ (65536u / 6u) >>
+ 16;
src_ptr += 8;
dst_ptr += 3;
}
@@ -700,22 +791,23 @@ void ScaleRowDown38_2_Box_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
- intptr_t stride = src_stride;
int i;
assert((dst_width % 3 == 0) && (dst_width > 0));
for (i = 0; i < dst_width; i += 3) {
- dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[stride + 0] +
- src_ptr[stride + 1] + src_ptr[stride + 2]) *
- (65536 / 6) >>
- 16;
- dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[stride + 3] +
- src_ptr[stride + 4] + src_ptr[stride + 5]) *
- (65536 / 6) >>
- 16;
- dst_ptr[2] =
- (src_ptr[6] + src_ptr[7] + src_ptr[stride + 6] + src_ptr[stride + 7]) *
- (65536 / 4) >>
+ dst_ptr[0] =
+ (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[src_stride + 0] +
+ src_ptr[src_stride + 1] + src_ptr[src_stride + 2]) *
+ (65536 / 6) >>
+ 16;
+ dst_ptr[1] =
+ (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride + 4] + src_ptr[src_stride + 5]) *
+ (65536 / 6) >>
16;
+ dst_ptr[2] = (src_ptr[6] + src_ptr[7] + src_ptr[src_stride + 6] +
+ src_ptr[src_stride + 7]) *
+ (65536 / 4) >>
+ 16;
src_ptr += 8;
dst_ptr += 3;
}
@@ -725,22 +817,23 @@ void ScaleRowDown38_2_Box_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int dst_width) {
- intptr_t stride = src_stride;
int i;
assert((dst_width % 3 == 0) && (dst_width > 0));
for (i = 0; i < dst_width; i += 3) {
- dst_ptr[0] = (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[stride + 0] +
- src_ptr[stride + 1] + src_ptr[stride + 2]) *
- (65536 / 6) >>
- 16;
- dst_ptr[1] = (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[stride + 3] +
- src_ptr[stride + 4] + src_ptr[stride + 5]) *
- (65536 / 6) >>
- 16;
- dst_ptr[2] =
- (src_ptr[6] + src_ptr[7] + src_ptr[stride + 6] + src_ptr[stride + 7]) *
- (65536 / 4) >>
+ dst_ptr[0] =
+ (src_ptr[0] + src_ptr[1] + src_ptr[2] + src_ptr[src_stride + 0] +
+ src_ptr[src_stride + 1] + src_ptr[src_stride + 2]) *
+ (65536u / 6u) >>
+ 16;
+ dst_ptr[1] =
+ (src_ptr[3] + src_ptr[4] + src_ptr[5] + src_ptr[src_stride + 3] +
+ src_ptr[src_stride + 4] + src_ptr[src_stride + 5]) *
+ (65536u / 6u) >>
16;
+ dst_ptr[2] = (src_ptr[6] + src_ptr[7] + src_ptr[src_stride + 6] +
+ src_ptr[src_stride + 7]) *
+ (65536u / 4u) >>
+ 16;
src_ptr += 8;
dst_ptr += 3;
}
@@ -776,22 +869,21 @@ void ScaleAddRow_16_C(const uint16_t* src_ptr,
}
}
+// ARGB scale row functions
+
void ScaleARGBRowDown2_C(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int x;
(void)src_stride;
- for (x = 0; x < dst_width - 1; x += 2) {
- dst[0] = src[1];
- dst[1] = src[3];
- src += 4;
- dst += 2;
- }
- if (dst_width & 1) {
- dst[0] = src[1];
+ for (x = 0; x < dst_width; ++x) {
+ dst_argb[0] = src_argb[4];
+ dst_argb[1] = src_argb[5];
+ dst_argb[2] = src_argb[6];
+ dst_argb[3] = src_argb[7];
+ src_argb += 8;
+ dst_argb += 4;
}
}
@@ -839,18 +931,15 @@ void ScaleARGBRowDownEven_C(const uint8_t* src_argb,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
- (void)src_stride;
int x;
- for (x = 0; x < dst_width - 1; x += 2) {
- dst[0] = src[0];
- dst[1] = src[src_stepx];
- src += src_stepx * 2;
- dst += 2;
- }
- if (dst_width & 1) {
- dst[0] = src[0];
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_argb[0] = src_argb[0];
+ dst_argb[1] = src_argb[1];
+ dst_argb[2] = src_argb[2];
+ dst_argb[3] = src_argb[3];
+ src_argb += src_stepx * 4;
+ dst_argb += 4;
}
}
@@ -884,18 +973,15 @@ void ScaleARGBCols_C(uint8_t* dst_argb,
int dst_width,
int x,
int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int j;
- for (j = 0; j < dst_width - 1; j += 2) {
- dst[0] = src[x >> 16];
- x += dx;
- dst[1] = src[x >> 16];
+ for (j = 0; j < dst_width; ++j) {
+ const uint8_t* src = src_argb + (x >> 16) * 4;
+ dst_argb[0] = src[0];
+ dst_argb[1] = src[1];
+ dst_argb[2] = src[2];
+ dst_argb[3] = src[3];
+ dst_argb += 4;
x += dx;
- dst += 2;
- }
- if (dst_width & 1) {
- dst[0] = src[x >> 16];
}
}
@@ -905,18 +991,15 @@ void ScaleARGBCols64_C(uint8_t* dst_argb,
int x32,
int dx) {
int64_t x = (int64_t)(x32);
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int j;
- for (j = 0; j < dst_width - 1; j += 2) {
- dst[0] = src[x >> 16];
- x += dx;
- dst[1] = src[x >> 16];
+ for (j = 0; j < dst_width; ++j) {
+ const uint8_t* src = src_argb + (x >> 16) * 4;
+ dst_argb[0] = src[0];
+ dst_argb[1] = src[1];
+ dst_argb[2] = src[2];
+ dst_argb[3] = src[3];
+ dst_argb += 4;
x += dx;
- dst += 2;
- }
- if (dst_width & 1) {
- dst[0] = src[x >> 16];
}
}
@@ -926,59 +1009,46 @@ void ScaleARGBColsUp2_C(uint8_t* dst_argb,
int dst_width,
int x,
int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int j;
(void)x;
(void)dx;
for (j = 0; j < dst_width - 1; j += 2) {
- dst[1] = dst[0] = src[0];
- src += 1;
- dst += 2;
+ dst_argb[0] = dst_argb[4] = src_argb[0];
+ dst_argb[1] = dst_argb[5] = src_argb[1];
+ dst_argb[2] = dst_argb[6] = src_argb[2];
+ dst_argb[3] = dst_argb[7] = src_argb[3];
+ src_argb += 4;
+ dst_argb += 8;
}
if (dst_width & 1) {
- dst[0] = src[0];
+ dst_argb[0] = src_argb[0];
+ dst_argb[1] = src_argb[1];
+ dst_argb[2] = src_argb[2];
+ dst_argb[3] = src_argb[3];
}
}
// TODO(fbarchard): Replace 0x7f ^ f with 128-f. bug=607.
// Mimics SSSE3 blender
-#define BLENDER1(a, b, f) ((a) * (0x7f ^ f) + (b)*f) >> 7
-#define BLENDERC(a, b, f, s) \
- (uint32_t)(BLENDER1(((a) >> s) & 255, ((b) >> s) & 255, f) << s)
-#define BLENDER(a, b, f) \
- BLENDERC(a, b, f, 24) | BLENDERC(a, b, f, 16) | BLENDERC(a, b, f, 8) | \
- BLENDERC(a, b, f, 0)
+#define BLENDER1(a, b, f) (((a) * (0x7f ^ (f)) + (b) * (f)) >> 7)
void ScaleARGBFilterCols_C(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int j;
- for (j = 0; j < dst_width - 1; j += 2) {
- int xi = x >> 16;
+ for (j = 0; j < dst_width; ++j) {
+ int xi = (x >> 16) * 4;
int xf = (x >> 9) & 0x7f;
- uint32_t a = src[xi];
- uint32_t b = src[xi + 1];
- dst[0] = BLENDER(a, b, xf);
- x += dx;
- xi = x >> 16;
- xf = (x >> 9) & 0x7f;
- a = src[xi];
- b = src[xi + 1];
- dst[1] = BLENDER(a, b, xf);
+ const uint8_t* a = src_argb + xi;
+ const uint8_t* b = a + 4;
+ dst_argb[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
+ dst_argb[1] = (uint8_t)BLENDER1(a[1], b[1], xf);
+ dst_argb[2] = (uint8_t)BLENDER1(a[2], b[2], xf);
+ dst_argb[3] = (uint8_t)BLENDER1(a[3], b[3], xf);
+ dst_argb += 4;
x += dx;
- dst += 2;
- }
- if (dst_width & 1) {
- int xi = x >> 16;
- int xf = (x >> 9) & 0x7f;
- uint32_t a = src[xi];
- uint32_t b = src[xi + 1];
- dst[0] = BLENDER(a, b, xf);
}
}
@@ -988,35 +1058,521 @@ void ScaleARGBFilterCols64_C(uint8_t* dst_argb,
int x32,
int dx) {
int64_t x = (int64_t)(x32);
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
int j;
- for (j = 0; j < dst_width - 1; j += 2) {
- int64_t xi = x >> 16;
+ for (j = 0; j < dst_width; ++j) {
+ int64_t xi = (x >> 16) * 4;
int xf = (x >> 9) & 0x7f;
- uint32_t a = src[xi];
- uint32_t b = src[xi + 1];
- dst[0] = BLENDER(a, b, xf);
+ const uint8_t* a = src_argb + xi;
+ const uint8_t* b = a + 4;
+ dst_argb[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
+ dst_argb[1] = (uint8_t)BLENDER1(a[1], b[1], xf);
+ dst_argb[2] = (uint8_t)BLENDER1(a[2], b[2], xf);
+ dst_argb[3] = (uint8_t)BLENDER1(a[3], b[3], xf);
+ dst_argb += 4;
x += dx;
- xi = x >> 16;
- xf = (x >> 9) & 0x7f;
- a = src[xi];
- b = src[xi + 1];
- dst[1] = BLENDER(a, b, xf);
+ }
+}
+#undef BLENDER1
+
+// UV scale row functions
+// same as ARGB but 2 channels
+
+void ScaleUVRowDown2_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = src_uv[2]; // Store the 2nd UV
+ dst_uv[1] = src_uv[3];
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDown2Linear_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (src_uv[0] + src_uv[2] + 1) >> 1;
+ dst_uv[1] = (src_uv[1] + src_uv[3] + 1) >> 1;
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDown2Box_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int x;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (src_uv[0] + src_uv[2] + src_uv[src_stride] +
+ src_uv[src_stride + 2] + 2) >>
+ 2;
+ dst_uv[1] = (src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
+ src_uv[src_stride + 3] + 2) >>
+ 2;
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDownEven_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = src_uv[0];
+ dst_uv[1] = src_uv[1];
+ src_uv += src_stepx * 2;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int x;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (src_uv[0] + src_uv[2] + src_uv[src_stride] +
+ src_uv[src_stride + 2] + 2) >>
+ 2;
+ dst_uv[1] = (src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
+ src_uv[src_stride + 3] + 2) >>
+ 2;
+ src_uv += src_stepx * 2;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDown2_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = src_uv[2]; // Store the 2nd UV
+ dst_uv[1] = src_uv[3];
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + 1) >> 1);
+ dst_uv[1] = (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + 1) >> 1);
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ uint16_t* dst_uv,
+ int dst_width) {
+ int x;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] +
+ src_uv[src_stride] + src_uv[src_stride + 2] + 2) >>
+ 2);
+ dst_uv[1] =
+ (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
+ src_uv[src_stride + 3] + 2) >>
+ 2);
+ src_uv += 4;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDownEven_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint16_t* dst_uv,
+ int dst_width) {
+ int x;
+ (void)src_stride;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = src_uv[0];
+ dst_uv[1] = src_uv[1];
+ src_uv += src_stepx * 2;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint16_t* dst_uv,
+ int dst_width) {
+ int x;
+ for (x = 0; x < dst_width; ++x) {
+ dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] +
+ src_uv[src_stride] + src_uv[src_stride + 2] + 2) >>
+ 2);
+ dst_uv[1] =
+ (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
+ src_uv[src_stride + 3] + 2) >>
+ 2);
+ src_uv += src_stepx * 2;
+ dst_uv += 2;
+ }
+}
+
+void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ dst_ptr[4 * x + 0] =
+ (src_ptr[2 * x + 0] * 3 + src_ptr[2 * x + 2] * 1 + 2) >> 2;
+ dst_ptr[4 * x + 1] =
+ (src_ptr[2 * x + 1] * 3 + src_ptr[2 * x + 3] * 1 + 2) >> 2;
+ dst_ptr[4 * x + 2] =
+ (src_ptr[2 * x + 0] * 1 + src_ptr[2 * x + 2] * 3 + 2) >> 2;
+ dst_ptr[4 * x + 3] =
+ (src_ptr[2 * x + 1] * 1 + src_ptr[2 * x + 3] * 3 + 2) >> 2;
+ }
+}
+
+void ScaleUVRowUp2_Bilinear_C(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ uint8_t* d = dst_ptr;
+ uint8_t* e = dst_ptr + dst_stride;
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ d[4 * x + 0] = (s[2 * x + 0] * 9 + s[2 * x + 2] * 3 + t[2 * x + 0] * 3 +
+ t[2 * x + 2] * 1 + 8) >>
+ 4;
+ d[4 * x + 1] = (s[2 * x + 1] * 9 + s[2 * x + 3] * 3 + t[2 * x + 1] * 3 +
+ t[2 * x + 3] * 1 + 8) >>
+ 4;
+ d[4 * x + 2] = (s[2 * x + 0] * 3 + s[2 * x + 2] * 9 + t[2 * x + 0] * 1 +
+ t[2 * x + 2] * 3 + 8) >>
+ 4;
+ d[4 * x + 3] = (s[2 * x + 1] * 3 + s[2 * x + 3] * 9 + t[2 * x + 1] * 1 +
+ t[2 * x + 3] * 3 + 8) >>
+ 4;
+ e[4 * x + 0] = (s[2 * x + 0] * 3 + s[2 * x + 2] * 1 + t[2 * x + 0] * 9 +
+ t[2 * x + 2] * 3 + 8) >>
+ 4;
+ e[4 * x + 1] = (s[2 * x + 1] * 3 + s[2 * x + 3] * 1 + t[2 * x + 1] * 9 +
+ t[2 * x + 3] * 3 + 8) >>
+ 4;
+ e[4 * x + 2] = (s[2 * x + 0] * 1 + s[2 * x + 2] * 3 + t[2 * x + 0] * 3 +
+ t[2 * x + 2] * 9 + 8) >>
+ 4;
+ e[4 * x + 3] = (s[2 * x + 1] * 1 + s[2 * x + 3] * 3 + t[2 * x + 1] * 3 +
+ t[2 * x + 3] * 9 + 8) >>
+ 4;
+ }
+}
+
+void ScaleUVRowUp2_Linear_16_C(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ dst_ptr[4 * x + 0] =
+ (src_ptr[2 * x + 0] * 3 + src_ptr[2 * x + 2] * 1 + 2) >> 2;
+ dst_ptr[4 * x + 1] =
+ (src_ptr[2 * x + 1] * 3 + src_ptr[2 * x + 3] * 1 + 2) >> 2;
+ dst_ptr[4 * x + 2] =
+ (src_ptr[2 * x + 0] * 1 + src_ptr[2 * x + 2] * 3 + 2) >> 2;
+ dst_ptr[4 * x + 3] =
+ (src_ptr[2 * x + 1] * 1 + src_ptr[2 * x + 3] * 3 + 2) >> 2;
+ }
+}
+
+void ScaleUVRowUp2_Bilinear_16_C(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* s = src_ptr;
+ const uint16_t* t = src_ptr + src_stride;
+ uint16_t* d = dst_ptr;
+ uint16_t* e = dst_ptr + dst_stride;
+ int src_width = dst_width >> 1;
+ int x;
+ assert((dst_width % 2 == 0) && (dst_width >= 0));
+ for (x = 0; x < src_width; ++x) {
+ d[4 * x + 0] = (s[2 * x + 0] * 9 + s[2 * x + 2] * 3 + t[2 * x + 0] * 3 +
+ t[2 * x + 2] * 1 + 8) >>
+ 4;
+ d[4 * x + 1] = (s[2 * x + 1] * 9 + s[2 * x + 3] * 3 + t[2 * x + 1] * 3 +
+ t[2 * x + 3] * 1 + 8) >>
+ 4;
+ d[4 * x + 2] = (s[2 * x + 0] * 3 + s[2 * x + 2] * 9 + t[2 * x + 0] * 1 +
+ t[2 * x + 2] * 3 + 8) >>
+ 4;
+ d[4 * x + 3] = (s[2 * x + 1] * 3 + s[2 * x + 3] * 9 + t[2 * x + 1] * 1 +
+ t[2 * x + 3] * 3 + 8) >>
+ 4;
+ e[4 * x + 0] = (s[2 * x + 0] * 3 + s[2 * x + 2] * 1 + t[2 * x + 0] * 9 +
+ t[2 * x + 2] * 3 + 8) >>
+ 4;
+ e[4 * x + 1] = (s[2 * x + 1] * 3 + s[2 * x + 3] * 1 + t[2 * x + 1] * 9 +
+ t[2 * x + 3] * 3 + 8) >>
+ 4;
+ e[4 * x + 2] = (s[2 * x + 0] * 1 + s[2 * x + 2] * 3 + t[2 * x + 0] * 3 +
+ t[2 * x + 2] * 9 + 8) >>
+ 4;
+ e[4 * x + 3] = (s[2 * x + 1] * 1 + s[2 * x + 3] * 3 + t[2 * x + 1] * 3 +
+ t[2 * x + 3] * 9 + 8) >>
+ 4;
+ }
+}
+
+// Scales a single row of pixels using point sampling.
+void ScaleUVCols_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ const uint8_t* src = src_uv + (x >> 16) * 2;
+ dst_uv[0] = src[0];
+ dst_uv[1] = src[1];
+ dst_uv += 2;
x += dx;
- dst += 2;
+ }
+}
+
+void ScaleUVCols64_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx) {
+ int64_t x = (int64_t)(x32);
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ const uint8_t* src = src_uv + (x >> 16) * 2;
+ dst_uv[0] = src[0];
+ dst_uv[1] = src[1];
+ dst_uv += 2;
+ x += dx;
+ }
+}
+
+// Scales a single row of pixels up by 2x using point sampling.
+void ScaleUVColsUp2_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ (void)x;
+ (void)dx;
+ for (j = 0; j < dst_width - 1; j += 2) {
+ dst_uv[0] = dst_uv[2] = src_uv[0];
+ dst_uv[1] = dst_uv[3] = src_uv[1];
+ src_uv += 2;
+ dst_uv += 4;
}
if (dst_width & 1) {
- int64_t xi = x >> 16;
+ dst_uv[0] = src_uv[0];
+ dst_uv[1] = src_uv[1];
+ }
+}
+
+// Performs (a + ((f * (b - a) + 64) >> 7)) which is equivalent of
+// ((a * (128 - f) + b * f + 64) >> 7).
+#define BLENDER1(a, b, f) ((a) + (((f) * ((b) - (a)) + 64) >> 7))
+
+void ScaleUVFilterCols_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ int xi = (x >> 16) * 2;
+ int xf = (x >> 9) & 0x7f;
+ const uint8_t* a = src_uv + xi;
+ const uint8_t* b = a + 2;
+ dst_uv[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
+ dst_uv[1] = (uint8_t)BLENDER1(a[1], b[1], xf);
+ dst_uv += 2;
+ x += dx;
+ }
+}
+
+void ScaleUVFilterCols64_C(uint8_t* dst_uv,
+ const uint8_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx) {
+ int64_t x = (int64_t)(x32);
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ int64_t xi = (x >> 16) * 2;
int xf = (x >> 9) & 0x7f;
- uint32_t a = src[xi];
- uint32_t b = src[xi + 1];
- dst[0] = BLENDER(a, b, xf);
+ const uint8_t* a = src_uv + xi;
+ const uint8_t* b = a + 2;
+ dst_uv[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
+ dst_uv[1] = (uint8_t)BLENDER1(a[1], b[1], xf);
+ dst_uv += 2;
+ x += dx;
}
}
#undef BLENDER1
-#undef BLENDERC
-#undef BLENDER
+
+// Scale UV columns by 3 box filter.
+void ScaleUVAddCols3_C(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ int i;
+ for (i = 0; i < dst_width; ++i) {
+ uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
+ uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
+ src_uv += 6;
+ dst_uv[0] = (uint8_t)((u * scaleval) >> 16);
+ dst_uv[1] = (uint8_t)((v * scaleval) >> 16);
+ dst_uv += 2;
+ }
+}
+
+// Scales a single row of 16-bit UV pixels using point sampling.
+void ScaleUVCols_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ const uint16_t* src = src_uv + (x >> 16) * 2;
+ dst_uv[0] = src[0];
+ dst_uv[1] = src[1];
+ dst_uv += 2;
+ x += dx;
+ }
+}
+
+void ScaleUVCols64_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx) {
+ int64_t x = (int64_t)(x32);
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ const uint16_t* src = src_uv + (x >> 16) * 2;
+ dst_uv[0] = src[0];
+ dst_uv[1] = src[1];
+ dst_uv += 2;
+ x += dx;
+ }
+}
+
+// Scales a single row of 16-bit UV pixels up by 2x using point sampling.
+void ScaleUVColsUp2_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ (void)x;
+ (void)dx;
+ for (j = 0; j < dst_width - 1; j += 2) {
+ dst_uv[0] = dst_uv[2] = src_uv[0];
+ dst_uv[1] = dst_uv[3] = src_uv[1];
+ src_uv += 2;
+ dst_uv += 4;
+ }
+ if (dst_width & 1) {
+ dst_uv[0] = src_uv[0];
+ dst_uv[1] = src_uv[1];
+ }
+}
+
+#define BLENDER_16(a, b, f) \
+ (uint16_t)((int)(a) + \
+ (int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> \
+ 16))
+
+void ScaleUVFilterCols_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x,
+ int dx) {
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ int xi = (x >> 16) * 2;
+ int xf = x & 0xffff;
+ const uint16_t* a = src_uv + xi;
+ const uint16_t* b = a + 2;
+ dst_uv[0] = BLENDER_16(a[0], b[0], xf);
+ dst_uv[1] = BLENDER_16(a[1], b[1], xf);
+ dst_uv += 2;
+ x += dx;
+ }
+}
+
+void ScaleUVFilterCols64_16_C(uint16_t* dst_uv,
+ const uint16_t* src_uv,
+ int dst_width,
+ int x32,
+ int dx) {
+ int64_t x = (int64_t)(x32);
+ int j;
+ for (j = 0; j < dst_width; ++j) {
+ int64_t xi = (x >> 16) * 2;
+ int xf = x & 0xffff;
+ const uint16_t* a = src_uv + xi;
+ const uint16_t* b = a + 2;
+ dst_uv[0] = BLENDER_16(a[0], b[0], xf);
+ dst_uv[1] = BLENDER_16(a[1], b[1], xf);
+ dst_uv += 2;
+ x += dx;
+ }
+}
+#undef BLENDER_16
+
+// Scale 16-bit UV columns by 3 box filter.
+void ScaleUVAddCols3_16_C(int dst_width,
+ int scaleval,
+ const uint32_t* src_uv,
+ uint16_t* dst_uv) {
+ int i;
+ for (i = 0; i < dst_width; ++i) {
+ uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
+ uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
+ src_uv += 6;
+ dst_uv[0] = (uint16_t)((u * (uint32_t)scaleval) >> 16);
+ dst_uv[1] = (uint16_t)((v * (uint32_t)scaleval) >> 16);
+ dst_uv += 2;
+ }
+}
// Scale plane vertically with bilinear interpolation.
void ScalePlaneVertical(int src_height,
@@ -1029,28 +1585,32 @@ void ScalePlaneVertical(int src_height,
int x,
int y,
int dy,
- int bpp,
+ int bpp, // bytes per pixel. 4 for ARGB.
enum FilterMode filtering) {
// TODO(fbarchard): Allow higher bpp.
int dst_width_bytes = dst_width * bpp;
- void (*InterpolateRow)(uint8_t * dst_argb, const uint8_t* src_argb,
+ void (*InterpolateRow)(uint8_t* dst_argb, const uint8_t* src_argb,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
- const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
+ const bool interpolate = filtering == kFilterBilinear;
+ int64_t max_y = 0;
+ if (src_height > 1) {
+ max_y = ((int64_t)src_height - 1) << 16;
+ if (interpolate) {
+ --max_y;
+ }
+ }
+ // Without vertical interpolation, InterpolateRow() copies one row. A zero
+ // stride keeps it from forming a second source row pointer past the end when
+ // y is clamped to the last row.
+ const ptrdiff_t interpolate_stride = interpolate ? src_stride : 0;
+ int64_t y64 = y;
int j;
assert(bpp >= 1 && bpp <= 4);
assert(src_height != 0);
assert(dst_width > 0);
assert(dst_height > 0);
src_argb += (x >> 16) * bpp;
-#if defined(HAS_INTERPOLATEROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_SSSE3;
- if (IS_ALIGNED(dst_width_bytes, 16)) {
- InterpolateRow = InterpolateRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@@ -1059,6 +1619,11 @@ void ScalePlaneVertical(int src_height,
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@@ -1067,36 +1632,66 @@ void ScalePlaneVertical(int src_height,
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MSA)
- if (TestCpuFlag(kCpuHasMSA)) {
- InterpolateRow = InterpolateRow_Any_MSA;
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(dst_width_bytes, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(dst_width_bytes, 32)) {
- InterpolateRow = InterpolateRow_MSA;
+ InterpolateRow = InterpolateRow_LASX;
}
}
#endif
-#if defined(HAS_INTERPOLATEROW_MMI)
- if (TestCpuFlag(kCpuHasMMI)) {
- InterpolateRow = InterpolateRow_Any_MMI;
- if (IS_ALIGNED(dst_width_bytes, 8)) {
- InterpolateRow = InterpolateRow_MMI;
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width_bytes, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
+
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
- if (y > max_y) {
- y = max_y;
+ if (y64 > max_y) {
+ y64 = max_y;
}
- yi = y >> 16;
- yf = filtering ? ((y >> 8) & 255) : 0;
- InterpolateRow(dst_argb, src_argb + yi * src_stride, src_stride,
- dst_width_bytes, yf);
+ yi = (int)(y64 >> 16);
+ yf = interpolate ? (int)((y64 >> 8) & 255) : 0;
+ InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
+ interpolate_stride, dst_width_bytes, yf);
dst_argb += dst_stride;
- y += dy;
+ y64 += dy;
}
}
+
void ScalePlaneVertical_16(int src_height,
int dst_width,
int dst_height,
@@ -1107,64 +1702,83 @@ void ScalePlaneVertical_16(int src_height,
int x,
int y,
int dy,
- int wpp,
+ int wpp, /* words per pixel. normally 1 */
enum FilterMode filtering) {
// TODO(fbarchard): Allow higher wpp.
int dst_width_words = dst_width * wpp;
- void (*InterpolateRow)(uint16_t * dst_argb, const uint16_t* src_argb,
+ void (*InterpolateRow)(uint16_t* dst_argb, const uint16_t* src_argb,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_16_C;
- const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
+ const bool interpolate = filtering == kFilterBilinear;
+ int64_t max_y = 0;
+ if (src_height > 1) {
+ max_y = ((int64_t)src_height - 1) << 16;
+ if (interpolate) {
+ --max_y;
+ }
+ }
+ // Without vertical interpolation, InterpolateRow() copies one row. A zero
+ // stride keeps it from forming a second source row pointer past the end when
+ // y is clamped to the last row.
+ const ptrdiff_t interpolate_stride = interpolate ? src_stride : 0;
+ int64_t y64 = y;
int j;
assert(wpp >= 1 && wpp <= 2);
assert(src_height != 0);
assert(dst_width > 0);
assert(dst_height > 0);
src_argb += (x >> 16) * wpp;
-#if defined(HAS_INTERPOLATEROW_16_SSE2)
- if (TestCpuFlag(kCpuHasSSE2)) {
- InterpolateRow = InterpolateRow_Any_16_SSE2;
- if (IS_ALIGNED(dst_width_bytes, 16)) {
- InterpolateRow = InterpolateRow_16_SSE2;
- }
- }
-#endif
-#if defined(HAS_INTERPOLATEROW_16_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- InterpolateRow = InterpolateRow_Any_16_SSSE3;
- if (IS_ALIGNED(dst_width_bytes, 16)) {
- InterpolateRow = InterpolateRow_16_SSSE3;
- }
- }
-#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
- InterpolateRow = InterpolateRow_Any_16_AVX2;
- if (IS_ALIGNED(dst_width_bytes, 32)) {
+ InterpolateRow = InterpolateRow_16_Any_AVX2;
+ if (IS_ALIGNED(dst_width_words, 32)) {
InterpolateRow = InterpolateRow_16_AVX2;
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_16_AVX512BW;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- InterpolateRow = InterpolateRow_Any_16_NEON;
- if (IS_ALIGNED(dst_width_bytes, 16)) {
+ InterpolateRow = InterpolateRow_16_Any_NEON;
+ if (IS_ALIGNED(dst_width_words, 8)) {
InterpolateRow = InterpolateRow_16_NEON;
}
}
+#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_16_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width_words, 8)) {
+ InterpolateRow = InterpolateRow_16_WASMSIMD;
+ }
+ }
#endif
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
- if (y > max_y) {
- y = max_y;
+ if (y64 > max_y) {
+ y64 = max_y;
}
- yi = y >> 16;
- yf = filtering ? ((y >> 8) & 255) : 0;
- InterpolateRow(dst_argb, src_argb + yi * src_stride, src_stride,
- dst_width_words, yf);
+ yi = (int)(y64 >> 16);
+ yf = interpolate ? (int)((y64 >> 8) & 255) : 0;
+ InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
+ interpolate_stride, dst_width_words, yf);
dst_argb += dst_stride;
- y += dy;
+ y64 += dy;
}
}
@@ -1181,8 +1795,9 @@ enum FilterMode ScaleFilterReduce(int src_width,
src_height = -src_height;
}
if (filtering == kFilterBox) {
- // If scaling both axis to 0.5 or larger, switch from Box to Bilinear.
- if (dst_width * 2 >= src_width && dst_height * 2 >= src_height) {
+ // If scaling either axis to 0.5 or larger, switch from Box to Bilinear.
+ if ((int64_t)dst_width * 2 >= src_width ||
+ (int64_t)dst_height * 2 >= src_height) {
filtering = kFilterBilinear;
}
}
@@ -1191,7 +1806,7 @@ enum FilterMode ScaleFilterReduce(int src_width,
filtering = kFilterLinear;
}
// TODO(fbarchard): Detect any odd scale factor and reduce to Linear.
- if (dst_height == src_height || dst_height * 3 == src_height) {
+ if (dst_height == src_height || (int64_t)dst_height * 3 == src_height) {
filtering = kFilterLinear;
}
// TODO(fbarchard): Remove 1 pixel wide filter restriction, which is to
@@ -1205,7 +1820,7 @@ enum FilterMode ScaleFilterReduce(int src_width,
filtering = kFilterNone;
}
// TODO(fbarchard): Detect any odd scale factor and reduce to None.
- if (dst_width == src_width || dst_width * 3 == src_width) {
+ if (dst_width == src_width || (int64_t)dst_width * 3 == src_width) {
filtering = kFilterNone;
}
}
@@ -1217,7 +1832,7 @@ int FixedDiv_C(int num, int div) {
return (int)(((int64_t)(num) << 16) / div);
}
-// Divide num by div and return as 16.16 fixed point result.
+// Divide num - 1 by div - 1 and return as 16.16 fixed point result.
int FixedDiv1_C(int num, int div) {
return (int)((((int64_t)(num) << 16) - 0x00010001) / (div - 1));
}
@@ -1243,8 +1858,10 @@ void ScaleSlope(int src_width,
assert(dst_width > 0);
assert(dst_height > 0);
// Check for 1 pixel and avoid FixedDiv overflow.
- if (dst_width == 1 && src_width >= 32768) {
- dst_width = src_width;
+ // TODO(b/573815252): Fix center coordinate calculation when dst == 1 and
+ // src == 32768.
+ if (dst_width == 1 && Abs(src_width) >= 32768) {
+ dst_width = Abs(src_width);
}
if (dst_height == 1 && src_height >= 32768) {
dst_height = src_height;
@@ -1260,14 +1877,14 @@ void ScaleSlope(int src_width,
if (dst_width <= Abs(src_width)) {
*dx = FixedDiv(Abs(src_width), dst_width);
*x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter.
- } else if (dst_width > 1) {
+ } else if (src_width > 1 && dst_width > 1) {
*dx = FixedDiv1(Abs(src_width), dst_width);
*x = 0;
}
if (dst_height <= src_height) {
*dy = FixedDiv(src_height, dst_height);
*y = CENTERSTART(*dy, -32768); // Subtract 0.5 (32768) to center filter.
- } else if (dst_height > 1) {
+ } else if (src_height > 1 && dst_height > 1) {
*dy = FixedDiv1(src_height, dst_height);
*y = 0;
}
@@ -1276,7 +1893,7 @@ void ScaleSlope(int src_width,
if (dst_width <= Abs(src_width)) {
*dx = FixedDiv(Abs(src_width), dst_width);
*x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter.
- } else if (dst_width > 1) {
+ } else if (src_width > 1 && dst_width > 1) {
*dx = FixedDiv1(Abs(src_width), dst_width);
*x = 0;
}
@@ -1298,35 +1915,6 @@ void ScaleSlope(int src_width,
}
#undef CENTERSTART
-// Read 8x2 upsample with filtering and write 16x1.
-// actually reads an extra pixel, so 9x2.
-void ScaleRowUp2_16_C(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- const uint16_t* src2 = src_ptr + src_stride;
-
- int x;
- for (x = 0; x < dst_width - 1; x += 2) {
- uint16_t p0 = src_ptr[0];
- uint16_t p1 = src_ptr[1];
- uint16_t p2 = src2[0];
- uint16_t p3 = src2[1];
- dst[0] = (p0 * 9 + p1 * 3 + p2 * 3 + p3 + 8) >> 4;
- dst[1] = (p0 * 3 + p1 * 9 + p2 + p3 * 3 + 8) >> 4;
- ++src_ptr;
- ++src2;
- dst += 2;
- }
- if (dst_width & 1) {
- uint16_t p0 = src_ptr[0];
- uint16_t p1 = src_ptr[1];
- uint16_t p2 = src2[0];
- uint16_t p3 = src2[1];
- dst[0] = (p0 * 9 + p1 * 3 + p2 * 3 + p3 + 8) >> 4;
- }
-}
-
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/libs/libyuv/source/scale_gcc.cc b/libs/libyuv/source/scale_gcc.cc
index 90a49f30d7..4c239b7d92 100644
--- a/libs/libyuv/source/scale_gcc.cc
+++ b/libs/libyuv/source/scale_gcc.cc
@@ -16,10 +16,19 @@ namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#ifdef HAS_SCALEROWDOWN34_AVX2
// Offsets for source bytes 0 to 9
static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
128, 128, 128, 128, 128, 128, 128, 128};
@@ -54,7 +63,9 @@ static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3};
// Coefficients for source bytes 21 to 31
static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2};
+#endif // HAS_SCALEROWDOWN34_AVX2
+#ifdef HAS_SCALEROWDOWN38_AVX2
static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128};
@@ -88,6 +99,7 @@ static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128,
// Scaling values for boxes of 3x2 and 2x2
static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,
65536 / 3, 65536 / 2, 0, 0};
+#endif // HAS_SCALEROWDOWN38_AVX2
// GCC versions of row functions are verbatim conversions from Visual C.
// Generated using gcc disassembly on Visual C object file:
@@ -101,22 +113,22 @@ void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
asm volatile(
// 16 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "psrlw $0x8,%%xmm0 \n"
- "psrlw $0x8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
@@ -125,30 +137,30 @@ void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
+
+ "pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pavgw %%xmm5,%%xmm0 \n"
- "pavgw %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pavgw %%xmm5,%%xmm0 \n"
+ "pavgw %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm4", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5");
}
void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
@@ -156,38 +168,37 @@ void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
+ "pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x00(%0,%3,1),%%xmm2 \n"
- "movdqu 0x10(%0,%3,1),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm3,%%xmm1 \n"
- "psrlw $0x1,%%xmm0 \n"
- "psrlw $0x1,%%xmm1 \n"
- "pavgw %%xmm5,%%xmm0 \n"
- "pavgw %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%3,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%3,1),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm3 \n"
+ "paddw %%xmm2,%%xmm0 \n"
+ "paddw %%xmm3,%%xmm1 \n"
+ "psrlw $0x1,%%xmm0 \n"
+ "psrlw $0x1,%%xmm1 \n"
+ "pavgw %%xmm5,%%xmm0 \n"
+ "pavgw %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
#ifdef HAS_SCALEROWDOWN2_AVX2
@@ -197,26 +208,24 @@ void ScaleRowDown2_AVX2(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
@@ -225,32 +234,31 @@ void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpavgw %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpavgw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm4", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5");
}
void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
@@ -258,125 +266,390 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
- "vmovdqu 0x20(%0,%3,1),%%ymm3 \n"
- "lea 0x40(%0),%0 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpsrlw $0x1,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x1,%%ymm1,%%ymm1 \n"
- "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpavgw %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%3,1),%%ymm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x1,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x1,%%ymm1,%%ymm1 \n"
+ "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpavgw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_SCALEROWDOWN2_AVX2
+#ifdef HAS_SCALEROWDOWN2_16_AVX2
+void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x10,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+
+void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $0x10,%%ymm4,%%ymm4 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x10,%%ymm1,%%ymm3 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpavgw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+
+void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $0x1f,%%ymm6,%%ymm7 \n"
+ "vpslld $0x1,%%ymm7,%%ymm7 \n"
+ "vpsrld $0x10,%%ymm6,%%ymm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x00(%0,%3,2),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%3,2),%%ymm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm4 \n"
+ "vpand %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x10,%%ymm2,%%ymm4 \n"
+ "vpand %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpaddd %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x10,%%ymm1,%%ymm4 \n"
+ "vpand %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpsrld $0x10,%%ymm3,%%ymm4 \n"
+ "vpand %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpaddd %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpaddd %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpaddd %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpaddd %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpsrld $0x2,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x2,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6", "xmm7");
+}
+#endif // HAS_SCALEROWDOWN2_16_AVX2
+
+#ifdef HAS_SCALEROWDOWN2_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovd %k3,%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrlw $0x8,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "=&r"(temp) // %3
+ :
+ : "memory", "cc", "xmm0", "k1");
+}
+
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
+ "vpabsb %%zmm4,%%zmm4 \n"
+ "vpxord %%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovd %k3,%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "=&r"(temp) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm4", "xmm5", "k1");
+}
+
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ uintptr_t temp;
+ asm volatile(
+ "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
+ "vpabsb %%zmm4,%%zmm4 \n"
+ "vpxord %%zmm5,%%zmm5,%%zmm5 \n"
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu8 (%0),%%zmm0 \n"
+ "vmovdqu8 0x00(%0,%4,1),%%zmm2 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x1,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovd %k3,%%k1 \n"
+ "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x00(%0,%4,1),%%zmm2%{%%k1%}%{z%}\n"
+ "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n"
+ "vpmaddubsw %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpsrlw $0x1,%%zmm0,%%zmm0 \n"
+ "vpavgw %%zmm5,%%zmm0,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "=&r"(temp) // %3
+ : "r"(src_stride) // %4
+ : "memory", "cc", "xmm0", "xmm2", "xmm4", "xmm5", "k1");
+}
+
+void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ dst_width -= 1;
+ if (dst_width > 0) {
+ ScaleRowDown2Box_AVX512BW(src_ptr, src_stride, dst_ptr, dst_width);
+ src_ptr += dst_width * 2;
+ dst_ptr += dst_width;
+ }
+ dst_ptr[0] = (src_ptr[0] + src_ptr[src_stride] + 1) >> 1;
+}
+#endif // HAS_SCALEROWDOWN2_AVX512BW
+
void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n"
- "psrld $0x18,%%xmm5 \n"
- "pslld $0x10,%%xmm5 \n"
+ "pcmpeqb %%xmm5,%%xmm5 \n"
+ "psrld $0x18,%%xmm5 \n"
+ "pslld $0x10,%%xmm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pand %%xmm5,%%xmm0 \n"
- "pand %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "psrlw $0x8,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pand %%xmm5,%%xmm0 \n"
+ "pand %%xmm5,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "psrlw $0x8,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
- intptr_t stridex3;
+ ptrdiff_t stridex3;
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "packuswb %%xmm4,%%xmm4 \n"
- "psllw $0x3,%%xmm5 \n"
- "lea 0x00(%4,%4,2),%3 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n"
+ "pabsw %%xmm4,%%xmm5 \n"
+ "pabsb %%xmm4,%%xmm4 \n" // 0x0101
+ "psllw $0x3,%%xmm5 \n" // 0x0008
+ "lea 0x00(%4,%4,2),%3 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x00(%0,%4,1),%%xmm2 \n"
- "movdqu 0x10(%0,%4,1),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm3,%%xmm1 \n"
- "movdqu 0x00(%0,%4,2),%%xmm2 \n"
- "movdqu 0x10(%0,%4,2),%%xmm3 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm3,%%xmm1 \n"
- "movdqu 0x00(%0,%3,1),%%xmm2 \n"
- "movdqu 0x10(%0,%3,1),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm3 \n"
- "paddw %%xmm2,%%xmm0 \n"
- "paddw %%xmm3,%%xmm1 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "psrlw $0x4,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width), // %2
- "=&r"(stridex3) // %3
- : "r"((intptr_t)(src_stride)) // %4
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x00(%0,%4,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%4,1),%%xmm3 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm3 \n"
+ "paddw %%xmm2,%%xmm0 \n"
+ "paddw %%xmm3,%%xmm1 \n"
+ "movdqu 0x00(%0,%4,2),%%xmm2 \n"
+ "movdqu 0x10(%0,%4,2),%%xmm3 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm3 \n"
+ "paddw %%xmm2,%%xmm0 \n"
+ "paddw %%xmm3,%%xmm1 \n"
+ "movdqu 0x00(%0,%3,1),%%xmm2 \n"
+ "movdqu 0x10(%0,%3,1),%%xmm3 \n"
+ "lea 0x20(%0),%0 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm3 \n"
+ "paddw %%xmm2,%%xmm0 \n"
+ "paddw %%xmm3,%%xmm1 \n"
+ "phaddw %%xmm1,%%xmm0 \n"
+ "paddw %%xmm5,%%xmm0 \n"
+ "psrlw $0x4,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "=&r"(stridex3) // %3
+ : "r"(src_stride) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
@@ -387,32 +660,32 @@ void ScaleRowDown4_AVX2(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- "vpsrld $0x18,%%ymm5,%%ymm5 \n"
- "vpslld $0x10,%%ymm5,%%ymm5 \n"
+ "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpsrld $0x18,%%ymm5,%%ymm5 \n"
+ "vpslld $0x10,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpand %%ymm5,%%ymm0,%%ymm0 \n"
- "vpand %%ymm5,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpand %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
@@ -420,806 +693,2556 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpsllw $0x3,%%ymm4,%%ymm5 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsw %%ymm4,%%ymm5 \n"
+ "vpabsb %%ymm4,%%ymm4 \n" // 0x0101
+ "vpsllw $0x3,%%ymm5,%%ymm5 \n" // 0x0008
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
- "vmovdqu 0x20(%0,%3,1),%%ymm3 \n"
- "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vmovdqu 0x00(%0,%3,2),%%ymm2 \n"
- "vmovdqu 0x20(%0,%3,2),%%ymm3 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vmovdqu 0x00(%0,%4,1),%%ymm2 \n"
- "vmovdqu 0x20(%0,%4,1),%%ymm3 \n"
- "lea 0x40(%0),%0 \n"
- "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
- "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
- "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpsrlw $0x4,%%ymm0,%%ymm0 \n"
- "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vmovdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "r"((intptr_t)(src_stride * 3)) // %4
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%3,1),%%ymm3 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x00(%0,%3,2),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%3,2),%%ymm3 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x00(%0,%4,1),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%4,1),%%ymm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpaddw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x4,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(src_stride * 3) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_SCALEROWDOWN4_AVX2
-void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
+#ifdef HAS_SCALEROWDOWN34_AVX2
+void ScaleRowDown34_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
(void)src_stride;
asm volatile(
- "movdqa %0,%%xmm3 \n"
- "movdqa %1,%%xmm4 \n"
- "movdqa %2,%%xmm5 \n"
- :
- : "m"(kShuf0), // %0
- "m"(kShuf1), // %1
- "m"(kShuf2) // %2
- );
- asm volatile(
+ "vbroadcasti128 %[kShuf0],%%ymm3 \n"
+ "vbroadcasti128 %[kShuf1],%%ymm4 \n"
+ "vbroadcasti128 %[kShuf2],%%ymm5 \n"
+ "sub $0x30,%[dst_width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm2 \n"
- "lea 0x20(%0),%0 \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "palignr $0x8,%%xmm0,%%xmm1 \n"
- "pshufb %%xmm3,%%xmm0 \n"
- "pshufb %%xmm4,%%xmm1 \n"
- "pshufb %%xmm5,%%xmm2 \n"
- "movq %%xmm0,(%1) \n"
- "movq %%xmm1,0x8(%1) \n"
- "movq %%xmm2,0x10(%1) \n"
- "lea 0x18(%1),%1 \n"
- "sub $0x18,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ "1: \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n"
+ "lea 0x40(%[src_ptr]),%[src_ptr] \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm6 \n"
+ "vpalignr $0x8,%%ymm2,%%ymm6,%%ymm1 \n"
+ "vpshufb %%ymm3,%%ymm2,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm5,%%ymm6,%%ymm2 \n"
+ "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%[dst_ptr]) \n"
+ "vmovq %%xmm2,0x10(%[dst_ptr]) \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm0 \n"
+ "vextracti128 $0x1,%%ymm2,%%xmm2 \n"
+ "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n"
+ "vmovq %%xmm2,0x28(%[dst_ptr]) \n"
+ "lea 0x30(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x30,%[dst_width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x30,%[dst_width] \n"
+ "jle 99f \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm2 \n"
+ "vpalignr $0x8,%%xmm0,%%xmm2,%%xmm1 \n"
+ "vpshufb %%xmm3,%%xmm0,%%xmm0 \n"
+ "vpshufb %%xmm4,%%xmm1,%%xmm1 \n"
+ "vpshufb %%xmm5,%%xmm2,%%xmm2 \n"
+ "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%[dst_ptr]) \n"
+ "vmovq %%xmm2,0x10(%[dst_ptr]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr),
+ [dst_width] "+r"(dst_width)
+ : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
-void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- asm volatile(
- "movdqa %0,%%xmm2 \n" // kShuf01
- "movdqa %1,%%xmm3 \n" // kShuf11
- "movdqa %2,%%xmm4 \n" // kShuf21
- :
- : "m"(kShuf01), // %0
- "m"(kShuf11), // %1
- "m"(kShuf21) // %2
- );
- asm volatile(
- "movdqa %0,%%xmm5 \n" // kMadd01
- "movdqa %1,%%xmm0 \n" // kMadd11
- "movdqa %2,%%xmm1 \n" // kRound34
- :
- : "m"(kMadd01), // %0
- "m"(kMadd11), // %1
- "m"(kRound34) // %2
- );
+void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
asm volatile(
+ "vbroadcasti128 %[kRound34],%%ymm5 \n"
+ "sub $0x30,%[dst_width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu 0x00(%0,%3,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm5,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,(%1) \n"
- "movdqu 0x8(%0),%%xmm6 \n"
- "movdqu 0x8(%0,%3,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm3,%%xmm6 \n"
- "pmaddubsw %%xmm0,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,0x8(%1) \n"
- "movdqu 0x10(%0),%%xmm6 \n"
- "movdqu 0x10(%0,%3,1),%%xmm7 \n"
- "lea 0x20(%0),%0 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm4,%%xmm6 \n"
- "pmaddubsw %4,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,0x10(%1) \n"
- "lea 0x18(%1),%1 \n"
- "sub $0x18,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "m"(kMadd21) // %4
+ "1: \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n"
+ "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n"
+ "lea 0x40(%[src_ptr]),%[src_ptr] \n"
+ "vpavgb %%ymm3,%%ymm1,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n"
+
+ "vbroadcasti128 %[kShuf01],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd01],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpsrlw $0x2,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n"
+
+ "vbroadcasti128 %[kShuf11],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd11],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm4,%%ymm4 \n"
+ "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n"
+ "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n"
+ "vpsrlw $0x2,%%ymm4,%%ymm4 \n"
+ "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+
+ "vbroadcasti128 %[kShuf21],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd21],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpsrlw $0x2,%%ymm3,%%ymm3 \n"
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n"
+
+ "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%xmm2,(%[dst_ptr]) \n"
+ "vmovq %%xmm3,0x10(%[dst_ptr]) \n"
+ "vextracti128 $0x1,%%ymm2,%%xmm0 \n"
+ "vextracti128 $0x1,%%ymm3,%%xmm1 \n"
+ "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n"
+ "vmovq %%xmm1,0x28(%[dst_ptr]) \n"
+ "lea 0x30(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x30,%[dst_width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x30,%[dst_width] \n"
+ "jle 99f \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n"
+ "vpavgb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm3 \n"
+ "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n"
+
+ "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n"
+ "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n"
+ "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x2,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+
+ "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n"
+ "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n"
+ "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n"
+ "vpsrlw $0x2,%%xmm4,%%xmm4 \n"
+ "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n"
+
+ "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n"
+ "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n"
+ "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n"
+ "vpsrlw $0x2,%%xmm3,%%xmm3 \n"
+ "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n"
+
+ "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%[dst_ptr]) \n"
+ "vmovq %%xmm3,0x10(%[dst_ptr]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr),
+ [dst_width] "+rm"(dst_width)
+ : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01),
+ [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01),
+ [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- asm volatile(
- "movdqa %0,%%xmm2 \n" // kShuf01
- "movdqa %1,%%xmm3 \n" // kShuf11
- "movdqa %2,%%xmm4 \n" // kShuf21
- :
- : "m"(kShuf01), // %0
- "m"(kShuf11), // %1
- "m"(kShuf21) // %2
- );
- asm volatile(
- "movdqa %0,%%xmm5 \n" // kMadd01
- "movdqa %1,%%xmm0 \n" // kMadd11
- "movdqa %2,%%xmm1 \n" // kRound34
- :
- : "m"(kMadd01), // %0
- "m"(kMadd11), // %1
- "m"(kRound34) // %2
- );
-
+void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
asm volatile(
+ "vbroadcasti128 %[kRound34],%%ymm5 \n"
+ "sub $0x30,%[dst_width] \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm6 \n"
- "movdqu 0x00(%0,%3,1),%%xmm7 \n"
- "pavgb %%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm5,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,(%1) \n"
- "movdqu 0x8(%0),%%xmm6 \n"
- "movdqu 0x8(%0,%3,1),%%xmm7 \n"
- "pavgb %%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm3,%%xmm6 \n"
- "pmaddubsw %%xmm0,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,0x8(%1) \n"
- "movdqu 0x10(%0),%%xmm6 \n"
- "movdqu 0x10(%0,%3,1),%%xmm7 \n"
- "lea 0x20(%0),%0 \n"
- "pavgb %%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
- "pshufb %%xmm4,%%xmm6 \n"
- "pmaddubsw %4,%%xmm6 \n"
- "paddsw %%xmm1,%%xmm6 \n"
- "psrlw $0x2,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movq %%xmm6,0x10(%1) \n"
- "lea 0x18(%1),%1 \n"
- "sub $0x18,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "m"(kMadd21) // %4
+ "1: \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n"
+ "vpavgb %%ymm0,%%ymm2,%%ymm2 \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n"
+ "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n"
+ "lea 0x40(%[src_ptr]),%[src_ptr] \n"
+ "vpavgb %%ymm1,%%ymm3,%%ymm3 \n"
+ "vpavgb %%ymm3,%%ymm1,%%ymm1 \n"
+ "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n"
+
+ "vbroadcasti128 %[kShuf01],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd01],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n"
+ "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n"
+ "vpsrlw $0x2,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n"
+
+ "vbroadcasti128 %[kShuf11],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd11],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm4,%%ymm4 \n"
+ "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n"
+ "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n"
+ "vpsrlw $0x2,%%ymm4,%%ymm4 \n"
+ "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+
+ "vbroadcasti128 %[kShuf21],%%ymm6 \n"
+ "vbroadcasti128 %[kMadd21],%%ymm7 \n"
+ "vpshufb %%ymm6,%%ymm3,%%ymm3 \n"
+ "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpsrlw $0x2,%%ymm3,%%ymm3 \n"
+ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n"
+
+ "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n"
+ "vmovdqu %%xmm2,(%[dst_ptr]) \n"
+ "vmovq %%xmm3,0x10(%[dst_ptr]) \n"
+ "vextracti128 $0x1,%%ymm2,%%xmm0 \n"
+ "vextracti128 $0x1,%%ymm3,%%xmm1 \n"
+ "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n"
+ "vmovq %%xmm1,0x28(%[dst_ptr]) \n"
+ "lea 0x30(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x30,%[dst_width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x30,%[dst_width] \n"
+ "jle 99f \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n"
+ "vpavgb %%ymm0,%%ymm1,%%ymm1 \n"
+ "vpavgb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm3 \n"
+ "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n"
+
+ "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n"
+ "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n"
+ "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x2,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+
+ "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n"
+ "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n"
+ "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n"
+ "vpsrlw $0x2,%%xmm4,%%xmm4 \n"
+ "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n"
+
+ "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n"
+ "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n"
+ "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n"
+ "vpsrlw $0x2,%%xmm3,%%xmm3 \n"
+ "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n"
+
+ "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%[dst_ptr]) \n"
+ "vmovq %%xmm3,0x10(%[dst_ptr]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr),
+ [dst_width] "+rm"(dst_width)
+ : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01),
+ [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01),
+ [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
+#endif // HAS_SCALEROWDOWN34_AVX2
-void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
+#ifdef HAS_SCALEROWDOWN38_AVX2
+void ScaleRowDown38_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
(void)src_stride;
asm volatile(
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
+ "vmovdqu %3,%%xmm4 \n"
+ "vinserti128 $0x1,%4,%%ymm4,%%ymm4 \n"
+ "sub $0x18,%2 \n"
+ "jl 2f \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "paddusb %%xmm1,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "movhlps %%xmm0,%%xmm1 \n"
- "movd %%xmm1,0x8(%1) \n"
- "lea 0xc(%1),%1 \n"
- "sub $0xc,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm2 \n"
+ "vextracti128 $0x1,%%ymm1,%%xmm3 \n"
+ "vpor %%xmm2,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm3,%%xmm1,%%xmm1 \n"
+ "vmovq %%xmm0,(%1) \n"
+ "vpsrldq $0x8,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,0x8(%1) \n"
+ "vmovq %%xmm1,0xc(%1) \n"
+ "vpsrldq $0x8,%%xmm1,%%xmm1 \n"
+ "vmovd %%xmm1,0x14(%1) \n"
+ "lea 0x18(%1),%1 \n"
+ "sub $0x18,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x18,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vextracti128 $0x1,%%ymm0,%%xmm2 \n"
+ "vpor %%xmm2,%%xmm0,%%xmm0 \n"
+ "vmovq %%xmm0,(%1) \n"
+ "vpsrldq $0x8,%%xmm0,%%xmm0 \n"
+ "vmovd %%xmm0,0x8(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
: "m"(kShuf38a), // %3
"m"(kShuf38b) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
-void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- asm volatile(
- "movdqa %0,%%xmm2 \n"
- "movdqa %1,%%xmm3 \n"
- "movdqa %2,%%xmm4 \n"
- "movdqa %3,%%xmm5 \n"
- :
- : "m"(kShufAb0), // %0
- "m"(kShufAb1), // %1
- "m"(kShufAb2), // %2
- "m"(kScaleAb2) // %3
- );
+void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
asm volatile(
+ "vbroadcasti128 %[kShufAb0],%%ymm2 \n"
+ "vbroadcasti128 %[kShufAb1],%%ymm3 \n"
+ "vbroadcasti128 %[kShufAb2],%%ymm4 \n"
+ "vbroadcasti128 %[kScaleAb2],%%ymm5 \n"
+ "vpxor %%ymm7,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%3,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "pavgb %%xmm1,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "pshufb %%xmm2,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm6 \n"
- "pshufb %%xmm3,%%xmm6 \n"
- "paddusw %%xmm6,%%xmm1 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "paddusw %%xmm0,%%xmm1 \n"
- "pmulhuw %%xmm5,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movd %%xmm1,(%1) \n"
- "psrlq $0x10,%%xmm1 \n"
- "movd %%xmm1,0x2(%1) \n"
- "lea 0x6(%1),%1 \n"
- "sub $0x6,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "1: \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n"
+ "lea 0x20(%[src_ptr]),%[src_ptr] \n"
+ "vpavgb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpshufb %%ymm3,%%ymm0,%%ymm6 \n"
+ "vpaddusw %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpaddusw %%ymm0,%%ymm1,%%ymm1 \n"
+ "vpmulhuw %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm7,%%ymm1,%%ymm1 \n"
+ "vextracti128 $0x1,%%ymm1,%%xmm0 \n"
+ "vpslldq $0x6,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm0,%%xmm1,%%xmm1 \n"
+ "vmovq %%xmm1,(%[dst_ptr]) \n"
+ "vpsrldq $0x8,%%xmm1,%%xmm1 \n"
+ "vmovd %%xmm1,0x8(%[dst_ptr]) \n"
+ "lea 0xc(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0xc,%[dst_width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr),
+ [dst_width] "+rm"(dst_width)
+ : [src_stride] "r"(src_stride), [kShufAb0] "m"(kShufAb0),
+ [kShufAb1] "m"(kShufAb1), [kShufAb2] "m"(kShufAb2),
+ [kScaleAb2] "m"(kScaleAb2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
-void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- asm volatile(
- "movdqa %0,%%xmm2 \n"
- "movdqa %1,%%xmm3 \n"
- "movdqa %2,%%xmm4 \n"
- "pxor %%xmm5,%%xmm5 \n"
- :
- : "m"(kShufAc), // %0
- "m"(kShufAc3), // %1
- "m"(kScaleAc33) // %2
- );
+void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
asm volatile(
+ "vbroadcasti128 %[kShufAc],%%ymm2 \n"
+ "vbroadcasti128 %[kShufAc3],%%ymm3 \n"
+ "vbroadcasti128 %[kScaleAc33],%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%3,1),%%xmm6 \n"
- "movhlps %%xmm0,%%xmm1 \n"
- "movhlps %%xmm6,%%xmm7 \n"
- "punpcklbw %%xmm5,%%xmm0 \n"
- "punpcklbw %%xmm5,%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm6 \n"
- "punpcklbw %%xmm5,%%xmm7 \n"
- "paddusw %%xmm6,%%xmm0 \n"
- "paddusw %%xmm7,%%xmm1 \n"
- "movdqu 0x00(%0,%3,2),%%xmm6 \n"
- "lea 0x10(%0),%0 \n"
- "movhlps %%xmm6,%%xmm7 \n"
- "punpcklbw %%xmm5,%%xmm6 \n"
- "punpcklbw %%xmm5,%%xmm7 \n"
- "paddusw %%xmm6,%%xmm0 \n"
- "paddusw %%xmm7,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm6 \n"
- "psrldq $0x2,%%xmm0 \n"
- "paddusw %%xmm0,%%xmm6 \n"
- "psrldq $0x2,%%xmm0 \n"
- "paddusw %%xmm0,%%xmm6 \n"
- "pshufb %%xmm2,%%xmm6 \n"
- "movdqa %%xmm1,%%xmm7 \n"
- "psrldq $0x2,%%xmm1 \n"
- "paddusw %%xmm1,%%xmm7 \n"
- "psrldq $0x2,%%xmm1 \n"
- "paddusw %%xmm1,%%xmm7 \n"
- "pshufb %%xmm3,%%xmm7 \n"
- "paddusw %%xmm7,%%xmm6 \n"
- "pmulhuw %%xmm4,%%xmm6 \n"
- "packuswb %%xmm6,%%xmm6 \n"
- "movd %%xmm6,(%1) \n"
- "psrlq $0x10,%%xmm6 \n"
- "movd %%xmm6,0x2(%1) \n"
- "lea 0x6(%1),%1 \n"
- "sub $0x6,%2 \n"
- "jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
+ "1: \n"
+ "vmovdqu (%[src_ptr]),%%ymm0 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm6 \n"
+ "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n"
+ "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n"
+ "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n"
+ "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n"
+ "vmovdqu 0x00(%[src_ptr],%[src_stride],2),%%ymm6 \n"
+ "lea 0x20(%[src_ptr]),%[src_ptr] \n"
+ "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n"
+ "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n"
+ "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n"
+ "vpsrldq $0x2,%%ymm0,%%ymm6 \n"
+ "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n"
+ "vpsrldq $0x4,%%ymm0,%%ymm0 \n"
+ "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n"
+ "vpshufb %%ymm2,%%ymm6,%%ymm6 \n"
+ "vpsrldq $0x2,%%ymm1,%%ymm7 \n"
+ "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n"
+ "vpsrldq $0x4,%%ymm1,%%ymm1 \n"
+ "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n"
+ "vpshufb %%ymm3,%%ymm7,%%ymm7 \n"
+ "vpaddusw %%ymm7,%%ymm6,%%ymm6 \n"
+ "vpmulhuw %%ymm4,%%ymm6,%%ymm6 \n"
+ "vpackuswb %%ymm5,%%ymm6,%%ymm6 \n"
+ "vextracti128 $0x1,%%ymm6,%%xmm0 \n"
+ "vpslldq $0x6,%%xmm0,%%xmm0 \n"
+ "vpor %%xmm0,%%xmm6,%%xmm6 \n"
+ "vmovq %%xmm6,(%[dst_ptr]) \n"
+ "vpsrldq $0x8,%%xmm6,%%xmm6 \n"
+ "vmovd %%xmm6,0x8(%[dst_ptr]) \n"
+ "lea 0xc(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0xc,%[dst_width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr),
+ [dst_width] "+rm"(dst_width)
+ : [src_stride] "r"(src_stride), [kShufAc] "m"(kShufAc),
+ [kShufAc3] "m"(kShufAc3), [kScaleAc33] "m"(kScaleAc33)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
+#endif // HAS_SCALEROWDOWN38_AVX2
-// Reads 16xN bytes and produces 16 shorts at a time.
-void ScaleAddRow_SSE2(const uint8_t* src_ptr,
- uint16_t* dst_ptr,
- int src_width) {
- asm volatile(
+static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5,
+ 10, 11, 8, 9, 14, 15, 12, 13};
- "pxor %%xmm5,%%xmm5 \n"
+static const uvec8 kLinearMadd31 = {3, 1, 1, 3, 3, 1, 1, 3,
+ 3, 1, 1, 3, 3, 1, 1, 3};
+
+#ifdef HAS_SCALEROWUP2_LINEAR_SSE2
+void ScaleRowUp2_Linear_SSE2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "pxor %%xmm0,%%xmm0 \n" // 0
+ "pcmpeqw %%xmm6,%%xmm6 \n"
+ "psrlw $15,%%xmm6 \n"
+ "psllw $1,%%xmm6 \n" // all 2
- // 16 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm3 \n"
- "lea 0x10(%0),%0 \n" // src_ptr += 16
- "movdqu (%1),%%xmm0 \n"
- "movdqu 0x10(%1),%%xmm1 \n"
- "movdqa %%xmm3,%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpckhbw %%xmm5,%%xmm3 \n"
- "paddusw %%xmm2,%%xmm0 \n"
- "paddusw %%xmm3,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movq (%0),%%xmm1 \n" // 01234567
+ "movq 1(%0),%%xmm2 \n" // 12345678
+ "movdqa %%xmm1,%%xmm3 \n"
+ "punpcklbw %%xmm2,%%xmm3 \n" // 0112233445566778
+ "punpcklbw %%xmm1,%%xmm1 \n" // 0011223344556677
+ "punpcklbw %%xmm2,%%xmm2 \n" // 1122334455667788
+ "movdqa %%xmm1,%%xmm4 \n"
+ "punpcklbw %%xmm0,%%xmm4 \n" // 00112233 (16)
+ "movdqa %%xmm2,%%xmm5 \n"
+ "punpcklbw %%xmm0,%%xmm5 \n" // 11223344 (16)
+ "paddw %%xmm5,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm5 \n"
+ "paddw %%xmm6,%%xmm4 \n"
+ "punpcklbw %%xmm0,%%xmm5 \n" // 01122334 (16)
+ "paddw %%xmm5,%%xmm5 \n"
+ "paddw %%xmm4,%%xmm5 \n" // 3*near+far+2 (lo)
+ "psrlw $2,%%xmm5 \n" // 3/4*near+1/4*far (lo)
+
+ "punpckhbw %%xmm0,%%xmm1 \n" // 44556677 (16)
+ "punpckhbw %%xmm0,%%xmm2 \n" // 55667788 (16)
+ "paddw %%xmm2,%%xmm1 \n"
+ "punpckhbw %%xmm0,%%xmm3 \n" // 45566778 (16)
+ "paddw %%xmm6,%%xmm1 \n"
+ "paddw %%xmm3,%%xmm3 \n"
+ "paddw %%xmm3,%%xmm1 \n" // 3*near+far+2 (hi)
+ "psrlw $2,%%xmm1 \n" // 3/4*near+1/4*far (hi)
+
+ "packuswb %%xmm1,%%xmm5 \n"
+ "movdqu %%xmm5,(%1) \n"
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
- "+r"(src_width) // %2
+ "+r"(dst_width) // %2
:
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
+#endif
-#ifdef HAS_SCALEADDROW_AVX2
-// Reads 32 bytes and accumulates to 32 shorts at a time.
-void ScaleAddRow_AVX2(const uint8_t* src_ptr,
- uint16_t* dst_ptr,
- int src_width) {
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSE2
+void ScaleRowUp2_Bilinear_SSE2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
asm volatile(
+ "1: \n"
+ "pxor %%xmm0,%%xmm0 \n" // 0
+ // above line
+ "movq (%0),%%xmm1 \n" // 01234567
+ "movq 1(%0),%%xmm2 \n" // 12345678
+ "movdqa %%xmm1,%%xmm3 \n"
+ "punpcklbw %%xmm2,%%xmm3 \n" // 0112233445566778
+ "punpcklbw %%xmm1,%%xmm1 \n" // 0011223344556677
+ "punpcklbw %%xmm2,%%xmm2 \n" // 1122334455667788
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "punpcklbw %%xmm0,%%xmm4 \n" // 00112233 (16)
+ "movdqa %%xmm2,%%xmm5 \n"
+ "punpcklbw %%xmm0,%%xmm5 \n" // 11223344 (16)
+ "paddw %%xmm5,%%xmm4 \n" // near+far
+ "movdqa %%xmm3,%%xmm5 \n"
+ "punpcklbw %%xmm0,%%xmm5 \n" // 01122334 (16)
+ "paddw %%xmm5,%%xmm5 \n" // 2*near
+ "paddw %%xmm5,%%xmm4 \n" // 3*near+far (1, lo)
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm3 \n"
- "lea 0x20(%0),%0 \n" // src_ptr += 32
- "vpermq $0xd8,%%ymm3,%%ymm3 \n"
- "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
- "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
- "vpaddusw (%1),%%ymm2,%%ymm0 \n"
- "vpaddusw 0x20(%1),%%ymm3,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(src_width) // %2
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
-}
-#endif // HAS_SCALEADDROW_AVX2
+ "punpckhbw %%xmm0,%%xmm1 \n" // 44556677 (16)
+ "punpckhbw %%xmm0,%%xmm2 \n" // 55667788 (16)
+ "paddw %%xmm2,%%xmm1 \n"
+ "punpckhbw %%xmm0,%%xmm3 \n" // 45566778 (16)
+ "paddw %%xmm3,%%xmm3 \n" // 2*near
+ "paddw %%xmm3,%%xmm1 \n" // 3*near+far (1, hi)
-// Constant for making pixels signed to avoid pmaddubsw
-// saturation.
-static const uvec8 kFsub80 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
- 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+ // below line
+ "movq (%0,%3),%%xmm6 \n" // 01234567
+ "movq 1(%0,%3),%%xmm2 \n" // 12345678
+ "movdqa %%xmm6,%%xmm3 \n"
+ "punpcklbw %%xmm2,%%xmm3 \n" // 0112233445566778
+ "punpcklbw %%xmm6,%%xmm6 \n" // 0011223344556677
+ "punpcklbw %%xmm2,%%xmm2 \n" // 1122334455667788
-// Constant for making pixels unsigned and adding .5 for rounding.
-static const uvec16 kFadd40 = {0x4040, 0x4040, 0x4040, 0x4040,
- 0x4040, 0x4040, 0x4040, 0x4040};
+ "movdqa %%xmm6,%%xmm5 \n"
+ "punpcklbw %%xmm0,%%xmm5 \n" // 00112233 (16)
+ "movdqa %%xmm2,%%xmm7 \n"
+ "punpcklbw %%xmm0,%%xmm7 \n" // 11223344 (16)
+ "paddw %%xmm7,%%xmm5 \n" // near+far
+ "movdqa %%xmm3,%%xmm7 \n"
+ "punpcklbw %%xmm0,%%xmm7 \n" // 01122334 (16)
+ "paddw %%xmm7,%%xmm7 \n" // 2*near
+ "paddw %%xmm7,%%xmm5 \n" // 3*near+far (2, lo)
-// Bilinear column filtering. SSSE3 version.
-void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- intptr_t x0, x1, temp_pixel;
- asm volatile(
- "movd %6,%%xmm2 \n"
- "movd %7,%%xmm3 \n"
- "movl $0x04040000,%k2 \n"
- "movd %k2,%%xmm5 \n"
- "pcmpeqb %%xmm6,%%xmm6 \n"
- "psrlw $0x9,%%xmm6 \n" // 0x007f007f
- "pcmpeqb %%xmm7,%%xmm7 \n"
- "psrlw $15,%%xmm7 \n" // 0x00010001
-
- "pextrw $0x1,%%xmm2,%k3 \n"
- "subl $0x2,%5 \n"
- "jl 29f \n"
- "movdqa %%xmm2,%%xmm0 \n"
- "paddd %%xmm3,%%xmm0 \n"
- "punpckldq %%xmm0,%%xmm2 \n"
- "punpckldq %%xmm3,%%xmm3 \n"
- "paddd %%xmm3,%%xmm3 \n"
- "pextrw $0x3,%%xmm2,%k4 \n"
+ "punpckhbw %%xmm0,%%xmm6 \n" // 44556677 (16)
+ "punpckhbw %%xmm0,%%xmm2 \n" // 55667788 (16)
+ "paddw %%xmm6,%%xmm2 \n" // near+far
+ "punpckhbw %%xmm0,%%xmm3 \n" // 45566778 (16)
+ "paddw %%xmm3,%%xmm3 \n" // 2*near
+ "paddw %%xmm3,%%xmm2 \n" // 3*near+far (2, hi)
- LABELALIGN
- "2: \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "paddd %%xmm3,%%xmm2 \n"
- "movzwl 0x00(%1,%3,1),%k2 \n"
- "movd %k2,%%xmm0 \n"
- "psrlw $0x9,%%xmm1 \n"
- "movzwl 0x00(%1,%4,1),%k2 \n"
- "movd %k2,%%xmm4 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "punpcklwd %%xmm4,%%xmm0 \n"
- "psubb %8,%%xmm0 \n" // make pixels signed.
- "pxor %%xmm6,%%xmm1 \n" // 128 - f = (f ^ 127 ) +
- // 1
- "paddusb %%xmm7,%%xmm1 \n"
- "pmaddubsw %%xmm0,%%xmm1 \n"
- "pextrw $0x1,%%xmm2,%k3 \n"
- "pextrw $0x3,%%xmm2,%k4 \n"
- "paddw %9,%%xmm1 \n" // make pixels unsigned.
- "psrlw $0x7,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm1 \n"
- "movd %%xmm1,%k2 \n"
- "mov %w2,(%0) \n"
- "lea 0x2(%0),%0 \n"
- "subl $0x2,%5 \n"
- "jge 2b \n"
+ // xmm4 xmm1
+ // xmm5 xmm2
+ "pcmpeqw %%xmm0,%%xmm0 \n"
+ "psrlw $15,%%xmm0 \n"
+ "psllw $3,%%xmm0 \n" // all 8
- LABELALIGN
- "29: \n"
- "addl $0x1,%5 \n"
- "jl 99f \n"
- "movzwl 0x00(%1,%3,1),%k2 \n"
- "movd %k2,%%xmm0 \n"
- "psrlw $0x9,%%xmm2 \n"
- "pshufb %%xmm5,%%xmm2 \n"
- "psubb %8,%%xmm0 \n" // make pixels signed.
- "pxor %%xmm6,%%xmm2 \n"
- "paddusb %%xmm7,%%xmm2 \n"
- "pmaddubsw %%xmm0,%%xmm2 \n"
- "paddw %9,%%xmm2 \n" // make pixels unsigned.
- "psrlw $0x7,%%xmm2 \n"
- "packuswb %%xmm2,%%xmm2 \n"
- "movd %%xmm2,%k2 \n"
- "mov %b2,(%0) \n"
- "99: \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "=&a"(temp_pixel), // %2
- "=&r"(x0), // %3
- "=&r"(x1), // %4
-#if defined(__x86_64__)
- "+rm"(dst_width) // %5
-#else
- "+m"(dst_width) // %5
-#endif
- : "rm"(x), // %6
- "rm"(dx), // %7
-#if defined(__x86_64__)
- "x"(kFsub80), // %8
- "x"(kFadd40) // %9
-#else
- "m"(kFsub80), // %8
- "m"(kFadd40) // %9
-#endif
+ "movdqa %%xmm4,%%xmm3 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "paddw %%xmm3,%%xmm3 \n" // 6*near+2*far (1, lo)
+ "paddw %%xmm0,%%xmm6 \n" // 3*near+far+8 (2, lo)
+ "paddw %%xmm4,%%xmm3 \n" // 9*near+3*far (1, lo)
+ "paddw %%xmm6,%%xmm3 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrlw $4,%%xmm3 \n" // ^ div by 16
+
+ "movdqa %%xmm1,%%xmm7 \n"
+ "movdqa %%xmm2,%%xmm6 \n"
+ "paddw %%xmm7,%%xmm7 \n" // 6*near+2*far (1, hi)
+ "paddw %%xmm0,%%xmm6 \n" // 3*near+far+8 (2, hi)
+ "paddw %%xmm1,%%xmm7 \n" // 9*near+3*far (1, hi)
+ "paddw %%xmm6,%%xmm7 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrlw $4,%%xmm7 \n" // ^ div by 16
+
+ "packuswb %%xmm7,%%xmm3 \n"
+ "movdqu %%xmm3,(%1) \n" // save above line
+
+ "movdqa %%xmm5,%%xmm3 \n"
+ "paddw %%xmm0,%%xmm4 \n" // 3*near+far+8 (1, lo)
+ "paddw %%xmm3,%%xmm3 \n" // 6*near+2*far (2, lo)
+ "paddw %%xmm3,%%xmm5 \n" // 9*near+3*far (2, lo)
+ "paddw %%xmm4,%%xmm5 \n" // 9 3 3 1 + 8 (lo)
+ "psrlw $4,%%xmm5 \n" // ^ div by 16
+
+ "movdqa %%xmm2,%%xmm3 \n"
+ "paddw %%xmm0,%%xmm1 \n" // 3*near+far+8 (1, hi)
+ "paddw %%xmm3,%%xmm3 \n" // 6*near+2*far (2, hi)
+ "paddw %%xmm3,%%xmm2 \n" // 9*near+3*far (2, hi)
+ "paddw %%xmm1,%%xmm2 \n" // 9 3 3 1 + 8 (hi)
+ "psrlw $4,%%xmm2 \n" // ^ div by 16
+
+ "packuswb %%xmm2,%%xmm5 \n"
+ "movdqu %%xmm5,(%1,%4) \n" // save below line
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
+#endif
-// Reads 4 pixels, duplicates them and writes 8 pixels.
-// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned.
-void ScaleColsUp2_SSE2(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- (void)x;
- (void)dx;
+#ifdef HAS_SCALEROWUP2_LINEAR_12_SSSE3
+void ScaleRowUp2_Linear_12_SSSE3(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
asm volatile(
+ "movdqa %3,%%xmm5 \n"
+ "pcmpeqw %%xmm4,%%xmm4 \n"
+ "psrlw $15,%%xmm4 \n"
+ "psllw $1,%%xmm4 \n" // all 2
LABELALIGN
- "1: \n"
- "movdqu (%1),%%xmm0 \n"
- "lea 0x10(%1),%1 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "movdqu %%xmm0,(%0) \n"
- "movdqu %%xmm1,0x10(%0) \n"
- "lea 0x20(%0),%0 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n" // 01234567 (16)
+ "movdqu 2(%0),%%xmm1 \n" // 12345678 (16)
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
-}
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhwd %%xmm1,%%xmm2 \n" // 45566778 (16)
+ "punpcklwd %%xmm1,%%xmm0 \n" // 01122334 (16)
-void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- (void)src_stride;
- asm volatile(
+ "movdqa %%xmm2,%%xmm3 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pshufb %%xmm5,%%xmm3 \n" // 54657687 (far)
+ "pshufb %%xmm5,%%xmm1 \n" // 10213243 (far)
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "shufps $0xdd,%%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
-}
+ "paddw %%xmm4,%%xmm1 \n" // far+2
+ "paddw %%xmm4,%%xmm3 \n" // far+2
+ "paddw %%xmm0,%%xmm1 \n" // near+far+2
+ "paddw %%xmm2,%%xmm3 \n" // near+far+2
+ "paddw %%xmm0,%%xmm0 \n" // 2*near
+ "paddw %%xmm2,%%xmm2 \n" // 2*near
+ "paddw %%xmm1,%%xmm0 \n" // 3*near+far+2 (lo)
+ "paddw %%xmm3,%%xmm2 \n" // 3*near+far+2 (hi)
-void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- (void)src_stride;
- asm volatile(
+ "psrlw $2,%%xmm0 \n" // 3/4*near+1/4*far
+ "psrlw $2,%%xmm2 \n" // 3/4*near+1/4*far
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm2,16(%1) \n"
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm2 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kLinearShuffleFar) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
+#endif
-void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_SSSE3
+void ScaleRowUp2_Bilinear_12_SSSE3(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
asm volatile(
+ "pcmpeqw %%xmm7,%%xmm7 \n"
+ "psrlw $15,%%xmm7 \n"
+ "psllw $3,%%xmm7 \n" // all 8
+ "movdqa %5,%%xmm6 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x00(%0,%3,1),%%xmm2 \n"
- "movdqu 0x10(%0,%3,1),%%xmm3 \n"
- "lea 0x20(%0),%0 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "pavgb %%xmm3,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm2 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
-}
+ "1: \n"
+ // above line
+ "movdqu (%0),%%xmm0 \n" // 01234567 (16)
+ "movdqu 2(%0),%%xmm1 \n" // 12345678 (16)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhwd %%xmm1,%%xmm2 \n" // 45566778 (16)
+ "punpcklwd %%xmm1,%%xmm0 \n" // 01122334 (16)
+ "movdqa %%xmm2,%%xmm3 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "pshufb %%xmm6,%%xmm3 \n" // 54657687 (far)
+ "pshufb %%xmm6,%%xmm1 \n" // 10213243 (far)
+ "paddw %%xmm0,%%xmm1 \n" // near+far
+ "paddw %%xmm2,%%xmm3 \n" // near+far
+ "paddw %%xmm0,%%xmm0 \n" // 2*near
+ "paddw %%xmm2,%%xmm2 \n" // 2*near
+ "paddw %%xmm1,%%xmm0 \n" // 3*near+far (1, lo)
+ "paddw %%xmm3,%%xmm2 \n" // 3*near+far (1, hi)
-// Reads 4 pixels at a time.
-// Alignment requirement: dst_argb 16 byte aligned.
-void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
+ // below line
+ "movdqu (%0,%3,2),%%xmm1 \n" // 01234567 (16)
+ "movdqu 2(%0,%3,2),%%xmm4 \n" // 12345678 (16)
+ "movdqa %%xmm1,%%xmm3 \n"
+ "punpckhwd %%xmm4,%%xmm3 \n" // 45566778 (16)
+ "punpcklwd %%xmm4,%%xmm1 \n" // 01122334 (16)
+ "movdqa %%xmm3,%%xmm5 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "pshufb %%xmm6,%%xmm5 \n" // 54657687 (far)
+ "pshufb %%xmm6,%%xmm4 \n" // 10213243 (far)
+ "paddw %%xmm1,%%xmm4 \n" // near+far
+ "paddw %%xmm3,%%xmm5 \n" // near+far
+ "paddw %%xmm1,%%xmm1 \n" // 2*near
+ "paddw %%xmm3,%%xmm3 \n" // 2*near
+ "paddw %%xmm4,%%xmm1 \n" // 3*near+far (2, lo)
+ "paddw %%xmm5,%%xmm3 \n" // 3*near+far (2, hi)
+
+ // xmm0 xmm2
+ // xmm1 xmm3
+
+ "movdqa %%xmm0,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm5 \n"
+ "paddw %%xmm4,%%xmm4 \n" // 6*near+2*far (1, lo)
+ "paddw %%xmm7,%%xmm5 \n" // 3*near+far+8 (2, lo)
+ "paddw %%xmm0,%%xmm4 \n" // 9*near+3*far (1, lo)
+ "paddw %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrlw $4,%%xmm4 \n" // ^ div by 16
+ "movdqu %%xmm4,(%1) \n"
+
+ "movdqa %%xmm2,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm5 \n"
+ "paddw %%xmm4,%%xmm4 \n" // 6*near+2*far (1, hi)
+ "paddw %%xmm7,%%xmm5 \n" // 3*near+far+8 (2, hi)
+ "paddw %%xmm2,%%xmm4 \n" // 9*near+3*far (1, hi)
+ "paddw %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrlw $4,%%xmm4 \n" // ^ div by 16
+ "movdqu %%xmm4,0x10(%1) \n"
+
+ "movdqa %%xmm1,%%xmm4 \n"
+ "paddw %%xmm7,%%xmm0 \n" // 3*near+far+8 (1, lo)
+ "paddw %%xmm4,%%xmm4 \n" // 6*near+2*far (2, lo)
+ "paddw %%xmm4,%%xmm1 \n" // 9*near+3*far (2, lo)
+ "paddw %%xmm0,%%xmm1 \n" // 9 3 3 1 + 8 (2, lo)
+ "psrlw $4,%%xmm1 \n" // ^ div by 16
+ "movdqu %%xmm1,(%1,%4,2) \n"
+
+ "movdqa %%xmm3,%%xmm4 \n"
+ "paddw %%xmm7,%%xmm2 \n" // 3*near+far+8 (1, hi)
+ "paddw %%xmm4,%%xmm4 \n" // 6*near+2*far (2, hi)
+ "paddw %%xmm4,%%xmm3 \n" // 9*near+3*far (2, hi)
+ "paddw %%xmm2,%%xmm3 \n" // 9 3 3 1 + 8 (2, hi)
+ "psrlw $4,%%xmm3 \n" // ^ div by 16
+ "movdqu %%xmm3,0x10(%1,%4,2) \n"
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kLinearShuffleFar) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_SSE2
+void ScaleRowUp2_Linear_16_SSE2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "pxor %%xmm5,%%xmm5 \n"
+ "pcmpeqd %%xmm4,%%xmm4 \n"
+ "psrld $31,%%xmm4 \n"
+ "pslld $1,%%xmm4 \n" // all 2
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 0123 (16b)
+ "movq 2(%0),%%xmm1 \n" // 1234 (16b)
+
+ "punpcklwd %%xmm5,%%xmm0 \n" // 0123 (32b)
+ "punpcklwd %%xmm5,%%xmm1 \n" // 1234 (32b)
+
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+
+ "pshufd $0b10110001,%%xmm2,%%xmm2 \n" // 1032 (even, far)
+ "pshufd $0b10110001,%%xmm3,%%xmm3 \n" // 2143 (odd, far)
+
+ "paddd %%xmm4,%%xmm2 \n" // far+2 (lo)
+ "paddd %%xmm4,%%xmm3 \n" // far+2 (hi)
+ "paddd %%xmm0,%%xmm2 \n" // near+far+2 (lo)
+ "paddd %%xmm1,%%xmm3 \n" // near+far+2 (hi)
+ "paddd %%xmm0,%%xmm0 \n" // 2*near (lo)
+ "paddd %%xmm1,%%xmm1 \n" // 2*near (hi)
+ "paddd %%xmm2,%%xmm0 \n" // 3*near+far+2 (lo)
+ "paddd %%xmm3,%%xmm1 \n" // 3*near+far+2 (hi)
+
+ "psrld $2,%%xmm0 \n" // 3/4*near+1/4*far (lo)
+ "psrld $2,%%xmm1 \n" // 3/4*near+1/4*far (hi)
+ "pshuflw $0x88,%%xmm0,%%xmm0 \n"
+ "pshufhw $0x88,%%xmm0,%%xmm0 \n"
+ "pshufd $0x88,%%xmm0,%%xmm0 \n"
+ "pshuflw $0x88,%%xmm1,%%xmm1 \n"
+ "pshufhw $0x88,%%xmm1,%%xmm1 \n"
+ "pshufd $0x88,%%xmm1,%%xmm1 \n"
+ "punpckldq %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 4 pixel to 8 pixel
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_SSE2
+void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "pxor %%xmm7,%%xmm7 \n"
+ "pcmpeqd %%xmm6,%%xmm6 \n"
+ "psrld $31,%%xmm6 \n"
+ "pslld $3,%%xmm6 \n" // all 8
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 0123 (16b)
+ "movq 2(%0),%%xmm1 \n" // 1234 (16b)
+ "punpcklwd %%xmm7,%%xmm0 \n" // 0123 (32b)
+ "punpcklwd %%xmm7,%%xmm1 \n" // 1234 (32b)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "pshufd $0b10110001,%%xmm2,%%xmm2 \n" // 1032 (even, far)
+ "pshufd $0b10110001,%%xmm3,%%xmm3 \n" // 2143 (odd, far)
+ "paddd %%xmm0,%%xmm2 \n" // near+far (lo)
+ "paddd %%xmm1,%%xmm3 \n" // near+far (hi)
+ "paddd %%xmm0,%%xmm0 \n" // 2*near (lo)
+ "paddd %%xmm1,%%xmm1 \n" // 2*near (hi)
+ "paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo)
+ "paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi)
+
+ "movq (%0,%3,2),%%xmm2 \n"
+ "movq 2(%0,%3,2),%%xmm3 \n"
+ "punpcklwd %%xmm7,%%xmm2 \n" // 0123 (32b)
+ "punpcklwd %%xmm7,%%xmm3 \n" // 1234 (32b)
+ "movdqa %%xmm2,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm5 \n"
+ "pshufd $0b10110001,%%xmm4,%%xmm4 \n" // 1032 (even, far)
+ "pshufd $0b10110001,%%xmm5,%%xmm5 \n" // 2143 (odd, far)
+ "paddd %%xmm2,%%xmm4 \n" // near+far (lo)
+ "paddd %%xmm3,%%xmm5 \n" // near+far (hi)
+ "paddd %%xmm2,%%xmm2 \n" // 2*near (lo)
+ "paddd %%xmm3,%%xmm3 \n" // 2*near (hi)
+ "paddd %%xmm4,%%xmm2 \n" // 3*near+far (2, lo)
+ "paddd %%xmm5,%%xmm3 \n" // 3*near+far (2, hi)
+
+ "movdqa %%xmm0,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "paddd %%xmm0,%%xmm4 \n" // 6*near+2*far (1, lo)
+ "paddd %%xmm6,%%xmm5 \n" // 3*near+far+8 (2, lo)
+ "paddd %%xmm0,%%xmm4 \n" // 9*near+3*far (1, lo)
+ "paddd %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrld $4,%%xmm4 \n" // ^ div by 16 (1, lo)
+
+ "movdqa %%xmm2,%%xmm5 \n"
+ "paddd %%xmm2,%%xmm5 \n" // 6*near+2*far (2, lo)
+ "paddd %%xmm6,%%xmm0 \n" // 3*near+far+8 (1, lo)
+ "paddd %%xmm2,%%xmm5 \n" // 9*near+3*far (2, lo)
+ "paddd %%xmm0,%%xmm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "psrld $4,%%xmm5 \n" // ^ div by 16 (2, lo)
+
+ "movdqa %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "paddd %%xmm1,%%xmm0 \n" // 6*near+2*far (1, hi)
+ "paddd %%xmm6,%%xmm2 \n" // 3*near+far+8 (2, hi)
+ "paddd %%xmm1,%%xmm0 \n" // 9*near+3*far (1, hi)
+ "paddd %%xmm2,%%xmm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrld $4,%%xmm0 \n" // ^ div by 16 (1, hi)
+
+ "movdqa %%xmm3,%%xmm2 \n"
+ "paddd %%xmm3,%%xmm2 \n" // 6*near+2*far (2, hi)
+ "paddd %%xmm6,%%xmm1 \n" // 3*near+far+8 (1, hi)
+ "paddd %%xmm3,%%xmm2 \n" // 9*near+3*far (2, hi)
+ "paddd %%xmm1,%%xmm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "psrld $4,%%xmm2 \n" // ^ div by 16 (2, hi)
+
+ "pshuflw $0x88,%%xmm4,%%xmm4 \n"
+ "pshufhw $0x88,%%xmm4,%%xmm4 \n"
+ "pshufd $0x88,%%xmm4,%%xmm4 \n"
+ "pshuflw $0x88,%%xmm0,%%xmm0 \n"
+ "pshufhw $0x88,%%xmm0,%%xmm0 \n"
+ "pshufd $0x88,%%xmm0,%%xmm0 \n"
+ "punpckldq %%xmm0,%%xmm4 \n"
+ "movdqu %%xmm4,(%1) \n" // store above
+ "pshuflw $0x88,%%xmm5,%%xmm5 \n"
+ "pshufhw $0x88,%%xmm5,%%xmm5 \n"
+ "pshufd $0x88,%%xmm5,%%xmm5 \n"
+ "pshuflw $0x88,%%xmm2,%%xmm2 \n"
+ "pshufhw $0x88,%%xmm2,%%xmm2 \n"
+ "pshufd $0x88,%%xmm2,%%xmm2 \n"
+ "punpckldq %%xmm2,%%xmm5 \n"
+ "movdqu %%xmm5,(%1,%4,2) \n" // store below
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 4 pixel to 8 pixel
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_SSSE3
+void ScaleRowUp2_Linear_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "pcmpeqw %%xmm4,%%xmm4 \n"
+ "psrlw $15,%%xmm4 \n"
+ "psllw $1,%%xmm4 \n" // all 2
+ "movdqa %3,%%xmm3 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 01234567
+ "movq 1(%0),%%xmm1 \n" // 12345678
+ "punpcklwd %%xmm0,%%xmm0 \n" // 0101232345456767
+ "punpcklwd %%xmm1,%%xmm1 \n" // 1212343456567878
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhdq %%xmm1,%%xmm2 \n" // 4545565667677878
+ "punpckldq %%xmm1,%%xmm0 \n" // 0101121223233434
+ "pmaddubsw %%xmm3,%%xmm2 \n" // 3*near+far (hi)
+ "pmaddubsw %%xmm3,%%xmm0 \n" // 3*near+far (lo)
+ "paddw %%xmm4,%%xmm0 \n" // 3*near+far+2 (lo)
+ "paddw %%xmm4,%%xmm2 \n" // 3*near+far+2 (hi)
+ "psrlw $2,%%xmm0 \n" // 3/4*near+1/4*far (lo)
+ "psrlw $2,%%xmm2 \n" // 3/4*near+1/4*far (hi)
+ "packuswb %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kLinearMadd31) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_SSSE3
+void ScaleRowUp2_Bilinear_SSSE3(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "pcmpeqw %%xmm6,%%xmm6 \n"
+ "psrlw $15,%%xmm6 \n"
+ "psllw $3,%%xmm6 \n" // all 8
+ "movdqa %5,%%xmm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 01234567
+ "movq 1(%0),%%xmm1 \n" // 12345678
+ "punpcklwd %%xmm0,%%xmm0 \n" // 0101232345456767
+ "punpcklwd %%xmm1,%%xmm1 \n" // 1212343456567878
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhdq %%xmm1,%%xmm2 \n" // 4545565667677878
+ "punpckldq %%xmm1,%%xmm0 \n" // 0101121223233434
+ "pmaddubsw %%xmm7,%%xmm2 \n" // 3*near+far (1, hi)
+ "pmaddubsw %%xmm7,%%xmm0 \n" // 3*near+far (1, lo)
+
+ "movq (%0,%3),%%xmm1 \n"
+ "movq 1(%0,%3),%%xmm4 \n"
+ "punpcklwd %%xmm1,%%xmm1 \n"
+ "punpcklwd %%xmm4,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "punpckhdq %%xmm4,%%xmm3 \n"
+ "punpckldq %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm7,%%xmm3 \n" // 3*near+far (2, hi)
+ "pmaddubsw %%xmm7,%%xmm1 \n" // 3*near+far (2, lo)
+
+ // xmm0 xmm2
+ // xmm1 xmm3
+
+ "movdqa %%xmm0,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm5 \n"
+ "paddw %%xmm0,%%xmm4 \n" // 6*near+2*far (1, lo)
+ "paddw %%xmm6,%%xmm5 \n" // 3*near+far+8 (2, lo)
+ "paddw %%xmm0,%%xmm4 \n" // 9*near+3*far (1, lo)
+ "paddw %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrlw $4,%%xmm4 \n" // ^ div by 16 (1, lo)
+
+ "movdqa %%xmm1,%%xmm5 \n"
+ "paddw %%xmm1,%%xmm5 \n" // 6*near+2*far (2, lo)
+ "paddw %%xmm6,%%xmm0 \n" // 3*near+far+8 (1, lo)
+ "paddw %%xmm1,%%xmm5 \n" // 9*near+3*far (2, lo)
+ "paddw %%xmm0,%%xmm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "psrlw $4,%%xmm5 \n" // ^ div by 16 (2, lo)
+
+ "movdqa %%xmm2,%%xmm0 \n"
+ "movdqa %%xmm3,%%xmm1 \n"
+ "paddw %%xmm2,%%xmm0 \n" // 6*near+2*far (1, hi)
+ "paddw %%xmm6,%%xmm1 \n" // 3*near+far+8 (2, hi)
+ "paddw %%xmm2,%%xmm0 \n" // 9*near+3*far (1, hi)
+ "paddw %%xmm1,%%xmm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrlw $4,%%xmm0 \n" // ^ div by 16 (1, hi)
+
+ "movdqa %%xmm3,%%xmm1 \n"
+ "paddw %%xmm3,%%xmm1 \n" // 6*near+2*far (2, hi)
+ "paddw %%xmm6,%%xmm2 \n" // 3*near+far+8 (1, hi)
+ "paddw %%xmm3,%%xmm1 \n" // 9*near+3*far (2, hi)
+ "paddw %%xmm2,%%xmm1 \n" // 9 3 3 1 + 8 (2, hi)
+ "psrlw $4,%%xmm1 \n" // ^ div by 16 (2, hi)
+
+ "packuswb %%xmm0,%%xmm4 \n"
+ "movdqu %%xmm4,(%1) \n" // store above
+ "packuswb %%xmm1,%%xmm5 \n"
+ "movdqu %%xmm5,(%1,%4) \n" // store below
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kLinearMadd31) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_AVX2
+void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrlw $15,%%ymm4,%%ymm4 \n"
+ "vpsllw $1,%%ymm4,%%ymm4 \n" // all 2
+ "vbroadcastf128 %3,%%ymm3 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // 0123456789ABCDEF
+ "vmovdqu 1(%0),%%xmm1 \n" // 123456789ABCDEF0
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n"
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n"
+ "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm1 \n" // 3*near+far (hi)
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" // 3*near+far (lo)
+ "vpaddw %%ymm4,%%ymm0,%%ymm0 \n" // 3*near+far+2 (lo)
+ "vpaddw %%ymm4,%%ymm1,%%ymm1 \n" // 3*near+far+2 (hi)
+ "vpsrlw $2,%%ymm0,%%ymm0 \n" // 3/4*near+1/4*far (lo)
+ "vpsrlw $2,%%ymm1,%%ymm1 \n" // 3/4*near+1/4*far (hi)
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 16 sample to 32 sample
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kLinearMadd31) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2
+void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqw %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $15,%%ymm6,%%ymm6 \n"
+ "vpsllw $3,%%ymm6,%%ymm6 \n" // all 8
+ "vbroadcastf128 %5,%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // 0123456789ABCDEF
+ "vmovdqu 1(%0),%%xmm1 \n" // 123456789ABCDEF0
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n"
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n"
+ "vpunpcklwd %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpunpcklwd %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpunpckhdq %%ymm1,%%ymm0,%%ymm2 \n"
+ "vpunpckldq %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm1 \n" // 3*near+far (1, hi)
+ "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n" // 3*near+far (1, lo)
+
+ "vmovdqu (%0,%3),%%xmm2 \n" // 0123456789ABCDEF
+ "vmovdqu 1(%0,%3),%%xmm3 \n" // 123456789ABCDEF0
+ "vpermq $0b11011000,%%ymm2,%%ymm2 \n"
+ "vpermq $0b11011000,%%ymm3,%%ymm3 \n"
+ "vpunpcklwd %%ymm2,%%ymm2,%%ymm2 \n"
+ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpunpckhdq %%ymm3,%%ymm2,%%ymm4 \n"
+ "vpunpckldq %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm7,%%ymm4,%%ymm3 \n" // 3*near+far (2, hi)
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" // 3*near+far (2, lo)
+
+ // ymm0 ymm1
+ // ymm2 ymm3
+
+ "vpaddw %%ymm0,%%ymm0,%%ymm4 \n" // 6*near+2*far (1, lo)
+ "vpaddw %%ymm6,%%ymm2,%%ymm5 \n" // 3*near+far+8 (2, lo)
+ "vpaddw %%ymm4,%%ymm0,%%ymm4 \n" // 9*near+3*far (1, lo)
+ "vpaddw %%ymm4,%%ymm5,%%ymm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" // ^ div by 16 (1, lo)
+
+ "vpaddw %%ymm2,%%ymm2,%%ymm5 \n" // 6*near+2*far (2, lo)
+ "vpaddw %%ymm6,%%ymm0,%%ymm0 \n" // 3*near+far+8 (1, lo)
+ "vpaddw %%ymm5,%%ymm2,%%ymm5 \n" // 9*near+3*far (2, lo)
+ "vpaddw %%ymm5,%%ymm0,%%ymm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "vpsrlw $4,%%ymm5,%%ymm5 \n" // ^ div by 16 (2, lo)
+
+ "vpaddw %%ymm1,%%ymm1,%%ymm0 \n" // 6*near+2*far (1, hi)
+ "vpaddw %%ymm6,%%ymm3,%%ymm2 \n" // 3*near+far+8 (2, hi)
+ "vpaddw %%ymm0,%%ymm1,%%ymm0 \n" // 9*near+3*far (1, hi)
+ "vpaddw %%ymm0,%%ymm2,%%ymm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "vpsrlw $4,%%ymm0,%%ymm0 \n" // ^ div by 16 (1, hi)
+
+ "vpaddw %%ymm3,%%ymm3,%%ymm2 \n" // 6*near+2*far (2, hi)
+ "vpaddw %%ymm6,%%ymm1,%%ymm1 \n" // 3*near+far+8 (1, hi)
+ "vpaddw %%ymm2,%%ymm3,%%ymm2 \n" // 9*near+3*far (2, hi)
+ "vpaddw %%ymm2,%%ymm1,%%ymm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "vpsrlw $4,%%ymm2,%%ymm2 \n" // ^ div by 16 (2, hi)
+
+ "vpackuswb %%ymm0,%%ymm4,%%ymm4 \n"
+ "vmovdqu %%ymm4,(%1) \n" // store above
+ "vpackuswb %%ymm2,%%ymm5,%%ymm5 \n"
+ "vmovdqu %%ymm5,(%1,%4) \n" // store below
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 16 sample to 32 sample
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kLinearMadd31) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2
+void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vbroadcastf128 %3,%%ymm5 \n"
+ "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrlw $15,%%ymm4,%%ymm4 \n"
+ "vpsllw $1,%%ymm4,%%ymm4 \n" // all 2
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // 0123456789ABCDEF (16b)
+ "vmovdqu 2(%0),%%ymm1 \n" // 123456789ABCDEF0 (16b)
+
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n" // 012389AB4567CDEF
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n" // 12349ABC5678DEF0
+
+ "vpunpckhwd %%ymm1,%%ymm0,%%ymm2 \n" // 899AABBCCDDEEFF0 (near)
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n" // 0112233445566778 (near)
+ "vpshufb %%ymm5,%%ymm2,%%ymm3 \n" // 98A9BACBDCEDFE0F (far)
+ "vpshufb %%ymm5,%%ymm0,%%ymm1 \n" // 1021324354657687 (far)
+
+ "vpaddw %%ymm4,%%ymm1,%%ymm1 \n" // far+2
+ "vpaddw %%ymm4,%%ymm3,%%ymm3 \n" // far+2
+ "vpaddw %%ymm0,%%ymm1,%%ymm1 \n" // near+far+2
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" // near+far+2
+ "vpaddw %%ymm0,%%ymm0,%%ymm0 \n" // 2*near
+ "vpaddw %%ymm2,%%ymm2,%%ymm2 \n" // 2*near
+ "vpaddw %%ymm0,%%ymm1,%%ymm0 \n" // 3*near+far+2
+ "vpaddw %%ymm2,%%ymm3,%%ymm2 \n" // 3*near+far+2
+
+ "vpsrlw $2,%%ymm0,%%ymm0 \n" // 3/4*near+1/4*far
+ "vpsrlw $2,%%ymm2,%%ymm2 \n" // 3/4*near+1/4*far
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm2,32(%1) \n"
+
+ "lea 0x20(%0),%0 \n"
+ "lea 0x40(%1),%1 \n" // 16 sample to 32 sample
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kLinearShuffleFar) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2
+void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "vbroadcastf128 %5,%%ymm5 \n"
+ "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrlw $15,%%ymm4,%%ymm4 \n"
+ "vpsllw $3,%%ymm4,%%ymm4 \n" // all 8
+
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0),%%xmm0 \n" // 01234567 (16b)
+ "vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b)
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n" // 0123000045670000
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n" // 1234000056780000
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n" // 0112233445566778 (near)
+ "vpshufb %%ymm5,%%ymm0,%%ymm1 \n" // 1021324354657687 (far)
+ "vpaddw %%ymm0,%%ymm1,%%ymm1 \n" // near+far
+ "vpaddw %%ymm0,%%ymm0,%%ymm0 \n" // 2*near
+ "vpaddw %%ymm0,%%ymm1,%%ymm2 \n" // 3*near+far (1)
+
+ "vmovdqu (%0,%3,2),%%xmm0 \n" // 01234567 (16b)
+ "vmovdqu 2(%0,%3,2),%%xmm1 \n" // 12345678 (16b)
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n" // 0123000045670000
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n" // 1234000056780000
+ "vpunpcklwd %%ymm1,%%ymm0,%%ymm0 \n" // 0112233445566778 (near)
+ "vpshufb %%ymm5,%%ymm0,%%ymm1 \n" // 1021324354657687 (far)
+ "vpaddw %%ymm0,%%ymm1,%%ymm1 \n" // near+far
+ "vpaddw %%ymm0,%%ymm0,%%ymm0 \n" // 2*near
+ "vpaddw %%ymm0,%%ymm1,%%ymm3 \n" // 3*near+far (2)
+
+ "vpaddw %%ymm2,%%ymm2,%%ymm0 \n" // 6*near+2*far (1)
+ "vpaddw %%ymm4,%%ymm3,%%ymm1 \n" // 3*near+far+8 (2)
+ "vpaddw %%ymm0,%%ymm2,%%ymm0 \n" // 9*near+3*far (1)
+ "vpaddw %%ymm0,%%ymm1,%%ymm0 \n" // 9 3 3 1 + 8 (1)
+ "vpsrlw $4,%%ymm0,%%ymm0 \n" // ^ div by 16
+ "vmovdqu %%ymm0,(%1) \n" // store above
+
+ "vpaddw %%ymm3,%%ymm3,%%ymm0 \n" // 6*near+2*far (2)
+ "vpaddw %%ymm4,%%ymm2,%%ymm1 \n" // 3*near+far+8 (1)
+ "vpaddw %%ymm0,%%ymm3,%%ymm0 \n" // 9*near+3*far (2)
+ "vpaddw %%ymm0,%%ymm1,%%ymm0 \n" // 9 3 3 1 + 8 (2)
+ "vpsrlw $4,%%ymm0,%%ymm0 \n" // ^ div by 16
+ "vmovdqu %%ymm0,(%1,%4,2) \n" // store below
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 sample to 16 sample
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kLinearShuffleFar) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2
+void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $31,%%ymm4,%%ymm4 \n"
+ "vpslld $1,%%ymm4,%%ymm4 \n" // all 2
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // 01234567 (16b, 1u1v)
+ "vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b, 1u1v)
+
+ "vpmovzxwd %%xmm0,%%ymm0 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm1,%%ymm1 \n" // 12345678 (32b, 1u1v)
+
+ "vpshufd $0b10110001,%%ymm0,%%ymm2 \n" // 10325476 (lo, far)
+ "vpshufd $0b10110001,%%ymm1,%%ymm3 \n" // 21436587 (hi, far)
+
+ "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" // far+2 (lo)
+ "vpaddd %%ymm4,%%ymm3,%%ymm3 \n" // far+2 (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" // near+far+2 (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm3 \n" // near+far+2 (hi)
+ "vpaddd %%ymm0,%%ymm0,%%ymm0 \n" // 2*near (lo)
+ "vpaddd %%ymm1,%%ymm1,%%ymm1 \n" // 2*near (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 3*near+far+2 (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm1 \n" // 3*near+far+2 (hi)
+
+ "vpsrld $2,%%ymm0,%%ymm0 \n" // 3/4*near+1/4*far (lo)
+ "vpsrld $2,%%ymm1,%%ymm1 \n" // 3/4*near+1/4*far (hi)
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpshufd $0b11011000,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 pixel to 16 pixel
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2
+void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $31,%%ymm6,%%ymm6 \n"
+ "vpslld $3,%%ymm6,%%ymm6 \n" // all 8
+
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0),%%xmm0 \n" // 01234567 (16b, 1u1v)
+ "vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b, 1u1v)
+ "vpmovzxwd %%xmm0,%%ymm0 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm1,%%ymm1 \n" // 12345678 (32b, 1u1v)
+ "vpshufd $0b10110001,%%ymm0,%%ymm2 \n" // 10325476 (lo, far)
+ "vpshufd $0b10110001,%%ymm1,%%ymm3 \n" // 21436587 (hi, far)
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" // near+far (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm3 \n" // near+far (hi)
+ "vpaddd %%ymm0,%%ymm0,%%ymm0 \n" // 2*near (lo)
+ "vpaddd %%ymm1,%%ymm1,%%ymm1 \n" // 2*near (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 3*near+far (1, lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm1 \n" // 3*near+far (1, hi)
+
+ "vmovdqu (%0,%3,2),%%xmm2 \n" // 01234567 (16b, 1u1v)
+ "vmovdqu 2(%0,%3,2),%%xmm3 \n" // 12345678 (16b, 1u1v)
+ "vpmovzxwd %%xmm2,%%ymm2 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm3,%%ymm3 \n" // 12345678 (32b, 1u1v)
+ "vpshufd $0b10110001,%%ymm2,%%ymm4 \n" // 10325476 (lo, far)
+ "vpshufd $0b10110001,%%ymm3,%%ymm5 \n" // 21436587 (hi, far)
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n" // near+far (lo)
+ "vpaddd %%ymm3,%%ymm5,%%ymm5 \n" // near+far (hi)
+ "vpaddd %%ymm2,%%ymm2,%%ymm2 \n" // 2*near (lo)
+ "vpaddd %%ymm3,%%ymm3,%%ymm3 \n" // 2*near (hi)
+ "vpaddd %%ymm2,%%ymm4,%%ymm2 \n" // 3*near+far (2, lo)
+ "vpaddd %%ymm3,%%ymm5,%%ymm3 \n" // 3*near+far (2, hi)
+
+ "vpaddd %%ymm0,%%ymm0,%%ymm4 \n" // 6*near+2*far (1, lo)
+ "vpaddd %%ymm6,%%ymm2,%%ymm5 \n" // 3*near+far+8 (2, lo)
+ "vpaddd %%ymm4,%%ymm0,%%ymm4 \n" // 9*near+3*far (1, lo)
+ "vpaddd %%ymm4,%%ymm5,%%ymm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "vpsrld $4,%%ymm4,%%ymm4 \n" // ^ div by 16 (1, lo)
+
+ "vpaddd %%ymm2,%%ymm2,%%ymm5 \n" // 6*near+2*far (2, lo)
+ "vpaddd %%ymm6,%%ymm0,%%ymm0 \n" // 3*near+far+8 (1, lo)
+ "vpaddd %%ymm5,%%ymm2,%%ymm5 \n" // 9*near+3*far (2, lo)
+ "vpaddd %%ymm5,%%ymm0,%%ymm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "vpsrld $4,%%ymm5,%%ymm5 \n" // ^ div by 16 (2, lo)
+
+ "vpaddd %%ymm1,%%ymm1,%%ymm0 \n" // 6*near+2*far (1, hi)
+ "vpaddd %%ymm6,%%ymm3,%%ymm2 \n" // 3*near+far+8 (2, hi)
+ "vpaddd %%ymm0,%%ymm1,%%ymm0 \n" // 9*near+3*far (1, hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "vpsrld $4,%%ymm0,%%ymm0 \n" // ^ div by 16 (1, hi)
+
+ "vpaddd %%ymm3,%%ymm3,%%ymm2 \n" // 6*near+2*far (2, hi)
+ "vpaddd %%ymm6,%%ymm1,%%ymm1 \n" // 3*near+far+8 (1, hi)
+ "vpaddd %%ymm2,%%ymm3,%%ymm2 \n" // 9*near+3*far (2, hi)
+ "vpaddd %%ymm2,%%ymm1,%%ymm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "vpsrld $4,%%ymm2,%%ymm2 \n" // ^ div by 16 (2, hi)
+
+ "vpackusdw %%ymm0,%%ymm4,%%ymm4 \n"
+ "vpshufd $0b11011000,%%ymm4,%%ymm4 \n"
+ "vmovdqu %%ymm4,(%1) \n" // store above
+ "vpackusdw %%ymm2,%%ymm5,%%ymm5 \n"
+ "vpshufd $0b11011000,%%ymm5,%%ymm5 \n"
+ "vmovdqu %%ymm5,(%1,%4,2) \n" // store below
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 pixel to 16 pixel
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif
+
+// Reads 16xN bytes and produces 16 shorts at a time.
+void ScaleAddRow_SSE2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ asm volatile("pxor %%xmm5,%%xmm5 \n"
+
+ // 16 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm3 \n"
+ "lea 0x10(%0),%0 \n" // src_ptr += 16
+ "movdqu (%1),%%xmm0 \n"
+ "movdqu 0x10(%1),%%xmm1 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "punpcklbw %%xmm5,%%xmm2 \n"
+ "punpckhbw %%xmm5,%%xmm3 \n"
+ "paddusw %%xmm2,%%xmm0 \n"
+ "paddusw %%xmm3,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(src_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+}
+
+#ifdef HAS_SCALEADDROW_AVX2
+// Reads 32 bytes and accumulates to 32 shorts at a time.
+void ScaleAddRow_AVX2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm3 \n"
+ "lea 0x20(%0),%0 \n" // src_ptr += 32
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+ "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
+ "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpaddusw (%1),%%ymm2,%%ymm0 \n"
+ "vpaddusw 0x20(%1),%%ymm3,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(src_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+}
+#endif // HAS_SCALEADDROW_AVX2
+
+#ifdef HAS_SCALEADDROW_AVX512BW
+// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail.
+LIBYUV_TARGET_AVX512BW
+void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ uintptr_t temp;
+ asm volatile(
+ "sub $0x40,%2 \n"
+ "jl 2f \n"
+
+ // 64 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "vpmovzxbw 0x20(%0),%%zmm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpaddusw (%1),%%zmm0,%%zmm0 \n"
+ "vpaddusw 0x40(%1),%%zmm1,%%zmm1 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "vmovdqu16 %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x40,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jl 3f \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpaddusw (%1),%%zmm0,%%zmm0 \n"
+ "vmovdqu16 %%zmm0,(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+
+ "3: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "mov $-1,%k3 \n"
+ "bzhi %k2,%k3,%k3 \n"
+ "kmovd %k3,%%k1 \n"
+ "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n"
+ "vpaddusw (%1),%%zmm0,%%zmm0%{%%k1%}%{z%}\n"
+ "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(src_width), // %2
+ "=&r"(temp) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "k1");
+}
+#endif // HAS_SCALEADDROW_AVX512BW
+
+#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW)
+static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15};
+#endif
+
+#ifdef HAS_SCALEADDCOLS1_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleAddCols1_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr) {
+ int boxwidth = dx >> 16;
+ if (boxwidth < 1) {
+ boxwidth = 1;
+ }
+ int scaleval = 65536 / (boxwidth * boxheight);
+ src_ptr += (x >> 16);
+ intptr_t temp, mask;
+ asm volatile(
+ "mov $0x0000ffff,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm6 \n"
+ "vpbroadcastd %k[scaleval],%%zmm7 \n"
+ "cmp $4,%k[bw] \n"
+ "jg 20f \n"
+
+ // Path 1: 1 <= boxwidth <= 4 (16 output pixels per loop).
+ "vpmovzxbd %[kSeq],%%zmm3 \n"
+ "vpslld $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpbroadcastw %k[bw],%%zmm2 \n"
+ "vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
+ "mov $0x00010000,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm0 \n"
+ "mov $0x00030002,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm1 \n"
+ "vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm0,%%zmm2,%%k2 \n"
+ "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
+ "mov %k[bw],%k[temp] \n"
+ "shl $4,%k[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k4 \n"
+ "shl $1,%q[temp] \n"
+ "sub $0x10,%[dst_width] \n"
+ "jl 12f \n"
+
+ LABELALIGN
+ "11: \n"
+ "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
+ "add %q[temp],%[src_ptr] \n"
+ "vmovdqa64 %%zmm4,%%zmm2 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
+ "vmovdqa64 %%zmm5,%%zmm3 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm2,%%zmm0 \n"
+ "vpandd %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpmovdb %%zmm2,(%[dst_ptr]) \n"
+ "lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x10,%[dst_width] \n"
+ "jge 11b \n"
+
+ "12: \n"
+ "add $0x10,%[dst_width] \n"
+ "jle 99f \n"
+ "mov %k[dst_width],%k[temp] \n"
+ "imul %k[bw],%k[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k4 \n"
+ "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
+ "vmovdqa64 %%zmm4,%%zmm2 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
+ "vmovdqa64 %%zmm5,%%zmm3 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm2,%%zmm0 \n"
+ "vpandd %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[dst_width],%q[mask],%q[mask]\n"
+ "kmovw %k[mask],%%k1 \n"
+ "vpmovdb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Path 2: 5 <= boxwidth <= 8 (8 output pixels per loop).
+ "20: \n"
+ "cmp $8,%k[bw] \n"
+ "jg 30f \n"
+ "vpmovzxbq %[kSeq],%%zmm3 \n"
+ "vpsllq $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpsllq $32,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpbroadcastw %k[bw],%%zmm2 \n"
+ "vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
+ "movabs $0x0003000200010000,%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm0 \n"
+ "movabs $0x0007000600050004,%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm1 \n"
+ "vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
+ "mov %k[bw],%k[temp] \n"
+ "shl $3,%k[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k4 \n"
+ "shl $1,%q[temp] \n"
+ "sub $0x8,%[dst_width] \n"
+ "jl 22f \n"
+
+ LABELALIGN
+ "21: \n"
+ "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
+ "add %q[temp],%[src_ptr] \n"
+ "vmovdqa64 %%zmm4,%%zmm2 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
+ "vmovdqa64 %%zmm5,%%zmm3 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm2,%%zmm0 \n"
+ "vpandd %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlq $32,%%zmm2,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpmovqb %%zmm2,(%[dst_ptr]) \n"
+ "lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x8,%[dst_width] \n"
+ "jge 21b \n"
+
+ "22: \n"
+ "add $0x8,%[dst_width] \n"
+ "jle 99f \n"
+ "mov %k[dst_width],%k[temp] \n"
+ "imul %k[bw],%k[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k4 \n"
+ "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
+ "vmovdqa64 %%zmm4,%%zmm2 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
+ "vmovdqa64 %%zmm5,%%zmm3 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm2,%%zmm0 \n"
+ "vpandd %%zmm6,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
+ "vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpsrlq $32,%%zmm2,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[dst_width],%q[mask],%q[mask]\n"
+ "kmovw %k[mask],%%k1 \n"
+ "vpmovqb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Path 3: boxwidth >= 9 (ZMM 32-word reduction per output pixel).
+ "30: \n"
+ "mov %k[bw],%k[temp] \n"
+ "and $0x1f,%k[temp] \n"
+ "mov $-1,%k[mask] \n"
+ "bzhi %k[temp],%k[mask],%k[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+
+ LABELALIGN
+ "31: \n"
+ "vpxord %%zmm2,%%zmm2,%%zmm2 \n"
+ "mov %[src_ptr],%q[mask] \n"
+ "lea (%[src_ptr],%q[bw],2),%[src_ptr] \n"
+ "mov %q[bw],%q[temp] \n"
+ "sub $0x20,%q[temp] \n"
+ "jl 33f \n"
+ "32: \n"
+ "vmovdqu16 (%q[mask]),%%zmm0 \n"
+ "lea 0x40(%q[mask]),%q[mask] \n"
+ "vpsrld $16,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "sub $0x20,%q[temp] \n"
+ "jge 32b \n"
+ "33: \n"
+ "add $0x20,%q[temp] \n"
+ "jle 34f \n"
+ "vmovdqu16 (%q[mask]),%%zmm0%{%%k1%}%{z%}\n"
+ "vpsrld $16,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm6,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "34: \n"
+ "vextracti32x8 $1,%%zmm2,%%ymm0 \n"
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
+ "vextracti128 $1,%%ymm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpshufd $0xee,%%xmm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpshufd $0x55,%%xmm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpmulld %%xmm7,%%xmm2,%%xmm2 \n"
+ "vpsrld $16,%%xmm2,%%xmm2 \n"
+ "vmovd %%xmm2,%k[temp] \n"
+ "mov %b[temp],(%[dst_ptr]) \n"
+ "inc %[dst_ptr] \n"
+ "dec %[dst_width] \n"
+ "jg 31b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [temp] "=&r"(temp), // %[temp]
+ [mask] "=&r"(mask) // %[mask]
+ : [bw] "r"((intptr_t)boxwidth), // %[bw]
+ [scaleval] "r"(scaleval), // %[scaleval]
+ [kSeq] "m"(kSequence0_15) // %[kSeq]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "k1", "k2", "k3", "k4");
+}
+#endif // HAS_SCALEADDCOLS1_AVX512BW
+
+#ifdef HAS_SCALEADDCOLS2_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleAddCols2_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr) {
+ int minboxwidth = dx >> 16;
+ int scaletbl[2];
+ scaletbl[0] = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight);
+ scaletbl[1] =
+ 65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight);
+ intptr_t base_ix, temp, mask;
+ asm volatile(
+ "mov $0x0000ffff,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm20 \n"
+ "vpbroadcastd %[scale0],%%zmm22 \n"
+ "vpbroadcastd %[scale1],%%zmm23 \n"
+ "cmpl $1,%[minbw] \n"
+ "jl 30f \n"
+ "cmpl $3,%[minbw] \n"
+ "jg 20f \n"
+
+ // Path 1: 1 <= minboxwidth <= 3 (16 output pixels per loop).
+ "vpmovzxbd %[kSeq],%%zmm16 \n"
+ "vpbroadcastd %k[dx],%%zmm17 \n"
+ "vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
+ "vpaddd %%zmm17,%%zmm16,%%zmm17 \n"
+ "mov $0x00010000,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm18 \n"
+ "mov $0x00030002,%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm19 \n"
+ "vpbroadcastd %[minbw],%%zmm21 \n"
+ "sub $0x10,%[dst_width] \n"
+ "jl 12f \n"
+
+ LABELALIGN
+ "11: \n"
+ "movslq %k[x],%q[base_ix] \n"
+ "movzwl %w[x],%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm2 \n"
+ "sar $16,%q[base_ix] \n"
+ "lea (%k[x],%k[dx],8),%k[temp] \n"
+ "lea (%k[temp],%k[dx],8),%k[x] \n"
+ "movslq %k[x],%q[temp] \n"
+ "sar $16,%q[temp] \n"
+ "sub %q[base_ix],%q[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k2 \n"
+ "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
+
+ "vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
+ "vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm3 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
+
+ "vpslld $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpslld $16,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
+ "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
+
+ "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm4,%%zmm0 \n"
+ "vpandd %%zmm20,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm5,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
+
+ "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
+ "vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm4,%%zmm4 \n"
+ "vpmovdb %%zmm4,(%[dst_ptr]) \n"
+ "lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x10,%[dst_width] \n"
+ "jge 11b \n"
+
+ "12: \n"
+ "add $0x10,%[dst_width] \n"
+ "jle 99f \n"
+ "movslq %k[x],%q[base_ix] \n"
+ "movzwl %w[x],%k[temp] \n"
+ "vpbroadcastd %k[temp],%%zmm2 \n"
+ "sar $16,%q[base_ix] \n"
+ "mov %k[dst_width],%k[temp] \n"
+ "imul %k[dx],%k[temp] \n"
+ "add %k[x],%k[temp] \n"
+ "sar $16,%k[temp] \n"
+ "movslq %k[temp],%q[temp] \n"
+ "sub %q[base_ix],%q[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k2 \n"
+ "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
+
+ "vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
+ "vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm3 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
+
+ "vpslld $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpslld $16,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
+ "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
+
+ "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm4,%%zmm0 \n"
+ "vpandd %%zmm20,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm5,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
+
+ "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
+ "vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm4,%%zmm4 \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[dst_width],%q[mask],%q[mask]\n"
+ "kmovw %k[mask],%%k1 \n"
+ "vpmovdb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Path 2: 4 <= minboxwidth <= 7 (8 output pixels per loop).
+ "20: \n"
+ "cmpl $7,%[minbw] \n"
+ "jg 30f \n"
+ "vpmovzxbq %[kSeq],%%zmm16 \n"
+ "movslq %k[dx],%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm17 \n"
+ "vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
+ "vpaddq %%zmm17,%%zmm16,%%zmm17 \n"
+ "movabs $0x0003000200010000,%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm18 \n"
+ "movabs $0x0007000600050004,%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm19 \n"
+ "movslq %[minbw],%q[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm21 \n"
+ "sub $0x8,%[dst_width] \n"
+ "jl 22f \n"
+
+ LABELALIGN
+ "21: \n"
+ "movslq %k[x],%q[base_ix] \n"
+ "movzwl %w[x],%k[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm2 \n"
+ "sar $16,%q[base_ix] \n"
+ "lea (%k[x],%k[dx],8),%k[x] \n"
+ "movslq %k[x],%q[temp] \n"
+ "sar $16,%q[temp] \n"
+ "sub %q[base_ix],%q[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k2 \n"
+ "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
+
+ "vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
+ "vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm3 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
+
+ "vpsllq $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpsllq $32,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpsllq $16,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpsllq $32,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
+
+ "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm4,%%zmm0 \n"
+ "vpandd %%zmm20,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm5,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpsrlq $32,%%zmm4,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+
+ "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
+ "vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm4,%%zmm4 \n"
+ "vpmovqb %%zmm4,(%[dst_ptr]) \n"
+ "lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
+ "sub $0x8,%[dst_width] \n"
+ "jge 21b \n"
+
+ "22: \n"
+ "add $0x8,%[dst_width] \n"
+ "jle 99f \n"
+ "movslq %k[x],%q[base_ix] \n"
+ "movzwl %w[x],%k[temp] \n"
+ "vpbroadcastq %q[temp],%%zmm2 \n"
+ "sar $16,%q[base_ix] \n"
+ "mov %k[dst_width],%k[temp] \n"
+ "imul %k[dx],%k[temp] \n"
+ "add %k[x],%k[temp] \n"
+ "sar $16,%k[temp] \n"
+ "movslq %k[temp],%q[temp] \n"
+ "sub %q[base_ix],%q[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "shr $32,%q[mask] \n"
+ "kmovd %k[mask],%%k2 \n"
+ "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
+
+ "vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
+ "vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
+ "vpsrld $16,%%zmm3,%%zmm3 \n"
+ "vpsrld $16,%%zmm2,%%zmm2 \n"
+ "vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
+ "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
+
+ "vpsllq $16,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpsllq $32,%%zmm3,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpsllq $16,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpsllq $32,%%zmm2,%%zmm4 \n"
+ "vpord %%zmm4,%%zmm2,%%zmm2 \n"
+ "vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
+ "vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
+ "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
+
+ "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
+ "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
+ "vpsrld $16,%%zmm4,%%zmm0 \n"
+ "vpandd %%zmm20,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm5,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+ "vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
+ "vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
+ "vpsrlq $32,%%zmm4,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
+
+ "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
+ "vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
+ "vpsrld $16,%%zmm4,%%zmm4 \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[dst_width],%q[mask],%q[mask]\n"
+ "kmovw %k[mask],%%k1 \n"
+ "vpmovqb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
+ "jmp 99f \n"
+
+ // Path 3: minboxwidth >= 8 or minboxwidth < 1.
+ LABELALIGN
+ "30: \n"
+ "movslq %k[x],%q[base_ix] \n"
+ "sar $16,%q[base_ix] \n"
+ "add %k[dx],%k[x] \n"
+ "movslq %k[x],%q[temp] \n"
+ "sar $16,%q[temp] \n"
+ "sub %q[base_ix],%q[temp] \n"
+ "cmp $1,%q[temp] \n"
+ "jge 31f \n"
+ "mov $1,%q[temp] \n"
+ "31: \n"
+ "vmovdqa32 %%xmm22,%%xmm3 \n"
+ "cmp %k[temp],%[minbw] \n"
+ "jge 32f \n"
+ "vmovdqa32 %%xmm23,%%xmm3 \n"
+ "32: \n"
+ "vpxord %%zmm2,%%zmm2,%%zmm2 \n"
+ "lea (%[src_ptr],%q[base_ix],2),%q[base_ix] \n"
+ "sub $0x20,%q[temp] \n"
+ "jl 34f \n"
+ "33: \n"
+ "vmovdqu16 (%q[base_ix]),%%zmm0 \n"
+ "lea 0x40(%q[base_ix]),%q[base_ix] \n"
+ "vpsrld $16,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "sub $0x20,%q[temp] \n"
+ "jge 33b \n"
+ "34: \n"
+ "add $0x20,%q[temp] \n"
+ "jle 35f \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovd %k[mask],%%k1 \n"
+ "vmovdqu16 (%q[base_ix]),%%zmm0%{%%k1%}%{z%} \n"
+ "vpsrld $16,%%zmm0,%%zmm1 \n"
+ "vpandd %%zmm20,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
+ "vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
+ "35: \n"
+ "vextracti32x8 $1,%%zmm2,%%ymm0 \n"
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
+ "vextracti128 $1,%%ymm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpshufd $0xee,%%xmm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpshufd $0x55,%%xmm2,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpmulld %%xmm3,%%xmm2,%%xmm2 \n"
+ "vpsrld $16,%%xmm2,%%xmm2 \n"
+ "vmovd %%xmm2,%k[temp] \n"
+ "mov %b[temp],(%[dst_ptr]) \n"
+ "inc %[dst_ptr] \n"
+ "dec %[dst_width] \n"
+ "jg 30b \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [x] "+r"(x), // %[x]
+ [base_ix] "=&r"(base_ix), // %[base_ix]
+ [temp] "=&r"(temp), // %[temp]
+ [mask] "=&r"(mask) // %[mask]
+ : [dx] "r"(dx), // %[dx]
+ [minbw] "m"(minboxwidth), // %[minbw]
+ [scale0] "m"(scaletbl[0]), // %[scale0]
+ [scale1] "m"(scaletbl[1]), // %[scale1]
+ [kSeq] "m"(kSequence0_15) // %[kSeq]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
+ "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1",
+ "k2", "k3", "k4");
+}
+#endif // HAS_SCALEADDCOLS2_AVX512BW
+
+#ifdef HAS_SCALEFILTERCOLS_AVX2
+static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+
+// Bilinear column filtering. AVX2 version.
+void ScaleFilterCols_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ int n = dst_width & ~7;
+ int r = dst_width & 7;
+ if (n > 0) {
+ SIMD_ALIGNED(int32_t xi_buf[8]);
+ int32_t* xi_ptr = xi_buf;
+ uintptr_t tmp;
+ int loop_n = n;
+ asm volatile(
+ "vpbroadcastd %[x],%%ymm2 \n"
+ "vpbroadcastd %[dx],%%ymm3 \n"
+ "vpmulld %[kStep],%%ymm3,%%ymm0 \n"
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
+ "vpslld $0x3,%%ymm3,%%ymm3 \n"
+ "vbroadcasti128 %[kShufFrac],%%ymm5 \n"
+ "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n"
+ "vpsrlw $0x8,%%xmm6,%%xmm7 \n" // 0x00ff
+ "vpsrlw $0x9,%%xmm6,%%xmm6 \n" // 0x007f
+
+ LABELALIGN
+ "1: \n"
+ "vpsrad $0x10,%%ymm2,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[xi]) \n"
+ "vpsrlw $0x9,%%ymm2,%%ymm1 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpermq $0xd8,%%ymm1,%%ymm1 \n"
+ "vpxor %%xmm6,%%xmm1,%%xmm1 \n"
+#if defined(__x86_64__)
+ "movslq 0x00(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x0,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x04(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x1,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x08(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x2,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x0c(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x3,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x10(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x4,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x14(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x5,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x18(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x6,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+ "movslq 0x1c(%[xi]),%q[tmp] \n"
+ "vpinsrw $0x7,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n"
+#else
+ "mov 0x00(%[xi]),%[tmp] \n"
+ "vpinsrw $0x0,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x04(%[xi]),%[tmp] \n"
+ "vpinsrw $0x1,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x08(%[xi]),%[tmp] \n"
+ "vpinsrw $0x2,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x0c(%[xi]),%[tmp] \n"
+ "vpinsrw $0x3,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x10(%[xi]),%[tmp] \n"
+ "vpinsrw $0x4,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x14(%[xi]),%[tmp] \n"
+ "vpinsrw $0x5,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x18(%[xi]),%[tmp] \n"
+ "vpinsrw $0x6,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+ "mov 0x1c(%[xi]),%[tmp] \n"
+ "vpinsrw $0x7,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n"
+#endif
+ "vpand %%xmm7,%%xmm0,%%xmm4 \n"
+ "vpmaddubsw %%xmm1,%%xmm0,%%xmm0 \n"
+ "vpaddw %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpsubw %%xmm6,%%xmm7,%%xmm4 \n"
+ "vpsrlw $0x1,%%xmm4,%%xmm4 \n" // 0x0040
+ "vpaddw %%xmm4,%%xmm0,%%xmm0 \n"
+ "vpsrlw $0x7,%%xmm0,%%xmm0 \n"
+ "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovq %%xmm0,(%[dst]) \n"
+ "lea 0x8(%[dst]),%[dst] \n"
+ "subl $0x8,%[n] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_ptr), [n] "+rm"(loop_n), [tmp] "=&r"(tmp)
+ : [src] "r"(src_ptr), [xi] "r"(xi_ptr), [x] "m"(x), [dx] "m"(dx),
+ [kStep] "m"(kStep0_7_Filter), [kShufFrac] "m"(kShufFracCols)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ int xi = x >> 16;
+ int a = src_ptr[xi];
+ int b = src_ptr[xi + 1];
+ dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7));
+ x = (int)((uint32_t)x + (uint32_t)dx);
+ }
+}
+#endif // HAS_SCALEFILTERCOLS_AVX2
+
+#ifdef HAS_SCALECOLSUP2_AVX2
+// Reads 16 pixels, duplicates them and writes 32 pixels (plus 16/8 tails).
+void ScaleColsUp2_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ (void)x;
+ (void)dx;
+ asm volatile(
+ "sub $0x20,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%1),%%xmm0 \n"
+ "lea 0x10(%1),%1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x20,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%2 \n"
+ "jle 99f \n"
+ "test $0x10,%2 \n"
+ "je 3f \n"
+ "vmovq (%1),%%xmm0 \n"
+ "lea 0x8(%1),%1 \n"
+ "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovdqu %%xmm0,(%0) \n"
+ "lea 0x10(%0),%0 \n"
+
+ "3: \n"
+ "test $0x8,%2 \n"
+ "je 99f \n"
+ "vmovd (%1),%%xmm0 \n"
+ "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n"
+ "vmovq %%xmm0,(%0) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_SCALECOLSUP2_AVX2
+
+#ifdef HAS_SCALEARGBROWDOWN2_AVX2
+void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovups (%0),%%ymm0 \n"
+ "vmovups 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovups %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+
+void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm0,%%ymm2,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+
+void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ asm volatile(
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
+ "vmovdqu 0x20(%0,%3,1),%%ymm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm3,%%ymm1,%%ymm1 \n"
+ "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm0,%%ymm2,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+}
+#endif // HAS_SCALEARGBROWDOWN2_AVX2
+
+#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2
+static const int32_t kStep0_7_Even[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+
+// Reads 8 pixels at a time.
+void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
int dst_width) {
- intptr_t src_stepx_x4 = (intptr_t)(src_stepx);
- intptr_t src_stepx_x12;
+ ptrdiff_t src_stepx_x32 = (ptrdiff_t)src_stepx * 32;
(void)src_stride;
asm volatile(
- "lea 0x00(,%1,4),%1 \n"
- "lea 0x00(%1,%1,2),%4 \n"
+ "vpbroadcastd %[stepx],%%ymm2 \n"
+ "vpmulld %[kStep],%%ymm2,%%ymm2 \n"
LABELALIGN
- "1: \n"
- "movd (%0),%%xmm0 \n"
- "movd 0x00(%0,%1,1),%%xmm1 \n"
- "punpckldq %%xmm1,%%xmm0 \n"
- "movd 0x00(%0,%1,2),%%xmm2 \n"
- "movd 0x00(%0,%4,1),%%xmm3 \n"
- "lea 0x00(%0,%1,4),%0 \n"
- "punpckldq %%xmm3,%%xmm2 \n"
- "punpcklqdq %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_stepx_x4), // %1
- "+r"(dst_argb), // %2
- "+r"(dst_width), // %3
- "=&r"(src_stepx_x12) // %4
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3");
-}
-
-// Blends four 2x2 to 4x1.
-// Alignment requirement: dst_argb 16 byte aligned.
-void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb,
+ "1: \n"
+ "vpcmpeqd %%ymm3,%%ymm3,%%ymm3 \n"
+ "vpgatherdd %%ymm3,(%[src],%%ymm2,4),%%ymm0 \n"
+ "add %[step32],%[src] \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+rm"(dst_width)
+ : [step32] "r"(src_stepx_x32), [stepx] "m"(src_stepx),
+ [kStep] "m"(kStep0_7_Even)
+ : "memory", "cc", "xmm0", "xmm2", "xmm3");
+}
+
+// Blends eight 2x2 to 8x1.
+void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
- intptr_t src_stepx_x4 = (intptr_t)(src_stepx);
- intptr_t src_stepx_x12;
- intptr_t row1 = (intptr_t)(src_stride);
+ const uint8_t* row1 = src_argb + src_stride;
+ ptrdiff_t step4_bytes = (ptrdiff_t)src_stepx * 16;
asm volatile(
- "lea 0x00(,%1,4),%1 \n"
- "lea 0x00(%1,%1,2),%4 \n"
- "lea 0x00(%0,%5,1),%5 \n"
+ "vmovd %[stepx],%%xmm4 \n"
+ "vpshufd $0x0,%%xmm4,%%xmm4 \n"
+ "vpmulld %[kStep],%%xmm4,%%xmm4 \n"
LABELALIGN
- "1: \n"
- "movq (%0),%%xmm0 \n"
- "movhps 0x00(%0,%1,1),%%xmm0 \n"
- "movq 0x00(%0,%1,2),%%xmm1 \n"
- "movhps 0x00(%0,%4,1),%%xmm1 \n"
- "lea 0x00(%0,%1,4),%0 \n"
- "movq (%5),%%xmm2 \n"
- "movhps 0x00(%5,%1,1),%%xmm2 \n"
- "movq 0x00(%5,%1,2),%%xmm3 \n"
- "movhps 0x00(%5,%4,1),%%xmm3 \n"
- "lea 0x00(%5,%1,4),%5 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "pavgb %%xmm3,%%xmm1 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm2 \n"
- "pavgb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_stepx_x4), // %1
- "+r"(dst_argb), // %2
- "+rm"(dst_width), // %3
- "=&r"(src_stepx_x12), // %4
- "+r"(row1) // %5
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3");
-}
-
-void ScaleARGBCols_SSE2(uint8_t* dst_argb,
+ "1: \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm0 \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm2 \n"
+ "add %[step4],%[row0] \n"
+ "add %[step4],%[row1] \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm1 \n"
+ "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n"
+ "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm3 \n"
+ "add %[step4],%[row0] \n"
+ "add %[step4],%[row1] \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm3,%%ymm1,%%ymm1 \n"
+ "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm0,%%ymm2,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [row0] "+r"(src_argb), [row1] "+r"(row1), [dst] "+r"(dst_argb),
+ [width] "+rm"(dst_width)
+ : [step4] "r"(step4_bytes), [stepx] "m"(src_stepx),
+ [kStep] "m"(kStep0_7_Even)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2
+
+#ifdef HAS_SCALEARGBCOLS_AVX2
+static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+
+void ScaleARGBCols_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
int dx) {
- intptr_t x0, x1;
- asm volatile(
- "movd %5,%%xmm2 \n"
- "movd %6,%%xmm3 \n"
- "pshufd $0x0,%%xmm2,%%xmm2 \n"
- "pshufd $0x11,%%xmm3,%%xmm0 \n"
- "paddd %%xmm0,%%xmm2 \n"
- "paddd %%xmm3,%%xmm3 \n"
- "pshufd $0x5,%%xmm3,%%xmm0 \n"
- "paddd %%xmm0,%%xmm2 \n"
- "paddd %%xmm3,%%xmm3 \n"
- "pshufd $0x0,%%xmm3,%%xmm3 \n"
- "pextrw $0x1,%%xmm2,%k0 \n"
- "pextrw $0x3,%%xmm2,%k1 \n"
- "cmp $0x0,%4 \n"
- "jl 99f \n"
- "sub $0x4,%4 \n"
- "jl 49f \n"
+ int n = dst_width & ~7;
+ int r = dst_width & 7;
+ if (n > 0) {
+ int loop_n = n;
+ asm volatile(
+ "vpbroadcastd %[x],%%ymm2 \n"
+ "vpbroadcastd %[dx],%%ymm3 \n"
+ "vpmulld %[kStep],%%ymm3,%%ymm0 \n"
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
+ "vpslld $0x3,%%ymm3,%%ymm3 \n"
- LABELALIGN
- "40: \n"
- "movd 0x00(%3,%0,4),%%xmm0 \n"
- "movd 0x00(%3,%1,4),%%xmm1 \n"
- "pextrw $0x5,%%xmm2,%k0 \n"
- "pextrw $0x7,%%xmm2,%k1 \n"
- "paddd %%xmm3,%%xmm2 \n"
- "punpckldq %%xmm1,%%xmm0 \n"
- "movd 0x00(%3,%0,4),%%xmm1 \n"
- "movd 0x00(%3,%1,4),%%xmm4 \n"
- "pextrw $0x1,%%xmm2,%k0 \n"
- "pextrw $0x3,%%xmm2,%k1 \n"
- "punpckldq %%xmm4,%%xmm1 \n"
- "punpcklqdq %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%4 \n"
- "jge 40b \n"
-
- "49: \n"
- "test $0x2,%4 \n"
- "je 29f \n"
- "movd 0x00(%3,%0,4),%%xmm0 \n"
- "movd 0x00(%3,%1,4),%%xmm1 \n"
- "pextrw $0x5,%%xmm2,%k0 \n"
- "punpckldq %%xmm1,%%xmm0 \n"
- "movq %%xmm0,(%2) \n"
- "lea 0x8(%2),%2 \n"
- "29: \n"
- "test $0x1,%4 \n"
- "je 99f \n"
- "movd 0x00(%3,%0,4),%%xmm0 \n"
- "movd %%xmm0,(%2) \n"
- "99: \n"
- : "=&a"(x0), // %0
- "=&d"(x1), // %1
- "+r"(dst_argb), // %2
- "+r"(src_argb), // %3
- "+r"(dst_width) // %4
- : "rm"(x), // %5
- "rm"(dx) // %6
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+ LABELALIGN
+ "1: \n"
+ "vpsrad $0x10,%%ymm2,%%ymm0 \n"
+ "vpaddd %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpgatherdd %%ymm4,(%[src],%%ymm0,4),%%ymm1 \n"
+ "vmovdqu %%ymm1,(%[dst]) \n"
+ "lea 0x20(%[dst]),%[dst] \n"
+ "subl $0x8,%[n] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [n] "+rm"(loop_n)
+ : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx),
+ [kStep] "m"(kStep0_7_Cols)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4;
+ dst_argb[0] = src[0];
+ dst_argb[1] = src[1];
+ dst_argb[2] = src[2];
+ dst_argb[3] = src[3];
+ dst_argb += 4;
+ x = (int)((uint32_t)x + (uint32_t)dx);
+ }
}
+#endif // HAS_SCALEARGBCOLS_AVX2
+#ifdef HAS_SCALEARGBCOLSUP2_AVX2
// Reads 4 pixels, duplicates them and writes 8 pixels.
-// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned.
-void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
+void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb,
const uint8_t* src_argb,
int dst_width,
int x,
@@ -1227,26 +3250,27 @@ void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
(void)x;
(void)dx;
asm volatile(
-
LABELALIGN
- "1: \n"
- "movdqu (%1),%%xmm0 \n"
- "lea 0x10(%1),%1 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpckldq %%xmm0,%%xmm0 \n"
- "punpckhdq %%xmm1,%%xmm1 \n"
- "movdqu %%xmm0,(%0) \n"
- "movdqu %%xmm1,0x10(%0) \n"
- "lea 0x20(%0),%0 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
-
+ "1: \n"
+ "vmovdqu (%1),%%xmm0 \n"
+ "lea 0x10(%1),%1 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpunpckldq %%ymm0,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0) \n"
+ "lea 0x20(%0),%0 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(dst_argb), // %0
"+r"(src_argb), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ :
+ : "memory", "cc", "xmm0");
}
+#endif // HAS_SCALEARGBCOLSUP2_AVX2
+
+#ifdef HAS_SCALEARGBFILTERCOLS_AVX2
+static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3};
// Shuffle table for arranging 2 pixels into pairs for pmaddubsw
static const uvec8 kShuffleColARGB = {
@@ -1254,95 +3278,86 @@ static const uvec8 kShuffleColARGB = {
8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel
};
-// Shuffle table for duplicating 2 fractions into 8 bytes each
-static const uvec8 kShuffleFractions = {
- 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
+// Shuffle table for duplicating 4 fractions across 2x128-bit lanes
+static const uint8_t kShuffleFractionsAVX2[32] = {
+ 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
+ 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u,
};
-// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version
-void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- intptr_t x0, x1;
- asm volatile(
- "movdqa %0,%%xmm4 \n"
- "movdqa %1,%%xmm5 \n"
- :
- : "m"(kShuffleColARGB), // %0
- "m"(kShuffleFractions) // %1
- );
-
- asm volatile(
- "movd %5,%%xmm2 \n"
- "movd %6,%%xmm3 \n"
- "pcmpeqb %%xmm6,%%xmm6 \n"
- "psrlw $0x9,%%xmm6 \n"
- "pextrw $0x1,%%xmm2,%k3 \n"
- "sub $0x2,%2 \n"
- "jl 29f \n"
- "movdqa %%xmm2,%%xmm0 \n"
- "paddd %%xmm3,%%xmm0 \n"
- "punpckldq %%xmm0,%%xmm2 \n"
- "punpckldq %%xmm3,%%xmm3 \n"
- "paddd %%xmm3,%%xmm3 \n"
- "pextrw $0x3,%%xmm2,%k4 \n"
-
- LABELALIGN
- "2: \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "paddd %%xmm3,%%xmm2 \n"
- "movq 0x00(%1,%3,4),%%xmm0 \n"
- "psrlw $0x9,%%xmm1 \n"
- "movhps 0x00(%1,%4,4),%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "pxor %%xmm6,%%xmm1 \n"
- "pmaddubsw %%xmm1,%%xmm0 \n"
- "psrlw $0x7,%%xmm0 \n"
- "pextrw $0x1,%%xmm2,%k3 \n"
- "pextrw $0x3,%%xmm2,%k4 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%0) \n"
- "lea 0x8(%0),%0 \n"
- "sub $0x2,%2 \n"
- "jge 2b \n"
+// Bilinear row filtering combines 4x2 -> 4x1. AVX2 version.
+void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ int n = dst_width & ~3;
+ int r = dst_width & 3;
+ if (n > 0) {
+ int loop_n = n;
+ asm volatile(
+ "vbroadcasti128 %[kShufARGB],%%ymm4 \n"
+ "vmovdqu %[kShufFrac],%%ymm5 \n"
+ "vmovd %[x],%%xmm2 \n"
+ "vpshufd $0x0,%%xmm2,%%xmm2 \n"
+ "vmovd %[dx],%%xmm3 \n"
+ "vpshufd $0x0,%%xmm3,%%xmm3 \n"
+ "vpmulld %[kStep],%%xmm3,%%xmm0 \n"
+ "vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
+ "vpslld $0x2,%%xmm3,%%xmm3 \n"
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $0x9,%%ymm6,%%ymm6 \n"
- LABELALIGN
- "29: \n"
- "add $0x1,%2 \n"
- "jl 99f \n"
- "psrlw $0x9,%%xmm2 \n"
- "movq 0x00(%1,%3,4),%%xmm0 \n"
- "pshufb %%xmm5,%%xmm2 \n"
- "pshufb %%xmm4,%%xmm0 \n"
- "pxor %%xmm6,%%xmm2 \n"
- "pmaddubsw %%xmm2,%%xmm0 \n"
- "psrlw $0x7,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movd %%xmm0,(%0) \n"
-
- LABELALIGN "99: \n" // clang-format error.
-
- : "+r"(dst_argb), // %0
- "+r"(src_argb), // %1
- "+rm"(dst_width), // %2
- "=&r"(x0), // %3
- "=&r"(x1) // %4
- : "rm"(x), // %5
- "rm"(dx) // %6
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ LABELALIGN
+ "1: \n"
+ "vpsrad $0x10,%%xmm2,%%xmm1 \n"
+ "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n"
+ "vpgatherdq %%ymm7,(%[src],%%xmm1,4),%%ymm0 \n"
+ "vpsrlw $0x9,%%xmm2,%%xmm1 \n"
+ "vpaddd %%xmm3,%%xmm2,%%xmm2 \n"
+ "vinserti128 $0x1,%%xmm1,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpxor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x7,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "subl $0x4,%[n] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [dst] "+r"(dst_argb), [n] "+rm"(loop_n)
+ : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx),
+ [kStep] "m"(kStep0_3_ARGB), [kShufARGB] "m"(kShuffleColARGB),
+ [kShufFrac] "m"(kShuffleFractionsAVX2)
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6", "xmm7");
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4;
+ int xf = (x >> 9) & 0x7f;
+ const uint8_t* a = src_argb + xi;
+ const uint8_t* b = a + 4;
+ dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7);
+ dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7);
+ dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7);
+ dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7);
+ dst_argb += 4;
+ x = (int)((uint32_t)x + (uint32_t)dx);
+ }
}
+#endif // HAS_SCALEARGBFILTERCOLS_AVX2
// Divide num by div and return as 16.16 fixed point result.
int FixedDiv_X86(int num, int div) {
asm volatile(
- "cdq \n"
- "shld $0x10,%%eax,%%edx \n"
- "shl $0x10,%%eax \n"
- "idiv %1 \n"
- "mov %0, %%eax \n"
+ "cdq \n"
+ "shld $0x10,%%eax,%%edx \n"
+ "shl $0x10,%%eax \n"
+ "idiv %1 \n"
+ "mov %0, %%eax \n"
: "+a"(num) // %0
: "c"(div) // %1
: "memory", "cc", "edx");
@@ -1352,20 +3367,1032 @@ int FixedDiv_X86(int num, int div) {
// Divide num - 1 by div - 1 and return as 16.16 fixed point result.
int FixedDiv1_X86(int num, int div) {
asm volatile(
- "cdq \n"
- "shld $0x10,%%eax,%%edx \n"
- "shl $0x10,%%eax \n"
- "sub $0x10001,%%eax \n"
- "sbb $0x0,%%edx \n"
- "sub $0x1,%1 \n"
- "idiv %1 \n"
- "mov %0, %%eax \n"
- : "+a"(num) // %0
- : "c"(div) // %1
+ "cdq \n"
+ "shld $0x10,%%eax,%%edx \n"
+ "shl $0x10,%%eax \n"
+ "sub $0x10001,%%eax \n"
+ "sbb $0x0,%%edx \n"
+ "sub $0x1,%1 \n"
+ "idiv %1 \n"
+ "mov %0, %%eax \n"
+ : "+a"(num), // %0
+ "+c"(div) // %1
+ :
: "memory", "cc", "edx");
return num;
}
+#ifdef HAS_SCALEUVROWDOWN2_AVX2
+void ScaleUVRowDown2_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "sub $0x10,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm0 \n"
+ "vpsrld $0x10,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_SCALEUVROWDOWN2_AVX2
+
+#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2
+void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $0x10,%%ymm4,%%ymm4 \n"
+ "sub $0x10,%2 \n"
+ "jl 2f \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm2 \n"
+ "vpsrld $0x10,%%ymm1,%%ymm3 \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%2 \n"
+ "jle 99f \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpsrld $0x10,%%ymm0,%%ymm2 \n"
+ "vpavgb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpand %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%xmm0,(%1) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2
+
+#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2
+// Shuffle table for splitting UV into upper and lower part of register.
+static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u,
+ 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u};
+static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u,
+ 6u, 14u, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80};
+
+void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" // 01010101
+ "vpabsb %%ymm4,%%ymm4 \n"
+ "vpxor %%ymm5,%%ymm5,%%ymm5 \n" // zero
+ "vbroadcastf128 %4,%%ymm1 \n" // split shuffler
+ "vbroadcastf128 %5,%%ymm3 \n" // merge shuffler
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n" // 16 UV row 0
+ "vmovdqu 0x00(%0,%3,1),%%ymm2 \n" // 16 UV row 1
+ "lea 0x20(%0),%0 \n"
+ "vpshufb %%ymm1,%%ymm0,%%ymm0 \n" // uuuuvvvv
+ "vpshufb %%ymm1,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" // horizontal add
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" // vertical add
+ "vpsrlw $0x1,%%ymm0,%%ymm0 \n" // round
+ "vpavgw %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm3,%%ymm0,%%ymm0 \n" // merge uv
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n" // combine qwords
+ "vmovdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n" // 8 UV
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "m"(kShuffleSplitUV), // %4
+ "m"(kShuffleMergeUV) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif // HAS_SCALEUVROWDOWN2BOX_AVX2
+
+#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2
+void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ ptrdiff_t step = (ptrdiff_t)src_stepx * 2;
+ ptrdiff_t step3 = step * 3;
+ (void)src_stride;
+ asm volatile(
+ "vpxor %%xmm0,%%xmm0,%%xmm0 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vpinsrw $0x0,(%[src]),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x1,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x2,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x3,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n"
+ "lea 0x00(%[src],%[step],4),%[src] \n"
+ "vpinsrw $0x4,(%[src]),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x5,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x6,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n"
+ "vpinsrw $0x7,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n"
+ "lea 0x00(%[src],%[step],4),%[src] \n"
+ "vmovdqu %%xmm0,(%[dst]) \n"
+ "lea 0x10(%[dst]),%[dst] \n"
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [width] "+rm"(dst_width)
+ : [step] "r"(step), [step3] "r"(step3)
+ : "memory", "cc", "xmm0");
+}
+#endif // HAS_SCALEUVROWDOWNEVEN_AVX2
+
+static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3,
+ 3, 1, 3, 1, 1, 3, 1, 3};
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_SSSE3
+void ScaleUVRowUp2_Linear_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "pcmpeqw %%xmm4,%%xmm4 \n"
+ "psrlw $15,%%xmm4 \n"
+ "psllw $1,%%xmm4 \n" // all 2
+ "movdqa %3,%%xmm3 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 00112233 (1u1v)
+ "movq 2(%0),%%xmm1 \n" // 11223344 (1u1v)
+ "punpcklbw %%xmm1,%%xmm0 \n" // 0101121223233434 (2u2v)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhdq %%xmm0,%%xmm2 \n" // 2323232334343434 (2u2v)
+ "punpckldq %%xmm0,%%xmm0 \n" // 0101010112121212 (2u2v)
+ "pmaddubsw %%xmm3,%%xmm2 \n" // 3*near+far (1u1v16, hi)
+ "pmaddubsw %%xmm3,%%xmm0 \n" // 3*near+far (1u1v16, lo)
+ "paddw %%xmm4,%%xmm0 \n" // 3*near+far+2 (lo)
+ "paddw %%xmm4,%%xmm2 \n" // 3*near+far+2 (hi)
+ "psrlw $2,%%xmm0 \n" // 3/4*near+1/4*far (lo)
+ "psrlw $2,%%xmm2 \n" // 3/4*near+1/4*far (hi)
+ "packuswb %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 4 uv to 8 uv
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kUVLinearMadd31) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_SSSE3
+void ScaleUVRowUp2_Bilinear_SSSE3(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "pcmpeqw %%xmm6,%%xmm6 \n"
+ "psrlw $15,%%xmm6 \n"
+ "psllw $3,%%xmm6 \n" // all 8
+ "movdqa %5,%%xmm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 00112233 (1u1v)
+ "movq 2(%0),%%xmm1 \n" // 11223344 (1u1v)
+ "punpcklbw %%xmm1,%%xmm0 \n" // 0101121223233434 (2u2v)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpckhdq %%xmm0,%%xmm2 \n" // 2323232334343434 (2u2v)
+ "punpckldq %%xmm0,%%xmm0 \n" // 0101010112121212 (2u2v)
+ "pmaddubsw %%xmm7,%%xmm2 \n" // 3*near+far (1u1v16, hi)
+ "pmaddubsw %%xmm7,%%xmm0 \n" // 3*near+far (1u1v16, lo)
+
+ "movq (%0,%3),%%xmm1 \n"
+ "movq 2(%0,%3),%%xmm4 \n"
+ "punpcklbw %%xmm4,%%xmm1 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "punpckhdq %%xmm1,%%xmm3 \n"
+ "punpckldq %%xmm1,%%xmm1 \n"
+ "pmaddubsw %%xmm7,%%xmm3 \n" // 3*near+far (2, hi)
+ "pmaddubsw %%xmm7,%%xmm1 \n" // 3*near+far (2, lo)
+
+ // xmm0 xmm2
+ // xmm1 xmm3
+
+ "movdqa %%xmm0,%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm5 \n"
+ "paddw %%xmm0,%%xmm4 \n" // 6*near+2*far (1, lo)
+ "paddw %%xmm6,%%xmm5 \n" // 3*near+far+8 (2, lo)
+ "paddw %%xmm0,%%xmm4 \n" // 9*near+3*far (1, lo)
+ "paddw %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrlw $4,%%xmm4 \n" // ^ div by 16 (1, lo)
+
+ "movdqa %%xmm1,%%xmm5 \n"
+ "paddw %%xmm1,%%xmm5 \n" // 6*near+2*far (2, lo)
+ "paddw %%xmm6,%%xmm0 \n" // 3*near+far+8 (1, lo)
+ "paddw %%xmm1,%%xmm5 \n" // 9*near+3*far (2, lo)
+ "paddw %%xmm0,%%xmm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "psrlw $4,%%xmm5 \n" // ^ div by 16 (2, lo)
+
+ "movdqa %%xmm2,%%xmm0 \n"
+ "movdqa %%xmm3,%%xmm1 \n"
+ "paddw %%xmm2,%%xmm0 \n" // 6*near+2*far (1, hi)
+ "paddw %%xmm6,%%xmm1 \n" // 3*near+far+8 (2, hi)
+ "paddw %%xmm2,%%xmm0 \n" // 9*near+3*far (1, hi)
+ "paddw %%xmm1,%%xmm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrlw $4,%%xmm0 \n" // ^ div by 16 (1, hi)
+
+ "movdqa %%xmm3,%%xmm1 \n"
+ "paddw %%xmm3,%%xmm1 \n" // 6*near+2*far (2, hi)
+ "paddw %%xmm6,%%xmm2 \n" // 3*near+far+8 (1, hi)
+ "paddw %%xmm3,%%xmm1 \n" // 9*near+3*far (2, hi)
+ "paddw %%xmm2,%%xmm1 \n" // 9 3 3 1 + 8 (2, hi)
+ "psrlw $4,%%xmm1 \n" // ^ div by 16 (2, hi)
+
+ "packuswb %%xmm0,%%xmm4 \n"
+ "movdqu %%xmm4,(%1) \n" // store above
+ "packuswb %%xmm1,%%xmm5 \n"
+ "movdqu %%xmm5,(%1,%4) \n" // store below
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 4 uv to 8 uv
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kUVLinearMadd31) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
+
+void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrlw $15,%%ymm4,%%ymm4 \n"
+ "vpsllw $1,%%ymm4,%%ymm4 \n" // all 2
+ "vbroadcastf128 %3,%%ymm3 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vmovdqu 2(%0),%%xmm1 \n"
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n"
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpunpckhdq %%ymm0,%%ymm0,%%ymm2 \n"
+ "vpunpckldq %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm1 \n" // 3*near+far (hi)
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" // 3*near+far (lo)
+ "vpaddw %%ymm4,%%ymm0,%%ymm0 \n" // 3*near+far+2 (lo)
+ "vpaddw %%ymm4,%%ymm1,%%ymm1 \n" // 3*near+far+2 (hi)
+ "vpsrlw $2,%%ymm0,%%ymm0 \n" // 3/4*near+1/4*far (lo)
+ "vpsrlw $2,%%ymm1,%%ymm1 \n" // 3/4*near+1/4*far (hi)
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 uv to 16 uv
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "m"(kUVLinearMadd31) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
+void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqw %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrlw $15,%%ymm6,%%ymm6 \n"
+ "vpsllw $3,%%ymm6,%%ymm6 \n" // all 8
+ "vbroadcastf128 %5,%%ymm7 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n"
+ "vmovdqu 2(%0),%%xmm1 \n"
+ "vpermq $0b11011000,%%ymm0,%%ymm0 \n"
+ "vpermq $0b11011000,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpunpckhdq %%ymm0,%%ymm0,%%ymm2 \n"
+ "vpunpckldq %%ymm0,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm1 \n" // 3*near+far (1, hi)
+ "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n" // 3*near+far (1, lo)
+
+ "vmovdqu (%0,%3),%%xmm2 \n" // 0123456789ABCDEF
+ "vmovdqu 2(%0,%3),%%xmm3 \n" // 123456789ABCDEF0
+ "vpermq $0b11011000,%%ymm2,%%ymm2 \n"
+ "vpermq $0b11011000,%%ymm3,%%ymm3 \n"
+ "vpunpcklbw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpunpckhdq %%ymm2,%%ymm2,%%ymm4 \n"
+ "vpunpckldq %%ymm2,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm7,%%ymm4,%%ymm3 \n" // 3*near+far (2, hi)
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" // 3*near+far (2, lo)
+
+ // ymm0 ymm1
+ // ymm2 ymm3
+
+ "vpaddw %%ymm0,%%ymm0,%%ymm4 \n" // 6*near+2*far (1, lo)
+ "vpaddw %%ymm6,%%ymm2,%%ymm5 \n" // 3*near+far+8 (2, lo)
+ "vpaddw %%ymm4,%%ymm0,%%ymm4 \n" // 9*near+3*far (1, lo)
+ "vpaddw %%ymm4,%%ymm5,%%ymm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "vpsrlw $4,%%ymm4,%%ymm4 \n" // ^ div by 16 (1, lo)
+
+ "vpaddw %%ymm2,%%ymm2,%%ymm5 \n" // 6*near+2*far (2, lo)
+ "vpaddw %%ymm6,%%ymm0,%%ymm0 \n" // 3*near+far+8 (1, lo)
+ "vpaddw %%ymm5,%%ymm2,%%ymm5 \n" // 9*near+3*far (2, lo)
+ "vpaddw %%ymm5,%%ymm0,%%ymm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "vpsrlw $4,%%ymm5,%%ymm5 \n" // ^ div by 16 (2, lo)
+
+ "vpaddw %%ymm1,%%ymm1,%%ymm0 \n" // 6*near+2*far (1, hi)
+ "vpaddw %%ymm6,%%ymm3,%%ymm2 \n" // 3*near+far+8 (2, hi)
+ "vpaddw %%ymm0,%%ymm1,%%ymm0 \n" // 9*near+3*far (1, hi)
+ "vpaddw %%ymm0,%%ymm2,%%ymm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "vpsrlw $4,%%ymm0,%%ymm0 \n" // ^ div by 16 (1, hi)
+
+ "vpaddw %%ymm3,%%ymm3,%%ymm2 \n" // 6*near+2*far (2, hi)
+ "vpaddw %%ymm6,%%ymm1,%%ymm1 \n" // 3*near+far+8 (1, hi)
+ "vpaddw %%ymm2,%%ymm3,%%ymm2 \n" // 9*near+3*far (2, hi)
+ "vpaddw %%ymm2,%%ymm1,%%ymm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "vpsrlw $4,%%ymm2,%%ymm2 \n" // ^ div by 16 (2, hi)
+
+ "vpackuswb %%ymm0,%%ymm4,%%ymm4 \n"
+ "vmovdqu %%ymm4,(%1) \n" // store above
+ "vpackuswb %%ymm2,%%ymm5,%%ymm5 \n"
+ "vmovdqu %%ymm5,(%1,%4) \n" // store below
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 8 uv to 16 uv
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride), // %4
+ "m"(kUVLinearMadd31) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+void ScaleUVRowUp2_Linear_16_SSE41(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "pxor %%xmm5,%%xmm5 \n"
+ "pcmpeqd %%xmm4,%%xmm4 \n"
+ "psrld $31,%%xmm4 \n"
+ "pslld $1,%%xmm4 \n" // all 2
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
+ "movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
+
+ "punpcklwd %%xmm5,%%xmm0 \n" // 0011 (32b, 1u1v)
+ "punpcklwd %%xmm5,%%xmm1 \n" // 1122 (32b, 1u1v)
+
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+
+ "pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (lo, far)
+ "pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (hi, far)
+
+ "paddd %%xmm4,%%xmm2 \n" // far+2 (lo)
+ "paddd %%xmm4,%%xmm3 \n" // far+2 (hi)
+ "paddd %%xmm0,%%xmm2 \n" // near+far+2 (lo)
+ "paddd %%xmm1,%%xmm3 \n" // near+far+2 (hi)
+ "paddd %%xmm0,%%xmm0 \n" // 2*near (lo)
+ "paddd %%xmm1,%%xmm1 \n" // 2*near (hi)
+ "paddd %%xmm2,%%xmm0 \n" // 3*near+far+2 (lo)
+ "paddd %%xmm3,%%xmm1 \n" // 3*near+far+2 (hi)
+
+ "psrld $2,%%xmm0 \n" // 3/4*near+1/4*far (lo)
+ "psrld $2,%%xmm1 \n" // 3/4*near+1/4*far (hi)
+ "packusdw %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 2 uv to 4 uv
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+void ScaleUVRowUp2_Bilinear_16_SSE41(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "pxor %%xmm7,%%xmm7 \n"
+ "pcmpeqd %%xmm6,%%xmm6 \n"
+ "psrld $31,%%xmm6 \n"
+ "pslld $3,%%xmm6 \n" // all 8
+
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
+ "movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
+ "punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v)
+ "punpcklwd %%xmm7,%%xmm1 \n" // 1122 (near) (32b, 1u1v)
+ "movdqa %%xmm0,%%xmm2 \n"
+ "movdqa %%xmm1,%%xmm3 \n"
+ "pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (far) (1, lo)
+ "pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (far) (1, hi)
+ "paddd %%xmm0,%%xmm2 \n" // near+far (1, lo)
+ "paddd %%xmm1,%%xmm3 \n" // near+far (1, hi)
+ "paddd %%xmm0,%%xmm0 \n" // 2*near (1, lo)
+ "paddd %%xmm1,%%xmm1 \n" // 2*near (1, hi)
+ "paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo)
+ "paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi)
+
+ "movq (%0,%3,2),%%xmm2 \n"
+ "movq 4(%0,%3,2),%%xmm3 \n"
+ "punpcklwd %%xmm7,%%xmm2 \n"
+ "punpcklwd %%xmm7,%%xmm3 \n"
+ "movdqa %%xmm2,%%xmm4 \n"
+ "movdqa %%xmm3,%%xmm5 \n"
+ "pshufd $0b01001110,%%xmm4,%%xmm4 \n" // 1100 (far) (2, lo)
+ "pshufd $0b01001110,%%xmm5,%%xmm5 \n" // 2211 (far) (2, hi)
+ "paddd %%xmm2,%%xmm4 \n" // near+far (2, lo)
+ "paddd %%xmm3,%%xmm5 \n" // near+far (2, hi)
+ "paddd %%xmm2,%%xmm2 \n" // 2*near (2, lo)
+ "paddd %%xmm3,%%xmm3 \n" // 2*near (2, hi)
+ "paddd %%xmm4,%%xmm2 \n" // 3*near+far (2, lo)
+ "paddd %%xmm5,%%xmm3 \n" // 3*near+far (2, hi)
+
+ "movdqa %%xmm0,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "paddd %%xmm0,%%xmm4 \n" // 6*near+2*far (1, lo)
+ "paddd %%xmm6,%%xmm5 \n" // 3*near+far+8 (2, lo)
+ "paddd %%xmm0,%%xmm4 \n" // 9*near+3*far (1, lo)
+ "paddd %%xmm5,%%xmm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "psrld $4,%%xmm4 \n" // ^ div by 16 (1, lo)
+
+ "movdqa %%xmm2,%%xmm5 \n"
+ "paddd %%xmm2,%%xmm5 \n" // 6*near+2*far (2, lo)
+ "paddd %%xmm6,%%xmm0 \n" // 3*near+far+8 (1, lo)
+ "paddd %%xmm2,%%xmm5 \n" // 9*near+3*far (2, lo)
+ "paddd %%xmm0,%%xmm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "psrld $4,%%xmm5 \n" // ^ div by 16 (2, lo)
+
+ "movdqa %%xmm1,%%xmm0 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "paddd %%xmm1,%%xmm0 \n" // 6*near+2*far (1, hi)
+ "paddd %%xmm6,%%xmm2 \n" // 3*near+far+8 (2, hi)
+ "paddd %%xmm1,%%xmm0 \n" // 9*near+3*far (1, hi)
+ "paddd %%xmm2,%%xmm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "psrld $4,%%xmm0 \n" // ^ div by 16 (1, hi)
+
+ "movdqa %%xmm3,%%xmm2 \n"
+ "paddd %%xmm3,%%xmm2 \n" // 6*near+2*far (2, hi)
+ "paddd %%xmm6,%%xmm1 \n" // 3*near+far+8 (1, hi)
+ "paddd %%xmm3,%%xmm2 \n" // 9*near+3*far (2, hi)
+ "paddd %%xmm1,%%xmm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "psrld $4,%%xmm2 \n" // ^ div by 16 (2, hi)
+
+ "packusdw %%xmm0,%%xmm4 \n"
+ "movdqu %%xmm4,(%1) \n" // store above
+ "packusdw %%xmm2,%%xmm5 \n"
+ "movdqu %%xmm5,(%1,%4,2) \n" // store below
+
+ "lea 0x8(%0),%0 \n"
+ "lea 0x10(%1),%1 \n" // 2 uv to 4 uv
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsrld $31,%%ymm4,%%ymm4 \n"
+ "vpslld $1,%%ymm4,%%ymm4 \n" // all 2
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // 00112233 (16b, 1u1v)
+ "vmovdqu 4(%0),%%xmm1 \n" // 11223344 (16b, 1u1v)
+
+ "vpmovzxwd %%xmm0,%%ymm0 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm1,%%ymm1 \n" // 12345678 (32b, 1u1v)
+
+ "vpshufd $0b01001110,%%ymm0,%%ymm2 \n" // 11003322 (lo, far)
+ "vpshufd $0b01001110,%%ymm1,%%ymm3 \n" // 22114433 (hi, far)
+
+ "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" // far+2 (lo)
+ "vpaddd %%ymm4,%%ymm3,%%ymm3 \n" // far+2 (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" // near+far+2 (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm3 \n" // near+far+2 (hi)
+ "vpaddd %%ymm0,%%ymm0,%%ymm0 \n" // 2*near (lo)
+ "vpaddd %%ymm1,%%ymm1,%%ymm1 \n" // 2*near (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 3*near+far+2 (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm1 \n" // 3*near+far+2 (hi)
+
+ "vpsrld $2,%%ymm0,%%ymm0 \n" // 3/4*near+1/4*far (lo)
+ "vpsrld $2,%%ymm1,%%ymm1 \n" // 3/4*near+1/4*far (hi)
+ "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 4 uv to 8 uv
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ asm volatile(
+ "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n"
+ "vpsrld $31,%%ymm6,%%ymm6 \n"
+ "vpslld $3,%%ymm6,%%ymm6 \n" // all 8
+
+ LABELALIGN
+ "1: \n"
+
+ "vmovdqu (%0),%%xmm0 \n" // 00112233 (16b, 1u1v)
+ "vmovdqu 4(%0),%%xmm1 \n" // 11223344 (16b, 1u1v)
+ "vpmovzxwd %%xmm0,%%ymm0 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm1,%%ymm1 \n" // 12345678 (32b, 1u1v)
+ "vpshufd $0b01001110,%%ymm0,%%ymm2 \n" // 11003322 (lo, far)
+ "vpshufd $0b01001110,%%ymm1,%%ymm3 \n" // 22114433 (hi, far)
+ "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" // near+far (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm3 \n" // near+far (hi)
+ "vpaddd %%ymm0,%%ymm0,%%ymm0 \n" // 2*near (lo)
+ "vpaddd %%ymm1,%%ymm1,%%ymm1 \n" // 2*near (hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 3*near+far (lo)
+ "vpaddd %%ymm1,%%ymm3,%%ymm1 \n" // 3*near+far (hi)
+
+ "vmovdqu (%0,%3,2),%%xmm2 \n" // 00112233 (16b, 1u1v)
+ "vmovdqu 4(%0,%3,2),%%xmm3 \n" // 11223344 (16b, 1u1v)
+ "vpmovzxwd %%xmm2,%%ymm2 \n" // 01234567 (32b, 1u1v)
+ "vpmovzxwd %%xmm3,%%ymm3 \n" // 12345678 (32b, 1u1v)
+ "vpshufd $0b01001110,%%ymm2,%%ymm4 \n" // 11003322 (lo, far)
+ "vpshufd $0b01001110,%%ymm3,%%ymm5 \n" // 22114433 (hi, far)
+ "vpaddd %%ymm2,%%ymm4,%%ymm4 \n" // near+far (lo)
+ "vpaddd %%ymm3,%%ymm5,%%ymm5 \n" // near+far (hi)
+ "vpaddd %%ymm2,%%ymm2,%%ymm2 \n" // 2*near (lo)
+ "vpaddd %%ymm3,%%ymm3,%%ymm3 \n" // 2*near (hi)
+ "vpaddd %%ymm2,%%ymm4,%%ymm2 \n" // 3*near+far (lo)
+ "vpaddd %%ymm3,%%ymm5,%%ymm3 \n" // 3*near+far (hi)
+
+ "vpaddd %%ymm0,%%ymm0,%%ymm4 \n" // 6*near+2*far (1, lo)
+ "vpaddd %%ymm6,%%ymm2,%%ymm5 \n" // 3*near+far+8 (2, lo)
+ "vpaddd %%ymm4,%%ymm0,%%ymm4 \n" // 9*near+3*far (1, lo)
+ "vpaddd %%ymm4,%%ymm5,%%ymm4 \n" // 9 3 3 1 + 8 (1, lo)
+ "vpsrld $4,%%ymm4,%%ymm4 \n" // ^ div by 16 (1, lo)
+
+ "vpaddd %%ymm2,%%ymm2,%%ymm5 \n" // 6*near+2*far (2, lo)
+ "vpaddd %%ymm6,%%ymm0,%%ymm0 \n" // 3*near+far+8 (1, lo)
+ "vpaddd %%ymm5,%%ymm2,%%ymm5 \n" // 9*near+3*far (2, lo)
+ "vpaddd %%ymm5,%%ymm0,%%ymm5 \n" // 9 3 3 1 + 8 (2, lo)
+ "vpsrld $4,%%ymm5,%%ymm5 \n" // ^ div by 16 (2, lo)
+
+ "vpaddd %%ymm1,%%ymm1,%%ymm0 \n" // 6*near+2*far (1, hi)
+ "vpaddd %%ymm6,%%ymm3,%%ymm2 \n" // 3*near+far+8 (2, hi)
+ "vpaddd %%ymm0,%%ymm1,%%ymm0 \n" // 9*near+3*far (1, hi)
+ "vpaddd %%ymm0,%%ymm2,%%ymm0 \n" // 9 3 3 1 + 8 (1, hi)
+ "vpsrld $4,%%ymm0,%%ymm0 \n" // ^ div by 16 (1, hi)
+
+ "vpaddd %%ymm3,%%ymm3,%%ymm2 \n" // 6*near+2*far (2, hi)
+ "vpaddd %%ymm6,%%ymm1,%%ymm1 \n" // 3*near+far+8 (1, hi)
+ "vpaddd %%ymm2,%%ymm3,%%ymm2 \n" // 9*near+3*far (2, hi)
+ "vpaddd %%ymm2,%%ymm1,%%ymm2 \n" // 9 3 3 1 + 8 (2, hi)
+ "vpsrld $4,%%ymm2,%%ymm2 \n" // ^ div by 16 (2, hi)
+
+ "vpackusdw %%ymm0,%%ymm4,%%ymm4 \n"
+ "vmovdqu %%ymm4,(%1) \n" // store above
+ "vpackusdw %%ymm2,%%ymm5,%%ymm5 \n"
+ "vmovdqu %%ymm5,(%1,%4,2) \n" // store below
+
+ "lea 0x10(%0),%0 \n"
+ "lea 0x20(%1),%1 \n" // 4 uv to 8 uv
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"(src_stride), // %3
+ "r"(dst_stride) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+#endif
+
+#ifdef HAS_SCALEUVADDCOLS3_AVX2
+static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
+static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
+static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
+
+// Scale UV columns by 3 box filter.
+// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
+void ScaleUVAddCols3_AVX2(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+#if defined(__x86_64__)
+ asm volatile(
+ "vmovd %k[scaleval],%%xmm7 \n"
+ "vpbroadcastw %%xmm7,%%ymm7 \n"
+ "vmovdqa %[kPerm0],%%ymm4 \n"
+ "vmovdqa %[kPerm1],%%ymm5 \n"
+ "vmovdqa %[kPerm2],%%ymm6 \n"
+ "sub $0x10,%[dst_width] \n"
+ "jl 2f \n"
+
+ // Process 16 output UV pixels (192 input bytes -> 32 output bytes).
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src_uv]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm0,%%ymm4,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpermd %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm0,%%ymm3,%%ymm8 \n"
+ "vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n"
+
+ "vmovdqu 0x60(%[src_uv]),%%ymm0 \n"
+ "vmovdqu 0x80(%[src_uv]),%%ymm1 \n"
+ "lea 0xc0(%[src_uv]),%[src_uv] \n"
+ "vpermd %%ymm0,%%ymm4,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu -0x20(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpermd %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
+ "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
+
+ "vpackuswb %%ymm3,%%ymm8,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%[dst_uv]) \n"
+ "lea 0x20(%[dst_uv]),%[dst_uv] \n"
+ "sub $0x10,%[dst_width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x10,%[dst_width] \n"
+ "jle 99f \n"
+
+ // 8-pixel remainder.
+ "vmovdqu (%[src_uv]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm0,%%ymm4,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpermd %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
+ "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vextracti128 $1,%%ymm3,%%xmm2 \n"
+ "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
+ "vmovdqu %%xmm3,(%[dst_uv]) \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ : [scaleval] "r"(scaleval), // %[scaleval]
+ [kPerm0] "m"(kPermUV3_0), // %[kPerm0]
+ [kPerm1] "m"(kPermUV3_1), // %[kPerm1]
+ [kPerm2] "m"(kPermUV3_2) // %[kPerm2]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7", "xmm8");
+#else // defined(__x86_64__)
+ asm volatile(
+ "vmovd %[scaleval],%%xmm7 \n"
+ "vpbroadcastw %%xmm7,%%ymm7 \n"
+ "vmovdqa %[kPerm0],%%ymm4 \n"
+ "vmovdqa %[kPerm1],%%ymm5 \n"
+ "vmovdqa %[kPerm2],%%ymm6 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%[src_uv]),%%ymm0 \n"
+ "vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
+ "vpermd %%ymm0,%%ymm4,%%ymm0 \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
+ "lea 0x60(%[src_uv]),%[src_uv] \n"
+ "vpermd %%ymm1,%%ymm4,%%ymm1 \n"
+ "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
+ "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
+ "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermd %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpermd %%ymm0,%%ymm6,%%ymm0 \n"
+ "vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
+ "vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
+ "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
+ "vextracti128 $1,%%ymm3,%%xmm2 \n"
+ "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
+ "vmovdqu %%xmm3,(%[dst_uv]) \n"
+ "lea 0x10(%[dst_uv]),%[dst_uv] \n"
+ "sub $0x8,%[dst_width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ : [scaleval] "rm"(scaleval), // %[scaleval]
+ [kPerm0] "m"(kPermUV3_0), // %[kPerm0]
+ [kPerm1] "m"(kPermUV3_1), // %[kPerm1]
+ [kPerm2] "m"(kPermUV3_2) // %[kPerm2]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+#endif // defined(__x86_64__)
+}
+#endif // HAS_SCALEUVADDCOLS3_AVX2
+
+#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
+static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
+ 24, 27, 30, 1, 4, 7, 10, 13};
+static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
+ 25, 28, 31, 2, 5, 8, 11, 14};
+static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
+ 26, 29, 0, 3, 6, 9, 12, 15};
+static const uint8_t kPackEvenBytes[64] = {
+ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
+ 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
+ 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
+ 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
+ 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
+
+// Scale UV columns by 3 box filter using AVX512VBMI.
+// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
+// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
+LIBYUV_TARGET_AVX512VBMI
+void ScaleUVAddCols3_AVX512VBMI(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ intptr_t temp, mask;
+ asm volatile(
+ "vpbroadcastw %k[scaleval],%%zmm20 \n"
+ "vpmovzxbd %[kCol0],%%zmm16 \n"
+ "vpmovzxbd %[kCol1],%%zmm17 \n"
+ "vpmovzxbd %[kCol2],%%zmm18 \n"
+ "vmovdqu8 %[kPack],%%zmm19 \n"
+ "mov $0xf800,%k[temp] \n"
+ "kmovw %k[temp],%%k5 \n"
+ "mov $0xfc00,%k[temp] \n"
+ "kmovw %k[temp],%%k6 \n"
+
+ "sub $0x20,%[dst_width] \n"
+ "jl 2f \n"
+
+ // Main loop: 32 output UV pixels (64 output bytes) per iteration.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu32 (%[src_uv]),%%zmm0 \n"
+ "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
+ "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
+ "vmovdqa32 %%zmm16,%%zmm3 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
+ "vmovdqa32 %%zmm17,%%zmm4 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
+ "vmovdqa32 %%zmm18,%%zmm5 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
+ "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
+ "vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
+
+ "vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n"
+ "vmovdqu32 0x100(%[src_uv]),%%zmm1 \n"
+ "vmovdqu32 0x140(%[src_uv]),%%zmm2 \n"
+ "lea 0x180(%[src_uv]),%[src_uv] \n"
+ "vmovdqa32 %%zmm16,%%zmm3 \n"
+ "vpermi2d %%zmm1,%%zmm6,%%zmm3 \n"
+ "vmovdqa32 %%zmm17,%%zmm4 \n"
+ "vpermi2d %%zmm1,%%zmm6,%%zmm4 \n"
+ "vmovdqa32 %%zmm18,%%zmm5 \n"
+ "vpermi2d %%zmm1,%%zmm6,%%zmm5 \n"
+ "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
+ "vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n"
+
+ "vmovdqa64 %%zmm19,%%zmm2 \n"
+ "vpermi2b %%zmm3,%%zmm0,%%zmm2 \n"
+ "vmovdqu8 %%zmm2,(%[dst_uv]) \n"
+ "lea 0x40(%[dst_uv]),%[dst_uv] \n"
+ "sub $0x20,%[dst_width] \n"
+ "jge 1b \n"
+
+ "2: \n"
+ "add $0x20,%[dst_width] \n"
+ "jle 99f \n"
+ "cmp $0x10,%[dst_width] \n"
+ "jl 3f \n"
+
+ // 16-pixel half-block (32 output bytes).
+ "vmovdqu32 (%[src_uv]),%%zmm0 \n"
+ "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
+ "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
+ "lea 0xc0(%[src_uv]),%[src_uv] \n"
+ "vmovdqa32 %%zmm16,%%zmm3 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
+ "vmovdqa32 %%zmm17,%%zmm4 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
+ "vmovdqa32 %%zmm18,%%zmm5 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
+ "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
+ "vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
+ "vpmovwb %%zmm0,(%[dst_uv]) \n"
+ "lea 0x20(%[dst_uv]),%[dst_uv] \n"
+ "sub $0x10,%[dst_width] \n"
+ "jle 99f \n"
+
+ // Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes).
+ "3: \n"
+ "lea (%q[dst_width],%q[dst_width],2),%q[temp] \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[temp],%q[mask],%q[mask] \n"
+ "kmovw %k[mask],%%k1 \n"
+ "shr $16,%q[mask] \n"
+ "kmovw %k[mask],%%k2 \n"
+ "shr $16,%q[mask] \n"
+ "kmovw %k[mask],%%k3 \n"
+ "mov $-1,%q[mask] \n"
+ "bzhi %q[dst_width],%q[mask],%q[mask] \n"
+ "kmovw %k[mask],%%k4 \n"
+
+ "vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n"
+ "vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n"
+ "vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n"
+ "vmovdqa32 %%zmm16,%%zmm3 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
+ "vmovdqa32 %%zmm17,%%zmm4 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
+ "vmovdqa32 %%zmm18,%%zmm5 \n"
+ "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
+ "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
+ "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
+ "vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
+ "vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
+ "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
+ "vpmovwb %%zmm0,%%ymm0 \n"
+ "vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n"
+
+ "99: \n"
+ "vzeroupper \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [temp] "=&r"(temp), // %[temp]
+ [mask] "=&r"(mask) // %[mask]
+ : [scaleval] "r"(scaleval), // %[scaleval]
+ [kCol0] "m"(kPerm512_Col0), // %[kCol0]
+ [kCol1] "m"(kPerm512_Col1), // %[kCol1]
+ [kCol2] "m"(kPerm512_Col2), // %[kCol2]
+ [kPack] "m"(kPackEvenBytes) // %[kPack]
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4",
+ "k5", "k6");
+}
+#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
+
#endif // defined(__x86_64__) || defined(__i386__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/scale_hvx.cc b/libs/libyuv/source/scale_hvx.cc
new file mode 100644
index 0000000000..0aab64a6a8
--- /dev/null
+++ b/libs/libyuv/source/scale_hvx.cc
@@ -0,0 +1,625 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/scale_row.h"
+
+#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
+#include <hexagon_protos.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <string.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+static __inline HVX_Vector vloada(const void* ptr) {
+ HVX_Vector v;
+ memcpy(&v, (const HVX_Vector*)ptr, sizeof(v));
+ return v;
+}
+
+static __inline HVX_UVector vloadu(const void* ptr) {
+ HVX_UVector v;
+ memcpy(&v, ptr, sizeof(v));
+ return v;
+}
+
+static __inline void vstoreu(void* ptr, HVX_UVector v) {
+ memcpy(ptr, &v, sizeof(v));
+}
+
+static __inline void vstoreu_partial(void* dst, HVX_Vector v, int n) {
+ if (n >= 128) {
+ vstoreu(dst, v);
+ } else if (n > 0) {
+ const uintptr_t addr = (uintptr_t)dst;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector value = Q6_V_vlalign_VVR(v, v, addr);
+ HVX_VectorPred ql_not = Q6_Q_vsetq_R(addr);
+ HVX_VectorPred qr = Q6_Q_vsetq2_R(offset + (uintptr_t)n);
+ if (offset + (uintptr_t)n > 128) {
+ Q6_vmem_QRIV(qr, (uint8_t*)dst + 128, value);
+ qr = Q6_Q_vcmp_eq_VbVb(value, value);
+ }
+ ql_not = Q6_Q_or_QQn(ql_not, qr);
+ Q6_vmem_QnRIV(ql_not, dst, value);
+ }
+}
+
+static __inline HVX_Vector vloadu_partial_zero(const void* src, int n) {
+ if (n >= 128) {
+ return vloadu(src);
+ }
+ if (n <= 0) {
+ return Q6_V_vzero();
+ }
+ const uintptr_t addr = (uintptr_t)src;
+ const uintptr_t offset = addr & 127;
+ HVX_Vector v0 = vloada(src);
+ HVX_Vector v1 =
+ (offset + (uintptr_t)n > 128) ? vloada((const uint8_t*)src + 128) : v0;
+ HVX_Vector v = Q6_V_valign_VVR(v1, v0, addr);
+ return Q6_V_vmux_QVV(Q6_Q_vsetq_R(n), v, Q6_V_vzero());
+}
+
+void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
+ while (src_width > 0) {
+ int w = src_width >= 128 ? 128 : src_width;
+ HVX_Vector s = vloadu_partial_zero(src_ptr, w);
+ HVX_VectorPair s16 = Q6_Wuh_vunpack_Vub(s);
+ int b0 = w > 64 ? 128 : w * 2;
+ HVX_Vector d0 = vloadu_partial_zero(dst_ptr, b0);
+ d0 = Q6_Vh_vadd_VhVh(d0, Q6_V_lo_W(s16));
+ vstoreu_partial(dst_ptr, d0, b0);
+ if (w > 64) {
+ int b1 = (w - 64) * 2;
+ HVX_Vector d1 = vloadu_partial_zero(dst_ptr + 64, b1);
+ d1 = Q6_Vh_vadd_VhVh(d1, Q6_V_hi_W(s16));
+ vstoreu_partial(dst_ptr + 64, d1, b1);
+ }
+ src_ptr += 128;
+ dst_ptr += 128;
+ src_width -= 128;
+ }
+}
+
+// vrdelta control vectors to compact 3 vectors of 3-column UV sums into 1
+// vector of 32 UV pairs (64 uint16_t):
+// kUVAddCols3P0 routes dwords 0, 3, ..., 30 -> dwords 0..10 (bytes 0..43)
+static const uint8_t kUVAddCols3P0[128] __attribute__((aligned(128))) = {
+ 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x08, 0x08, 0x08, 0x08,
+ 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
+ 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+};
+// kUVAddCols3P1 routes dwords 1, 4, ..., 31 -> dwords 11..21 (bytes 44..87)
+static const uint8_t kUVAddCols3P1[128] __attribute__((aligned(128))) = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10, 0x10, 0x10, 0x10,
+ 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
+ 0x10, 0x10, 0x10, 0x10, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ 0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+};
+// kUVAddCols3P2 routes dwords 2, 5, ..., 29 -> dwords 22..31 (bytes 88..127)
+static const uint8_t kUVAddCols3P2[128] __attribute__((aligned(128))) = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x08, 0x08, 0x08, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
+};
+
+void ScaleUVAddCols3_HVX(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ const HVX_Vector p0 = vloada(kUVAddCols3P0);
+ const HVX_Vector p1 = vloada(kUVAddCols3P1);
+ const HVX_Vector p2 = vloada(kUVAddCols3P2);
+ const HVX_VectorPred q44 = Q6_Q_vsetq_R(44);
+ const HVX_VectorPred q88 = Q6_Q_vsetq_R(88);
+ const uint32_t scale_splat =
+ ((uint32_t)(uint16_t)scaleval << 16) | (uint16_t)scaleval;
+ while (dst_width > 0) {
+ int w = dst_width >= 32 ? 32 : dst_width;
+ int src_bytes = w * 12;
+ HVX_Vector v0 =
+ vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector v1 =
+ src_bytes > 128
+ ? vloadu_partial_zero((const uint8_t*)src_uv + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector v2 =
+ src_bytes > 256
+ ? vloadu_partial_zero((const uint8_t*)src_uv + 256, src_bytes - 256)
+ : Q6_V_vzero();
+
+ HVX_Vector s0 =
+ Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v0, Q6_V_valign_VVR(v1, v0, 4)),
+ Q6_V_valign_VVR(v1, v0, 8));
+ HVX_Vector s1 =
+ Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v1, Q6_V_valign_VVR(v2, v1, 4)),
+ Q6_V_valign_VVR(v2, v1, 8));
+ HVX_Vector s2 =
+ Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v2, Q6_V_valign_VVR(v2, v2, 4)),
+ Q6_V_valign_VVR(v2, v2, 8));
+
+ HVX_Vector sum = Q6_V_vmux_QVV(
+ q44, Q6_V_vrdelta_VV(s0, p0),
+ Q6_V_vmux_QVV(q88, Q6_V_vrdelta_VV(s1, p1), Q6_V_vrdelta_VV(s2, p2)));
+
+ HVX_VectorPair prod = Q6_Wuw_vmpy_VuhRuh(sum, scale_splat);
+ HVX_Vector scaled = Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(prod), Q6_V_lo_W(prod));
+ HVX_Vector out = Q6_Vub_vpack_VhVh_sat(scaled, scaled);
+ vstoreu_partial(dst_uv, out, w * 2);
+ src_uv += 192;
+ dst_uv += 64;
+ dst_width -= 32;
+ }
+}
+
+void ScaleRowDown2_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 128 ? 128 : dst_width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -1);
+ vstoreu_partial(dst, Q6_V_hi_W(d), w);
+ src_ptr += 256;
+ dst += 128;
+ dst_width -= 128;
+ }
+}
+
+void ScaleRowDown2Linear_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 128 ? 128 : dst_width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_Vector avg = Q6_Vub_vavg_VubVub_rnd(Q6_V_lo_W(d), Q6_V_hi_W(d));
+ vstoreu_partial(dst, avg, w);
+ src_ptr += 256;
+ dst += 128;
+ dst_width -= 128;
+ }
+}
+
+void ScaleRowDown2Box_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ const int32_t kOnes = 0x01010101;
+ HVX_Vector vtwo = Q6_Vh_vsplat_R(2);
+ while (dst_width > 0) {
+ int w = dst_width >= 128 ? 128 : dst_width;
+ int src_bytes = w * 2;
+ HVX_Vector s0 = vloadu_partial_zero(s, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t0 = vloadu_partial_zero(t, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector sum0 = Q6_Vh_vdmpy_VubRb(s0, kOnes);
+ sum0 = Q6_Vh_vdmpyacc_VhVubRb(sum0, t0, kOnes);
+ sum0 = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum0, vtwo), 2);
+
+ HVX_Vector sum1 = Q6_V_vzero();
+ if (src_bytes > 128) {
+ HVX_Vector s1 = vloadu_partial_zero(s + 128, src_bytes - 128);
+ HVX_Vector t1 = vloadu_partial_zero(t + 128, src_bytes - 128);
+ sum1 = Q6_Vh_vdmpy_VubRb(s1, kOnes);
+ sum1 = Q6_Vh_vdmpyacc_VhVubRb(sum1, t1, kOnes);
+ sum1 = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum1, vtwo), 2);
+ }
+ HVX_Vector out = Q6_Vub_vpack_VhVh_sat(sum1, sum0);
+ vstoreu_partial(dst, out, w);
+ s += 256;
+ t += 256;
+ dst += 128;
+ dst_width -= 128;
+ }
+}
+
+void ScaleRowDown2Box_Odd_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ dst_width -= 1;
+ if (dst_width > 0) {
+ ScaleRowDown2Box_HVX(src_ptr, src_stride, dst, dst_width);
+ }
+ const uint8_t* s = src_ptr + dst_width * 2;
+ const uint8_t* t = s + src_stride;
+ dst[dst_width] = (s[0] + t[0] + 1) >> 1;
+}
+
+void ScaleRowDown2_16_HVX(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 64, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -2);
+ vstoreu_partial(dst, Q6_V_hi_W(d), w * 2);
+ src_ptr += 128;
+ dst += 64;
+ dst_width -= 64;
+ }
+}
+
+void ScaleRowDown2Linear_16_HVX(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 64, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -2);
+ HVX_Vector avg = Q6_Vuh_vavg_VuhVuh_rnd(Q6_V_lo_W(d), Q6_V_hi_W(d));
+ vstoreu_partial(dst, avg, w * 2);
+ src_ptr += 128;
+ dst += 64;
+ dst_width -= 64;
+ }
+}
+
+void ScaleRowDown2Box_16_HVX(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ const uint16_t* s = src_ptr;
+ const uint16_t* t = src_ptr + src_stride;
+ HVX_Vector vtwo = Q6_V_vsplat_R(2);
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 = vloadu_partial_zero(s, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t0 = vloadu_partial_zero(t, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(s + 64, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(t + 64, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair ds = Q6_W_vdeal_VVR(s1, s0, -2);
+ HVX_VectorPair dt = Q6_W_vdeal_VVR(t1, t0, -2);
+ HVX_VectorPair s_even = Q6_Wuw_vunpack_Vuh(Q6_V_lo_W(ds));
+ HVX_VectorPair s_odd = Q6_Wuw_vunpack_Vuh(Q6_V_hi_W(ds));
+ HVX_VectorPair t_even = Q6_Wuw_vunpack_Vuh(Q6_V_lo_W(dt));
+ HVX_VectorPair t_odd = Q6_Wuw_vunpack_Vuh(Q6_V_hi_W(dt));
+
+ HVX_Vector sum_lo = Q6_Vw_vadd_VwVw(Q6_V_lo_W(s_even), Q6_V_lo_W(s_odd));
+ sum_lo = Q6_Vw_vadd_VwVw(sum_lo, Q6_V_lo_W(t_even));
+ sum_lo = Q6_Vw_vadd_VwVw(sum_lo, Q6_V_lo_W(t_odd));
+ sum_lo = Q6_Vw_vasr_VwR(Q6_Vw_vadd_VwVw(sum_lo, vtwo), 2);
+
+ HVX_Vector sum_hi = Q6_Vw_vadd_VwVw(Q6_V_hi_W(s_even), Q6_V_hi_W(s_odd));
+ sum_hi = Q6_Vw_vadd_VwVw(sum_hi, Q6_V_hi_W(t_even));
+ sum_hi = Q6_Vw_vadd_VwVw(sum_hi, Q6_V_hi_W(t_odd));
+ sum_hi = Q6_Vw_vasr_VwR(Q6_Vw_vadd_VwVw(sum_hi, vtwo), 2);
+
+ HVX_Vector out = Q6_Vuh_vpack_VwVw_sat(sum_hi, sum_lo);
+ vstoreu_partial(dst, out, w * 2);
+ s += 128;
+ t += 128;
+ dst += 64;
+ dst_width -= 64;
+ }
+}
+
+void ScaleRowDown4_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 128 ? 128 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 =
+ src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 128,
+ src_bytes > 256 ? 128 : src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector s2 =
+ src_bytes > 256
+ ? vloadu_partial_zero(src_ptr + 256,
+ src_bytes > 384 ? 128 : src_bytes - 256)
+ : Q6_V_vzero();
+ HVX_Vector s3 = src_bytes > 384
+ ? vloadu_partial_zero(src_ptr + 384, src_bytes - 384)
+ : Q6_V_vzero();
+ HVX_VectorPair d01 = Q6_W_vdeal_VVR(s1, s0, -1);
+ HVX_VectorPair d23 = Q6_W_vdeal_VVR(s3, s2, -1);
+ HVX_VectorPair d_even = Q6_W_vdeal_VVR(Q6_V_lo_W(d23), Q6_V_lo_W(d01), -1);
+ vstoreu_partial(dst_ptr, Q6_V_hi_W(d_even), w);
+ src_ptr += 512;
+ dst_ptr += 128;
+ dst_width -= 128;
+ }
+}
+
+void ScaleRowDown4Box_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* r0 = src_ptr;
+ const uint8_t* r1 = r0 + src_stride;
+ const uint8_t* r2 = r1 + src_stride;
+ const uint8_t* r3 = r2 + src_stride;
+ const int32_t kOnes = 0x01010101;
+ HVX_Vector veight = Q6_Vh_vsplat_R(8);
+ while (dst_width > 0) {
+ int w = dst_width >= 128 ? 128 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector sum[4];
+ for (int c = 0; c < 4; ++c) {
+ int rem = src_bytes - c * 128;
+ if (rem > 0) {
+ int b = rem > 128 ? 128 : rem;
+ HVX_Vector v0 = vloadu_partial_zero(r0 + c * 128, b);
+ HVX_Vector v1 = vloadu_partial_zero(r1 + c * 128, b);
+ HVX_Vector v2 = vloadu_partial_zero(r2 + c * 128, b);
+ HVX_Vector v3 = vloadu_partial_zero(r3 + c * 128, b);
+ HVX_Vector s = Q6_Vw_vrmpy_VubRb(v0, kOnes);
+ s = Q6_Vw_vrmpyacc_VwVubRb(s, v1, kOnes);
+ s = Q6_Vw_vrmpyacc_VwVubRb(s, v2, kOnes);
+ s = Q6_Vw_vrmpyacc_VwVubRb(s, v3, kOnes);
+ sum[c] = s;
+ } else {
+ sum[c] = Q6_V_vzero();
+ }
+ }
+ HVX_Vector s01 = Q6_Vh_vpack_VwVw_sat(sum[1], sum[0]);
+ HVX_Vector s23 = Q6_Vh_vpack_VwVw_sat(sum[3], sum[2]);
+ s01 = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(s01, veight), 4);
+ s23 = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(s23, veight), 4);
+ HVX_Vector out = Q6_Vub_vpack_VhVh_sat(s23, s01);
+ vstoreu_partial(dst_ptr, out, w);
+ r0 += 512;
+ r1 += 512;
+ r2 += 512;
+ r3 += 512;
+ dst_ptr += 128;
+ dst_width -= 128;
+ }
+}
+
+void ScaleARGBRowDown2_HVX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 32 ? 32 : dst_width;
+ int src_bytes = w * 8;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -4);
+ vstoreu_partial(dst_argb, Q6_V_hi_W(d), w * 4);
+ src_argb += 256;
+ dst_argb += 128;
+ dst_width -= 32;
+ }
+}
+
+void ScaleARGBRowDown2Linear_HVX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 32 ? 32 : dst_width;
+ int src_bytes = w * 8;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_argb, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_argb + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -4);
+ HVX_Vector avg = Q6_Vub_vavg_VubVub_rnd(Q6_V_lo_W(d), Q6_V_hi_W(d));
+ vstoreu_partial(dst_argb, avg, w * 4);
+ src_argb += 256;
+ dst_argb += 128;
+ dst_width -= 32;
+ }
+}
+
+void ScaleARGBRowDown2Box_HVX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const uint8_t* s = src_argb;
+ const uint8_t* t = src_argb + src_stride;
+ HVX_Vector vtwo = Q6_Vh_vsplat_R(2);
+ while (dst_width > 0) {
+ int w = dst_width >= 32 ? 32 : dst_width;
+ int src_bytes = w * 8;
+ HVX_Vector s0 = vloadu_partial_zero(s, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t0 = vloadu_partial_zero(t, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(s + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(t + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair ds = Q6_W_vdeal_VVR(s1, s0, -4);
+ HVX_VectorPair dt = Q6_W_vdeal_VVR(t1, t0, -4);
+ HVX_VectorPair s_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(ds));
+ HVX_VectorPair s_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(ds));
+ HVX_VectorPair t_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(dt));
+ HVX_VectorPair t_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(dt));
+
+ HVX_Vector sum_lo = Q6_Vh_vadd_VhVh(Q6_V_lo_W(s_even), Q6_V_lo_W(s_odd));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_even));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_odd));
+ sum_lo = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_lo, vtwo), 2);
+
+ HVX_Vector sum_hi = Q6_Vh_vadd_VhVh(Q6_V_hi_W(s_even), Q6_V_hi_W(s_odd));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_even));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_odd));
+ sum_hi = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_hi, vtwo), 2);
+
+ HVX_Vector out = Q6_Vub_vpack_VhVh_sat(sum_hi, sum_lo);
+ vstoreu_partial(dst_argb, out, w * 4);
+ s += 256;
+ t += 256;
+ dst_argb += 128;
+ dst_width -= 32;
+ }
+}
+
+void ScaleUVRowDown2_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -2);
+ vstoreu_partial(dst_uv, Q6_V_hi_W(d), w * 2);
+ src_ptr += 256;
+ dst_uv += 128;
+ dst_width -= 64;
+ }
+}
+
+void ScaleUVRowDown2Linear_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 =
+ vloadu_partial_zero(src_ptr, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(src_ptr + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair d = Q6_W_vdeal_VVR(s1, s0, -2);
+ HVX_Vector avg = Q6_Vub_vavg_VubVub_rnd(Q6_V_lo_W(d), Q6_V_hi_W(d));
+ vstoreu_partial(dst_uv, avg, w * 2);
+ src_ptr += 256;
+ dst_uv += 128;
+ dst_width -= 64;
+ }
+}
+
+void ScaleUVRowDown2Box_HVX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ HVX_Vector vtwo = Q6_Vh_vsplat_R(2);
+ while (dst_width > 0) {
+ int w = dst_width >= 64 ? 64 : dst_width;
+ int src_bytes = w * 4;
+ HVX_Vector s0 = vloadu_partial_zero(s, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector t0 = vloadu_partial_zero(t, src_bytes > 128 ? 128 : src_bytes);
+ HVX_Vector s1 = src_bytes > 128
+ ? vloadu_partial_zero(s + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_Vector t1 = src_bytes > 128
+ ? vloadu_partial_zero(t + 128, src_bytes - 128)
+ : Q6_V_vzero();
+ HVX_VectorPair ds = Q6_W_vdeal_VVR(s1, s0, -2);
+ HVX_VectorPair dt = Q6_W_vdeal_VVR(t1, t0, -2);
+ HVX_VectorPair s_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(ds));
+ HVX_VectorPair s_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(ds));
+ HVX_VectorPair t_even = Q6_Wuh_vunpack_Vub(Q6_V_lo_W(dt));
+ HVX_VectorPair t_odd = Q6_Wuh_vunpack_Vub(Q6_V_hi_W(dt));
+
+ HVX_Vector sum_lo = Q6_Vh_vadd_VhVh(Q6_V_lo_W(s_even), Q6_V_lo_W(s_odd));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_even));
+ sum_lo = Q6_Vh_vadd_VhVh(sum_lo, Q6_V_lo_W(t_odd));
+ sum_lo = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_lo, vtwo), 2);
+
+ HVX_Vector sum_hi = Q6_Vh_vadd_VhVh(Q6_V_hi_W(s_even), Q6_V_hi_W(s_odd));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_even));
+ sum_hi = Q6_Vh_vadd_VhVh(sum_hi, Q6_V_hi_W(t_odd));
+ sum_hi = Q6_Vh_vasr_VhR(Q6_Vh_vadd_VhVh(sum_hi, vtwo), 2);
+
+ HVX_Vector out = Q6_Vub_vpack_VhVh_sat(sum_hi, sum_lo);
+ vstoreu_partial(dst_uv, out, w * 2);
+ s += 256;
+ t += 256;
+ dst_uv += 128;
+ dst_width -= 64;
+ }
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
diff --git a/libs/libyuv/source/scale_lsx.cc b/libs/libyuv/source/scale_lsx.cc
new file mode 100644
index 0000000000..cdb333ee42
--- /dev/null
+++ b/libs/libyuv/source/scale_lsx.cc
@@ -0,0 +1,975 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+
+#include "libyuv/scale_row.h"
+
+#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+static const uint64_t kScale38Shuf[2] = {0x13100e0b08060300ULL,
+ 0x000000001e1b1816ULL};
+static const uint64_t kScale38BoxShuf[2] = {0x0a08160604120200ULL,
+ 0x000000001e0e0c1aULL};
+static const uint64_t kScale34Shuf[2] = {0x0908070504030100ULL,
+ 0x000000000f0d0c0bULL};
+static const uint64_t kScale34BoxLo[2] = {0x0504030202010100ULL,
+ 0x0000000007060605ULL};
+static const uint64_t kScale34BoxHi[2] = {0x0d0c0b0a0a090908ULL,
+ 0x000000000f0e0e0dULL};
+static const uint64_t kScale34BoxWt[2] = {0x0103030102020103ULL,
+ 0x0000000003010202ULL};
+static const uint64_t kScale34Pack12[2] = {0x0908050403020100ULL,
+ 0x000000000d0c0b0aULL};
+static const uint64_t kRamp0123[2] = {0x0000000100000000ULL,
+ 0x0000000300000002ULL};
+
+void ScaleARGBRowDown2_LSX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vpickod.w $vr0, $vr1, $vr0 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void ScaleARGBRowDown2Linear_LSX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vpickev.w $vr2, $vr1, $vr0 \n"
+ "vpickod.w $vr3, $vr1, $vr0 \n"
+ "vavgr.bu $vr0, $vr3, $vr2 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const uint8_t* t = src_argb + src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[s], 0 \n"
+ "vld $vr1, %[s], 16 \n"
+ "vld $vr2, %[t], 0 \n"
+ "vld $vr3, %[t], 16 \n"
+ "vpickev.w $vr4, $vr1, $vr0 \n"
+ "vpickod.w $vr5, $vr1, $vr0 \n"
+ "vpickev.w $vr6, $vr3, $vr2 \n"
+ "vpickod.w $vr7, $vr3, $vr2 \n"
+ "vaddwev.h.bu $vr0, $vr4, $vr5 \n"
+ "vaddwod.h.bu $vr1, $vr4, $vr5 \n"
+ "vaddwev.h.bu $vr2, $vr6, $vr7 \n"
+ "vaddwod.h.bu $vr3, $vr6, $vr7 \n"
+ "vadd.h $vr0, $vr0, $vr2 \n"
+ "vadd.h $vr1, $vr1, $vr3 \n"
+ "vsrari.h $vr0, $vr0, 2 \n"
+ "vsrari.h $vr1, $vr1, 2 \n"
+ "vpackev.b $vr0, $vr1, $vr0 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[s], %[s], 32 \n"
+ "addi.d %[t], %[t], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [s] "+r"(src_argb), // %[s]
+ [t] "+r"(t), // %[t]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void ScaleARGBRowDownEven_LSX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int32_t src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ uint32_t r0, r1, r2, r3;
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "ld.w %[r0], %[src], 0 \n"
+ "add.d %[src], %[src], %[step] \n"
+ "ld.w %[r1], %[src], 0 \n"
+ "add.d %[src], %[src], %[step] \n"
+ "ld.w %[r2], %[src], 0 \n"
+ "add.d %[src], %[src], %[step] \n"
+ "ld.w %[r3], %[src], 0 \n"
+ "add.d %[src], %[src], %[step] \n"
+ "st.w %[r0], %[dst], 0 \n"
+ "st.w %[r1], %[dst], 4 \n"
+ "st.w %[r2], %[dst], 8 \n"
+ "st.w %[r3], %[dst], 12 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width), // %[w]
+ [r0] "=&r"(r0), // %[r0]
+ [r1] "=&r"(r1), // %[r1]
+ [r2] "=&r"(r2), // %[r2]
+ [r3] "=&r"(r3) // %[r3]
+ : [step] "r"(step) // %[step]
+ : "memory");
+}
+
+void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ const uint8_t* next_argb = src_argb + src_stride;
+ uint64_t r0, r1;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "ld.d %[r0], %[s], 0 \n"
+ "add.d %[s], %[s], %[step] \n"
+ "ld.d %[r1], %[s], 0 \n"
+ "add.d %[s], %[s], %[step] \n"
+ "vinsgr2vr.d $vr0, %[r0], 0 \n"
+ "vinsgr2vr.d $vr0, %[r1], 1 \n"
+ "ld.d %[r0], %[s], 0 \n"
+ "add.d %[s], %[s], %[step] \n"
+ "ld.d %[r1], %[s], 0 \n"
+ "add.d %[s], %[s], %[step] \n"
+ "vinsgr2vr.d $vr1, %[r0], 0 \n"
+ "vinsgr2vr.d $vr1, %[r1], 1 \n"
+
+ "ld.d %[r0], %[t], 0 \n"
+ "add.d %[t], %[t], %[step] \n"
+ "ld.d %[r1], %[t], 0 \n"
+ "add.d %[t], %[t], %[step] \n"
+ "vinsgr2vr.d $vr2, %[r0], 0 \n"
+ "vinsgr2vr.d $vr2, %[r1], 1 \n"
+ "ld.d %[r0], %[t], 0 \n"
+ "add.d %[t], %[t], %[step] \n"
+ "ld.d %[r1], %[t], 0 \n"
+ "add.d %[t], %[t], %[step] \n"
+ "vinsgr2vr.d $vr3, %[r0], 0 \n"
+ "vinsgr2vr.d $vr3, %[r1], 1 \n"
+
+ "vpickev.w $vr4, $vr1, $vr0 \n"
+ "vpickod.w $vr5, $vr1, $vr0 \n"
+ "vpickev.w $vr6, $vr3, $vr2 \n"
+ "vpickod.w $vr7, $vr3, $vr2 \n"
+ "vaddwev.h.bu $vr0, $vr4, $vr5 \n"
+ "vaddwod.h.bu $vr1, $vr4, $vr5 \n"
+ "vaddwev.h.bu $vr2, $vr6, $vr7 \n"
+ "vaddwod.h.bu $vr3, $vr6, $vr7 \n"
+ "vadd.h $vr0, $vr0, $vr2 \n"
+ "vadd.h $vr1, $vr1, $vr3 \n"
+ "vsrari.h $vr0, $vr0, 2 \n"
+ "vsrari.h $vr1, $vr1, 2 \n"
+ "vpackev.b $vr0, $vr1, $vr0 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [s] "+r"(src_argb), // %[s]
+ [t] "+r"(next_argb), // %[t]
+ [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width), // %[w]
+ [r0] "=&r"(r0), // %[r0]
+ [r1] "=&r"(r1) // %[r1]
+ : [step] "r"(step) // %[step]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void ScaleRowDown2_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vpickod.b $vr0, $vr1, $vr0 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void ScaleRowDown2Linear_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vpickev.b $vr2, $vr1, $vr0 \n"
+ "vpickod.b $vr3, $vr1, $vr0 \n"
+ "vavgr.bu $vr0, $vr3, $vr2 \n"
+ "vst $vr0, %[dst], 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void ScaleRowDown2Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ const uint8_t* src_nex = src_ptr + src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vld $vr2, %[nex], 0 \n"
+ "vld $vr3, %[nex], 16 \n"
+ "vaddwev.h.bu $vr4, $vr0, $vr2 \n"
+ "vaddwod.h.bu $vr5, $vr0, $vr2 \n"
+ "vaddwev.h.bu $vr6, $vr1, $vr3 \n"
+ "vaddwod.h.bu $vr7, $vr1, $vr3 \n"
+ "vadd.h $vr4, $vr4, $vr5 \n"
+ "vadd.h $vr6, $vr6, $vr7 \n"
+ "vsrarni.b.h $vr6, $vr4, 2 \n"
+ "vst $vr6, %[dst], 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[nex], %[nex], 32 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [nex] "+r"(src_nex), // %[nex]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void ScaleRowDown4_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vpickev.b $vr0, $vr1, $vr0 \n"
+ "vpickod.b $vr0, $vr0, $vr0 \n"
+ "vstelm.d $vr0, %[dst], 0, 0 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 8 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1");
+}
+
+void ScaleRowDown4Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ const uint8_t* ptr1 = src_ptr + src_stride;
+ const uint8_t* ptr2 = ptr1 + src_stride;
+ const uint8_t* ptr3 = ptr2 + src_stride;
+ asm volatile(
+ "srli.w %[w], %[w], 3 \n"
+ "beqz %[w], 2f \n"
+ "1: \n"
+ "vld $vr0, %[p0], 0 \n"
+ "vld $vr1, %[p0], 16 \n"
+ "vld $vr2, %[p1], 0 \n"
+ "vld $vr3, %[p1], 16 \n"
+ "vld $vr4, %[p2], 0 \n"
+ "vld $vr5, %[p2], 16 \n"
+ "vld $vr6, %[p3], 0 \n"
+ "vld $vr7, %[p3], 16 \n"
+ "vhaddw.hu.bu $vr0, $vr0, $vr0 \n"
+ "vhaddw.hu.bu $vr1, $vr1, $vr1 \n"
+ "vhaddw.hu.bu $vr2, $vr2, $vr2 \n"
+ "vhaddw.hu.bu $vr3, $vr3, $vr3 \n"
+ "vhaddw.hu.bu $vr4, $vr4, $vr4 \n"
+ "vhaddw.hu.bu $vr5, $vr5, $vr5 \n"
+ "vhaddw.hu.bu $vr6, $vr6, $vr6 \n"
+ "vhaddw.hu.bu $vr7, $vr7, $vr7 \n"
+ "vadd.h $vr0, $vr0, $vr2 \n"
+ "vadd.h $vr1, $vr1, $vr3 \n"
+ "vadd.h $vr4, $vr4, $vr6 \n"
+ "vadd.h $vr5, $vr5, $vr7 \n"
+ "vadd.h $vr0, $vr0, $vr4 \n"
+ "vadd.h $vr1, $vr1, $vr5 \n"
+ "vhaddw.wu.hu $vr0, $vr0, $vr0 \n"
+ "vhaddw.wu.hu $vr1, $vr1, $vr1 \n"
+ "vsrarni.h.w $vr1, $vr0, 4 \n"
+ "vsrlni.b.h $vr1, $vr1, 0 \n"
+ "vstelm.d $vr1, %[dst], 0, 0 \n"
+ "addi.d %[p0], %[p0], 32 \n"
+ "addi.d %[p1], %[p1], 32 \n"
+ "addi.d %[p2], %[p2], 32 \n"
+ "addi.d %[p3], %[p3], 32 \n"
+ "addi.d %[dst], %[dst], 8 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [p0] "+r"(src_ptr), // %[p0]
+ [p1] "+r"(ptr1), // %[p1]
+ [p2] "+r"(ptr2), // %[p2]
+ [p3] "+r"(ptr3), // %[p3]
+ [dst] "+r"(dst), // %[dst]
+ [w] "+r"(dst_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void ScaleRowDown38_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ int count = dst_width / 12;
+ (void)src_stride;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vld $vr2, %[shuf], 0 \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vshuf.b $vr0, $vr1, $vr0, $vr2 \n"
+ "vstelm.d $vr0, %[dst], 0, 0 \n"
+ "vstelm.w $vr0, %[dst], 8, 2 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [shuf] "r"(kScale38Shuf) // %[shuf]
+ : "memory", "$vr0", "$vr1", "$vr2");
+}
+
+void ScaleRowDown38_2_Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_nex = src_ptr + src_stride;
+ int count = dst_width / 12;
+ int c2aaa = 0x2aaa;
+ int c4000 = 0x4000;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vld $vr8, %[shuf], 0 \n"
+ "vreplgr2vr.h $vr9, %[c2aaa] \n"
+ "vreplgr2vr.w $vr10, %[c4000] \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vld $vr1, %[src], 16 \n"
+ "vld $vr2, %[nex], 0 \n"
+ "vld $vr3, %[nex], 16 \n"
+ "vaddwev.h.bu $vr4, $vr0, $vr2 \n"
+ "vaddwod.h.bu $vr5, $vr0, $vr2 \n"
+ "vaddwev.h.bu $vr6, $vr1, $vr3 \n"
+ "vaddwod.h.bu $vr7, $vr1, $vr3 \n"
+ "vpickev.h $vr0, $vr6, $vr4 \n"
+ "vpickev.h $vr1, $vr7, $vr5 \n"
+ "vpackod.h $vr2, $vr5, $vr4 \n"
+ "vpackod.h $vr3, $vr7, $vr6 \n"
+ "vpickev.w $vr4, $vr3, $vr2 \n"
+ "vadd.h $vr0, $vr0, $vr1 \n"
+ "vadd.h $vr0, $vr0, $vr4 \n"
+ "vmuh.h $vr0, $vr0, $vr9 \n"
+ "vpickod.w $vr2, $vr3, $vr2 \n"
+ "vhaddw.wu.hu $vr2, $vr2, $vr2 \n"
+ "vmul.w $vr2, $vr2, $vr10 \n"
+ "vshuf.b $vr0, $vr2, $vr0, $vr8 \n"
+ "vstelm.d $vr0, %[dst], 0, 0 \n"
+ "vstelm.w $vr0, %[dst], 8, 2 \n"
+ "addi.d %[src], %[src], 32 \n"
+ "addi.d %[nex], %[nex], 32 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [nex] "+r"(src_nex), // %[nex]
+ [dst] "+r"(dst_ptr), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [shuf] "r"(kScale38BoxShuf), // %[shuf]
+ [c2aaa] "r"(c2aaa), // %[c2aaa]
+ [c4000] "r"(c4000) // %[c4000]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10");
+}
+
+void ScaleRowDown38_3_Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* ptr1 = src_ptr + src_stride;
+ const uint8_t* ptr2 = ptr1 + src_stride;
+ int count = dst_width / 12;
+ int c1c71 = 0x1c71;
+ int c2aaa = 0x2aaa;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vldi $vr11, 0 \n"
+ "vld $vr8, %[shuf], 0 \n"
+ "vreplgr2vr.h $vr9, %[c1c71] \n"
+ "vreplgr2vr.w $vr10, %[c2aaa] \n"
+ "1: \n"
+ "vld $vr0, %[p0], 0 \n"
+ "vld $vr1, %[p0], 16 \n"
+ "vld $vr2, %[p1], 0 \n"
+ "vld $vr3, %[p1], 16 \n"
+ "vaddwev.h.bu $vr4, $vr0, $vr2 \n"
+ "vaddwod.h.bu $vr5, $vr0, $vr2 \n"
+ "vaddwev.h.bu $vr6, $vr1, $vr3 \n"
+ "vaddwod.h.bu $vr7, $vr1, $vr3 \n"
+ "vld $vr0, %[p2], 0 \n"
+ "vld $vr1, %[p2], 16 \n"
+ "vpackev.b $vr2, $vr11, $vr0 \n"
+ "vpackod.b $vr3, $vr11, $vr0 \n"
+ "vadd.h $vr4, $vr4, $vr2 \n"
+ "vadd.h $vr5, $vr5, $vr3 \n"
+ "vpackev.b $vr2, $vr11, $vr1 \n"
+ "vpackod.b $vr3, $vr11, $vr1 \n"
+ "vadd.h $vr6, $vr6, $vr2 \n"
+ "vadd.h $vr7, $vr7, $vr3 \n"
+ "vpickev.h $vr0, $vr6, $vr4 \n"
+ "vpickev.h $vr1, $vr7, $vr5 \n"
+ "vpackod.h $vr2, $vr5, $vr4 \n"
+ "vpackod.h $vr3, $vr7, $vr6 \n"
+ "vpickev.w $vr4, $vr3, $vr2 \n"
+ "vadd.h $vr0, $vr0, $vr1 \n"
+ "vadd.h $vr0, $vr0, $vr4 \n"
+ "vmuh.h $vr0, $vr0, $vr9 \n"
+ "vpickod.w $vr2, $vr3, $vr2 \n"
+ "vhaddw.wu.hu $vr2, $vr2, $vr2 \n"
+ "vmul.w $vr2, $vr2, $vr10 \n"
+ "vshuf.b $vr0, $vr2, $vr0, $vr8 \n"
+ "vstelm.d $vr0, %[dst], 0, 0 \n"
+ "vstelm.w $vr0, %[dst], 8, 2 \n"
+ "addi.d %[p0], %[p0], 32 \n"
+ "addi.d %[p1], %[p1], 32 \n"
+ "addi.d %[p2], %[p2], 32 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [p0] "+r"(src_ptr), // %[p0]
+ [p1] "+r"(ptr1), // %[p1]
+ [p2] "+r"(ptr2), // %[p2]
+ [dst] "+r"(dst_ptr), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [shuf] "r"(kScale38BoxShuf), // %[shuf]
+ [c1c71] "r"(c1c71), // %[c1c71]
+ [c2aaa] "r"(c2aaa) // %[c2aaa]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11");
+}
+
+void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
+ asm volatile(
+ "srli.w %[w], %[w], 4 \n"
+ "beqz %[w], 2f \n"
+ "vldi $vr0, 0 \n"
+ "1: \n"
+ "vld $vr1, %[src], 0 \n"
+ "vld $vr2, %[dst], 0 \n"
+ "vld $vr3, %[dst], 16 \n"
+ "vilvl.b $vr4, $vr0, $vr1 \n"
+ "vilvh.b $vr5, $vr0, $vr1 \n"
+ "vadd.h $vr2, $vr2, $vr4 \n"
+ "vadd.h $vr3, $vr3, $vr5 \n"
+ "vst $vr2, %[dst], 0 \n"
+ "vst $vr3, %[dst], 16 \n"
+ "addi.d %[src], %[src], 16 \n"
+ "addi.d %[dst], %[dst], 32 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst_ptr), // %[dst]
+ [w] "+r"(src_width) // %[w]
+ :
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
+}
+
+static const uint8_t kScaleUVAddCols3Shuf[] = {
+ 0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7,
+ 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11,
+ 8, 9, 10, 11, 20, 21, 22, 23, 0, 1, 2, 3, 12, 13, 14, 15,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 24, 25, 26, 27,
+ 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31,
+};
+
+void ScaleUVAddCols3_LSX(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ asm volatile(
+ "vld $vr6, %[shuf], 0 \n"
+ "vld $vr7, %[shuf], 16 \n"
+ "vld $vr8, %[shuf], 32 \n"
+ "vld $vr9, %[shuf], 48 \n"
+ "vld $vr10, %[shuf], 64 \n"
+ "vld $vr11, %[shuf], 80 \n"
+ "vreplgr2vr.h $vr12, %[scaleval] \n"
+ "1: \n"
+ "vld $vr0, %[src_uv], 0 \n"
+ "vld $vr1, %[src_uv], 16 \n"
+ "vld $vr2, %[src_uv], 32 \n"
+ "vshuf.b $vr3, $vr1, $vr0, $vr6 \n"
+ "vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
+ "vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
+ "vshuf.b $vr3, $vr2, $vr3, $vr9 \n"
+ "vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
+ "vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
+ "vadd.h $vr3, $vr3, $vr4 \n"
+ "vadd.h $vr3, $vr3, $vr5 \n"
+ "vmuh.hu $vr3, $vr3, $vr12 \n"
+ "vld $vr0, %[src_uv], 48 \n"
+ "vld $vr1, %[src_uv], 64 \n"
+ "vld $vr2, %[src_uv], 80 \n"
+ "addi.d %[src_uv], %[src_uv], 96 \n"
+ "vshuf.b $vr13, $vr1, $vr0, $vr6 \n"
+ "vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
+ "vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
+ "vshuf.b $vr13, $vr2, $vr13, $vr9 \n"
+ "vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
+ "vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
+ "vadd.h $vr13, $vr13, $vr4 \n"
+ "vadd.h $vr13, $vr13, $vr5 \n"
+ "vmuh.hu $vr13, $vr13, $vr12 \n"
+ "vpickev.b $vr0, $vr13, $vr3 \n"
+ "vst $vr0, %[dst_uv], 0 \n"
+ "addi.d %[dst_uv], %[dst_uv], 16 \n"
+ "addi.w %[dst_width], %[dst_width], -8 \n"
+ "bgtz %[dst_width], 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ : [scaleval] "r"(scaleval), // %[scaleval]
+ [shuf] "r"(kScaleUVAddCols3Shuf)
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13");
+}
+
+void ScaleFilterCols_LSX(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ ptrdiff_t i0, i1, i2, i3;
+ uint32_t p0, p1;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "vld $vr0, %[ramp], 0 \n"
+ "vreplgr2vr.w $vr1, %[dx] \n"
+ "vreplgr2vr.w $vr2, %[x] \n"
+ "vmul.w $vr0, $vr1, $vr0 \n"
+ "vslli.w $vr1, $vr1, 2 \n"
+ "vadd.w $vr2, $vr2, $vr0 \n"
+ "vldi $vr3, 0x7f \n"
+ "vsrli.w $vr3, $vr3, 24 \n"
+ "vaddi.wu $vr4, $vr3, 1 \n"
+ "1: \n"
+ "vsrai.w $vr5, $vr2, 16 \n"
+ "vsrai.w $vr6, $vr2, 9 \n"
+ "vand.v $vr6, $vr6, $vr3 \n"
+ "vsub.w $vr7, $vr4, $vr6 \n"
+ "vslli.w $vr6, $vr6, 8 \n"
+ "vor.v $vr6, $vr6, $vr7 \n"
+ "vadd.w $vr2, $vr2, $vr1 \n"
+ "vpickve2gr.w %[i0], $vr5, 0 \n"
+ "vpickve2gr.w %[i1], $vr5, 1 \n"
+ "vpickve2gr.w %[i2], $vr5, 2 \n"
+ "vpickve2gr.w %[i3], $vr5, 3 \n"
+ "ldx.h %[p0], %[src], %[i0] \n"
+ "ldx.h %[p1], %[src], %[i1] \n"
+ "vinsgr2vr.h $vr5, %[p0], 0 \n"
+ "vinsgr2vr.h $vr5, %[p1], 2 \n"
+ "ldx.h %[p0], %[src], %[i2] \n"
+ "ldx.h %[p1], %[src], %[i3] \n"
+ "vinsgr2vr.h $vr5, %[p0], 4 \n"
+ "vinsgr2vr.h $vr5, %[p1], 6 \n"
+ "vmulwev.h.bu $vr7, $vr5, $vr6 \n"
+ "vmaddwod.h.bu $vr7, $vr5, $vr6 \n"
+ "vsrari.h $vr5, $vr7, 7 \n"
+ "vpickev.b $vr5, $vr5, $vr5 \n"
+ "vpickev.b $vr5, $vr5, $vr5 \n"
+ "vstelm.w $vr5, %[dst], 0, 0 \n"
+ "addi.d %[dst], %[dst], 4 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [dst] "+r"(dst_ptr), // %[dst]
+ [w] "+r"(dst_width), // %[w]
+ [i0] "=&r"(i0), // %[i0]
+ [i1] "=&r"(i1), // %[i1]
+ [i2] "=&r"(i2), // %[i2]
+ [i3] "=&r"(i3), // %[i3]
+ [p0] "=&r"(p0), // %[p0]
+ [p1] "=&r"(p1) // %[p1]
+ : [src] "r"(src_ptr), // %[src]
+ [x] "r"(x), // %[x]
+ [dx] "r"(dx), // %[dx]
+ [ramp] "r"(kRamp0123) // %[ramp]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7");
+}
+
+void ScaleARGBCols_LSX(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ ptrdiff_t i0, i1, i2, i3;
+ uint32_t p0, p1;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "vld $vr0, %[ramp], 0 \n"
+ "vreplgr2vr.w $vr1, %[dx] \n"
+ "vreplgr2vr.w $vr2, %[x] \n"
+ "vmul.w $vr0, $vr1, $vr0 \n"
+ "vslli.w $vr1, $vr1, 2 \n"
+ "vadd.w $vr2, $vr2, $vr0 \n"
+ "1: \n"
+ "vsrai.w $vr3, $vr2, 16 \n"
+ "vslli.w $vr3, $vr3, 2 \n"
+ "vadd.w $vr2, $vr2, $vr1 \n"
+ "vpickve2gr.w %[i0], $vr3, 0 \n"
+ "vpickve2gr.w %[i1], $vr3, 1 \n"
+ "vpickve2gr.w %[i2], $vr3, 2 \n"
+ "vpickve2gr.w %[i3], $vr3, 3 \n"
+ "ldx.w %[p0], %[src], %[i0] \n"
+ "ldx.w %[p1], %[src], %[i1] \n"
+ "st.w %[p0], %[dst], 0 \n"
+ "st.w %[p1], %[dst], 4 \n"
+ "ldx.w %[p0], %[src], %[i2] \n"
+ "ldx.w %[p1], %[src], %[i3] \n"
+ "st.w %[p0], %[dst], 8 \n"
+ "st.w %[p1], %[dst], 12 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width), // %[w]
+ [i0] "=&r"(i0), // %[i0]
+ [i1] "=&r"(i1), // %[i1]
+ [i2] "=&r"(i2), // %[i2]
+ [i3] "=&r"(i3), // %[i3]
+ [p0] "=&r"(p0), // %[p0]
+ [p1] "=&r"(p1) // %[p1]
+ : [src] "r"(src_argb), // %[src]
+ [x] "r"(x), // %[x]
+ [dx] "r"(dx), // %[dx]
+ [ramp] "r"(kRamp0123) // %[ramp]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3");
+}
+
+void ScaleARGBFilterCols_LSX(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ ptrdiff_t i0, i1, i2, i3;
+ uint64_t p0, p1;
+ asm volatile(
+ "srli.w %[w], %[w], 2 \n"
+ "beqz %[w], 2f \n"
+ "vld $vr0, %[ramp], 0 \n"
+ "vreplgr2vr.w $vr1, %[dx] \n"
+ "vreplgr2vr.w $vr2, %[x] \n"
+ "vmul.w $vr0, $vr1, $vr0 \n"
+ "vslli.w $vr1, $vr1, 2 \n"
+ "vadd.w $vr2, $vr2, $vr0 \n"
+ "vldi $vr3, 0x7f \n"
+ "1: \n"
+ "vsrai.w $vr4, $vr2, 16 \n"
+ "vslli.w $vr4, $vr4, 2 \n"
+ "vsrai.w $vr5, $vr2, 9 \n"
+ "vadd.w $vr2, $vr2, $vr1 \n"
+ "vand.v $vr5, $vr5, $vr3 \n"
+ "vshuf4i.b $vr5, $vr5, 0x00 \n"
+ "vxor.v $vr6, $vr5, $vr3 \n"
+ "vilvl.b $vr7, $vr5, $vr6 \n"
+ "vilvh.b $vr8, $vr5, $vr6 \n"
+ "vpickve2gr.w %[i0], $vr4, 0 \n"
+ "vpickve2gr.w %[i1], $vr4, 1 \n"
+ "vpickve2gr.w %[i2], $vr4, 2 \n"
+ "vpickve2gr.w %[i3], $vr4, 3 \n"
+ "ldx.d %[p0], %[src], %[i0] \n"
+ "ldx.d %[p1], %[src], %[i1] \n"
+ "vinsgr2vr.d $vr4, %[p0], 0 \n"
+ "vinsgr2vr.d $vr4, %[p1], 1 \n"
+ "ldx.d %[p0], %[src], %[i2] \n"
+ "ldx.d %[p1], %[src], %[i3] \n"
+ "vinsgr2vr.d $vr5, %[p0], 0 \n"
+ "vinsgr2vr.d $vr5, %[p1], 1 \n"
+ "vpickev.w $vr6, $vr5, $vr4 \n"
+ "vpickod.w $vr4, $vr5, $vr4 \n"
+ "vilvl.b $vr5, $vr4, $vr6 \n"
+ "vilvh.b $vr4, $vr4, $vr6 \n"
+ "vmulwev.h.bu $vr6, $vr5, $vr7 \n"
+ "vmaddwod.h.bu $vr6, $vr5, $vr7 \n"
+ "vmulwev.h.bu $vr7, $vr4, $vr8 \n"
+ "vmaddwod.h.bu $vr7, $vr4, $vr8 \n"
+ "vsrani.b.h $vr7, $vr6, 7 \n"
+ "vst $vr7, %[dst], 0 \n"
+ "addi.d %[dst], %[dst], 16 \n"
+ "addi.w %[w], %[w], -1 \n"
+ "bnez %[w], 1b \n"
+ "2: \n"
+ : [dst] "+r"(dst_argb), // %[dst]
+ [w] "+r"(dst_width), // %[w]
+ [i0] "=&r"(i0), // %[i0]
+ [i1] "=&r"(i1), // %[i1]
+ [i2] "=&r"(i2), // %[i2]
+ [i3] "=&r"(i3), // %[i3]
+ [p0] "=&r"(p0), // %[p0]
+ [p1] "=&r"(p1) // %[p1]
+ : [src] "r"(src_argb), // %[src]
+ [x] "r"(x), // %[x]
+ [dx] "r"(dx), // %[dx]
+ [ramp] "r"(kRamp0123) // %[ramp]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8");
+}
+
+void ScaleRowDown34_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ int count = dst_width / 12;
+ (void)src_stride;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vld $vr1, %[shuf], 0 \n"
+ "1: \n"
+ "vld $vr0, %[src], 0 \n"
+ "vshuf.b $vr0, $vr0, $vr0, $vr1 \n"
+ "vstelm.d $vr0, %[dst], 0, 0 \n"
+ "vstelm.w $vr0, %[dst], 8, 2 \n"
+ "addi.d %[src], %[src], 16 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [shuf] "r"(kScale34Shuf) // %[shuf]
+ : "memory", "$vr0", "$vr1");
+}
+
+void ScaleRowDown34_0_Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* d,
+ int dst_width) {
+ const uint8_t* src_nex = src_ptr + src_stride;
+ int count = dst_width / 12;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vld $vr0, %[lo], 0 \n"
+ "vld $vr1, %[hi], 0 \n"
+ "vld $vr2, %[wt], 0 \n"
+ "vld $vr3, %[pack], 0 \n"
+ "1: \n"
+ "vld $vr4, %[src], 0 \n"
+ "vld $vr5, %[nex], 0 \n"
+ "vshuf.b $vr6, $vr4, $vr4, $vr0 \n"
+ "vshuf.b $vr7, $vr4, $vr4, $vr1 \n"
+ "vshuf.b $vr8, $vr5, $vr5, $vr0 \n"
+ "vshuf.b $vr9, $vr5, $vr5, $vr1 \n"
+ "vmulwev.h.bu $vr4, $vr6, $vr2 \n"
+ "vmaddwod.h.bu $vr4, $vr6, $vr2 \n"
+ "vmulwev.h.bu $vr5, $vr7, $vr2 \n"
+ "vmaddwod.h.bu $vr5, $vr7, $vr2 \n"
+ "vmulwev.h.bu $vr6, $vr8, $vr2 \n"
+ "vmaddwod.h.bu $vr6, $vr8, $vr2 \n"
+ "vmulwev.h.bu $vr7, $vr9, $vr2 \n"
+ "vmaddwod.h.bu $vr7, $vr9, $vr2 \n"
+ "vsrari.h $vr4, $vr4, 2 \n"
+ "vsrari.h $vr5, $vr5, 2 \n"
+ "vsrari.h $vr6, $vr6, 2 \n"
+ "vsrari.h $vr7, $vr7, 2 \n"
+ "vslli.h $vr8, $vr4, 1 \n"
+ "vslli.h $vr9, $vr5, 1 \n"
+ "vadd.h $vr4, $vr4, $vr8 \n"
+ "vadd.h $vr5, $vr5, $vr9 \n"
+ "vadd.h $vr4, $vr4, $vr6 \n"
+ "vadd.h $vr5, $vr5, $vr7 \n"
+ "vsrarni.b.h $vr5, $vr4, 2 \n"
+ "vshuf.b $vr5, $vr5, $vr5, $vr3 \n"
+ "vstelm.d $vr5, %[dst], 0, 0 \n"
+ "vstelm.w $vr5, %[dst], 8, 2 \n"
+ "addi.d %[src], %[src], 16 \n"
+ "addi.d %[nex], %[nex], 16 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [nex] "+r"(src_nex), // %[nex]
+ [dst] "+r"(d), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [lo] "r"(kScale34BoxLo), // %[lo]
+ [hi] "r"(kScale34BoxHi), // %[hi]
+ [wt] "r"(kScale34BoxWt), // %[wt]
+ [pack] "r"(kScale34Pack12) // %[pack]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9");
+}
+
+void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* d,
+ int dst_width) {
+ const uint8_t* src_nex = src_ptr + src_stride;
+ int count = dst_width / 12;
+ asm volatile(
+ "beqz %[cnt], 2f \n"
+ "vld $vr0, %[lo], 0 \n"
+ "vld $vr1, %[hi], 0 \n"
+ "vld $vr2, %[wt], 0 \n"
+ "vld $vr3, %[pack], 0 \n"
+ "1: \n"
+ "vld $vr4, %[src], 0 \n"
+ "vld $vr5, %[nex], 0 \n"
+ "vshuf.b $vr6, $vr4, $vr4, $vr0 \n"
+ "vshuf.b $vr7, $vr4, $vr4, $vr1 \n"
+ "vshuf.b $vr8, $vr5, $vr5, $vr0 \n"
+ "vshuf.b $vr9, $vr5, $vr5, $vr1 \n"
+ "vmulwev.h.bu $vr4, $vr6, $vr2 \n"
+ "vmaddwod.h.bu $vr4, $vr6, $vr2 \n"
+ "vmulwev.h.bu $vr5, $vr7, $vr2 \n"
+ "vmaddwod.h.bu $vr5, $vr7, $vr2 \n"
+ "vmulwev.h.bu $vr6, $vr8, $vr2 \n"
+ "vmaddwod.h.bu $vr6, $vr8, $vr2 \n"
+ "vmulwev.h.bu $vr7, $vr9, $vr2 \n"
+ "vmaddwod.h.bu $vr7, $vr9, $vr2 \n"
+ "vsrari.h $vr4, $vr4, 2 \n"
+ "vsrari.h $vr5, $vr5, 2 \n"
+ "vsrari.h $vr6, $vr6, 2 \n"
+ "vsrari.h $vr7, $vr7, 2 \n"
+ "vadd.h $vr4, $vr4, $vr6 \n"
+ "vadd.h $vr5, $vr5, $vr7 \n"
+ "vsrarni.b.h $vr5, $vr4, 1 \n"
+ "vshuf.b $vr5, $vr5, $vr5, $vr3 \n"
+ "vstelm.d $vr5, %[dst], 0, 0 \n"
+ "vstelm.w $vr5, %[dst], 8, 2 \n"
+ "addi.d %[src], %[src], 16 \n"
+ "addi.d %[nex], %[nex], 16 \n"
+ "addi.d %[dst], %[dst], 12 \n"
+ "addi.w %[cnt], %[cnt], -1 \n"
+ "bnez %[cnt], 1b \n"
+ "2: \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [nex] "+r"(src_nex), // %[nex]
+ [dst] "+r"(d), // %[dst]
+ [cnt] "+r"(count) // %[cnt]
+ : [lo] "r"(kScale34BoxLo), // %[lo]
+ [hi] "r"(kScale34BoxHi), // %[hi]
+ [wt] "r"(kScale34BoxWt), // %[wt]
+ [pack] "r"(kScale34Pack12) // %[pack]
+ : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+ "$vr7", "$vr8", "$vr9");
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
diff --git a/libs/libyuv/source/scale_mmi.cc b/libs/libyuv/source/scale_mmi.cc
deleted file mode 100644
index 990463c2a6..0000000000
--- a/libs/libyuv/source/scale_mmi.cc
+++ /dev/null
@@ -1,1113 +0,0 @@
-/*
- * Copyright 2013 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "libyuv/scale.h"
-
-#include <assert.h>
-#include <string.h>
-
-#include "libyuv/cpu_id.h"
-#include "libyuv/planar_functions.h" // For CopyARGB
-#include "libyuv/row.h"
-#include "libyuv/scale_row.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-// This module is for Mips MMI.
-#if !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-// clang-format off
-
-// CPU agnostic row functions
-void ScaleRowDown2_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1, dest;
- const uint64_t shift = 0x8ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "psrlh %[src0], %[src0], %[shift] \n\t"
-
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "psrlh %[src1], %[src1], %[shift] \n\t"
-
- "packushb %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [shift] "f"(shift)
- : "memory");
-}
-
-void ScaleRowDown2Linear_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest0, dest1;
-
- const uint64_t mask = 0x00ff00ff00ff00ffULL;
- const uint64_t shift = 0x8ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "and %[dest0], %[src0], %[mask] \n\t"
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "and %[dest1], %[src1], %[mask] \n\t"
- "packushb %[dest0], %[dest0], %[dest1] \n\t"
-
- "psrlh %[src0], %[src0], %[shift] \n\t"
- "psrlh %[src1], %[src1], %[shift] \n\t"
- "packushb %[dest1], %[src0], %[src1] \n\t"
-
- "pavgb %[dest], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest0] "=&f"(dest0),
- [dest1] "=&f"(dest1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [mask] "f"(mask),
- [shift] "f"(shift), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleRowDown2Box_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
-
- uint64_t s0, s1, t0, t1;
- uint64_t dest, dest0, dest1;
-
- const uint64_t ph = 0x0002000200020002ULL;
- const uint64_t mask = 0x00ff00ff00ff00ffULL;
- const uint64_t shift0 = 0x2ULL;
- const uint64_t shift1 = 0x8ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[s0], 0x00(%[s]) \n\t"
- "gsldlc1 %[s0], 0x07(%[s]) \n\t"
- "psrlh %[s1], %[s0], %[shift1] \n\t"
- "and %[s0], %[s0], %[mask] \n\t"
-
- "gsldrc1 %[t0], 0x00(%[t]) \n\t"
- "gsldlc1 %[t0], 0x07(%[t]) \n\t"
- "psrlh %[t1], %[t0], %[shift1] \n\t"
- "and %[t0], %[t0], %[mask] \n\t"
-
- "paddh %[dest0], %[s0], %[s1] \n\t"
- "paddh %[dest0], %[dest0], %[t0] \n\t"
- "paddh %[dest0], %[dest0], %[t1] \n\t"
- "paddh %[dest0], %[dest0], %[ph] \n\t"
- "psrlh %[dest0], %[dest0], %[shift0] \n\t"
-
- "gsldrc1 %[s0], 0x08(%[s]) \n\t"
- "gsldlc1 %[s0], 0x0f(%[s]) \n\t"
- "psrlh %[s1], %[s0], %[shift1] \n\t"
- "and %[s0], %[s0], %[mask] \n\t"
-
- "gsldrc1 %[t0], 0x08(%[t]) \n\t"
- "gsldlc1 %[t0], 0x0f(%[t]) \n\t"
- "psrlh %[t1], %[t0], %[shift1] \n\t"
- "and %[t0], %[t0], %[mask] \n\t"
-
- "paddh %[dest1], %[s0], %[s1] \n\t"
- "paddh %[dest1], %[dest1], %[t0] \n\t"
- "paddh %[dest1], %[dest1], %[t1] \n\t"
- "paddh %[dest1], %[dest1], %[ph] \n\t"
- "psrlh %[dest1], %[dest1], %[shift0] \n\t"
-
- "packushb %[dest], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[s], %[s], 0x10 \n\t"
- "daddiu %[t], %[t], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [s0] "=&f"(s0), [s1] "=&f"(s1), [t0] "=&f"(t0), [t1] "=&f"(t1),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest] "=&f"(dest)
- : [s] "r"(s), [t] "r"(t), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [shift0] "f"(shift0), [shift1] "f"(shift1), [ph] "f"(ph),
- [mask] "f"(mask)
- : "memory");
-}
-
-void ScaleARGBRowDown2_MMI(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- (void)src_stride;
-
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
-
- uint64_t src0, src1, dest;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "punpckhwd %[dest], %[src0], %[src1] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src), [dst_ptr] "r"(dst), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleARGBRowDown2Linear_MMI(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest_hi, dest_lo;
-
- __asm__ volatile(
- "1: \n\t"
- "lwc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "lwc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "punpcklwd %[dest_lo], %[src0], %[src1] \n\t"
- "lwc1 %[src0], 0x04(%[src_ptr]) \n\t"
- "lwc1 %[src1], 0x0c(%[src_ptr]) \n\t"
- "punpcklwd %[dest_hi], %[src0], %[src1] \n\t"
-
- "pavgb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleARGBRowDown2Box_MMI(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- const uint8_t* s = src_argb;
- const uint8_t* t = src_argb + src_stride;
-
- uint64_t s0, s_hi, s_lo;
- uint64_t t0, t_hi, t_lo;
- uint64_t dest, dest_hi, dest_lo;
-
- const uint64_t mask = 0x0ULL;
- const uint64_t ph = 0x0002000200020002ULL;
- const uint64_t shfit = 0x2ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[s0], 0x00(%[s]) \n\t"
- "gsldlc1 %[s0], 0x07(%[s]) \n\t"
- "punpcklbh %[s_lo], %[s0], %[mask] \n\t"
- "punpckhbh %[s_hi], %[s0], %[mask] \n\t"
- "paddh %[dest_lo], %[s_lo], %[s_hi] \n\t"
-
- "gsldrc1 %[t0], 0x00(%[t]) \n\t"
- "gsldlc1 %[t0], 0x07(%[t]) \n\t"
- "punpcklbh %[t_lo], %[t0], %[mask] \n\t"
- "punpckhbh %[t_hi], %[t0], %[mask] \n\t"
- "paddh %[dest_lo], %[dest_lo], %[t_lo] \n\t"
- "paddh %[dest_lo], %[dest_lo], %[t_hi] \n\t"
-
- "paddh %[dest_lo], %[dest_lo], %[ph] \n\t"
- "psrlh %[dest_lo], %[dest_lo], %[shfit] \n\t"
-
- "gsldrc1 %[s0], 0x08(%[s]) \n\t"
- "gsldlc1 %[s0], 0x0f(%[s]) \n\t"
- "punpcklbh %[s_lo], %[s0], %[mask] \n\t"
- "punpckhbh %[s_hi], %[s0], %[mask] \n\t"
- "paddh %[dest_hi], %[s_lo], %[s_hi] \n\t"
-
- "gsldrc1 %[t0], 0x08(%[t]) \n\t"
- "gsldlc1 %[t0], 0x0f(%[t]) \n\t"
- "punpcklbh %[t_lo], %[t0], %[mask] \n\t"
- "punpckhbh %[t_hi], %[t0], %[mask] \n\t"
- "paddh %[dest_hi], %[dest_hi], %[t_lo] \n\t"
- "paddh %[dest_hi], %[dest_hi], %[t_hi] \n\t"
-
- "paddh %[dest_hi], %[dest_hi], %[ph] \n\t"
- "psrlh %[dest_hi], %[dest_hi], %[shfit] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[s], %[s], 0x10 \n\t"
- "daddiu %[t], %[t], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [s0] "=&f"(s0), [t0] "=&f"(t0), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [s_hi] "=&f"(s_hi), [s_lo] "=&f"(s_lo),
- [t_hi] "=&f"(t_hi), [t_lo] "=&f"(t_lo), [dest] "=&f"(dest)
- : [s] "r"(s), [t] "r"(t), [dst_ptr] "r"(dst_argb), [width] "r"(dst_width),
- [mask] "f"(mask), [ph] "f"(ph), [shfit] "f"(shfit)
- : "memory");
-}
-
-void ScaleRowDown2_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1, dest;
- const uint64_t shift = 0x10ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "psrlw %[src0], %[src0], %[shift] \n\t"
-
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "psrlw %[src1], %[src1], %[shift] \n\t"
-
- "packsswh %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [shift] "f"(shift)
- : "memory");
-}
-
-void ScaleRowDown2Linear_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest_hi, dest_lo;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "punpcklhw %[dest_lo], %[src0], %[src1] \n\t"
- "punpckhhw %[dest_hi], %[src0], %[src1] \n\t"
-
- "punpcklhw %[src0], %[dest_lo], %[dest_hi] \n\t"
- "punpckhhw %[src1], %[dest_lo], %[dest_hi] \n\t"
-
- "pavgh %[dest], %[src0], %[src1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleRowDown2Box_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- const uint16_t* s = src_ptr;
- const uint16_t* t = src_ptr + src_stride;
-
- uint64_t s0, s1, s_hi, s_lo;
- uint64_t t0, t1, t_hi, t_lo;
- uint64_t dest, dest0, dest1;
-
- const uint64_t ph = 0x0000000200000002ULL;
- const uint64_t mask = 0x0000ffff0000ffffULL;
- const uint64_t shift0 = 0x10ULL;
- const uint64_t shift1 = 0x2ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[s0], 0x00(%[s]) \n\t"
- "gsldlc1 %[s0], 0x07(%[s]) \n\t"
- "psrlw %[s1], %[s0], %[shift0] \n\t"
- "and %[s0], %[s0], %[mask] \n\t"
-
- "gsldrc1 %[t0], 0x00(%[t]) \n\t"
- "gsldlc1 %[t0], 0x07(%[t]) \n\t"
- "psrlw %[t1], %[t0], %[shift0] \n\t"
- "and %[t0], %[t0], %[mask] \n\t"
-
- "paddw %[dest0], %[s0], %[s1] \n\t"
- "paddw %[dest0], %[dest0], %[t0] \n\t"
- "paddw %[dest0], %[dest0], %[t1] \n\t"
- "paddw %[dest0], %[dest0], %[ph] \n\t"
- "psrlw %[dest0], %[dest0], %[shift1] \n\t"
-
- "gsldrc1 %[s0], 0x08(%[s]) \n\t"
- "gsldlc1 %[s0], 0x0f(%[s]) \n\t"
- "psrlw %[s1], %[s0], %[shift0] \n\t"
- "and %[s0], %[s0], %[mask] \n\t"
-
- "gsldrc1 %[t0], 0x08(%[t]) \n\t"
- "gsldlc1 %[t0], 0x0f(%[t]) \n\t"
- "psrlw %[t1], %[t0], %[shift0] \n\t"
- "and %[t0], %[t0], %[mask] \n\t"
-
- "paddw %[dest1], %[s0], %[s1] \n\t"
- "paddw %[dest1], %[dest1], %[t0] \n\t"
- "paddw %[dest1], %[dest1], %[t1] \n\t"
- "paddw %[dest1], %[dest1], %[ph] \n\t"
- "psrlw %[dest1], %[dest1], %[shift1] \n\t"
-
- "packsswh %[dest], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[s], %[s], 0x10 \n\t"
- "daddiu %[t], %[t], 0x10 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [s0] "=&f"(s0), [s1] "=&f"(s1), [t0] "=&f"(t0), [t1] "=&f"(t1),
- [s_hi] "=&f"(s_hi), [s_lo] "=&f"(s_lo), [t_hi] "=&f"(t_hi),
- [t_lo] "=&f"(t_lo), [dest0] "=&f"(dest0), [dest1] "=&f"(dest1),
- [dest] "=&f"(dest)
- : [s] "r"(s), [t] "r"(t), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [shift0] "f"(shift0), [shift1] "f"(shift1), [ph] "f"(ph),
- [mask] "f"(mask)
- : "memory");
-}
-
-void ScaleRowDown4_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest_hi, dest_lo;
-
- const uint64_t shift = 0x10ULL;
- const uint64_t mask = 0x000000ff000000ffULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "psrlw %[src0], %[src0], %[shift] \n\t"
- "and %[src0], %[src0], %[mask] \n\t"
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "psrlw %[src1], %[src1], %[shift] \n\t"
- "and %[src1], %[src1], %[mask] \n\t"
- "packsswh %[dest_lo], %[src0], %[src1] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_ptr]) \n\t"
- "psrlw %[src0], %[src0], %[shift] \n\t"
- "and %[src0], %[src0], %[mask] \n\t"
- "gsldrc1 %[src1], 0x18(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_ptr]) \n\t"
- "psrlw %[src1], %[src1], %[shift] \n\t"
- "and %[src1], %[src1], %[mask] \n\t"
- "packsswh %[dest_hi], %[src0], %[src1] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x20 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [shift] "f"(shift), [mask] "f"(mask)
- : "memory");
-}
-
-void ScaleRowDown4_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest_hi, dest_lo;
-
- const uint64_t mask = 0x0ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src1], 0x08(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x0f(%[src_ptr]) \n\t"
- "punpckhhw %[dest_lo], %[src0], %[src1] \n\t"
- "punpcklhw %[dest_lo], %[dest_lo], %[mask] \n\t"
-
- "gsldrc1 %[src0], 0x10(%[src_ptr]) \n\t"
- "gsldlc1 %[src0], 0x17(%[src_ptr]) \n\t"
- "gsldrc1 %[src1], 0x18(%[src_ptr]) \n\t"
- "gsldlc1 %[src1], 0x1f(%[src_ptr]) \n\t"
- "punpckhhw %[dest_hi], %[src0], %[src1] \n\t"
- "punpcklhw %[dest_hi], %[dest_hi], %[mask] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x20 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest_hi] "=&f"(dest_hi),
- [dest_lo] "=&f"(dest_lo), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [mask] "f"(mask)
- : "memory");
-}
-
-#define DO_SCALEROWDOWN4BOX_PUNPCKADD() \
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t" \
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t" \
- "paddh %[dest_lo], %[dest_lo], %[src_lo] \n\t" \
- "paddh %[dest_hi], %[dest_hi], %[src_hi] \n\t"
-
-#define DO_SCALEROWDOWN4BOX_LOOP(reg) \
- "ldc1 %[src], 0x00(%[src0_ptr]) \n\t" \
- "punpcklbh %[dest_lo], %[src], %[mask0] \n\t" \
- "punpckhbh %[dest_hi], %[src], %[mask0] \n\t" \
- \
- "ldc1 %[src], 0x00(%[src1_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_PUNPCKADD() \
- \
- "ldc1 %[src], 0x00(%[src2_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_PUNPCKADD() \
- \
- "ldc1 %[src], 0x00(%[src3_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_PUNPCKADD() \
- \
- "pmaddhw %[dest_lo], %[dest_lo], %[mask1] \n\t" \
- "pmaddhw %[dest_hi], %[dest_hi], %[mask1] \n\t" \
- "packsswh " #reg ", %[dest_lo], %[dest_hi] \n\t" \
- "pmaddhw " #reg ", " #reg ", %[mask1] \n\t" \
- "paddh " #reg ", " #reg ", %[ph] \n\t" \
- "psrlh " #reg ", " #reg ", %[shift] \n\t" \
- \
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t" \
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t" \
- "daddiu %[src2_ptr], %[src2_ptr], 0x08 \n\t" \
- "daddiu %[src3_ptr], %[src3_ptr], 0x08 \n\t"
-
-/* LibYUVScaleTest.ScaleDownBy4_Box */
-void ScaleRowDown4Box_MMI(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- const uint8_t* src0_ptr = src_ptr;
- const uint8_t* src1_ptr = src_ptr + src_stride;
- const uint8_t* src2_ptr = src_ptr + src_stride * 2;
- const uint8_t* src3_ptr = src_ptr + src_stride * 3;
-
- uint64_t src, src_hi, src_lo;
- uint64_t dest, dest_hi, dest_lo, dest0, dest1, dest2, dest3;
-
- const uint64_t mask0 = 0x0ULL;
- const uint64_t mask1 = 0x0001000100010001ULL;
- const uint64_t ph = 0x0008000800080008ULL;
- const uint64_t shift = 0x4ULL;
-
- __asm__ volatile(
- "1: \n\t"
-
- DO_SCALEROWDOWN4BOX_LOOP(%[dest0])
- DO_SCALEROWDOWN4BOX_LOOP(%[dest1])
- DO_SCALEROWDOWN4BOX_LOOP(%[dest2])
- DO_SCALEROWDOWN4BOX_LOOP(%[dest3])
-
- "packsswh %[dest_lo], %[dest0], %[dest1] \n\t"
- "packsswh %[dest_hi], %[dest2], %[dest3] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3), [src] "=&f"(src), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src0_ptr), [src1_ptr] "r"(src1_ptr),
- [src2_ptr] "r"(src2_ptr), [src3_ptr] "r"(src3_ptr), [dst_ptr] "r"(dst),
- [width] "r"(dst_width), [shift] "f"(shift), [mask0] "f"(mask0),
- [ph] "f"(ph), [mask1] "f"(mask1)
- : "memory");
-}
-
-#define DO_SCALEROWDOWN4BOX_16_PUNPCKADD() \
- "punpcklbh %[src_lo], %[src], %[mask0] \n\t" \
- "punpckhbh %[src_hi], %[src], %[mask0] \n\t" \
- "paddh %[dest_lo], %[dest_lo], %[src_lo] \n\t" \
- "paddh %[dest_hi], %[dest_hi], %[src_hi] \n\t"
-
-#define DO_SCALEROWDOWN4BOX_16_LOOP(reg) \
- "ldc1 %[src], 0x00(%[src0_ptr]) \n\t" \
- "punpcklbh %[dest_lo], %[src], %[mask0] \n\t" \
- "punpckhbh %[dest_hi], %[src], %[mask0] \n\t" \
- \
- "ldc1 %[src], 0x00(%[src1_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_16_PUNPCKADD() \
- \
- "ldc1 %[src], 0x00(%[src2_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_16_PUNPCKADD() \
- \
- "ldc1 %[src], 0x00(%[src3_ptr]) \n\t" \
- DO_SCALEROWDOWN4BOX_16_PUNPCKADD() \
- \
- "paddw %[dest], %[dest_lo], %[dest_hi] \n\t" \
- "punpckhwd %[dest_hi], %[dest], %[dest] \n\t" \
- "paddw %[dest], %[dest_hi], %[dest] \n\t" \
- "paddw %[dest], %[dest], %[ph] \n\t" \
- "psraw %[dest], %[dest], %[shift] \n\t" \
- "and " #reg ", %[dest], %[mask1] \n\t" \
- \
- "daddiu %[src0_ptr], %[src0_ptr], 0x08 \n\t" \
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t" \
- "daddiu %[src2_ptr], %[src2_ptr], 0x08 \n\t" \
- "daddiu %[src3_ptr], %[src3_ptr], 0x08 \n\t"
-
-/* LibYUVScaleTest.ScaleDownBy4_Box_16 */
-void ScaleRowDown4Box_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- const uint16_t* src0_ptr = src_ptr;
- const uint16_t* src1_ptr = src_ptr + src_stride;
- const uint16_t* src2_ptr = src_ptr + src_stride * 2;
- const uint16_t* src3_ptr = src_ptr + src_stride * 3;
-
- uint64_t src, src_hi, src_lo;
- uint64_t dest, dest_hi, dest_lo, dest0, dest1, dest2, dest3;
-
- const uint64_t mask0 = 0x0ULL;
- const uint64_t mask1 = 0x00000000ffffffffULL;
- const uint64_t ph = 0x0000000800000008ULL;
- const uint64_t shift = 0x04ULL;
-
- __asm__ volatile(
- "1: \n\t"
-
- DO_SCALEROWDOWN4BOX_16_LOOP(%[dest0])
- DO_SCALEROWDOWN4BOX_16_LOOP(%[dest1])
- DO_SCALEROWDOWN4BOX_16_LOOP(%[dest2])
- DO_SCALEROWDOWN4BOX_16_LOOP(%[dest3])
- "punpcklwd %[dest_lo], %[dest0], %[dest1] \n\t"
- "punpcklwd %[dest_hi], %[dest2], %[dest3] \n\t"
-
- "packushb %[dest], %[dest_lo], %[dest_hi] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [dest2] "=&f"(dest2),
- [dest3] "=&f"(dest3), [src] "=&f"(src), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src0_ptr), [src1_ptr] "r"(src1_ptr),
- [src2_ptr] "r"(src2_ptr), [src3_ptr] "r"(src3_ptr), [dst_ptr] "r"(dst),
- [width] "r"(dst_width), [shift] "f"(shift), [mask0] "f"(mask0),
- [ph] "f"(ph), [mask1] "f"(mask1)
- : "memory");
-}
-
-// Scales a single row of pixels up by 2x using point sampling.
-void ScaleColsUp2_MMI(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- uint64_t src, dest;
-
- (void)x;
- (void)dx;
-
- __asm__ volatile(
- "1: \n\t"
- "lwc1 %[src], 0x00(%[src_ptr]) \n\t"
-
- "punpcklbh %[dest], %[src], %[src] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x04 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst_ptr), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleColsUp2_16_MMI(uint16_t* dst_ptr,
- const uint16_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- uint64_t src, dest;
-
- (void)x;
- (void)dx;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
-
- "punpcklhw %[dest], %[src], %[src] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "punpckhhw %[dest], %[src], %[src] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src] "=&f"(src), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst_ptr), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleAddRow_MMI(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
- uint64_t src, src_hi, src_lo, dest0, dest1;
- const uint64_t mask = 0x0ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src], %[mask] \n\t"
- "punpckhbh %[src_hi], %[src], %[mask] \n\t"
-
- "gsldrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
- "gsldlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "paddush %[dest0], %[dest0], %[src_lo] \n\t"
- "gsldrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
- "gsldlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "paddush %[dest1], %[dest1], %[src_hi] \n\t"
-
- "gssdlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
- "gssdlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [src_hi] "=&f"(src_hi),
- [src_lo] "=&f"(src_lo), [src] "=&f"(src)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst_ptr), [width] "r"(src_width),
- [mask] "f"(mask)
- : "memory");
-}
-
-void ScaleAddRow_16_MMI(const uint16_t* src_ptr,
- uint32_t* dst_ptr,
- int src_width) {
- uint64_t src, src_hi, src_lo, dest0, dest1;
- const uint64_t mask = 0x0ULL;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "punpcklhw %[src_lo], %[src], %[mask] \n\t"
- "punpckhhw %[src_hi], %[src], %[mask] \n\t"
-
- "gsldrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
- "gsldlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "paddw %[dest0], %[dest0], %[src_lo] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
-
- "gsldrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
- "gsldlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "paddw %[dest1], %[dest1], %[src_hi] \n\t"
- "gssdlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [src_hi] "=&f"(src_hi),
- [src_lo] "=&f"(src_lo), [src] "=&f"(src)
- : [src_ptr] "r"(src_ptr), [dst_ptr] "r"(dst_ptr), [width] "r"(src_width),
- [mask] "f"(mask)
- : "memory");
-}
-
-void ScaleARGBRowDownEven_MMI(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- (void)src_stride;
-
- uint64_t src0, src1, dest;
-
- __asm__ volatile(
- "1: \n\t"
- "lwc1 %[src0], 0x00(%[src_ptr]) \n\t"
- "dadd %[src_ptr], %[src_ptr], %[src_stepx_4]\n\t"
- "lwc1 %[src1], 0x00(%[src_ptr]) \n\t"
- "punpcklwd %[dest], %[src0], %[src1] \n\t"
-
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "dadd %[src_ptr], %[src_ptr], %[src_stepx_4]\n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb),
- [src_stepx_4] "r"(src_stepx << 2), [width] "r"(dst_width)
- : "memory");
-}
-
-void ScaleARGBRowDownEvenBox_MMI(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- const uint8_t* src0_ptr = src_argb;
- const uint8_t* src1_ptr = src_argb + src_stride;
-
- uint64_t src0, src1, src_hi, src_lo;
- uint64_t dest, dest_hi, dest_lo, dest0, dest1;
-
- const uint64_t mask = 0x0ULL;
- const uint64_t ph = 0x0002000200020002ULL;
- const uint64_t shift = 0x2ULL;
-
- __asm__ volatile(
- "1: \n\t"
-
- "lwc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "punpcklbh %[dest_lo], %[src0], %[mask] \n\t"
- "lwc1 %[src0], 0x04(%[src0_ptr]) \n\t"
- "punpcklbh %[dest_hi], %[src0], %[mask] \n\t"
-
- "lwc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src1], %[mask] \n\t"
- "lwc1 %[src1], 0x04(%[src1_ptr]) \n\t"
- "punpcklbh %[src_hi], %[src1], %[mask] \n\t"
- "paddh %[dest_lo], %[dest_lo], %[src_lo] \n\t"
- "paddh %[dest_hi], %[dest_hi], %[src_hi] \n\t"
- "paddh %[dest0], %[dest_hi], %[dest_lo] \n\t"
- "paddh %[dest0], %[dest0], %[ph] \n\t"
- "psrlh %[dest0], %[dest0], %[shift] \n\t"
-
- "dadd %[src0_ptr], %[src0_ptr], %[src_stepx_4] \n\t"
- "dadd %[src1_ptr], %[src1_ptr], %[src_stepx_4] \n\t"
-
- "lwc1 %[src0], 0x00(%[src0_ptr]) \n\t"
- "punpcklbh %[dest_lo], %[src0], %[mask] \n\t"
- "lwc1 %[src0], 0x04(%[src0_ptr]) \n\t"
- "punpcklbh %[dest_hi], %[src0], %[mask] \n\t"
-
- "lwc1 %[src1], 0x00(%[src1_ptr]) \n\t"
- "punpcklbh %[src_lo], %[src1], %[mask] \n\t"
- "lwc1 %[src1], 0x04(%[src1_ptr]) \n\t"
- "punpcklbh %[src_hi], %[src1], %[mask] \n\t"
- "paddh %[dest_lo], %[dest_lo], %[src_lo] \n\t"
- "paddh %[dest_hi], %[dest_hi], %[src_hi] \n\t"
- "paddh %[dest1], %[dest_hi], %[dest_lo] \n\t"
- "paddh %[dest1], %[dest1], %[ph] \n\t"
- "psrlh %[dest1], %[dest1], %[shift] \n\t"
-
- "packushb %[dest], %[dest0], %[dest1] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "dadd %[src0_ptr], %[src0_ptr], %[src_stepx_4] \n\t"
- "dadd %[src1_ptr], %[src1_ptr], %[src_stepx_4] \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x08 \n\t"
- "daddi %[width], %[width], -0x02 \n\t"
- "bnez %[width], 1b \n\t"
- : [src_hi] "=&f"(src_hi), [src_lo] "=&f"(src_lo),
- [dest_hi] "=&f"(dest_hi), [dest_lo] "=&f"(dest_lo),
- [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [src0] "=&f"(src0),
- [src1] "=&f"(src1), [dest] "=&f"(dest)
- : [src0_ptr] "r"(src0_ptr), [src1_ptr] "r"(src1_ptr),
- [dst_ptr] "r"(dst_argb), [width] "r"(dst_width),
- [src_stepx_4] "r"(src_stepx << 2), [shift] "f"(shift), [mask] "f"(mask),
- [ph] "f"(ph)
- : "memory");
-}
-
-// Scales a single row of pixels using point sampling.
-void ScaleARGBCols_MMI(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
-
- const uint32_t* src_tmp;
-
- uint64_t dest, offset;
-
- const uint64_t shift0 = 16;
- const uint64_t shift1 = 2;
-
- __asm__ volatile(
- "1: \n\t"
- "srav %[offset], %[x], %[shift0] \n\t"
- "sllv %[offset], %[offset], %[shift1] \n\t"
- "dadd %[src_tmp], %[src_ptr], %[offset] \n\t"
- "lwc1 %[dest], 0x00(%[src_tmp]) \n\t"
- "swc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- "dadd %[x], %[x], %[dx] \n\t"
-
- "daddiu %[dst_ptr], %[dst_ptr], 0x04 \n\t"
- "daddi %[width], %[width], -0x01 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest] "=&f"(dest), [offset] "=&r"(offset), [src_tmp] "=&r"(src_tmp)
- : [src_ptr] "r"(src), [dst_ptr] "r"(dst), [width] "r"(dst_width),
- [dx] "r"(dx), [x] "r"(x), [shift0] "r"(shift0), [shift1] "r"(shift1)
- : "memory");
-}
-
-// Scales a single row of pixels up by 2x using point sampling.
-void ScaleARGBColsUp2_MMI(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- uint64_t src, dest0, dest1;
- (void)x;
- (void)dx;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src], 0x00(%[src_ptr]) \n\t"
- "gsldlc1 %[src], 0x07(%[src_ptr]) \n\t"
- "punpcklwd %[dest0], %[src], %[src] \n\t"
- "gssdlc1 %[dest0], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest0], 0x00(%[dst_ptr]) \n\t"
- "punpckhwd %[dest1], %[src], %[src] \n\t"
- "gssdlc1 %[dest1], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest1], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src_ptr], %[src_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x04 \n\t"
- "bnez %[width], 1b \n\t"
- : [dest0] "=&f"(dest0), [dest1] "=&f"(dest1), [src] "=&f"(src)
- : [src_ptr] "r"(src_argb), [dst_ptr] "r"(dst_argb), [width] "r"(dst_width)
- : "memory");
-}
-
-// Divide num by div and return as 16.16 fixed point result.
-/* LibYUVBaseTest.TestFixedDiv */
-int FixedDiv_MIPS(int num, int div) {
- int quotient = 0;
- const int shift = 16;
-
- asm(
- "dsll %[num], %[num], %[shift] \n\t"
- "ddiv %[num], %[div] \t\n"
- "mflo %[quo] \t\n"
- : [quo] "+&r"(quotient)
- : [num] "r"(num), [div] "r"(div), [shift] "r"(shift));
-
- return quotient;
-}
-
-// Divide num by div and return as 16.16 fixed point result.
-/* LibYUVScaleTest.ARGBScaleTo320x240_Linear */
-int FixedDiv1_MIPS(int num, int div) {
- int quotient = 0;
- const int shift = 16;
- const int val1 = 1;
- const int64_t val11 = 0x00010001ULL;
-
- asm(
- "dsll %[num], %[num], %[shift] \n\t"
- "dsub %[num], %[num], %[val11] \n\t"
- "dsub %[div], %[div], %[val1] \n\t"
- "ddiv %[num], %[div] \t\n"
- "mflo %[quo] \t\n"
- : [quo] "+&r"(quotient)
- : [num] "r"(num), [div] "r"(div), [val1] "r"(val1), [val11] "r"(val11),
- [shift] "r"(shift));
-
- return quotient;
-}
-
-// Read 8x2 upsample with filtering and write 16x1.
-// actually reads an extra pixel, so 9x2.
-void ScaleRowUp2_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- const uint16_t* src2_ptr = src_ptr + src_stride;
-
- uint64_t src0, src1;
- uint64_t dest, dest04, dest15, dest26, dest37;
- uint64_t tmp0, tmp1, tmp2, tmp3;
-
- const uint64_t mask0 = 0x0003000900030009ULL;
- const uint64_t mask1 = 0x0001000300010003ULL;
- const uint64_t mask2 = 0x0009000300090003ULL;
- const uint64_t mask3 = 0x0003000100030001ULL;
- const uint64_t ph = 0x0000000800000008ULL;
- const uint64_t shift = 4;
-
- __asm__ volatile(
- "1: \n\t"
- "gsldrc1 %[src0], 0x00(%[src1_ptr]) \n\t"
- "gsldlc1 %[src0], 0x07(%[src1_ptr]) \n\t"
- "pmaddhw %[dest04], %[src0], %[mask0] \n\t"
- "gsldrc1 %[src1], 0x00(%[src2_ptr]) \n\t"
- "gsldlc1 %[src1], 0x07(%[src2_ptr]) \n\t"
- "pmaddhw %[dest], %[src1], %[mask1] \n\t"
- "paddw %[dest04], %[dest04], %[dest] \n\t"
- "paddw %[dest04], %[dest04], %[ph] \n\t"
- "psrlw %[dest04], %[dest04], %[shift] \n\t"
-
- "pmaddhw %[dest15], %[src0], %[mask2] \n\t"
- "pmaddhw %[dest], %[src1], %[mask3] \n\t"
- "paddw %[dest15], %[dest15], %[dest] \n\t"
- "paddw %[dest15], %[dest15], %[ph] \n\t"
- "psrlw %[dest15], %[dest15], %[shift] \n\t"
-
- "gsldrc1 %[src0], 0x02(%[src1_ptr]) \n\t"
- "gsldlc1 %[src0], 0x09(%[src1_ptr]) \n\t"
- "pmaddhw %[dest26], %[src0], %[mask0] \n\t"
- "gsldrc1 %[src1], 0x02(%[src2_ptr]) \n\t"
- "gsldlc1 %[src1], 0x09(%[src2_ptr]) \n\t"
- "pmaddhw %[dest], %[src1], %[mask1] \n\t"
- "paddw %[dest26], %[dest26], %[dest] \n\t"
- "paddw %[dest26], %[dest26], %[ph] \n\t"
- "psrlw %[dest26], %[dest26], %[shift] \n\t"
-
- "pmaddhw %[dest37], %[src0], %[mask2] \n\t"
- "pmaddhw %[dest], %[src1], %[mask3] \n\t"
- "paddw %[dest37], %[dest37], %[dest] \n\t"
- "paddw %[dest37], %[dest37], %[ph] \n\t"
- "psrlw %[dest37], %[dest37], %[shift] \n\t"
-
- /* tmp0 = ( 00 04 02 06 ) */
- "packsswh %[tmp0], %[dest04], %[dest26] \n\t"
- /* tmp1 = ( 01 05 03 07 ) */
- "packsswh %[tmp1], %[dest15], %[dest37] \n\t"
-
- /* tmp2 = ( 00 01 04 05 )*/
- "punpcklhw %[tmp2], %[tmp0], %[tmp1] \n\t"
- /* tmp3 = ( 02 03 06 07 )*/
- "punpckhhw %[tmp3], %[tmp0], %[tmp1] \n\t"
-
- /* ( 00 01 02 03 ) */
- "punpcklwd %[dest], %[tmp2], %[tmp3] \n\t"
- "gssdlc1 %[dest], 0x07(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x00(%[dst_ptr]) \n\t"
-
- /* ( 04 05 06 07 ) */
- "punpckhwd %[dest], %[tmp2], %[tmp3] \n\t"
- "gssdlc1 %[dest], 0x0f(%[dst_ptr]) \n\t"
- "gssdrc1 %[dest], 0x08(%[dst_ptr]) \n\t"
-
- "daddiu %[src1_ptr], %[src1_ptr], 0x08 \n\t"
- "daddiu %[src2_ptr], %[src2_ptr], 0x08 \n\t"
- "daddiu %[dst_ptr], %[dst_ptr], 0x10 \n\t"
- "daddi %[width], %[width], -0x08 \n\t"
- "bnez %[width], 1b \n\t"
- : [src0] "=&f"(src0), [src1] "=&f"(src1), [dest04] "=&f"(dest04),
- [dest15] "=&f"(dest15), [dest26] "=&f"(dest26), [dest37] "=&f"(dest37),
- [tmp0] "=&f"(tmp0), [tmp1] "=&f"(tmp1), [tmp2] "=&f"(tmp2),
- [tmp3] "=&f"(tmp3), [dest] "=&f"(dest)
- : [src1_ptr] "r"(src_ptr), [src2_ptr] "r"(src2_ptr), [dst_ptr] "r"(dst),
- [width] "r"(dst_width), [mask0] "f"(mask0), [mask1] "f"(mask1),
- [mask2] "f"(mask2), [mask3] "f"(mask3), [shift] "f"(shift), [ph] "f"(ph)
- : "memory");
-}
-
-// clang-format on
-
-#endif // !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
diff --git a/libs/libyuv/source/scale_msa.cc b/libs/libyuv/source/scale_msa.cc
deleted file mode 100644
index 482a521f0d..0000000000
--- a/libs/libyuv/source/scale_msa.cc
+++ /dev/null
@@ -1,949 +0,0 @@
-/*
- * Copyright 2016 The LibYuv Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <assert.h>
-
-#include "libyuv/scale_row.h"
-
-// This module is for GCC MSA
-#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
-#include "libyuv/macros_msa.h"
-
-#ifdef __cplusplus
-namespace libyuv {
-extern "C" {
-#endif
-
-#define LOAD_INDEXED_DATA(srcp, indx0, out0) \
- { \
- out0[0] = srcp[indx0[0]]; \
- out0[1] = srcp[indx0[1]]; \
- out0[2] = srcp[indx0[2]]; \
- out0[3] = srcp[indx0[3]]; \
- }
-
-void ScaleARGBRowDown2_MSA(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- int x;
- v16u8 src0, src1, dst0;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_argb, 16);
- dst0 = (v16u8)__msa_pckod_w((v4i32)src1, (v4i32)src0);
- ST_UB(dst0, dst_argb);
- src_argb += 32;
- dst_argb += 16;
- }
-}
-
-void ScaleARGBRowDown2Linear_MSA(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- int x;
- v16u8 src0, src1, vec0, vec1, dst0;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_argb, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_argb, 16);
- vec0 = (v16u8)__msa_pckev_w((v4i32)src1, (v4i32)src0);
- vec1 = (v16u8)__msa_pckod_w((v4i32)src1, (v4i32)src0);
- dst0 = (v16u8)__msa_aver_u_b((v16u8)vec0, (v16u8)vec1);
- ST_UB(dst0, dst_argb);
- src_argb += 32;
- dst_argb += 16;
- }
-}
-
-void ScaleARGBRowDown2Box_MSA(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- int x;
- const uint8_t* s = src_argb;
- const uint8_t* t = src_argb + src_stride;
- v16u8 src0, src1, src2, src3, vec0, vec1, vec2, vec3, dst0;
- v8u16 reg0, reg1, reg2, reg3;
- v16i8 shuffler = {0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15};
-
- for (x = 0; x < dst_width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)t, 0);
- src3 = (v16u8)__msa_ld_b((v16i8*)t, 16);
- vec0 = (v16u8)__msa_vshf_b(shuffler, (v16i8)src0, (v16i8)src0);
- vec1 = (v16u8)__msa_vshf_b(shuffler, (v16i8)src1, (v16i8)src1);
- vec2 = (v16u8)__msa_vshf_b(shuffler, (v16i8)src2, (v16i8)src2);
- vec3 = (v16u8)__msa_vshf_b(shuffler, (v16i8)src3, (v16i8)src3);
- reg0 = __msa_hadd_u_h(vec0, vec0);
- reg1 = __msa_hadd_u_h(vec1, vec1);
- reg2 = __msa_hadd_u_h(vec2, vec2);
- reg3 = __msa_hadd_u_h(vec3, vec3);
- reg0 += reg2;
- reg1 += reg3;
- reg0 = (v8u16)__msa_srari_h((v8i16)reg0, 2);
- reg1 = (v8u16)__msa_srari_h((v8i16)reg1, 2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- ST_UB(dst0, dst_argb);
- s += 32;
- t += 32;
- dst_argb += 16;
- }
-}
-
-void ScaleARGBRowDownEven_MSA(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int32_t src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- int x;
- int32_t stepx = src_stepx * 4;
- int32_t data0, data1, data2, data3;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 4) {
- data0 = LW(src_argb);
- data1 = LW(src_argb + stepx);
- data2 = LW(src_argb + stepx * 2);
- data3 = LW(src_argb + stepx * 3);
- SW(data0, dst_argb);
- SW(data1, dst_argb + 4);
- SW(data2, dst_argb + 8);
- SW(data3, dst_argb + 12);
- src_argb += stepx * 4;
- dst_argb += 16;
- }
-}
-
-void ScaleARGBRowDownEvenBox_MSA(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- int x;
- const uint8_t* nxt_argb = src_argb + src_stride;
- int32_t stepx = src_stepx * 4;
- int64_t data0, data1, data2, data3;
- v16u8 src0 = {0}, src1 = {0}, src2 = {0}, src3 = {0};
- v16u8 vec0, vec1, vec2, vec3;
- v8u16 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
- v16u8 dst0;
-
- for (x = 0; x < dst_width; x += 4) {
- data0 = LD(src_argb);
- data1 = LD(src_argb + stepx);
- data2 = LD(src_argb + stepx * 2);
- data3 = LD(src_argb + stepx * 3);
- src0 = (v16u8)__msa_insert_d((v2i64)src0, 0, data0);
- src0 = (v16u8)__msa_insert_d((v2i64)src0, 1, data1);
- src1 = (v16u8)__msa_insert_d((v2i64)src1, 0, data2);
- src1 = (v16u8)__msa_insert_d((v2i64)src1, 1, data3);
- data0 = LD(nxt_argb);
- data1 = LD(nxt_argb + stepx);
- data2 = LD(nxt_argb + stepx * 2);
- data3 = LD(nxt_argb + stepx * 3);
- src2 = (v16u8)__msa_insert_d((v2i64)src2, 0, data0);
- src2 = (v16u8)__msa_insert_d((v2i64)src2, 1, data1);
- src3 = (v16u8)__msa_insert_d((v2i64)src3, 0, data2);
- src3 = (v16u8)__msa_insert_d((v2i64)src3, 1, data3);
- vec0 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- vec1 = (v16u8)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- vec2 = (v16u8)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- vec3 = (v16u8)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- reg0 = __msa_hadd_u_h(vec0, vec0);
- reg1 = __msa_hadd_u_h(vec1, vec1);
- reg2 = __msa_hadd_u_h(vec2, vec2);
- reg3 = __msa_hadd_u_h(vec3, vec3);
- reg4 = (v8u16)__msa_pckev_d((v2i64)reg2, (v2i64)reg0);
- reg5 = (v8u16)__msa_pckev_d((v2i64)reg3, (v2i64)reg1);
- reg6 = (v8u16)__msa_pckod_d((v2i64)reg2, (v2i64)reg0);
- reg7 = (v8u16)__msa_pckod_d((v2i64)reg3, (v2i64)reg1);
- reg4 += reg6;
- reg5 += reg7;
- reg4 = (v8u16)__msa_srari_h((v8i16)reg4, 2);
- reg5 = (v8u16)__msa_srari_h((v8i16)reg5, 2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg5, (v16i8)reg4);
- ST_UB(dst0, dst_argb);
- src_argb += stepx * 4;
- nxt_argb += stepx * 4;
- dst_argb += 16;
- }
-}
-
-void ScaleRowDown2_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- v16u8 src0, src1, src2, src3, dst0, dst1;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 32) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 48);
- dst0 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- dst1 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- ST_UB2(dst0, dst1, dst, 16);
- src_ptr += 64;
- dst += 32;
- }
-}
-
-void ScaleRowDown2Linear_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- v16u8 src0, src1, src2, src3, vec0, vec1, vec2, vec3, dst0, dst1;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 32) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 48);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec2 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- vec1 = (v16u8)__msa_pckod_b((v16i8)src1, (v16i8)src0);
- vec3 = (v16u8)__msa_pckod_b((v16i8)src3, (v16i8)src2);
- dst0 = __msa_aver_u_b(vec1, vec0);
- dst1 = __msa_aver_u_b(vec3, vec2);
- ST_UB2(dst0, dst1, dst, 16);
- src_ptr += 64;
- dst += 32;
- }
-}
-
-void ScaleRowDown2Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7, dst0, dst1;
- v8u16 vec0, vec1, vec2, vec3;
-
- for (x = 0; x < dst_width; x += 32) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 48);
- src4 = (v16u8)__msa_ld_b((v16i8*)t, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t, 16);
- src6 = (v16u8)__msa_ld_b((v16i8*)t, 32);
- src7 = (v16u8)__msa_ld_b((v16i8*)t, 48);
- vec0 = __msa_hadd_u_h(src0, src0);
- vec1 = __msa_hadd_u_h(src1, src1);
- vec2 = __msa_hadd_u_h(src2, src2);
- vec3 = __msa_hadd_u_h(src3, src3);
- vec0 += __msa_hadd_u_h(src4, src4);
- vec1 += __msa_hadd_u_h(src5, src5);
- vec2 += __msa_hadd_u_h(src6, src6);
- vec3 += __msa_hadd_u_h(src7, src7);
- vec0 = (v8u16)__msa_srari_h((v8i16)vec0, 2);
- vec1 = (v8u16)__msa_srari_h((v8i16)vec1, 2);
- vec2 = (v8u16)__msa_srari_h((v8i16)vec2, 2);
- vec3 = (v8u16)__msa_srari_h((v8i16)vec3, 2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)vec3, (v16i8)vec2);
- ST_UB2(dst0, dst1, dst, 16);
- s += 64;
- t += 64;
- dst += 32;
- }
-}
-
-void ScaleRowDown4_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- v16u8 src0, src1, src2, src3, vec0, vec1, dst0;
- (void)src_stride;
-
- for (x = 0; x < dst_width; x += 16) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 48);
- vec0 = (v16u8)__msa_pckev_b((v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_pckev_b((v16i8)src3, (v16i8)src2);
- dst0 = (v16u8)__msa_pckod_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst);
- src_ptr += 64;
- dst += 16;
- }
-}
-
-void ScaleRowDown4Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- const uint8_t* s = src_ptr;
- const uint8_t* t0 = s + src_stride;
- const uint8_t* t1 = s + src_stride * 2;
- const uint8_t* t2 = s + src_stride * 3;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7, dst0;
- v8u16 vec0, vec1, vec2, vec3;
- v4u32 reg0, reg1, reg2, reg3;
-
- for (x = 0; x < dst_width; x += 16) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 48);
- src4 = (v16u8)__msa_ld_b((v16i8*)t0, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t0, 16);
- src6 = (v16u8)__msa_ld_b((v16i8*)t0, 32);
- src7 = (v16u8)__msa_ld_b((v16i8*)t0, 48);
- vec0 = __msa_hadd_u_h(src0, src0);
- vec1 = __msa_hadd_u_h(src1, src1);
- vec2 = __msa_hadd_u_h(src2, src2);
- vec3 = __msa_hadd_u_h(src3, src3);
- vec0 += __msa_hadd_u_h(src4, src4);
- vec1 += __msa_hadd_u_h(src5, src5);
- vec2 += __msa_hadd_u_h(src6, src6);
- vec3 += __msa_hadd_u_h(src7, src7);
- src0 = (v16u8)__msa_ld_b((v16i8*)t1, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)t1, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)t1, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)t1, 48);
- src4 = (v16u8)__msa_ld_b((v16i8*)t2, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t2, 16);
- src6 = (v16u8)__msa_ld_b((v16i8*)t2, 32);
- src7 = (v16u8)__msa_ld_b((v16i8*)t2, 48);
- vec0 += __msa_hadd_u_h(src0, src0);
- vec1 += __msa_hadd_u_h(src1, src1);
- vec2 += __msa_hadd_u_h(src2, src2);
- vec3 += __msa_hadd_u_h(src3, src3);
- vec0 += __msa_hadd_u_h(src4, src4);
- vec1 += __msa_hadd_u_h(src5, src5);
- vec2 += __msa_hadd_u_h(src6, src6);
- vec3 += __msa_hadd_u_h(src7, src7);
- reg0 = __msa_hadd_u_w(vec0, vec0);
- reg1 = __msa_hadd_u_w(vec1, vec1);
- reg2 = __msa_hadd_u_w(vec2, vec2);
- reg3 = __msa_hadd_u_w(vec3, vec3);
- reg0 = (v4u32)__msa_srari_w((v4i32)reg0, 4);
- reg1 = (v4u32)__msa_srari_w((v4i32)reg1, 4);
- reg2 = (v4u32)__msa_srari_w((v4i32)reg2, 4);
- reg3 = (v4u32)__msa_srari_w((v4i32)reg3, 4);
- vec0 = (v8u16)__msa_pckev_h((v8i16)reg1, (v8i16)reg0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)reg3, (v8i16)reg2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)vec1, (v16i8)vec0);
- ST_UB(dst0, dst);
- s += 64;
- t0 += 64;
- t1 += 64;
- t2 += 64;
- dst += 16;
- }
-}
-
-void ScaleRowDown38_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x, width;
- uint64_t dst0;
- uint32_t dst1;
- v16u8 src0, src1, vec0;
- v16i8 mask = {0, 3, 6, 8, 11, 14, 16, 19, 22, 24, 27, 30, 0, 0, 0, 0};
- (void)src_stride;
-
- assert(dst_width % 3 == 0);
- width = dst_width / 3;
-
- for (x = 0; x < width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 16);
- vec0 = (v16u8)__msa_vshf_b(mask, (v16i8)src1, (v16i8)src0);
- dst0 = __msa_copy_u_d((v2i64)vec0, 0);
- dst1 = __msa_copy_u_w((v4i32)vec0, 2);
- SD(dst0, dst);
- SW(dst1, dst + 8);
- src_ptr += 32;
- dst += 12;
- }
-}
-
-void ScaleRowDown38_2_Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- int x, width;
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
- uint64_t dst0;
- uint32_t dst1;
- v16u8 src0, src1, src2, src3, out;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v4u32 tmp0, tmp1, tmp2, tmp3, tmp4;
- v8i16 zero = {0};
- v8i16 mask = {0, 1, 2, 8, 3, 4, 5, 9};
- v16i8 dst_mask = {0, 2, 16, 4, 6, 18, 8, 10, 20, 12, 14, 22, 0, 0, 0, 0};
- v4u32 const_0x2AAA = (v4u32)__msa_fill_w(0x2AAA);
- v4u32 const_0x4000 = (v4u32)__msa_fill_w(0x4000);
-
- assert((dst_width % 3 == 0) && (dst_width > 0));
- width = dst_width / 3;
-
- for (x = 0; x < width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)t, 0);
- src3 = (v16u8)__msa_ld_b((v16i8*)t, 16);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- vec0 = __msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = __msa_hadd_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = __msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = __msa_hadd_u_h((v16u8)vec3, (v16u8)vec3);
- vec4 = (v8u16)__msa_vshf_h(mask, zero, (v8i16)vec0);
- vec5 = (v8u16)__msa_vshf_h(mask, zero, (v8i16)vec1);
- vec6 = (v8u16)__msa_vshf_h(mask, zero, (v8i16)vec2);
- vec7 = (v8u16)__msa_vshf_h(mask, zero, (v8i16)vec3);
- vec0 = (v8u16)__msa_pckod_w((v4i32)vec1, (v4i32)vec0);
- vec1 = (v8u16)__msa_pckod_w((v4i32)vec3, (v4i32)vec2);
- vec0 = (v8u16)__msa_pckod_w((v4i32)vec1, (v4i32)vec0);
- tmp0 = __msa_hadd_u_w(vec4, vec4);
- tmp1 = __msa_hadd_u_w(vec5, vec5);
- tmp2 = __msa_hadd_u_w(vec6, vec6);
- tmp3 = __msa_hadd_u_w(vec7, vec7);
- tmp4 = __msa_hadd_u_w(vec0, vec0);
- vec0 = (v8u16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- tmp0 = __msa_hadd_u_w(vec0, vec0);
- tmp1 = __msa_hadd_u_w(vec1, vec1);
- tmp0 *= const_0x2AAA;
- tmp1 *= const_0x2AAA;
- tmp4 *= const_0x4000;
- tmp0 = (v4u32)__msa_srai_w((v4i32)tmp0, 16);
- tmp1 = (v4u32)__msa_srai_w((v4i32)tmp1, 16);
- tmp4 = (v4u32)__msa_srai_w((v4i32)tmp4, 16);
- vec0 = (v8u16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)tmp4, (v8i16)tmp4);
- out = (v16u8)__msa_vshf_b(dst_mask, (v16i8)vec1, (v16i8)vec0);
- dst0 = __msa_copy_u_d((v2i64)out, 0);
- dst1 = __msa_copy_u_w((v4i32)out, 2);
- SD(dst0, dst_ptr);
- SW(dst1, dst_ptr + 8);
- s += 32;
- t += 32;
- dst_ptr += 12;
- }
-}
-
-void ScaleRowDown38_3_Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- int x, width;
- const uint8_t* s = src_ptr;
- const uint8_t* t0 = s + src_stride;
- const uint8_t* t1 = s + src_stride * 2;
- uint64_t dst0;
- uint32_t dst1;
- v16u8 src0, src1, src2, src3, src4, src5, out;
- v8u16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
- v4u32 tmp0, tmp1, tmp2, tmp3, tmp4;
- v8u16 zero = {0};
- v8i16 mask = {0, 1, 2, 8, 3, 4, 5, 9};
- v16i8 dst_mask = {0, 2, 16, 4, 6, 18, 8, 10, 20, 12, 14, 22, 0, 0, 0, 0};
- v4u32 const_0x1C71 = (v4u32)__msa_fill_w(0x1C71);
- v4u32 const_0x2AAA = (v4u32)__msa_fill_w(0x2AAA);
-
- assert((dst_width % 3 == 0) && (dst_width > 0));
- width = dst_width / 3;
-
- for (x = 0; x < width; x += 4) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)t0, 0);
- src3 = (v16u8)__msa_ld_b((v16i8*)t0, 16);
- src4 = (v16u8)__msa_ld_b((v16i8*)t1, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t1, 16);
- vec0 = (v8u16)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- vec1 = (v8u16)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- vec2 = (v8u16)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- vec3 = (v8u16)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- vec4 = (v8u16)__msa_ilvr_b((v16i8)zero, (v16i8)src4);
- vec5 = (v8u16)__msa_ilvl_b((v16i8)zero, (v16i8)src4);
- vec6 = (v8u16)__msa_ilvr_b((v16i8)zero, (v16i8)src5);
- vec7 = (v8u16)__msa_ilvl_b((v16i8)zero, (v16i8)src5);
- vec0 = __msa_hadd_u_h((v16u8)vec0, (v16u8)vec0);
- vec1 = __msa_hadd_u_h((v16u8)vec1, (v16u8)vec1);
- vec2 = __msa_hadd_u_h((v16u8)vec2, (v16u8)vec2);
- vec3 = __msa_hadd_u_h((v16u8)vec3, (v16u8)vec3);
- vec0 += __msa_hadd_u_h((v16u8)vec4, (v16u8)vec4);
- vec1 += __msa_hadd_u_h((v16u8)vec5, (v16u8)vec5);
- vec2 += __msa_hadd_u_h((v16u8)vec6, (v16u8)vec6);
- vec3 += __msa_hadd_u_h((v16u8)vec7, (v16u8)vec7);
- vec4 = (v8u16)__msa_vshf_h(mask, (v8i16)zero, (v8i16)vec0);
- vec5 = (v8u16)__msa_vshf_h(mask, (v8i16)zero, (v8i16)vec1);
- vec6 = (v8u16)__msa_vshf_h(mask, (v8i16)zero, (v8i16)vec2);
- vec7 = (v8u16)__msa_vshf_h(mask, (v8i16)zero, (v8i16)vec3);
- vec0 = (v8u16)__msa_pckod_w((v4i32)vec1, (v4i32)vec0);
- vec1 = (v8u16)__msa_pckod_w((v4i32)vec3, (v4i32)vec2);
- vec0 = (v8u16)__msa_pckod_w((v4i32)vec1, (v4i32)vec0);
- tmp0 = __msa_hadd_u_w(vec4, vec4);
- tmp1 = __msa_hadd_u_w(vec5, vec5);
- tmp2 = __msa_hadd_u_w(vec6, vec6);
- tmp3 = __msa_hadd_u_w(vec7, vec7);
- tmp4 = __msa_hadd_u_w(vec0, vec0);
- vec0 = (v8u16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- tmp0 = __msa_hadd_u_w(vec0, vec0);
- tmp1 = __msa_hadd_u_w(vec1, vec1);
- tmp0 *= const_0x1C71;
- tmp1 *= const_0x1C71;
- tmp4 *= const_0x2AAA;
- tmp0 = (v4u32)__msa_srai_w((v4i32)tmp0, 16);
- tmp1 = (v4u32)__msa_srai_w((v4i32)tmp1, 16);
- tmp4 = (v4u32)__msa_srai_w((v4i32)tmp4, 16);
- vec0 = (v8u16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- vec1 = (v8u16)__msa_pckev_h((v8i16)tmp4, (v8i16)tmp4);
- out = (v16u8)__msa_vshf_b(dst_mask, (v16i8)vec1, (v16i8)vec0);
- dst0 = __msa_copy_u_d((v2i64)out, 0);
- dst1 = __msa_copy_u_w((v4i32)out, 2);
- SD(dst0, dst_ptr);
- SW(dst1, dst_ptr + 8);
- s += 32;
- t0 += 32;
- t1 += 32;
- dst_ptr += 12;
- }
-}
-
-void ScaleAddRow_MSA(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
- int x;
- v16u8 src0;
- v8u16 dst0, dst1;
- v16i8 zero = {0};
-
- assert(src_width > 0);
-
- for (x = 0; x < src_width; x += 16) {
- src0 = LD_UB(src_ptr);
- dst0 = (v8u16)__msa_ld_h((v8i16*)dst_ptr, 0);
- dst1 = (v8u16)__msa_ld_h((v8i16*)dst_ptr, 16);
- dst0 += (v8u16)__msa_ilvr_b(zero, (v16i8)src0);
- dst1 += (v8u16)__msa_ilvl_b(zero, (v16i8)src0);
- ST_UH2(dst0, dst1, dst_ptr, 8);
- src_ptr += 16;
- dst_ptr += 16;
- }
-}
-
-void ScaleFilterCols_MSA(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- int j;
- v4i32 vec_x = __msa_fill_w(x);
- v4i32 vec_dx = __msa_fill_w(dx);
- v4i32 vec_const = {0, 1, 2, 3};
- v4i32 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
- v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
- v8u16 reg0, reg1;
- v16u8 dst0;
- v4i32 const_0xFFFF = __msa_fill_w(0xFFFF);
- v4i32 const_0x40 = __msa_fill_w(0x40);
-
- vec0 = vec_dx * vec_const;
- vec1 = vec_dx * 4;
- vec_x += vec0;
-
- for (j = 0; j < dst_width - 1; j += 16) {
- vec2 = vec_x >> 16;
- vec6 = vec_x & const_0xFFFF;
- vec_x += vec1;
- vec3 = vec_x >> 16;
- vec7 = vec_x & const_0xFFFF;
- vec_x += vec1;
- vec4 = vec_x >> 16;
- vec8 = vec_x & const_0xFFFF;
- vec_x += vec1;
- vec5 = vec_x >> 16;
- vec9 = vec_x & const_0xFFFF;
- vec_x += vec1;
- vec6 >>= 9;
- vec7 >>= 9;
- vec8 >>= 9;
- vec9 >>= 9;
- LOAD_INDEXED_DATA(src_ptr, vec2, tmp0);
- LOAD_INDEXED_DATA(src_ptr, vec3, tmp1);
- LOAD_INDEXED_DATA(src_ptr, vec4, tmp2);
- LOAD_INDEXED_DATA(src_ptr, vec5, tmp3);
- vec2 += 1;
- vec3 += 1;
- vec4 += 1;
- vec5 += 1;
- LOAD_INDEXED_DATA(src_ptr, vec2, tmp4);
- LOAD_INDEXED_DATA(src_ptr, vec3, tmp5);
- LOAD_INDEXED_DATA(src_ptr, vec4, tmp6);
- LOAD_INDEXED_DATA(src_ptr, vec5, tmp7);
- tmp4 -= tmp0;
- tmp5 -= tmp1;
- tmp6 -= tmp2;
- tmp7 -= tmp3;
- tmp4 *= vec6;
- tmp5 *= vec7;
- tmp6 *= vec8;
- tmp7 *= vec9;
- tmp4 += const_0x40;
- tmp5 += const_0x40;
- tmp6 += const_0x40;
- tmp7 += const_0x40;
- tmp4 >>= 7;
- tmp5 >>= 7;
- tmp6 >>= 7;
- tmp7 >>= 7;
- tmp0 += tmp4;
- tmp1 += tmp5;
- tmp2 += tmp6;
- tmp3 += tmp7;
- reg0 = (v8u16)__msa_pckev_h((v8i16)tmp1, (v8i16)tmp0);
- reg1 = (v8u16)__msa_pckev_h((v8i16)tmp3, (v8i16)tmp2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- __msa_st_b(dst0, dst_ptr, 0);
- dst_ptr += 16;
- }
-}
-
-void ScaleARGBCols_MSA(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- uint32_t* dst = (uint32_t*)(dst_argb);
- int j;
- v4i32 x_vec = __msa_fill_w(x);
- v4i32 dx_vec = __msa_fill_w(dx);
- v4i32 const_vec = {0, 1, 2, 3};
- v4i32 vec0, vec1, vec2;
- v4i32 dst0;
-
- vec0 = dx_vec * const_vec;
- vec1 = dx_vec * 4;
- x_vec += vec0;
-
- for (j = 0; j < dst_width; j += 4) {
- vec2 = x_vec >> 16;
- x_vec += vec1;
- LOAD_INDEXED_DATA(src, vec2, dst0);
- __msa_st_w(dst0, dst, 0);
- dst += 4;
- }
-}
-
-void ScaleARGBFilterCols_MSA(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- const uint32_t* src = (const uint32_t*)(src_argb);
- int j;
- v4u32 src0, src1, src2, src3;
- v4u32 vec0, vec1, vec2, vec3;
- v16u8 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
- v16u8 mult0, mult1, mult2, mult3;
- v8u16 tmp0, tmp1, tmp2, tmp3;
- v16u8 dst0, dst1;
- v4u32 vec_x = (v4u32)__msa_fill_w(x);
- v4u32 vec_dx = (v4u32)__msa_fill_w(dx);
- v4u32 vec_const = {0, 1, 2, 3};
- v16u8 const_0x7f = (v16u8)__msa_fill_b(0x7f);
-
- vec0 = vec_dx * vec_const;
- vec1 = vec_dx * 4;
- vec_x += vec0;
-
- for (j = 0; j < dst_width - 1; j += 8) {
- vec2 = vec_x >> 16;
- reg0 = (v16u8)(vec_x >> 9);
- vec_x += vec1;
- vec3 = vec_x >> 16;
- reg1 = (v16u8)(vec_x >> 9);
- vec_x += vec1;
- reg0 = reg0 & const_0x7f;
- reg1 = reg1 & const_0x7f;
- reg0 = (v16u8)__msa_shf_b((v16i8)reg0, 0);
- reg1 = (v16u8)__msa_shf_b((v16i8)reg1, 0);
- reg2 = reg0 ^ const_0x7f;
- reg3 = reg1 ^ const_0x7f;
- mult0 = (v16u8)__msa_ilvr_b((v16i8)reg0, (v16i8)reg2);
- mult1 = (v16u8)__msa_ilvl_b((v16i8)reg0, (v16i8)reg2);
- mult2 = (v16u8)__msa_ilvr_b((v16i8)reg1, (v16i8)reg3);
- mult3 = (v16u8)__msa_ilvl_b((v16i8)reg1, (v16i8)reg3);
- LOAD_INDEXED_DATA(src, vec2, src0);
- LOAD_INDEXED_DATA(src, vec3, src1);
- vec2 += 1;
- vec3 += 1;
- LOAD_INDEXED_DATA(src, vec2, src2);
- LOAD_INDEXED_DATA(src, vec3, src3);
- reg4 = (v16u8)__msa_ilvr_b((v16i8)src2, (v16i8)src0);
- reg5 = (v16u8)__msa_ilvl_b((v16i8)src2, (v16i8)src0);
- reg6 = (v16u8)__msa_ilvr_b((v16i8)src3, (v16i8)src1);
- reg7 = (v16u8)__msa_ilvl_b((v16i8)src3, (v16i8)src1);
- tmp0 = __msa_dotp_u_h(reg4, mult0);
- tmp1 = __msa_dotp_u_h(reg5, mult1);
- tmp2 = __msa_dotp_u_h(reg6, mult2);
- tmp3 = __msa_dotp_u_h(reg7, mult3);
- tmp0 >>= 7;
- tmp1 >>= 7;
- tmp2 >>= 7;
- tmp3 >>= 7;
- dst0 = (v16u8)__msa_pckev_b((v16i8)tmp1, (v16i8)tmp0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)tmp3, (v16i8)tmp2);
- __msa_st_b(dst0, dst_argb, 0);
- __msa_st_b(dst1, dst_argb, 16);
- dst_argb += 32;
- }
-}
-
-void ScaleRowDown34_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst,
- int dst_width) {
- int x;
- (void)src_stride;
- v16u8 src0, src1, src2, src3;
- v16u8 vec0, vec1, vec2;
- v16i8 mask0 = {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20};
- v16i8 mask1 = {5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25};
- v16i8 mask2 = {11, 12, 13, 15, 16, 17, 19, 20,
- 21, 23, 24, 25, 27, 28, 29, 31};
-
- assert((dst_width % 3 == 0) && (dst_width > 0));
-
- for (x = 0; x < dst_width; x += 48) {
- src0 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)src_ptr, 48);
- vec0 = (v16u8)__msa_vshf_b(mask0, (v16i8)src1, (v16i8)src0);
- vec1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src2, (v16i8)src1);
- vec2 = (v16u8)__msa_vshf_b(mask2, (v16i8)src3, (v16i8)src2);
- __msa_st_b((v16i8)vec0, dst, 0);
- __msa_st_b((v16i8)vec1, dst, 16);
- __msa_st_b((v16i8)vec2, dst, 32);
- src_ptr += 64;
- dst += 48;
- }
-}
-
-void ScaleRowDown34_0_Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* d,
- int dst_width) {
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
- int x;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7, dst0, dst1, dst2;
- v16u8 vec0, vec1, vec2, vec3, vec4, vec5;
- v16u8 vec6, vec7, vec8, vec9, vec10, vec11;
- v8i16 reg0, reg1, reg2, reg3, reg4, reg5;
- v8i16 reg6, reg7, reg8, reg9, reg10, reg11;
- v16u8 const0 = {3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1};
- v16u8 const1 = {1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1};
- v16u8 const2 = {1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3};
- v16i8 mask0 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
- v16i8 mask1 = {10, 11, 12, 13, 13, 14, 14, 15,
- 16, 17, 17, 18, 18, 19, 20, 21};
- v16i8 mask2 = {5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15};
- v8i16 shft0 = {2, 1, 2, 2, 1, 2, 2, 1};
- v8i16 shft1 = {2, 2, 1, 2, 2, 1, 2, 2};
- v8i16 shft2 = {1, 2, 2, 1, 2, 2, 1, 2};
-
- assert((dst_width % 3 == 0) && (dst_width > 0));
-
- for (x = 0; x < dst_width; x += 48) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 48);
- src4 = (v16u8)__msa_ld_b((v16i8*)t, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t, 16);
- src6 = (v16u8)__msa_ld_b((v16i8*)t, 32);
- src7 = (v16u8)__msa_ld_b((v16i8*)t, 48);
- vec0 = (v16u8)__msa_vshf_b(mask0, (v16i8)src0, (v16i8)src0);
- vec1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- vec2 = (v16u8)__msa_vshf_b(mask2, (v16i8)src1, (v16i8)src1);
- vec3 = (v16u8)__msa_vshf_b(mask0, (v16i8)src2, (v16i8)src2);
- vec4 = (v16u8)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
- vec5 = (v16u8)__msa_vshf_b(mask2, (v16i8)src3, (v16i8)src3);
- vec6 = (v16u8)__msa_vshf_b(mask0, (v16i8)src4, (v16i8)src4);
- vec7 = (v16u8)__msa_vshf_b(mask1, (v16i8)src5, (v16i8)src4);
- vec8 = (v16u8)__msa_vshf_b(mask2, (v16i8)src5, (v16i8)src5);
- vec9 = (v16u8)__msa_vshf_b(mask0, (v16i8)src6, (v16i8)src6);
- vec10 = (v16u8)__msa_vshf_b(mask1, (v16i8)src7, (v16i8)src6);
- vec11 = (v16u8)__msa_vshf_b(mask2, (v16i8)src7, (v16i8)src7);
- reg0 = (v8i16)__msa_dotp_u_h(vec0, const0);
- reg1 = (v8i16)__msa_dotp_u_h(vec1, const1);
- reg2 = (v8i16)__msa_dotp_u_h(vec2, const2);
- reg3 = (v8i16)__msa_dotp_u_h(vec3, const0);
- reg4 = (v8i16)__msa_dotp_u_h(vec4, const1);
- reg5 = (v8i16)__msa_dotp_u_h(vec5, const2);
- reg6 = (v8i16)__msa_dotp_u_h(vec6, const0);
- reg7 = (v8i16)__msa_dotp_u_h(vec7, const1);
- reg8 = (v8i16)__msa_dotp_u_h(vec8, const2);
- reg9 = (v8i16)__msa_dotp_u_h(vec9, const0);
- reg10 = (v8i16)__msa_dotp_u_h(vec10, const1);
- reg11 = (v8i16)__msa_dotp_u_h(vec11, const2);
- reg0 = __msa_srar_h(reg0, shft0);
- reg1 = __msa_srar_h(reg1, shft1);
- reg2 = __msa_srar_h(reg2, shft2);
- reg3 = __msa_srar_h(reg3, shft0);
- reg4 = __msa_srar_h(reg4, shft1);
- reg5 = __msa_srar_h(reg5, shft2);
- reg6 = __msa_srar_h(reg6, shft0);
- reg7 = __msa_srar_h(reg7, shft1);
- reg8 = __msa_srar_h(reg8, shft2);
- reg9 = __msa_srar_h(reg9, shft0);
- reg10 = __msa_srar_h(reg10, shft1);
- reg11 = __msa_srar_h(reg11, shft2);
- reg0 = reg0 * 3 + reg6;
- reg1 = reg1 * 3 + reg7;
- reg2 = reg2 * 3 + reg8;
- reg3 = reg3 * 3 + reg9;
- reg4 = reg4 * 3 + reg10;
- reg5 = reg5 * 3 + reg11;
- reg0 = __msa_srari_h(reg0, 2);
- reg1 = __msa_srari_h(reg1, 2);
- reg2 = __msa_srari_h(reg2, 2);
- reg3 = __msa_srari_h(reg3, 2);
- reg4 = __msa_srari_h(reg4, 2);
- reg5 = __msa_srari_h(reg5, 2);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)reg3, (v16i8)reg2);
- dst2 = (v16u8)__msa_pckev_b((v16i8)reg5, (v16i8)reg4);
- __msa_st_b((v16i8)dst0, d, 0);
- __msa_st_b((v16i8)dst1, d, 16);
- __msa_st_b((v16i8)dst2, d, 32);
- s += 64;
- t += 64;
- d += 48;
- }
-}
-
-void ScaleRowDown34_1_Box_MSA(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* d,
- int dst_width) {
- const uint8_t* s = src_ptr;
- const uint8_t* t = src_ptr + src_stride;
- int x;
- v16u8 src0, src1, src2, src3, src4, src5, src6, src7, dst0, dst1, dst2;
- v16u8 vec0, vec1, vec2, vec3, vec4, vec5;
- v16u8 vec6, vec7, vec8, vec9, vec10, vec11;
- v8i16 reg0, reg1, reg2, reg3, reg4, reg5;
- v8i16 reg6, reg7, reg8, reg9, reg10, reg11;
- v16u8 const0 = {3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1};
- v16u8 const1 = {1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1};
- v16u8 const2 = {1, 1, 1, 3, 3, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3};
- v16i8 mask0 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
- v16i8 mask1 = {10, 11, 12, 13, 13, 14, 14, 15,
- 16, 17, 17, 18, 18, 19, 20, 21};
- v16i8 mask2 = {5, 6, 6, 7, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15};
- v8i16 shft0 = {2, 1, 2, 2, 1, 2, 2, 1};
- v8i16 shft1 = {2, 2, 1, 2, 2, 1, 2, 2};
- v8i16 shft2 = {1, 2, 2, 1, 2, 2, 1, 2};
-
- assert((dst_width % 3 == 0) && (dst_width > 0));
-
- for (x = 0; x < dst_width; x += 48) {
- src0 = (v16u8)__msa_ld_b((v16i8*)s, 0);
- src1 = (v16u8)__msa_ld_b((v16i8*)s, 16);
- src2 = (v16u8)__msa_ld_b((v16i8*)s, 32);
- src3 = (v16u8)__msa_ld_b((v16i8*)s, 48);
- src4 = (v16u8)__msa_ld_b((v16i8*)t, 0);
- src5 = (v16u8)__msa_ld_b((v16i8*)t, 16);
- src6 = (v16u8)__msa_ld_b((v16i8*)t, 32);
- src7 = (v16u8)__msa_ld_b((v16i8*)t, 48);
- vec0 = (v16u8)__msa_vshf_b(mask0, (v16i8)src0, (v16i8)src0);
- vec1 = (v16u8)__msa_vshf_b(mask1, (v16i8)src1, (v16i8)src0);
- vec2 = (v16u8)__msa_vshf_b(mask2, (v16i8)src1, (v16i8)src1);
- vec3 = (v16u8)__msa_vshf_b(mask0, (v16i8)src2, (v16i8)src2);
- vec4 = (v16u8)__msa_vshf_b(mask1, (v16i8)src3, (v16i8)src2);
- vec5 = (v16u8)__msa_vshf_b(mask2, (v16i8)src3, (v16i8)src3);
- vec6 = (v16u8)__msa_vshf_b(mask0, (v16i8)src4, (v16i8)src4);
- vec7 = (v16u8)__msa_vshf_b(mask1, (v16i8)src5, (v16i8)src4);
- vec8 = (v16u8)__msa_vshf_b(mask2, (v16i8)src5, (v16i8)src5);
- vec9 = (v16u8)__msa_vshf_b(mask0, (v16i8)src6, (v16i8)src6);
- vec10 = (v16u8)__msa_vshf_b(mask1, (v16i8)src7, (v16i8)src6);
- vec11 = (v16u8)__msa_vshf_b(mask2, (v16i8)src7, (v16i8)src7);
- reg0 = (v8i16)__msa_dotp_u_h(vec0, const0);
- reg1 = (v8i16)__msa_dotp_u_h(vec1, const1);
- reg2 = (v8i16)__msa_dotp_u_h(vec2, const2);
- reg3 = (v8i16)__msa_dotp_u_h(vec3, const0);
- reg4 = (v8i16)__msa_dotp_u_h(vec4, const1);
- reg5 = (v8i16)__msa_dotp_u_h(vec5, const2);
- reg6 = (v8i16)__msa_dotp_u_h(vec6, const0);
- reg7 = (v8i16)__msa_dotp_u_h(vec7, const1);
- reg8 = (v8i16)__msa_dotp_u_h(vec8, const2);
- reg9 = (v8i16)__msa_dotp_u_h(vec9, const0);
- reg10 = (v8i16)__msa_dotp_u_h(vec10, const1);
- reg11 = (v8i16)__msa_dotp_u_h(vec11, const2);
- reg0 = __msa_srar_h(reg0, shft0);
- reg1 = __msa_srar_h(reg1, shft1);
- reg2 = __msa_srar_h(reg2, shft2);
- reg3 = __msa_srar_h(reg3, shft0);
- reg4 = __msa_srar_h(reg4, shft1);
- reg5 = __msa_srar_h(reg5, shft2);
- reg6 = __msa_srar_h(reg6, shft0);
- reg7 = __msa_srar_h(reg7, shft1);
- reg8 = __msa_srar_h(reg8, shft2);
- reg9 = __msa_srar_h(reg9, shft0);
- reg10 = __msa_srar_h(reg10, shft1);
- reg11 = __msa_srar_h(reg11, shft2);
- reg0 += reg6;
- reg1 += reg7;
- reg2 += reg8;
- reg3 += reg9;
- reg4 += reg10;
- reg5 += reg11;
- reg0 = __msa_srari_h(reg0, 1);
- reg1 = __msa_srari_h(reg1, 1);
- reg2 = __msa_srari_h(reg2, 1);
- reg3 = __msa_srari_h(reg3, 1);
- reg4 = __msa_srari_h(reg4, 1);
- reg5 = __msa_srari_h(reg5, 1);
- dst0 = (v16u8)__msa_pckev_b((v16i8)reg1, (v16i8)reg0);
- dst1 = (v16u8)__msa_pckev_b((v16i8)reg3, (v16i8)reg2);
- dst2 = (v16u8)__msa_pckev_b((v16i8)reg5, (v16i8)reg4);
- __msa_st_b((v16i8)dst0, d, 0);
- __msa_st_b((v16i8)dst1, d, 16);
- __msa_st_b((v16i8)dst2, d, 32);
- s += 64;
- t += 64;
- d += 48;
- }
-}
-
-#ifdef __cplusplus
-} // extern "C"
-} // namespace libyuv
-#endif
-
-#endif // !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
diff --git a/libs/libyuv/source/scale_neon.cc b/libs/libyuv/source/scale_neon.cc
index 366b155ba4..2d5be0af37 100644
--- a/libs/libyuv/source/scale_neon.cc
+++ b/libs/libyuv/source/scale_neon.cc
@@ -9,12 +9,20 @@
*/
#include "libyuv/row.h"
+#include "libyuv/scale_row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC Neon.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
@@ -29,17 +37,17 @@ void ScaleRowDown2_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into q0, odd into q1
- "vld2.8 {q0, q1}, [%0]! \n"
- "subs %2, %2, #16 \n" // 16 processed per loop
- "vst1.8 {q1}, [%1]! \n" // store odd pixels
- "bgt 1b \n"
+ "vld2.8 {q0, q1}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "vst1.8 {q1}, [%1]! \n" // store odd pixels
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "q0", "q1" // Clobber List
+ : "memory", "cc", "q0", "q1" // Clobber List
);
}
@@ -50,17 +58,17 @@ void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "vld2.8 {q0, q1}, [%0]! \n" // load 32 pixels
- "subs %2, %2, #16 \n" // 16 processed per loop
- "vrhadd.u8 q0, q0, q1 \n" // rounding half add
- "vst1.8 {q0}, [%1]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld2.8 {q0, q1}, [%0]! \n" // load 32 pixels
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "vrhadd.u8 q0, q0, q1 \n" // rounding half add
+ "vst1.8 {q0}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "q0", "q1" // Clobber List
+ : "memory", "cc", "q0", "q1" // Clobber List
);
}
@@ -71,27 +79,27 @@ void ScaleRowDown2Box_NEON(const uint8_t* src_ptr,
int dst_width) {
asm volatile(
// change the stride to row 2 pointer
- "add %1, %0 \n"
- "1: \n"
- "vld1.8 {q0, q1}, [%0]! \n" // load row 1 and post inc
- "vld1.8 {q2, q3}, [%1]! \n" // load row 2 and post inc
- "subs %3, %3, #16 \n" // 16 processed per loop
- "vpaddl.u8 q0, q0 \n" // row 1 add adjacent
- "vpaddl.u8 q1, q1 \n"
- "vpadal.u8 q0, q2 \n" // row 2 add adjacent +
+ "add %1, %0 \n"
+ "1: \n"
+ "vld1.8 {q0, q1}, [%0]! \n" // load row 1 and post inc
+ "vld1.8 {q2, q3}, [%1]! \n" // load row 2 and post inc
+ "subs %3, %3, #16 \n" // 16 processed per loop
+ "vpaddl.u8 q0, q0 \n" // row 1 add adjacent
+ "vpaddl.u8 q1, q1 \n"
+ "vpadal.u8 q0, q2 \n" // row 2 add adjacent +
// row1
- "vpadal.u8 q1, q3 \n"
- "vrshrn.u16 d0, q0, #2 \n" // downshift, round and
+ "vpadal.u8 q1, q3 \n"
+ "vrshrn.u16 d0, q0, #2 \n" // downshift, round and
// pack
- "vrshrn.u16 d1, q1, #2 \n"
- "vst1.8 {q0}, [%2]! \n"
- "bgt 1b \n"
+ "vrshrn.u16 d1, q1, #2 \n"
+ "vst1.8 {q0}, [%2]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_stride), // %1
"+r"(dst), // %2
"+r"(dst_width) // %3
:
- : "q0", "q1", "q2", "q3" // Clobber List
+ : "memory", "cc", "q0", "q1", "q2", "q3" // Clobber List
);
}
@@ -101,11 +109,11 @@ void ScaleRowDown4_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
- "subs %2, %2, #8 \n" // 8 processed per loop
- "vst1.8 {d2}, [%1]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
+ "subs %2, %2, #8 \n" // 8 processed per loop
+ "vst1.8 {d2}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -121,21 +129,21 @@ void ScaleRowDown4Box_NEON(const uint8_t* src_ptr,
const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
const uint8_t* src_ptr3 = src_ptr + src_stride * 3;
asm volatile(
- "1: \n"
- "vld1.8 {q0}, [%0]! \n" // load up 16x4
- "vld1.8 {q1}, [%3]! \n"
- "vld1.8 {q2}, [%4]! \n"
- "vld1.8 {q3}, [%5]! \n"
- "subs %2, %2, #4 \n"
- "vpaddl.u8 q0, q0 \n"
- "vpadal.u8 q0, q1 \n"
- "vpadal.u8 q0, q2 \n"
- "vpadal.u8 q0, q3 \n"
- "vpaddl.u16 q0, q0 \n"
- "vrshrn.u32 d0, q0, #4 \n" // divide by 16 w/rounding
- "vmovn.u16 d0, q0 \n"
- "vst1.32 {d0[0]}, [%1]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld1.8 {q0}, [%0]! \n" // load up 16x4
+ "vld1.8 {q1}, [%3]! \n"
+ "vld1.8 {q2}, [%4]! \n"
+ "vld1.8 {q3}, [%5]! \n"
+ "subs %2, %2, #4 \n"
+ "vpaddl.u8 q0, q0 \n"
+ "vpadal.u8 q0, q1 \n"
+ "vpadal.u8 q0, q2 \n"
+ "vpadal.u8 q0, q3 \n"
+ "vpaddl.u16 q0, q0 \n"
+ "vrshrn.u32 d0, q0, #4 \n" // divide by 16 w/rounding
+ "vmovn.u16 d0, q0 \n"
+ "vst1.32 {d0[0]}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
@@ -155,12 +163,12 @@ void ScaleRowDown34_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
- "subs %2, %2, #24 \n"
- "vmov d2, d3 \n" // order d0, d1, d2
- "vst3.8 {d0, d1, d2}, [%1]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
+ "subs %2, %2, #24 \n"
+ "vmov d2, d3 \n" // order d0, d1, d2
+ "vst3.8 {d0, d1, d2}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -173,49 +181,49 @@ void ScaleRowDown34_0_Box_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "vmov.u8 d24, #3 \n"
- "add %3, %0 \n"
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
- "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
- "subs %2, %2, #24 \n"
+ "vmov.u8 d24, #3 \n"
+ "add %3, %0 \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
+ "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
+ "subs %2, %2, #24 \n"
// filter src line 0 with src line 1
// expand chars to shorts to allow for room
// when adding lines together
- "vmovl.u8 q8, d4 \n"
- "vmovl.u8 q9, d5 \n"
- "vmovl.u8 q10, d6 \n"
- "vmovl.u8 q11, d7 \n"
+ "vmovl.u8 q8, d4 \n"
+ "vmovl.u8 q9, d5 \n"
+ "vmovl.u8 q10, d6 \n"
+ "vmovl.u8 q11, d7 \n"
// 3 * line_0 + line_1
- "vmlal.u8 q8, d0, d24 \n"
- "vmlal.u8 q9, d1, d24 \n"
- "vmlal.u8 q10, d2, d24 \n"
- "vmlal.u8 q11, d3, d24 \n"
+ "vmlal.u8 q8, d0, d24 \n"
+ "vmlal.u8 q9, d1, d24 \n"
+ "vmlal.u8 q10, d2, d24 \n"
+ "vmlal.u8 q11, d3, d24 \n"
- // (3 * line_0 + line_1) >> 2
- "vqrshrn.u16 d0, q8, #2 \n"
- "vqrshrn.u16 d1, q9, #2 \n"
- "vqrshrn.u16 d2, q10, #2 \n"
- "vqrshrn.u16 d3, q11, #2 \n"
+ // (3 * line_0 + line_1 + 2) >> 2
+ "vqrshrn.u16 d0, q8, #2 \n"
+ "vqrshrn.u16 d1, q9, #2 \n"
+ "vqrshrn.u16 d2, q10, #2 \n"
+ "vqrshrn.u16 d3, q11, #2 \n"
- // a0 = (src[0] * 3 + s[1] * 1) >> 2
- "vmovl.u8 q8, d1 \n"
- "vmlal.u8 q8, d0, d24 \n"
- "vqrshrn.u16 d0, q8, #2 \n"
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "vmovl.u8 q8, d1 \n"
+ "vmlal.u8 q8, d0, d24 \n"
+ "vqrshrn.u16 d0, q8, #2 \n"
- // a1 = (src[1] * 1 + s[2] * 1) >> 1
- "vrhadd.u8 d1, d1, d2 \n"
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "vrhadd.u8 d1, d1, d2 \n"
- // a2 = (src[2] * 1 + s[3] * 3) >> 2
- "vmovl.u8 q8, d2 \n"
- "vmlal.u8 q8, d3, d24 \n"
- "vqrshrn.u16 d2, q8, #2 \n"
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "vmovl.u8 q8, d2 \n"
+ "vmlal.u8 q8, d3, d24 \n"
+ "vqrshrn.u16 d2, q8, #2 \n"
- "vst3.8 {d0, d1, d2}, [%1]! \n"
+ "vst3.8 {d0, d1, d2}, [%1]! \n"
- "bgt 1b \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
@@ -230,31 +238,31 @@ void ScaleRowDown34_1_Box_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "vmov.u8 d24, #3 \n"
- "add %3, %0 \n"
- "1: \n"
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
- "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
- "subs %2, %2, #24 \n"
+ "vmov.u8 d24, #3 \n"
+ "add %3, %0 \n"
+ "1: \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
+ "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
+ "subs %2, %2, #24 \n"
// average src line 0 with src line 1
- "vrhadd.u8 q0, q0, q2 \n"
- "vrhadd.u8 q1, q1, q3 \n"
+ "vrhadd.u8 q0, q0, q2 \n"
+ "vrhadd.u8 q1, q1, q3 \n"
- // a0 = (src[0] * 3 + s[1] * 1) >> 2
- "vmovl.u8 q3, d1 \n"
- "vmlal.u8 q3, d0, d24 \n"
- "vqrshrn.u16 d0, q3, #2 \n"
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "vmovl.u8 q3, d1 \n"
+ "vmlal.u8 q3, d0, d24 \n"
+ "vqrshrn.u16 d0, q3, #2 \n"
- // a1 = (src[1] * 1 + s[2] * 1) >> 1
- "vrhadd.u8 d1, d1, d2 \n"
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "vrhadd.u8 d1, d1, d2 \n"
- // a2 = (src[2] * 1 + s[3] * 3) >> 2
- "vmovl.u8 q3, d2 \n"
- "vmlal.u8 q3, d3, d24 \n"
- "vqrshrn.u16 d2, q3, #2 \n"
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "vmovl.u8 q3, d2 \n"
+ "vmlal.u8 q3, d3, d24 \n"
+ "vqrshrn.u16 d2, q3, #2 \n"
- "vst3.8 {d0, d1, d2}, [%1]! \n"
- "bgt 1b \n"
+ "vst3.8 {d0, d1, d2}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
@@ -282,20 +290,20 @@ void ScaleRowDown38_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "vld1.8 {q3}, [%3] \n"
- "1: \n"
- "vld1.8 {d0, d1, d2, d3}, [%0]! \n"
- "subs %2, %2, #12 \n"
- "vtbl.u8 d4, {d0, d1, d2, d3}, d6 \n"
- "vtbl.u8 d5, {d0, d1, d2, d3}, d7 \n"
- "vst1.8 {d4}, [%1]! \n"
- "vst1.32 {d5[0]}, [%1]! \n"
- "bgt 1b \n"
+ "vld1.8 {q3}, [%3] \n"
+ "1: \n"
+ "vld1.8 {d0, d1, d2, d3}, [%0]! \n"
+ "subs %2, %2, #12 \n"
+ "vtbl.u8 d4, {d0, d1, d2, d3}, d6 \n"
+ "vtbl.u8 d5, {d0, d1, d2, d3}, d7 \n"
+ "vst1.8 {d4}, [%1]! \n"
+ "vst1.32 {d5[0]}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
: "r"(&kShuf38) // %3
- : "d0", "d1", "d2", "d3", "d4", "d5", "memory", "cc");
+ : "memory", "cc", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7");
}
// 32x3 -> 12x1
@@ -306,57 +314,57 @@ void OMITFP ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
const uint8_t* src_ptr1 = src_ptr + src_stride * 2;
asm volatile(
- "vld1.16 {q13}, [%5] \n"
- "vld1.8 {q14}, [%6] \n"
- "vld1.8 {q15}, [%7] \n"
- "add %3, %0 \n"
- "1: \n"
+ "vld1.16 {q13}, [%5] \n"
+ "vld1.8 {q14}, [%6] \n"
+ "vld1.8 {q15}, [%7] \n"
+ "add %3, %0 \n"
+ "1: \n"
// d0 = 00 40 01 41 02 42 03 43
// d1 = 10 50 11 51 12 52 13 53
// d2 = 20 60 21 61 22 62 23 63
// d3 = 30 70 31 71 32 72 33 73
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
- "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
- "vld4.8 {d16, d17, d18, d19}, [%4]! \n"
- "subs %2, %2, #12 \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
+ "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
+ "vld4.8 {d16, d17, d18, d19}, [%4]! \n"
+ "subs %2, %2, #12 \n"
// Shuffle the input data around to get align the data
// so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
// d0 = 00 10 01 11 02 12 03 13
// d1 = 40 50 41 51 42 52 43 53
- "vtrn.u8 d0, d1 \n"
- "vtrn.u8 d4, d5 \n"
- "vtrn.u8 d16, d17 \n"
+ "vtrn.u8 d0, d1 \n"
+ "vtrn.u8 d4, d5 \n"
+ "vtrn.u8 d16, d17 \n"
// d2 = 20 30 21 31 22 32 23 33
// d3 = 60 70 61 71 62 72 63 73
- "vtrn.u8 d2, d3 \n"
- "vtrn.u8 d6, d7 \n"
- "vtrn.u8 d18, d19 \n"
+ "vtrn.u8 d2, d3 \n"
+ "vtrn.u8 d6, d7 \n"
+ "vtrn.u8 d18, d19 \n"
// d0 = 00+10 01+11 02+12 03+13
// d2 = 40+50 41+51 42+52 43+53
- "vpaddl.u8 q0, q0 \n"
- "vpaddl.u8 q2, q2 \n"
- "vpaddl.u8 q8, q8 \n"
+ "vpaddl.u8 q0, q0 \n"
+ "vpaddl.u8 q2, q2 \n"
+ "vpaddl.u8 q8, q8 \n"
// d3 = 60+70 61+71 62+72 63+73
- "vpaddl.u8 d3, d3 \n"
- "vpaddl.u8 d7, d7 \n"
- "vpaddl.u8 d19, d19 \n"
+ "vpaddl.u8 d3, d3 \n"
+ "vpaddl.u8 d7, d7 \n"
+ "vpaddl.u8 d19, d19 \n"
// combine source lines
- "vadd.u16 q0, q2 \n"
- "vadd.u16 q0, q8 \n"
- "vadd.u16 d4, d3, d7 \n"
- "vadd.u16 d4, d19 \n"
+ "vadd.u16 q0, q2 \n"
+ "vadd.u16 q0, q8 \n"
+ "vadd.u16 d4, d3, d7 \n"
+ "vadd.u16 d4, d19 \n"
// dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0]
// + s[6 + st * 1] + s[7 + st * 1]
// + s[6 + st * 2] + s[7 + st * 2]) / 6
"vqrdmulh.s16 q2, q2, q13 \n"
- "vmovn.u16 d4, q2 \n"
+ "vmovn.u16 d4, q2 \n"
// Shuffle 2,3 reg around so that 2 can be added to the
// 0,1 reg and 3 can be added to the 4,5 reg. This
@@ -364,24 +372,24 @@ void OMITFP ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
// registers are already expanded. Then do transposes
// to get aligned.
// q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
- "vmovl.u8 q1, d2 \n"
- "vmovl.u8 q3, d6 \n"
- "vmovl.u8 q9, d18 \n"
+ "vmovl.u8 q1, d2 \n"
+ "vmovl.u8 q3, d6 \n"
+ "vmovl.u8 q9, d18 \n"
// combine source lines
- "vadd.u16 q1, q3 \n"
- "vadd.u16 q1, q9 \n"
+ "vadd.u16 q1, q3 \n"
+ "vadd.u16 q1, q9 \n"
// d4 = xx 20 xx 30 xx 22 xx 32
// d5 = xx 21 xx 31 xx 23 xx 33
- "vtrn.u32 d2, d3 \n"
+ "vtrn.u32 d2, d3 \n"
// d4 = xx 20 xx 21 xx 22 xx 23
// d5 = xx 30 xx 31 xx 32 xx 33
- "vtrn.u16 d2, d3 \n"
+ "vtrn.u16 d2, d3 \n"
// 0+1+2, 3+4+5
- "vadd.u16 q0, q1 \n"
+ "vadd.u16 q0, q1 \n"
// Need to divide, but can't downshift as the the value
// isn't a power of 2. So multiply by 65536 / n
@@ -390,14 +398,14 @@ void OMITFP ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
// Align for table lookup, vtbl requires registers to
// be adjacent
- "vmov.u8 d2, d4 \n"
+ "vmov.u8 d2, d4 \n"
- "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
- "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
+ "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
+ "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
- "vst1.8 {d3}, [%1]! \n"
- "vst1.32 {d4[0]}, [%1]! \n"
- "bgt 1b \n"
+ "vst1.8 {d3}, [%1]! \n"
+ "vst1.32 {d4[0]}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
@@ -416,46 +424,46 @@ void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "vld1.16 {q13}, [%4] \n"
- "vld1.8 {q14}, [%5] \n"
- "add %3, %0 \n"
- "1: \n"
+ "vld1.16 {q13}, [%4] \n"
+ "vld1.8 {q14}, [%5] \n"
+ "add %3, %0 \n"
+ "1: \n"
// d0 = 00 40 01 41 02 42 03 43
// d1 = 10 50 11 51 12 52 13 53
// d2 = 20 60 21 61 22 62 23 63
// d3 = 30 70 31 71 32 72 33 73
- "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
- "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
- "subs %2, %2, #12 \n"
+ "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
+ "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
+ "subs %2, %2, #12 \n"
// Shuffle the input data around to get align the data
// so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
// d0 = 00 10 01 11 02 12 03 13
// d1 = 40 50 41 51 42 52 43 53
- "vtrn.u8 d0, d1 \n"
- "vtrn.u8 d4, d5 \n"
+ "vtrn.u8 d0, d1 \n"
+ "vtrn.u8 d4, d5 \n"
// d2 = 20 30 21 31 22 32 23 33
// d3 = 60 70 61 71 62 72 63 73
- "vtrn.u8 d2, d3 \n"
- "vtrn.u8 d6, d7 \n"
+ "vtrn.u8 d2, d3 \n"
+ "vtrn.u8 d6, d7 \n"
// d0 = 00+10 01+11 02+12 03+13
// d2 = 40+50 41+51 42+52 43+53
- "vpaddl.u8 q0, q0 \n"
- "vpaddl.u8 q2, q2 \n"
+ "vpaddl.u8 q0, q0 \n"
+ "vpaddl.u8 q2, q2 \n"
// d3 = 60+70 61+71 62+72 63+73
- "vpaddl.u8 d3, d3 \n"
- "vpaddl.u8 d7, d7 \n"
+ "vpaddl.u8 d3, d3 \n"
+ "vpaddl.u8 d7, d7 \n"
// combine source lines
- "vadd.u16 q0, q2 \n"
- "vadd.u16 d4, d3, d7 \n"
+ "vadd.u16 q0, q2 \n"
+ "vadd.u16 d4, d3, d7 \n"
// dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4
- "vqrshrn.u16 d4, q2, #2 \n"
+ "vqrshrn.u16 d4, q2, #2 \n"
// Shuffle 2,3 reg around so that 2 can be added to the
// 0,1 reg and 3 can be added to the 4,5 reg. This
@@ -463,22 +471,22 @@ void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
// registers are already expanded. Then do transposes
// to get aligned.
// q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
- "vmovl.u8 q1, d2 \n"
- "vmovl.u8 q3, d6 \n"
+ "vmovl.u8 q1, d2 \n"
+ "vmovl.u8 q3, d6 \n"
// combine source lines
- "vadd.u16 q1, q3 \n"
+ "vadd.u16 q1, q3 \n"
// d4 = xx 20 xx 30 xx 22 xx 32
// d5 = xx 21 xx 31 xx 23 xx 33
- "vtrn.u32 d2, d3 \n"
+ "vtrn.u32 d2, d3 \n"
// d4 = xx 20 xx 21 xx 22 xx 23
// d5 = xx 30 xx 31 xx 32 xx 33
- "vtrn.u16 d2, d3 \n"
+ "vtrn.u16 d2, d3 \n"
// 0+1+2, 3+4+5
- "vadd.u16 q0, q1 \n"
+ "vadd.u16 q0, q1 \n"
// Need to divide, but can't downshift as the the value
// isn't a power of 2. So multiply by 65536 / n
@@ -487,14 +495,14 @@ void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
// Align for table lookup, vtbl requires registers to
// be adjacent
- "vmov.u8 d2, d4 \n"
+ "vmov.u8 d2, d4 \n"
- "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
- "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
+ "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
+ "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
- "vst1.8 {d3}, [%1]! \n"
- "vst1.32 {d4[0]}, [%1]! \n"
- "bgt 1b \n"
+ "vst1.8 {d3}, [%1]! \n"
+ "vst1.32 {d4[0]}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
@@ -504,20 +512,498 @@ void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
: "q0", "q1", "q2", "q3", "q13", "q14", "memory", "cc");
}
+void ScaleRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "vmov.u8 d30, #3 \n"
+
+ "1: \n"
+ "vld1.8 {d4}, [%0]! \n" // 01234567
+ "vld1.8 {d5}, [%3]! \n" // 12345678
+
+ "vmovl.u8 q0, d4 \n" // 01234567 (16b)
+ "vmovl.u8 q1, d5 \n" // 12345678 (16b)
+ "vmlal.u8 q0, d5, d30 \n" // 3*near+far (odd)
+ "vmlal.u8 q1, d4, d30 \n" // 3*near+far (even)
+
+ "vrshrn.u16 d1, q0, #2 \n" // 3/4*near+1/4*far (odd)
+ "vrshrn.u16 d0, q1, #2 \n" // 3/4*near+1/4*far (even)
+
+ "vst2.8 {d0, d1}, [%1]! \n" // store
+ "subs %2, %2, #16 \n" // 8 sample -> 16 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "+r"(src_temp) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q15" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ uint8_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint8_t* src_temp = src_ptr + 1;
+ const uint8_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "vmov.u16 q15, #3 \n"
+ "vmov.u8 d28, #3 \n"
+
+ "1: \n"
+ "vld1.8 {d4}, [%0]! \n" // 01234567
+ "vld1.8 {d5}, [%5]! \n" // 12345678
+
+ "vmovl.u8 q0, d4 \n" // 01234567 (16b)
+ "vmovl.u8 q1, d5 \n" // 12345678 (16b)
+ "vmlal.u8 q0, d5, d28 \n" // 3*near+far (1, odd)
+ "vmlal.u8 q1, d4, d28 \n" // 3*near+far (1, even)
+
+ "vld1.8 {d8}, [%1]! \n"
+ "vld1.8 {d9}, [%6]! \n"
+
+ "vmovl.u8 q2, d8 \n"
+ "vmovl.u8 q3, d9 \n"
+ "vmlal.u8 q2, d9, d28 \n" // 3*near+far (2, odd)
+ "vmlal.u8 q3, d8, d28 \n" // 3*near+far (2, even)
+
+ // e o
+ // q1 q0
+ // q3 q2
+
+ "vmovq q4, q2 \n"
+ "vmovq q5, q3 \n"
+ "vmla.u16 q4, q0, q15 \n" // 9 3 3 1 (1, odd)
+ "vmla.u16 q5, q1, q15 \n" // 9 3 3 1 (1, even)
+ "vmla.u16 q0, q2, q15 \n" // 9 3 3 1 (2, odd)
+ "vmla.u16 q1, q3, q15 \n" // 9 3 3 1 (2, even)
+
+ // e o
+ // q5 q4
+ // q1 q0
+
+ "vrshrn.u16 d2, q1, #4 \n" // 2, even
+ "vrshrn.u16 d3, q0, #4 \n" // 2, odd
+ "vrshrn.u16 d0, q5, #4 \n" // 1, even
+ "vrshrn.u16 d1, q4, #4 \n" // 1, odd
+
+ "vst2.8 {d0, d1}, [%2]! \n" // store
+ "vst2.8 {d2, d3}, [%3]! \n" // store
+ "subs %4, %4, #16 \n" // 8 sample -> 16 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_ptr1), // %3
+ "+r"(dst_width), // %4
+ "+r"(src_temp), // %5
+ "+r"(src_temp1) // %6
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "d28",
+ "q15" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Linear_12_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "vmov.u16 q15, #3 \n"
+
+ "1: \n"
+ "vld1.16 {q1}, [%0]! \n" // 01234567 (16b)
+ "vld1.16 {q0}, [%3]! \n" // 12345678 (16b)
+
+ "vmovq q2, q0 \n"
+ "vmla.u16 q0, q1, q15 \n" // 3*near+far (odd)
+ "vmla.u16 q1, q2, q15 \n" // 3*near+far (even)
+
+ "vrshr.u16 q0, q0, #2 \n" // 3/4*near+1/4*far (odd)
+ "vrshr.u16 q1, q1, #2 \n" // 3/4*near+1/4*far (even)
+
+ "vst2.16 {d0, d1, d2, d3}, [%1]! \n" // store
+ "subs %2, %2, #16 \n" // 8 sample -> 16 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "+r"(src_temp) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q15" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_12_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 1;
+ const uint16_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "vmov.u16 q15, #3 \n"
+
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n" // 01234567 (16b)
+ "vld1.16 {q1}, [%5]! \n" // 12345678 (16b)
+
+ "vmovq q2, q0 \n"
+ "vmla.u16 q0, q1, q15 \n" // 3*near+far (odd)
+ "vmla.u16 q1, q2, q15 \n" // 3*near+far (even)
+
+ "vld1.16 {q2}, [%1]! \n" // 01234567 (16b)
+ "vld1.16 {q3}, [%6]! \n" // 12345678 (16b)
+
+ "vmovq q4, q2 \n"
+ "vmla.u16 q2, q3, q15 \n" // 3*near+far (odd)
+ "vmla.u16 q3, q4, q15 \n" // 3*near+far (even)
+
+ "vmovq q4, q2 \n"
+ "vmovq q5, q3 \n"
+ "vmla.u16 q4, q0, q15 \n" // 9 3 3 1 (1, odd)
+ "vmla.u16 q5, q1, q15 \n" // 9 3 3 1 (1, even)
+ "vmla.u16 q0, q2, q15 \n" // 9 3 3 1 (2, odd)
+ "vmla.u16 q1, q3, q15 \n" // 9 3 3 1 (2, even)
+
+ "vrshr.u16 q2, q1, #4 \n" // 2, even
+ "vrshr.u16 q3, q0, #4 \n" // 2, odd
+ "vrshr.u16 q0, q5, #4 \n" // 1, even
+ "vrshr.u16 q1, q4, #4 \n" // 1, odd
+
+ "vst2.16 {d0, d1, d2, d3}, [%2]! \n" // store
+ "vst2.16 {d4, d5, d6, d7}, [%3]! \n" // store
+ "subs %4, %4, #16 \n" // 8 sample -> 16 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_ptr1), // %3
+ "+r"(dst_width), // %4
+ "+r"(src_temp), // %5
+ "+r"(src_temp1) // %6
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5",
+ "q15" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "vmov.u16 d31, #3 \n"
+
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n" // 01234567 (16b)
+ "vld1.16 {q1}, [%3]! \n" // 12345678 (16b)
+
+ "vmovl.u16 q2, d0 \n" // 0123 (32b)
+ "vmovl.u16 q3, d1 \n" // 4567 (32b)
+ "vmovl.u16 q4, d2 \n" // 1234 (32b)
+ "vmovl.u16 q5, d3 \n" // 5678 (32b)
+
+ "vmlal.u16 q2, d2, d31 \n"
+ "vmlal.u16 q3, d3, d31 \n"
+ "vmlal.u16 q4, d0, d31 \n"
+ "vmlal.u16 q5, d1, d31 \n"
+
+ "vrshrn.u32 d0, q4, #2 \n"
+ "vrshrn.u32 d1, q5, #2 \n"
+ "vrshrn.u32 d2, q2, #2 \n"
+ "vrshrn.u32 d3, q3, #2 \n"
+
+ "vst2.16 {q0, q1}, [%1]! \n" // store
+ "subs %2, %2, #16 \n" // 8 sample -> 16 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "+r"(src_temp) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q15" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 1;
+ const uint16_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "vmov.u16 d31, #3 \n"
+ "vmov.u32 q14, #3 \n"
+
+ "1: \n"
+ "vld1.16 {d0}, [%0]! \n" // 0123 (16b)
+ "vld1.16 {d1}, [%5]! \n" // 1234 (16b)
+ "vmovl.u16 q2, d0 \n" // 0123 (32b)
+ "vmovl.u16 q3, d1 \n" // 1234 (32b)
+ "vmlal.u16 q2, d1, d31 \n"
+ "vmlal.u16 q3, d0, d31 \n"
+
+ "vld1.16 {d0}, [%1]! \n" // 0123 (16b)
+ "vld1.16 {d1}, [%6]! \n" // 1234 (16b)
+ "vmovl.u16 q4, d0 \n" // 0123 (32b)
+ "vmovl.u16 q5, d1 \n" // 1234 (32b)
+ "vmlal.u16 q4, d1, d31 \n"
+ "vmlal.u16 q5, d0, d31 \n"
+
+ "vmovq q0, q4 \n"
+ "vmovq q1, q5 \n"
+ "vmla.u32 q4, q2, q14 \n"
+ "vmla.u32 q5, q3, q14 \n"
+ "vmla.u32 q2, q0, q14 \n"
+ "vmla.u32 q3, q1, q14 \n"
+
+ "vrshrn.u32 d1, q4, #4 \n"
+ "vrshrn.u32 d0, q5, #4 \n"
+ "vrshrn.u32 d3, q2, #4 \n"
+ "vrshrn.u32 d2, q3, #4 \n"
+
+ "vst2.16 {d0, d1}, [%2]! \n" // store
+ "vst2.16 {d2, d3}, [%3]! \n" // store
+ "subs %4, %4, #8 \n" // 4 sample -> 8 sample
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_ptr1), // %3
+ "+r"(dst_width), // %4
+ "+r"(src_temp), // %5
+ "+r"(src_temp1) // %6
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q14",
+ "d31" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_temp = src_ptr + 2;
+ asm volatile(
+ "vmov.u8 d30, #3 \n"
+
+ "1: \n"
+ "vld1.8 {d4}, [%0]! \n" // 00112233 (1u1v)
+ "vld1.8 {d5}, [%3]! \n" // 11223344 (1u1v)
+
+ "vmovl.u8 q0, d4 \n" // 00112233 (1u1v, 16b)
+ "vmovl.u8 q1, d5 \n" // 11223344 (1u1v, 16b)
+ "vmlal.u8 q0, d5, d30 \n" // 3*near+far (odd)
+ "vmlal.u8 q1, d4, d30 \n" // 3*near+far (even)
+
+ "vrshrn.u16 d1, q0, #2 \n" // 3/4*near+1/4*far (odd)
+ "vrshrn.u16 d0, q1, #2 \n" // 3/4*near+1/4*far (even)
+
+ "vst2.16 {d0, d1}, [%1]! \n" // store
+ "subs %2, %2, #8 \n" // 4 uv -> 8 uv
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "+r"(src_temp) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "d30" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ uint8_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint8_t* src_temp = src_ptr + 2;
+ const uint8_t* src_temp1 = src_ptr1 + 2;
+
+ asm volatile(
+ "vmov.u16 q15, #3 \n"
+ "vmov.u8 d28, #3 \n"
+
+ "1: \n"
+ "vld1.8 {d4}, [%0]! \n" // 00112233 (1u1v)
+ "vld1.8 {d5}, [%5]! \n" // 11223344 (1u1v)
+
+ "vmovl.u8 q0, d4 \n" // 00112233 (1u1v, 16b)
+ "vmovl.u8 q1, d5 \n" // 11223344 (1u1v, 16b)
+ "vmlal.u8 q0, d5, d28 \n" // 3*near+far (1, odd)
+ "vmlal.u8 q1, d4, d28 \n" // 3*near+far (1, even)
+
+ "vld1.8 {d8}, [%1]! \n" // 00112233 (1u1v)
+ "vld1.8 {d9}, [%6]! \n" // 11223344 (1u1v)
+
+ "vmovl.u8 q2, d8 \n" // 00112233 (1u1v, 16b)
+ "vmovl.u8 q3, d9 \n" // 11223344 (1u1v, 16b)
+ "vmlal.u8 q2, d9, d28 \n" // 3*near+far (2, odd)
+ "vmlal.u8 q3, d8, d28 \n" // 3*near+far (2, even)
+
+ // e o
+ // q1 q0
+ // q3 q2
+
+ "vmovq q4, q2 \n"
+ "vmovq q5, q3 \n"
+ "vmla.u16 q4, q0, q15 \n" // 9 3 3 1 (1, odd)
+ "vmla.u16 q5, q1, q15 \n" // 9 3 3 1 (1, even)
+ "vmla.u16 q0, q2, q15 \n" // 9 3 3 1 (2, odd)
+ "vmla.u16 q1, q3, q15 \n" // 9 3 3 1 (2, even)
+
+ // e o
+ // q5 q4
+ // q1 q0
+
+ "vrshrn.u16 d2, q1, #4 \n" // 2, even
+ "vrshrn.u16 d3, q0, #4 \n" // 2, odd
+ "vrshrn.u16 d0, q5, #4 \n" // 1, even
+ "vrshrn.u16 d1, q4, #4 \n" // 1, odd
+
+ "vst2.16 {d0, d1}, [%2]! \n" // store
+ "vst2.16 {d2, d3}, [%3]! \n" // store
+ "subs %4, %4, #8 \n" // 4 uv -> 8 uv
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_ptr1), // %3
+ "+r"(dst_width), // %4
+ "+r"(src_temp), // %5
+ "+r"(src_temp1) // %6
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "d28",
+ "q15" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 2;
+ asm volatile(
+ "vmov.u16 d30, #3 \n"
+
+ "1: \n"
+ "vld1.16 {q0}, [%0]! \n" // 00112233 (1u1v, 16)
+ "vld1.16 {q1}, [%3]! \n" // 11223344 (1u1v, 16)
+
+ "vmovl.u16 q2, d0 \n" // 0011 (1u1v, 32b)
+ "vmovl.u16 q3, d2 \n" // 1122 (1u1v, 32b)
+ "vmovl.u16 q4, d1 \n" // 2233 (1u1v, 32b)
+ "vmovl.u16 q5, d3 \n" // 3344 (1u1v, 32b)
+ "vmlal.u16 q2, d2, d30 \n" // 3*near+far (odd)
+ "vmlal.u16 q3, d0, d30 \n" // 3*near+far (even)
+ "vmlal.u16 q4, d3, d30 \n" // 3*near+far (odd)
+ "vmlal.u16 q5, d1, d30 \n" // 3*near+far (even)
+
+ "vrshrn.u32 d1, q2, #2 \n" // 3/4*near+1/4*far (odd)
+ "vrshrn.u32 d0, q3, #2 \n" // 3/4*near+1/4*far (even)
+ "vrshrn.u32 d3, q4, #2 \n" // 3/4*near+1/4*far (odd)
+ "vrshrn.u32 d2, q5, #2 \n" // 3/4*near+1/4*far (even)
+
+ "vst2.32 {d0, d1}, [%1]! \n" // store
+ "vst2.32 {d2, d3}, [%1]! \n" // store
+ "subs %2, %2, #8 \n" // 4 uv -> 8 uv
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width), // %2
+ "+r"(src_temp) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5",
+ "d30" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 2;
+ const uint16_t* src_temp1 = src_ptr1 + 2;
+
+ asm volatile(
+ "vmov.u16 d30, #3 \n"
+ "vmov.u32 q14, #3 \n"
+
+ "1: \n"
+ "vld1.8 {d0}, [%0]! \n" // 0011 (1u1v)
+ "vld1.8 {d1}, [%5]! \n" // 1122 (1u1v)
+ "vmovl.u16 q2, d0 \n" // 0011 (1u1v, 32b)
+ "vmovl.u16 q3, d1 \n" // 1122 (1u1v, 32b)
+ "vmlal.u16 q2, d1, d30 \n" // 3*near+far (1, odd)
+ "vmlal.u16 q3, d0, d30 \n" // 3*near+far (1, even)
+
+ "vld1.8 {d0}, [%1]! \n" // 0011 (1u1v)
+ "vld1.8 {d1}, [%6]! \n" // 1122 (1u1v)
+ "vmovl.u16 q4, d0 \n" // 0011 (1u1v, 32b)
+ "vmovl.u16 q5, d1 \n" // 1122 (1u1v, 32b)
+ "vmlal.u16 q4, d1, d30 \n" // 3*near+far (2, odd)
+ "vmlal.u16 q5, d0, d30 \n" // 3*near+far (2, even)
+
+ "vmovq q0, q4 \n"
+ "vmovq q1, q5 \n"
+ "vmla.u32 q4, q2, q14 \n" // 9 3 3 1 (1, odd)
+ "vmla.u32 q5, q3, q14 \n" // 9 3 3 1 (1, even)
+ "vmla.u32 q2, q0, q14 \n" // 9 3 3 1 (2, odd)
+ "vmla.u32 q3, q1, q14 \n" // 9 3 3 1 (2, even)
+
+ "vrshrn.u32 d1, q4, #4 \n" // 1, odd
+ "vrshrn.u32 d0, q5, #4 \n" // 1, even
+ "vrshrn.u32 d3, q2, #4 \n" // 2, odd
+ "vrshrn.u32 d2, q3, #4 \n" // 2, even
+
+ "vst2.32 {d0, d1}, [%2]! \n" // store
+ "vst2.32 {d2, d3}, [%3]! \n" // store
+ "subs %4, %4, #4 \n" // 2 uv -> 4 uv
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_ptr1), // %3
+ "+r"(dst_width), // %4
+ "+r"(src_temp), // %5
+ "+r"(src_temp1) // %6
+ :
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q14",
+ "d30" // Clobber List
+ );
+}
+
// Add a row of bytes to a row of shorts. Used for box filter.
// Reads 16 bytes and accumulates to 16 shorts at a time.
void ScaleAddRow_NEON(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
asm volatile(
- "1: \n"
- "vld1.16 {q1, q2}, [%1] \n" // load accumulator
- "vld1.8 {q0}, [%0]! \n" // load 16 bytes
- "vaddw.u8 q2, q2, d1 \n" // add
- "vaddw.u8 q1, q1, d0 \n"
- "vst1.16 {q1, q2}, [%1]! \n" // store accumulator
- "subs %2, %2, #16 \n" // 16 processed per loop
- "bgt 1b \n"
+ "1: \n"
+ "vld1.16 {q1, q2}, [%1] \n" // load accumulator
+ "vld1.8 {q0}, [%0]! \n" // load 16 bytes
+ "vaddw.u8 q2, q2, d1 \n" // add
+ "vaddw.u8 q1, q1, d0 \n"
+ "vst1.16 {q1, q2}, [%1]! \n" // store accumulator
+ "subs %2, %2, #16 \n" // 16 processed per loop
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_width) // %2
@@ -547,17 +1033,17 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
int* tmp = dx_offset;
const uint8_t* src_tmp = src_ptr;
asm volatile (
- "vdup.32 q0, %3 \n" // x
- "vdup.32 q1, %4 \n" // dx
- "vld1.32 {q2}, [%5] \n" // 0 1 2 3
- "vshl.i32 q3, q1, #2 \n" // 4 * dx
- "vmul.s32 q1, q1, q2 \n"
+ "vdup.32 q0, %3 \n" // x
+ "vdup.32 q1, %4 \n" // dx
+ "vld1.32 {q2}, [%5] \n" // 0 1 2 3
+ "vshl.i32 q3, q1, #2 \n" // 4 * dx
+ "vmul.s32 q1, q1, q2 \n"
// x , x + 1 * dx, x + 2 * dx, x + 3 * dx
- "vadd.s32 q1, q1, q0 \n"
+ "vadd.s32 q1, q1, q0 \n"
// x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
- "vadd.s32 q2, q1, q3 \n"
- "vshl.i32 q0, q3, #1 \n" // 8 * dx
- "1: \n"
+ "vadd.s32 q2, q1, q3 \n"
+ "vshl.i32 q0, q3, #1 \n" // 8 * dx
+ "1: \n"
LOAD2_DATA8_LANE(0)
LOAD2_DATA8_LANE(1)
LOAD2_DATA8_LANE(2)
@@ -566,27 +1052,27 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
LOAD2_DATA8_LANE(5)
LOAD2_DATA8_LANE(6)
LOAD2_DATA8_LANE(7)
- "vmov q10, q1 \n"
- "vmov q11, q2 \n"
- "vuzp.16 q10, q11 \n"
- "vmovl.u8 q8, d6 \n"
- "vmovl.u8 q9, d7 \n"
- "vsubl.s16 q11, d18, d16 \n"
- "vsubl.s16 q12, d19, d17 \n"
- "vmovl.u16 q13, d20 \n"
- "vmovl.u16 q10, d21 \n"
- "vmul.s32 q11, q11, q13 \n"
- "vmul.s32 q12, q12, q10 \n"
- "vrshrn.s32 d18, q11, #16 \n"
- "vrshrn.s32 d19, q12, #16 \n"
- "vadd.s16 q8, q8, q9 \n"
- "vmovn.s16 d6, q8 \n"
-
- "vst1.8 {d6}, [%0]! \n" // store pixels
- "vadd.s32 q1, q1, q0 \n"
- "vadd.s32 q2, q2, q0 \n"
- "subs %2, %2, #8 \n" // 8 processed per loop
- "bgt 1b \n"
+ "vmov q10, q1 \n"
+ "vmov q11, q2 \n"
+ "vuzp.16 q10, q11 \n"
+ "vmovl.u8 q8, d6 \n"
+ "vmovl.u8 q9, d7 \n"
+ "vsubl.s16 q11, d18, d16 \n"
+ "vsubl.s16 q12, d19, d17 \n"
+ "vmovl.u16 q13, d20 \n"
+ "vmovl.u16 q10, d21 \n"
+ "vmul.s32 q11, q11, q13 \n"
+ "vmul.s32 q12, q12, q10 \n"
+ "vrshrn.s32 d18, q11, #16 \n"
+ "vrshrn.s32 d19, q12, #16 \n"
+ "vadd.s16 q8, q8, q9 \n"
+ "vmovn.s16 d6, q8 \n"
+
+ "vst1.8 {d6}, [%0]! \n" // store pixels
+ "vadd.s32 q1, q1, q0 \n"
+ "vadd.s32 q2, q2, q0 \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop
+ "bgt 1b \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(dst_width), // %2
@@ -602,104 +1088,19 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
#undef LOAD2_DATA8_LANE
-// 16x2 -> 16x1
-void ScaleFilterRows_NEON(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- asm volatile(
- "cmp %4, #0 \n"
- "beq 100f \n"
- "add %2, %1 \n"
- "cmp %4, #64 \n"
- "beq 75f \n"
- "cmp %4, #128 \n"
- "beq 50f \n"
- "cmp %4, #192 \n"
- "beq 25f \n"
-
- "vdup.8 d5, %4 \n"
- "rsb %4, #256 \n"
- "vdup.8 d4, %4 \n"
- // General purpose row blend.
- "1: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vmull.u8 q13, d0, d4 \n"
- "vmull.u8 q14, d1, d4 \n"
- "vmlal.u8 q13, d2, d5 \n"
- "vmlal.u8 q14, d3, d5 \n"
- "vrshrn.u16 d0, q13, #8 \n"
- "vrshrn.u16 d1, q14, #8 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 1b \n"
- "b 99f \n"
-
- // Blend 25 / 75.
- "25: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 25b \n"
- "b 99f \n"
-
- // Blend 50 / 50.
- "50: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 50b \n"
- "b 99f \n"
-
- // Blend 75 / 25.
- "75: \n"
- "vld1.8 {q1}, [%1]! \n"
- "vld1.8 {q0}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 75b \n"
- "b 99f \n"
-
- // Blend 100 / 0 - Copy row unchanged.
- "100: \n"
- "vld1.8 {q0}, [%1]! \n"
- "subs %3, %3, #16 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 100b \n"
-
- "99: \n"
- "vst1.8 {d1[7]}, [%0] \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(src_stride), // %2
- "+r"(dst_width), // %3
- "+r"(source_y_fraction) // %4
- :
- : "q0", "q1", "d4", "d5", "q13", "q14", "memory", "cc");
-}
-
void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
- "subs %2, %2, #8 \n" // 8 processed per loop
- "vmov q2, q1 \n" // load next 8 ARGB
- "vst2.32 {q2, q3}, [%1]! \n" // store odd pixels
- "bgt 1b \n"
+ "1: \n"
+ "vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
+ "subs %2, %2, #8 \n" // 8 processed per loop
+ "vmov q2, q1 \n" // load next 8 ARGB
+ "vst2.32 {q2, q3}, [%1]! \n" // store odd pixels
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst), // %1
"+r"(dst_width) // %2
@@ -721,14 +1122,14 @@ void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
- "subs %2, %2, #8 \n" // 8 processed per loop
- "vrhadd.u8 q0, q0, q1 \n" // rounding half add
- "vrhadd.u8 q1, q2, q3 \n" // rounding half add
- "vst2.32 {q0, q1}, [%1]! \n"
- "bgt 1b \n"
+ "1: \n"
+ "vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
+ "subs %2, %2, #8 \n" // 8 processed per loop
+ "vrhadd.u8 q0, q0, q1 \n" // rounding half add
+ "vrhadd.u8 q1, q2, q3 \n" // rounding half add
+ "vst2.32 {q0, q1}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(dst_width) // %2
@@ -743,27 +1144,27 @@ void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr,
int dst_width) {
asm volatile(
// change the stride to row 2 pointer
- "add %1, %1, %0 \n"
- "1: \n"
- "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
- "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
- "subs %3, %3, #8 \n" // 8 processed per loop.
- "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
- "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
- "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
- "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts.
- "vld4.8 {d16, d18, d20, d22}, [%1]! \n" // load 8 more ARGB
- "vld4.8 {d17, d19, d21, d23}, [%1]! \n" // load last 8 ARGB
- "vpadal.u8 q0, q8 \n" // B 16 bytes -> 8 shorts.
- "vpadal.u8 q1, q9 \n" // G 16 bytes -> 8 shorts.
- "vpadal.u8 q2, q10 \n" // R 16 bytes -> 8 shorts.
- "vpadal.u8 q3, q11 \n" // A 16 bytes -> 8 shorts.
- "vrshrn.u16 d0, q0, #2 \n" // round and pack to bytes
- "vrshrn.u16 d1, q1, #2 \n"
- "vrshrn.u16 d2, q2, #2 \n"
- "vrshrn.u16 d3, q3, #2 \n"
- "vst4.8 {d0, d1, d2, d3}, [%2]! \n"
- "bgt 1b \n"
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
+ "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
+ "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
+ "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts.
+ "vld4.8 {d16, d18, d20, d22}, [%1]! \n" // load 8 more ARGB
+ "vld4.8 {d17, d19, d21, d23}, [%1]! \n" // load last 8 ARGB
+ "vpadal.u8 q0, q8 \n" // B 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q9 \n" // G 16 bytes -> 8 shorts.
+ "vpadal.u8 q2, q10 \n" // R 16 bytes -> 8 shorts.
+ "vpadal.u8 q3, q11 \n" // A 16 bytes -> 8 shorts.
+ "vrshrn.u16 d0, q0, #2 \n" // round and pack to bytes
+ "vrshrn.u16 d1, q1, #2 \n"
+ "vrshrn.u16 d2, q2, #2 \n"
+ "vrshrn.u16 d3, q3, #2 \n"
+ "vst4.8 {d0, d1, d2, d3}, [%2]! \n"
+ "bgt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_stride), // %1
"+r"(dst), // %2
@@ -781,15 +1182,15 @@ void ScaleARGBRowDownEven_NEON(const uint8_t* src_argb,
int dst_width) {
(void)src_stride;
asm volatile(
- "mov r12, %3, lsl #2 \n"
- "1: \n"
- "vld1.32 {d0[0]}, [%0], r12 \n"
- "vld1.32 {d0[1]}, [%0], r12 \n"
- "vld1.32 {d1[0]}, [%0], r12 \n"
- "vld1.32 {d1[1]}, [%0], r12 \n"
- "subs %2, %2, #4 \n" // 4 pixels per loop.
- "vst1.8 {q0}, [%1]! \n"
- "bgt 1b \n"
+ "mov r12, %3, lsl #2 \n"
+ "1: \n"
+ "vld1.32 {d0[0]}, [%0], r12 \n"
+ "vld1.32 {d0[1]}, [%0], r12 \n"
+ "vld1.32 {d1[0]}, [%0], r12 \n"
+ "vld1.32 {d1[1]}, [%0], r12 \n"
+ "subs %2, %2, #4 \n" // 4 pixels per loop.
+ "vst1.8 {q0}, [%1]! \n"
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(dst_width) // %2
@@ -805,30 +1206,30 @@ void ScaleARGBRowDownEvenBox_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int dst_width) {
asm volatile(
- "mov r12, %4, lsl #2 \n"
- "add %1, %1, %0 \n"
- "1: \n"
- "vld1.8 {d0}, [%0], r12 \n" // 4 2x2 blocks -> 2x1
- "vld1.8 {d1}, [%1], r12 \n"
- "vld1.8 {d2}, [%0], r12 \n"
- "vld1.8 {d3}, [%1], r12 \n"
- "vld1.8 {d4}, [%0], r12 \n"
- "vld1.8 {d5}, [%1], r12 \n"
- "vld1.8 {d6}, [%0], r12 \n"
- "vld1.8 {d7}, [%1], r12 \n"
- "vaddl.u8 q0, d0, d1 \n"
- "vaddl.u8 q1, d2, d3 \n"
- "vaddl.u8 q2, d4, d5 \n"
- "vaddl.u8 q3, d6, d7 \n"
- "vswp.8 d1, d2 \n" // ab_cd -> ac_bd
- "vswp.8 d5, d6 \n" // ef_gh -> eg_fh
- "vadd.u16 q0, q0, q1 \n" // (a+b)_(c+d)
- "vadd.u16 q2, q2, q3 \n" // (e+f)_(g+h)
- "vrshrn.u16 d0, q0, #2 \n" // first 2 pixels.
- "vrshrn.u16 d1, q2, #2 \n" // next 2 pixels.
- "subs %3, %3, #4 \n" // 4 pixels per loop.
- "vst1.8 {q0}, [%2]! \n"
- "bgt 1b \n"
+ "mov r12, %4, lsl #2 \n"
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "vld1.8 {d0}, [%0], r12 \n" // 4 2x2 blocks -> 2x1
+ "vld1.8 {d1}, [%1], r12 \n"
+ "vld1.8 {d2}, [%0], r12 \n"
+ "vld1.8 {d3}, [%1], r12 \n"
+ "vld1.8 {d4}, [%0], r12 \n"
+ "vld1.8 {d5}, [%1], r12 \n"
+ "vld1.8 {d6}, [%0], r12 \n"
+ "vld1.8 {d7}, [%1], r12 \n"
+ "vaddl.u8 q0, d0, d1 \n"
+ "vaddl.u8 q1, d2, d3 \n"
+ "vaddl.u8 q2, d4, d5 \n"
+ "vaddl.u8 q3, d6, d7 \n"
+ "vswp.8 d1, d2 \n" // ab_cd -> ac_bd
+ "vswp.8 d5, d6 \n" // ef_gh -> eg_fh
+ "vadd.u16 q0, q0, q1 \n" // (a+b)_(c+d)
+ "vadd.u16 q2, q2, q3 \n" // (e+f)_(g+h)
+ "vrshrn.u16 d0, q0, #2 \n" // first 2 pixels.
+ "vrshrn.u16 d1, q2, #2 \n" // next 2 pixels.
+ "subs %3, %3, #4 \n" // 4 pixels per loop.
+ "vst1.8 {q0}, [%2]! \n"
+ "bgt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_stride), // %1
"+r"(dst_argb), // %2
@@ -853,7 +1254,7 @@ void ScaleARGBCols_NEON(uint8_t* dst_argb,
int tmp;
const uint8_t* src_tmp = src_argb;
asm volatile(
- "1: \n"
+ "1: \n"
// clang-format off
LOAD1_DATA32_LANE(d0, 0)
LOAD1_DATA32_LANE(d0, 1)
@@ -865,8 +1266,8 @@ void ScaleARGBCols_NEON(uint8_t* dst_argb,
LOAD1_DATA32_LANE(d3, 1)
// clang-format on
"vst1.32 {q0, q1}, [%0]! \n" // store pixels
- "subs %2, %2, #8 \n" // 8 processed per loop
- "bgt 1b \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop
+ "bgt 1b \n"
: "+r"(dst_argb), // %0
"+r"(src_argb), // %1
"+r"(dst_width), // %2
@@ -897,16 +1298,16 @@ void ScaleARGBFilterCols_NEON(uint8_t* dst_argb,
int* tmp = dx_offset;
const uint8_t* src_tmp = src_argb;
asm volatile (
- "vdup.32 q0, %3 \n" // x
- "vdup.32 q1, %4 \n" // dx
- "vld1.32 {q2}, [%5] \n" // 0 1 2 3
- "vshl.i32 q9, q1, #2 \n" // 4 * dx
- "vmul.s32 q1, q1, q2 \n"
- "vmov.i8 q3, #0x7f \n" // 0x7F
- "vmov.i16 q15, #0x7f \n" // 0x7F
+ "vdup.32 q0, %3 \n" // x
+ "vdup.32 q1, %4 \n" // dx
+ "vld1.32 {q2}, [%5] \n" // 0 1 2 3
+ "vshl.i32 q9, q1, #2 \n" // 4 * dx
+ "vmul.s32 q1, q1, q2 \n"
+ "vmov.i8 q3, #0x7f \n" // 0x7F
+ "vmov.i16 q15, #0x7f \n" // 0x7F
// x , x + 1 * dx, x + 2 * dx, x + 3 * dx
- "vadd.s32 q8, q1, q0 \n"
- "1: \n"
+ "vadd.s32 q8, q1, q0 \n"
+ "1: \n"
// d0, d1: a
// d2, d3: b
LOAD2_DATA32_LANE(d0, d2, 0)
@@ -950,6 +1351,140 @@ void ScaleARGBFilterCols_NEON(uint8_t* dst_argb,
#undef LOAD2_DATA32_LANE
+void ScaleUVRowDown2_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "vld2.16 {d0, d2}, [%0]! \n" // load 8 UV pixels.
+ "vld2.16 {d1, d3}, [%0]! \n" // load next 8 UV
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vst1.16 {q1}, [%1]! \n" // store 8 UV
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "q0", "q1");
+}
+
+void ScaleUVRowDown2Linear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "vld2.16 {d0, d2}, [%0]! \n" // load 8 UV pixels.
+ "vld2.16 {d1, d3}, [%0]! \n" // load next 8 UV
+ "subs %2, %2, #8 \n" // 8 processed per loop.
+ "vrhadd.u8 q0, q0, q1 \n" // rounding half add
+ "vst1.16 {q0}, [%1]! \n" // store 8 UV
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "q0", "q1");
+}
+
+void ScaleUVRowDown2Box_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ asm volatile(
+ // change the stride to row 2 pointer
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "vld2.8 {d0, d2}, [%0]! \n" // load 8 UV pixels.
+ "vld2.8 {d1, d3}, [%0]! \n" // load next 8 UV
+ "subs %3, %3, #8 \n" // 8 processed per loop.
+ "vpaddl.u8 q0, q0 \n" // U 16 bytes -> 8 shorts.
+ "vpaddl.u8 q1, q1 \n" // V 16 bytes -> 8 shorts.
+ "vld2.8 {d16, d18}, [%1]! \n" // load 8 more UV
+ "vld2.8 {d17, d19}, [%1]! \n" // load last 8 UV
+ "vpadal.u8 q0, q8 \n" // U 16 bytes -> 8 shorts.
+ "vpadal.u8 q1, q9 \n" // V 16 bytes -> 8 shorts.
+ "vrshrn.u16 d0, q0, #2 \n" // round and pack to bytes
+ "vrshrn.u16 d1, q1, #2 \n"
+ "vst2.8 {d0, d1}, [%2]! \n"
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_stride), // %1
+ "+r"(dst), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "q0", "q1", "q8", "q9");
+}
+
+// Reads 4 pixels at a time.
+void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx, // pixel step
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src1_ptr = src_ptr + src_stepx * 2;
+ const uint8_t* src2_ptr = src_ptr + src_stepx * 4;
+ const uint8_t* src3_ptr = src_ptr + src_stepx * 6;
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "vld1.16 {d0[0]}, [%0], %6 \n"
+ "vld1.16 {d0[1]}, [%1], %6 \n"
+ "vld1.16 {d0[2]}, [%2], %6 \n"
+ "vld1.16 {d0[3]}, [%3], %6 \n"
+ "subs %5, %5, #4 \n" // 4 pixels per loop.
+ "vst1.8 {d0}, [%4]! \n"
+ "bgt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src1_ptr), // %1
+ "+r"(src2_ptr), // %2
+ "+r"(src3_ptr), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_width) // %5
+ : "r"(src_stepx * 8) // %6
+ : "memory", "cc", "d0");
+}
+
+// Scale UV columns by 3 box filter.
+// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
+void ScaleUVAddCols3_NEON(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ asm volatile(
+ "vdup.16 d16, %[scaleval] \n"
+ "1: \n"
+ "vld3.32 {d0, d2, d4}, [%[src_uv]]! \n"
+ "vld3.32 {d1, d3, d5}, [%[src_uv]]! \n"
+ "vadd.u16 q0, q0, q1 \n"
+ "vadd.u16 q0, q0, q2 \n"
+ "vld3.32 {d2, d4, d6}, [%[src_uv]]! \n"
+ "vld3.32 {d3, d5, d7}, [%[src_uv]]! \n"
+ "vadd.u16 q1, q1, q2 \n"
+ "vadd.u16 q1, q1, q3 \n"
+ "vmull.u16 q2, d0, d16 \n"
+ "vmull.u16 q3, d1, d16 \n"
+ "vmull.u16 q9, d2, d16 \n"
+ "vmull.u16 q10, d3, d16 \n"
+ "vshrn.i32 d0, q2, #16 \n"
+ "vshrn.i32 d1, q3, #16 \n"
+ "vshrn.i32 d2, q9, #16 \n"
+ "vshrn.i32 d3, q10, #16 \n"
+ "vmovn.i16 d0, q0 \n"
+ "vmovn.i16 d1, q1 \n"
+ "subs %[dst_width], %[dst_width], #8 \n"
+ "vst1.8 {q0}, [%[dst_uv]]! \n"
+ "bgt 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ : [scaleval] "r"(scaleval) // %[scaleval]
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10");
+}
+
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus
diff --git a/libs/libyuv/source/scale_neon64.cc b/libs/libyuv/source/scale_neon64.cc
index 0a7b80ce1d..2bb04b6e9b 100644
--- a/libs/libyuv/source/scale_neon64.cc
+++ b/libs/libyuv/source/scale_neon64.cc
@@ -17,6 +17,13 @@ namespace libyuv {
extern "C" {
#endif
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
@@ -27,17 +34,18 @@ void ScaleRowDown2_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into v0, odd into v1
- "ld2 {v0.16b,v1.16b}, [%0], #32 \n"
- "subs %w2, %w2, #16 \n" // 16 processed per loop
- "st1 {v1.16b}, [%1], #16 \n" // store odd pixels
- "b.gt 1b \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "st1 {v1.16b}, [%1], #16 \n" // store odd pixels
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1" // Clobber List
+ : "memory", "cc", "v0", "v1" // Clobber List
);
}
@@ -48,18 +56,19 @@ void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into v0, odd into v1
- "ld2 {v0.16b,v1.16b}, [%0], #32 \n"
- "subs %w2, %w2, #16 \n" // 16 processed per loop
- "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
- "st1 {v0.16b}, [%1], #16 \n"
- "b.gt 1b \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "st1 {v0.16b}, [%1], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1" // Clobber List
+ : "memory", "cc", "v0", "v1" // Clobber List
);
}
@@ -70,25 +79,27 @@ void ScaleRowDown2Box_NEON(const uint8_t* src_ptr,
int dst_width) {
asm volatile(
// change the stride to row 2 pointer
- "add %1, %1, %0 \n"
- "1: \n"
- "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load row 1 and post inc
- "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load row 2 and post inc
- "subs %w3, %w3, #16 \n" // 16 processed per loop
- "uaddlp v0.8h, v0.16b \n" // row 1 add adjacent
- "uaddlp v1.8h, v1.16b \n"
- "uadalp v0.8h, v2.16b \n" // += row 2 add adjacent
- "uadalp v1.8h, v3.16b \n"
- "rshrn v0.8b, v0.8h, #2 \n" // round and pack
- "rshrn2 v0.16b, v1.8h, #2 \n"
- "st1 {v0.16b}, [%2], #16 \n"
- "b.gt 1b \n"
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load row 1 and post inc
+ "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load row 2 and post inc
+ "subs %w3, %w3, #16 \n" // 16 processed per loop
+ "uaddlp v0.8h, v0.16b \n" // row 1 add adjacent
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "uaddlp v1.8h, v1.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v0.8h, v2.16b \n" // += row 2 add adjacent
+ "uadalp v1.8h, v3.16b \n"
+ "rshrn v0.8b, v0.8h, #2 \n" // round and pack
+ "rshrn2 v0.16b, v1.8h, #2 \n"
+ "st1 {v0.16b}, [%2], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_stride), // %1
"+r"(dst), // %2
"+r"(dst_width) // %3
:
- : "v0", "v1", "v2", "v3" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
);
}
@@ -98,16 +109,17 @@ void ScaleRowDown4_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "st1 {v2.8b}, [%1], #8 \n"
- "b.gt 1b \n"
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "st1 {v2.16b}, [%1], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleRowDown4Box_NEON(const uint8_t* src_ptr,
@@ -118,20 +130,28 @@ void ScaleRowDown4Box_NEON(const uint8_t* src_ptr,
const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
const uint8_t* src_ptr3 = src_ptr + src_stride * 3;
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load up 16x4
- "ld1 {v1.16b}, [%2], #16 \n"
- "ld1 {v2.16b}, [%3], #16 \n"
- "ld1 {v3.16b}, [%4], #16 \n"
- "subs %w5, %w5, #4 \n"
- "uaddlp v0.8h, v0.16b \n"
- "uadalp v0.8h, v1.16b \n"
- "uadalp v0.8h, v2.16b \n"
- "uadalp v0.8h, v3.16b \n"
- "addp v0.8h, v0.8h, v0.8h \n"
- "rshrn v0.8b, v0.8h, #4 \n" // divide by 16 w/rounding
- "st1 {v0.s}[0], [%1], #4 \n"
- "b.gt 1b \n"
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load up 16x8
+ "ldp q1, q5, [%2], #32 \n"
+ "ldp q2, q6, [%3], #32 \n"
+ "ldp q3, q7, [%4], #32 \n"
+ "subs %w5, %w5, #8 \n"
+ "uaddlp v0.8h, v0.16b \n"
+ "uaddlp v4.8h, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "uadalp v0.8h, v1.16b \n"
+ "uadalp v4.8h, v5.16b \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "uadalp v0.8h, v2.16b \n"
+ "uadalp v4.8h, v6.16b \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "uadalp v0.8h, v3.16b \n"
+ "uadalp v4.8h, v7.16b \n"
+ "prfm pldl1keep, [%4, 448] \n"
+ "addp v0.8h, v0.8h, v4.8h \n"
+ "rshrn v0.8b, v0.8h, #4 \n" // divide by 16 w/rounding
+ "str d0, [%1], #8 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_ptr1), // %2
@@ -139,29 +159,45 @@ void ScaleRowDown4Box_NEON(const uint8_t* src_ptr,
"+r"(src_ptr3), // %4
"+r"(dst_width) // %5
:
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
-// Down scale from 4 to 3 pixels. Use the neon multilane read/write
-// to load up the every 4th pixel into a 4 different registers.
-// Point samples 32 pixels to 24 pixels.
+static const uvec8 kShuf34_0 = {
+ 0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20,
+};
+static const uvec8 kShuf34_1 = {
+ 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25,
+};
+static const uvec8 kShuf34_2 = {
+ 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25, 27, 28, 29, 31,
+};
+
+// Down scale from 4 to 3 pixels. Point samples 64 pixels to 48 pixels.
void ScaleRowDown34_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "subs %w2, %w2, #24 \n"
- "orr v2.16b, v3.16b, v3.16b \n" // order v0,v1,v2
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- :
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ "ld1 {v29.16b}, [%[kShuf34_0]] \n"
+ "ld1 {v30.16b}, [%[kShuf34_1]] \n"
+ "ld1 {v31.16b}, [%[kShuf34_2]] \n"
+ "1: \n"
+ "ld1 {v0.16b,v1.16b,v2.16b,v3.16b}, [%[src_ptr]], #64 \n"
+ "subs %w[width], %w[width], #48 \n"
+ "tbl v0.16b, {v0.16b, v1.16b}, v29.16b \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n"
+ "tbl v1.16b, {v1.16b, v2.16b}, v30.16b \n"
+ "tbl v2.16b, {v2.16b, v3.16b}, v31.16b \n"
+ "st1 {v0.16b,v1.16b,v2.16b}, [%[dst_ptr]], #48 \n"
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width) // %[width]
+ : [kShuf34_0] "r"(&kShuf34_0), // %[kShuf34_0]
+ [kShuf34_1] "r"(&kShuf34_1), // %[kShuf34_1]
+ [kShuf34_2] "r"(&kShuf34_2) // %[kShuf34_2]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v29", "v30", "v31");
}
void ScaleRowDown34_0_Box_NEON(const uint8_t* src_ptr,
@@ -169,56 +205,77 @@ void ScaleRowDown34_0_Box_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "movi v20.8b, #3 \n"
- "add %3, %3, %0 \n"
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1
- "subs %w2, %w2, #24 \n"
+ "movi v24.16b, #3 \n"
+ "add %3, %3, %0 \n"
+
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1
+ "subs %w2, %w2, #48 \n"
// filter src line 0 with src line 1
// expand chars to shorts to allow for room
// when adding lines together
- "ushll v16.8h, v4.8b, #0 \n"
- "ushll v17.8h, v5.8b, #0 \n"
- "ushll v18.8h, v6.8b, #0 \n"
- "ushll v19.8h, v7.8b, #0 \n"
+ "ushll v16.8h, v4.8b, #0 \n"
+ "ushll v17.8h, v5.8b, #0 \n"
+ "ushll v18.8h, v6.8b, #0 \n"
+ "ushll v19.8h, v7.8b, #0 \n"
+ "ushll2 v20.8h, v4.16b, #0 \n"
+ "ushll2 v21.8h, v5.16b, #0 \n"
+ "ushll2 v22.8h, v6.16b, #0 \n"
+ "ushll2 v23.8h, v7.16b, #0 \n"
// 3 * line_0 + line_1
- "umlal v16.8h, v0.8b, v20.8b \n"
- "umlal v17.8h, v1.8b, v20.8b \n"
- "umlal v18.8h, v2.8b, v20.8b \n"
- "umlal v19.8h, v3.8b, v20.8b \n"
-
- // (3 * line_0 + line_1) >> 2
- "uqrshrn v0.8b, v16.8h, #2 \n"
- "uqrshrn v1.8b, v17.8h, #2 \n"
- "uqrshrn v2.8b, v18.8h, #2 \n"
- "uqrshrn v3.8b, v19.8h, #2 \n"
-
- // a0 = (src[0] * 3 + s[1] * 1) >> 2
- "ushll v16.8h, v1.8b, #0 \n"
- "umlal v16.8h, v0.8b, v20.8b \n"
- "uqrshrn v0.8b, v16.8h, #2 \n"
-
- // a1 = (src[1] * 1 + s[2] * 1) >> 1
- "urhadd v1.8b, v1.8b, v2.8b \n"
-
- // a2 = (src[2] * 1 + s[3] * 3) >> 2
- "ushll v16.8h, v2.8b, #0 \n"
- "umlal v16.8h, v3.8b, v20.8b \n"
- "uqrshrn v2.8b, v16.8h, #2 \n"
-
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
-
- "b.gt 1b \n"
+ "umlal v16.8h, v0.8b, v24.8b \n"
+ "umlal v17.8h, v1.8b, v24.8b \n"
+ "umlal v18.8h, v2.8b, v24.8b \n"
+ "umlal v19.8h, v3.8b, v24.8b \n"
+ "umlal2 v20.8h, v0.16b, v24.16b \n"
+ "umlal2 v21.8h, v1.16b, v24.16b \n"
+ "umlal2 v22.8h, v2.16b, v24.16b \n"
+ "umlal2 v23.8h, v3.16b, v24.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ // (3 * line_0 + line_1 + 2) >> 2
+ "uqrshrn v0.8b, v16.8h, #2 \n"
+ "uqrshrn v1.8b, v17.8h, #2 \n"
+ "uqrshrn v2.8b, v18.8h, #2 \n"
+ "uqrshrn v3.8b, v19.8h, #2 \n"
+ "uqrshrn2 v0.16b, v20.8h, #2 \n"
+ "uqrshrn2 v1.16b, v21.8h, #2 \n"
+ "uqrshrn2 v2.16b, v22.8h, #2 \n"
+ "uqrshrn2 v3.16b, v23.8h, #2 \n"
+ "prfm pldl1keep, [%3, 448] \n"
+
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "ushll v16.8h, v1.8b, #0 \n"
+ "ushll2 v17.8h, v1.16b, #0 \n"
+ "umlal v16.8h, v0.8b, v24.8b \n"
+ "umlal2 v17.8h, v0.16b, v24.16b \n"
+ "uqrshrn v0.8b, v16.8h, #2 \n"
+ "uqrshrn2 v0.16b, v17.8h, #2 \n"
+
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "urhadd v1.16b, v1.16b, v2.16b \n"
+
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "ushll v16.8h, v2.8b, #0 \n"
+ "ushll2 v17.8h, v2.16b, #0 \n"
+ "umlal v16.8h, v3.8b, v24.8b \n"
+ "umlal2 v17.8h, v3.16b, v24.16b \n"
+ "uqrshrn v2.8b, v16.8h, #2 \n"
+ "uqrshrn2 v2.16b, v17.8h, #2 \n"
+
+ "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n"
+
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
"+r"(src_stride) // %3
:
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v20", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24");
}
void ScaleRowDown34_1_Box_NEON(const uint8_t* src_ptr,
@@ -226,51 +283,57 @@ void ScaleRowDown34_1_Box_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "movi v20.8b, #3 \n"
- "add %3, %3, %0 \n"
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1
- "subs %w2, %w2, #24 \n"
+ "movi v20.16b, #3 \n"
+ "add %3, %3, %0 \n"
+
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1
+ "subs %w2, %w2, #48 \n"
// average src line 0 with src line 1
- "urhadd v0.8b, v0.8b, v4.8b \n"
- "urhadd v1.8b, v1.8b, v5.8b \n"
- "urhadd v2.8b, v2.8b, v6.8b \n"
- "urhadd v3.8b, v3.8b, v7.8b \n"
-
- // a0 = (src[0] * 3 + s[1] * 1) >> 2
- "ushll v4.8h, v1.8b, #0 \n"
- "umlal v4.8h, v0.8b, v20.8b \n"
- "uqrshrn v0.8b, v4.8h, #2 \n"
-
- // a1 = (src[1] * 1 + s[2] * 1) >> 1
- "urhadd v1.8b, v1.8b, v2.8b \n"
-
- // a2 = (src[2] * 1 + s[3] * 3) >> 2
- "ushll v4.8h, v2.8b, #0 \n"
- "umlal v4.8h, v3.8b, v20.8b \n"
- "uqrshrn v2.8b, v4.8h, #2 \n"
-
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
- "b.gt 1b \n"
+ "urhadd v0.16b, v0.16b, v4.16b \n"
+ "urhadd v1.16b, v1.16b, v5.16b \n"
+ "urhadd v2.16b, v2.16b, v6.16b \n"
+ "urhadd v3.16b, v3.16b, v7.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "ushll v4.8h, v1.8b, #0 \n"
+ "ushll2 v5.8h, v1.16b, #0 \n"
+ "umlal v4.8h, v0.8b, v20.8b \n"
+ "umlal2 v5.8h, v0.16b, v20.16b \n"
+ "uqrshrn v0.8b, v4.8h, #2 \n"
+ "uqrshrn2 v0.16b, v5.8h, #2 \n"
+ "prfm pldl1keep, [%3, 448] \n"
+
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "urhadd v1.16b, v1.16b, v2.16b \n"
+
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "ushll v4.8h, v2.8b, #0 \n"
+ "ushll2 v5.8h, v2.16b, #0 \n"
+ "umlal v4.8h, v3.8b, v20.8b \n"
+ "umlal2 v5.8h, v3.16b, v20.16b \n"
+ "uqrshrn v2.8b, v4.8h, #2 \n"
+ "uqrshrn2 v2.16b, v5.8h, #2 \n"
+
+ "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
"+r"(src_stride) // %3
:
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20");
}
static const uvec8 kShuf38 = {0, 3, 6, 8, 11, 14, 16, 19,
22, 24, 27, 30, 0, 0, 0, 0};
-static const uvec8 kShuf38_2 = {0, 16, 32, 2, 18, 33, 4, 20,
- 34, 6, 22, 35, 0, 0, 0, 0};
-static const vec16 kMult38_Div6 = {65536 / 12, 65536 / 12, 65536 / 12,
- 65536 / 12, 65536 / 12, 65536 / 12,
- 65536 / 12, 65536 / 12};
-static const vec16 kMult38_Div9 = {65536 / 18, 65536 / 18, 65536 / 18,
- 65536 / 18, 65536 / 18, 65536 / 18,
- 65536 / 18, 65536 / 18};
+static const vec16 kMult38_Div664 = {
+ 65536 / 12, 65536 / 12, 65536 / 8, 65536 / 12, 65536 / 12, 65536 / 8, 0, 0};
+static const vec16 kMult38_Div996 = {65536 / 18, 65536 / 18, 65536 / 12,
+ 65536 / 18, 65536 / 18, 65536 / 12,
+ 0, 0};
// 32 -> 12
void ScaleRowDown38_NEON(const uint8_t* src_ptr,
@@ -279,240 +342,651 @@ void ScaleRowDown38_NEON(const uint8_t* src_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "ld1 {v3.16b}, [%3] \n"
- "1: \n"
- "ld1 {v0.16b,v1.16b}, [%0], #32 \n"
- "subs %w2, %w2, #12 \n"
- "tbl v2.16b, {v0.16b,v1.16b}, v3.16b \n"
- "st1 {v2.8b}, [%1], #8 \n"
- "st1 {v2.s}[2], [%1], #4 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"(&kShuf38) // %3
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ "ld1 {v3.16b}, [%[kShuf38]] \n"
+ "subs %w[width], %w[width], #12 \n"
+ "b.eq 2f \n"
+
+ "1: \n"
+ "ldp q0, q1, [%[src_ptr]], #32 \n"
+ "subs %w[width], %w[width], #12 \n"
+ "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n" // prefetch 7 lines ahead
+ "str q2, [%[dst_ptr]] \n"
+ "add %[dst_ptr], %[dst_ptr], #12 \n"
+ "b.gt 1b \n"
+
+ // Store exactly 12 bytes on the final iteration to avoid writing past
+ // the end of the array.
+ "2: \n"
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n"
+ "st1 {v2.8b}, [%[dst_ptr]], #8 \n"
+ "st1 {v2.s}[2], [%[dst_ptr]] \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width) // %[width]
+ : [kShuf38] "r"(&kShuf38) // %[kShuf38]
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
-// 32x3 -> 12x1
-void OMITFP ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- const uint8_t* src_ptr1 = src_ptr + src_stride * 2;
- ptrdiff_t tmp_src_stride = src_stride;
+static const uvec8 kScaleRowDown38_3_BoxIndices1[] = {
+ 0, 1, 6, 7, 12, 13, 16, 17, 22, 23, 28, 29, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_3_BoxIndices2[] = {
+ 2, 3, 8, 9, 14, 15, 18, 19, 24, 25, 30, 31, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_3_BoxIndices3[] = {
+ 4, 5, 10, 11, 255, 255, 20, 21, 26, 27, 255, 255, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_NarrowIndices[] = {
+ 0, 2, 4, 6, 8, 10, 16, 18, 20, 22, 24, 26, 255, 255, 255, 255};
+void ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
asm volatile(
- "ld1 {v29.8h}, [%5] \n"
- "ld1 {v30.16b}, [%6] \n"
- "ld1 {v31.8h}, [%7] \n"
- "add %2, %2, %0 \n"
- "1: \n"
-
- // 00 40 01 41 02 42 03 43
- // 10 50 11 51 12 52 13 53
- // 20 60 21 61 22 62 23 63
- // 30 70 31 71 32 72 33 73
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n"
- "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%3], #32 \n"
- "subs %w4, %w4, #12 \n"
-
- // Shuffle the input data around to get align the data
- // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
- // 00 10 01 11 02 12 03 13
- // 40 50 41 51 42 52 43 53
- "trn1 v20.8b, v0.8b, v1.8b \n"
- "trn2 v21.8b, v0.8b, v1.8b \n"
- "trn1 v22.8b, v4.8b, v5.8b \n"
- "trn2 v23.8b, v4.8b, v5.8b \n"
- "trn1 v24.8b, v16.8b, v17.8b \n"
- "trn2 v25.8b, v16.8b, v17.8b \n"
-
- // 20 30 21 31 22 32 23 33
- // 60 70 61 71 62 72 63 73
- "trn1 v0.8b, v2.8b, v3.8b \n"
- "trn2 v1.8b, v2.8b, v3.8b \n"
- "trn1 v4.8b, v6.8b, v7.8b \n"
- "trn2 v5.8b, v6.8b, v7.8b \n"
- "trn1 v16.8b, v18.8b, v19.8b \n"
- "trn2 v17.8b, v18.8b, v19.8b \n"
-
- // 00+10 01+11 02+12 03+13
- // 40+50 41+51 42+52 43+53
- "uaddlp v20.4h, v20.8b \n"
- "uaddlp v21.4h, v21.8b \n"
- "uaddlp v22.4h, v22.8b \n"
- "uaddlp v23.4h, v23.8b \n"
- "uaddlp v24.4h, v24.8b \n"
- "uaddlp v25.4h, v25.8b \n"
-
- // 60+70 61+71 62+72 63+73
- "uaddlp v1.4h, v1.8b \n"
- "uaddlp v5.4h, v5.8b \n"
- "uaddlp v17.4h, v17.8b \n"
-
- // combine source lines
- "add v20.4h, v20.4h, v22.4h \n"
- "add v21.4h, v21.4h, v23.4h \n"
- "add v20.4h, v20.4h, v24.4h \n"
- "add v21.4h, v21.4h, v25.4h \n"
- "add v2.4h, v1.4h, v5.4h \n"
- "add v2.4h, v2.4h, v17.4h \n"
-
- // dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0]
- // + s[6 + st * 1] + s[7 + st * 1]
- // + s[6 + st * 2] + s[7 + st * 2]) / 6
- "sqrdmulh v2.8h, v2.8h, v29.8h \n"
- "xtn v2.8b, v2.8h \n"
-
- // Shuffle 2,3 reg around so that 2 can be added to the
- // 0,1 reg and 3 can be added to the 4,5 reg. This
- // requires expanding from u8 to u16 as the 0,1 and 4,5
- // registers are already expanded. Then do transposes
- // to get aligned.
- // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
- "ushll v16.8h, v16.8b, #0 \n"
- "uaddl v0.8h, v0.8b, v4.8b \n"
-
- // combine source lines
- "add v0.8h, v0.8h, v16.8h \n"
-
- // xx 20 xx 21 xx 22 xx 23
- // xx 30 xx 31 xx 32 xx 33
- "trn1 v1.8h, v0.8h, v0.8h \n"
- "trn2 v4.8h, v0.8h, v0.8h \n"
- "xtn v0.4h, v1.4s \n"
- "xtn v4.4h, v4.4s \n"
-
- // 0+1+2, 3+4+5
- "add v20.8h, v20.8h, v0.8h \n"
- "add v21.8h, v21.8h, v4.8h \n"
-
- // Need to divide, but can't downshift as the the value
- // isn't a power of 2. So multiply by 65536 / n
- // and take the upper 16 bits.
- "sqrdmulh v0.8h, v20.8h, v31.8h \n"
- "sqrdmulh v1.8h, v21.8h, v31.8h \n"
-
- // Align for table lookup, vtbl requires registers to be adjacent
- "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v30.16b \n"
-
- "st1 {v3.8b}, [%1], #8 \n"
- "st1 {v3.s}[2], [%1], #4 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(tmp_src_stride), // %2
- "+r"(src_ptr1), // %3
- "+r"(dst_width) // %4
- : "r"(&kMult38_Div6), // %5
- "r"(&kShuf38_2), // %6
- "r"(&kMult38_Div9) // %7
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v29", "v30", "v31",
- "memory", "cc");
+ "ld1 {v27.16b}, [%[tblArray1]] \n"
+ "ld1 {v28.16b}, [%[tblArray2]] \n"
+ "ld1 {v29.16b}, [%[tblArray3]] \n"
+ "ld1 {v31.16b}, [%[tblArray4]] \n"
+ "ld1 {v30.16b}, [%[div996]] \n"
+
+ "1: \n"
+ "ldp q20, q0, [%[src_ptr]], #32 \n"
+ "ldp q21, q1, [%[src_ptr1]], #32 \n"
+ "ldp q22, q2, [%[src_ptr2]], #32 \n"
+
+ "subs %w[width], %w[width], #12 \n"
+
+ // Add across strided rows first.
+ "uaddl v23.8h, v20.8b, v21.8b \n"
+ "uaddl v3.8h, v0.8b, v1.8b \n"
+ "uaddl2 v24.8h, v20.16b, v21.16b \n"
+ "uaddl2 v4.8h, v0.16b, v1.16b \n"
+
+ "uaddw v23.8h, v23.8h, v22.8b \n"
+ "uaddw v3.8h, v3.8h, v2.8b \n"
+ "uaddw2 v24.8h, v24.8h, v22.16b \n" // abcdefgh ...
+ "uaddw2 v4.8h, v4.8h, v2.16b \n"
+
+ // Permute groups of {three,three,two} into separate vectors to sum.
+ "tbl v20.16b, {v23.16b, v24.16b}, v27.16b \n" // a d g ...
+ "tbl v0.16b, {v3.16b, v4.16b}, v27.16b \n"
+ "tbl v21.16b, {v23.16b, v24.16b}, v28.16b \n" // b e h ...
+ "tbl v1.16b, {v3.16b, v4.16b}, v28.16b \n"
+ "tbl v22.16b, {v23.16b, v24.16b}, v29.16b \n" // c f 0...
+ "tbl v2.16b, {v3.16b, v4.16b}, v29.16b \n"
+
+ "add v23.8h, v20.8h, v21.8h \n"
+ "add v3.8h, v0.8h, v1.8h \n"
+ "add v24.8h, v23.8h, v22.8h \n" // a+b+c d+e+f g+h
+ "add v4.8h, v3.8h, v2.8h \n"
+
+ "sqrdmulh v24.8h, v24.8h, v30.8h \n" // v /= {9,9,6}
+ "sqrdmulh v25.8h, v4.8h, v30.8h \n"
+ "tbl v21.16b, {v24.16b, v25.16b}, v31.16b \n" // Narrow.
+ "st1 {v21.d}[0], [%[dst_ptr]], #8 \n"
+ "st1 {v21.s}[2], [%[dst_ptr]], #4 \n"
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [src_ptr2] "+r"(src_ptr2), // %[src_ptr2]
+ [width] "+r"(dst_width) // %[width]
+ : [div996] "r"(&kMult38_Div996), // %[div996]
+ [tblArray1] "r"(kScaleRowDown38_3_BoxIndices1), // %[tblArray1]
+ [tblArray2] "r"(kScaleRowDown38_3_BoxIndices2), // %[tblArray2]
+ [tblArray3] "r"(kScaleRowDown38_3_BoxIndices3), // %[tblArray3]
+ [tblArray4] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray4]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "v22",
+ "v23", "v24", "v25", "v27", "v28", "v29", "v30", "v31");
}
-// 32x2 -> 12x1
+static const uvec8 kScaleRowDown38_2_BoxIndices1[] = {
+ 0, 1, 3, 4, 6, 7, 8, 9, 11, 12, 14, 15, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_2_BoxIndices2[] = {
+ 2, 18, 5, 21, 255, 255, 10, 26, 13, 29, 255, 255, 255, 255, 255, 255};
+
void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
- // TODO(fbarchard): use src_stride directly for clang 3.5+.
- ptrdiff_t tmp_src_stride = src_stride;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ asm volatile(
+ "ld1 {v28.16b}, [%[tblArray1]] \n"
+ "ld1 {v29.16b}, [%[tblArray2]] \n"
+ "ld1 {v31.16b}, [%[tblArray3]] \n"
+ "ld1 {v30.8h}, [%[div664]] \n"
+
+ "1: \n"
+ "ldp q20, q0, [%[src_ptr]], #32 \n" // abcdefgh ...
+ "ldp q21, q1, [%[src_ptr1]], #32 \n" // ijklmnop ...
+ "subs %w[width], %w[width], #12 \n"
+
+ // Permute into groups of six values (three pairs) to be summed.
+ "tbl v22.16b, {v20.16b}, v28.16b \n" // abdegh ...
+ "tbl v2.16b, {v0.16b}, v28.16b \n"
+ "tbl v23.16b, {v21.16b}, v28.16b \n" // ijlmop ...
+ "tbl v3.16b, {v1.16b}, v28.16b \n"
+ "tbl v24.16b, {v20.16b, v21.16b}, v29.16b \n" // ckfn00 ...
+ "tbl v4.16b, {v0.16b, v1.16b}, v29.16b \n"
+
+ "uaddlp v22.8h, v22.16b \n" // a+b d+e g+h ...
+ "uaddlp v2.8h, v2.16b \n"
+ "uaddlp v23.8h, v23.16b \n" // i+j l+m o+p ...
+ "uaddlp v3.8h, v3.16b \n"
+ "uaddlp v24.8h, v24.16b \n" // c+k f+n 0 ...
+ "uaddlp v4.8h, v4.16b \n"
+ "add v20.8h, v22.8h, v23.8h \n"
+ "add v0.8h, v2.8h, v3.8h \n"
+ "add v21.8h, v20.8h, v24.8h \n" // a+b+i+j+c+k ...
+ "add v1.8h, v0.8h, v4.8h \n"
+
+ "sqrdmulh v21.8h, v21.8h, v30.8h \n" // v /= {6,6,4}
+ "sqrdmulh v22.8h, v1.8h, v30.8h \n"
+ "tbl v21.16b, {v21.16b, v22.16b}, v31.16b \n" // Narrow.
+ "st1 {v21.d}[0], [%[dst_ptr]], #8 \n"
+ "st1 {v21.s}[2], [%[dst_ptr]], #4 \n"
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [width] "+r"(dst_width) // %[width]
+ : [div664] "r"(&kMult38_Div664), // %[div664]
+ [tblArray1] "r"(kScaleRowDown38_2_BoxIndices1), // %[tblArray1]
+ [tblArray2] "r"(kScaleRowDown38_2_BoxIndices2), // %[tblArray2]
+ [tblArray3] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray3]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "v22",
+ "v23", "v24", "v28", "v29", "v30", "v31");
+}
+
+void ScaleRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "movi v31.16b, #3 \n"
+
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // 0123456789abcdef
+ "ldr q1, [%1], #16 \n" // 123456789abcdefg
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.8h, v0.8b, #0 \n" // 01234567 (16b)
+ "ushll v3.8h, v1.8b, #0 \n" // 12345678 (16b)
+ "ushll2 v4.8h, v0.16b, #0 \n" // 89abcdef (16b)
+ "ushll2 v5.8h, v1.16b, #0 \n" // 9abcdefg (16b)
+
+ "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (odd)
+ "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (even)
+ "umlal2 v4.8h, v1.16b, v31.16b \n" // 3*near+far (odd)
+ "umlal2 v5.8h, v0.16b, v31.16b \n" // 3*near+far (even)
+
+ "rshrn v2.8b, v2.8h, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn v1.8b, v3.8h, #2 \n" // 3/4*near+1/4*far (even)
+ "rshrn2 v2.16b, v4.8h, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn2 v1.16b, v5.8h, #2 \n" // 3/4*near+1/4*far (even)
+
+ "st2 {v1.16b, v2.16b}, [%2], #32 \n"
+ "subs %w3, %w3, #32 \n"
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_temp), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ uint8_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint8_t* src_temp = src_ptr + 1;
+ const uint8_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "movi v31.8b, #3 \n"
+ "movi v30.8h, #3 \n"
+
+ "1: \n"
+ "ldr d0, [%0], #8 \n" // 01234567
+ "ldr d1, [%2], #8 \n" // 12345678
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.8h, v0.8b, #0 \n" // 01234567 (16b)
+ "ushll v3.8h, v1.8b, #0 \n" // 12345678 (16b)
+ "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (1, odd)
+ "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (1, even)
+
+ "ldr d0, [%1], #8 \n"
+ "ldr d1, [%3], #8 \n"
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v4.8h, v0.8b, #0 \n" // 01234567 (16b)
+ "ushll v5.8h, v1.8b, #0 \n" // 12345678 (16b)
+ "umlal v4.8h, v1.8b, v31.8b \n" // 3*near+far (2, odd)
+ "umlal v5.8h, v0.8b, v31.8b \n" // 3*near+far (2, even)
+
+ "mov v0.16b, v4.16b \n"
+ "mov v1.16b, v5.16b \n"
+ "mla v4.8h, v2.8h, v30.8h \n" // 9 3 3 1 (1, odd)
+ "mla v5.8h, v3.8h, v30.8h \n" // 9 3 3 1 (1, even)
+ "mla v2.8h, v0.8h, v30.8h \n" // 9 3 3 1 (2, odd)
+ "mla v3.8h, v1.8h, v30.8h \n" // 9 3 3 1 (2, even)
+
+ "rshrn v2.8b, v2.8h, #4 \n" // 2, odd
+ "rshrn v1.8b, v3.8h, #4 \n" // 2, even
+ "rshrn v4.8b, v4.8h, #4 \n" // 1, odd
+ "rshrn v3.8b, v5.8h, #4 \n" // 1, even
+
+ "st2 {v1.8b, v2.8b}, [%5], #16 \n" // store 1
+ "st2 {v3.8b, v4.8b}, [%4], #16 \n" // store 2
+ "subs %w6, %w6, #16 \n" // 8 sample -> 16 sample
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(src_temp), // %2
+ "+r"(src_temp1), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_ptr1), // %5
+ "+r"(dst_width) // %6
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Linear_12_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "movi v31.8h, #3 \n"
+
+ "1: \n"
+ "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
+ "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "mov v2.16b, v0.16b \n"
+ "mla v0.8h, v1.8h, v31.8h \n" // 3*near+far (odd)
+ "mla v1.8h, v2.8h, v31.8h \n" // 3*near+far (even)
+
+ "urshr v2.8h, v0.8h, #2 \n" // 3/4*near+1/4*far (odd)
+ "urshr v1.8h, v1.8h, #2 \n" // 3/4*near+1/4*far (even)
+
+ "st2 {v1.8h, v2.8h}, [%2], #32 \n" // store
+ "subs %w3, %w3, #16 \n" // 8 sample -> 16 sample
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_temp), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v31" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_12_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 1;
+ const uint16_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "movi v31.8h, #3 \n"
+
+ "1: \n"
+ "ld1 {v2.8h}, [%0], #16 \n" // 01234567 (16b)
+ "ld1 {v3.8h}, [%2], #16 \n" // 12345678 (16b)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "mov v0.16b, v2.16b \n"
+ "mla v2.8h, v3.8h, v31.8h \n" // 3*near+far (odd)
+ "mla v3.8h, v0.8h, v31.8h \n" // 3*near+far (even)
+
+ "ld1 {v4.8h}, [%1], #16 \n" // 01234567 (16b)
+ "ld1 {v5.8h}, [%3], #16 \n" // 12345678 (16b)
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+
+ "mov v0.16b, v4.16b \n"
+ "mla v4.8h, v5.8h, v31.8h \n" // 3*near+far (odd)
+ "mla v5.8h, v0.8h, v31.8h \n" // 3*near+far (even)
+
+ "mov v0.16b, v4.16b \n"
+ "mov v1.16b, v5.16b \n"
+ "mla v4.8h, v2.8h, v31.8h \n" // 9 3 3 1 (1, odd)
+ "mla v5.8h, v3.8h, v31.8h \n" // 9 3 3 1 (1, even)
+ "mla v2.8h, v0.8h, v31.8h \n" // 9 3 3 1 (2, odd)
+ "mla v3.8h, v1.8h, v31.8h \n" // 9 3 3 1 (2, even)
+
+ "urshr v2.8h, v2.8h, #4 \n" // 2, odd
+ "urshr v1.8h, v3.8h, #4 \n" // 2, even
+ "urshr v4.8h, v4.8h, #4 \n" // 1, odd
+ "urshr v3.8h, v5.8h, #4 \n" // 1, even
+
+ "st2 {v3.8h, v4.8h}, [%4], #32 \n" // store 1
+ "st2 {v1.8h, v2.8h}, [%5], #32 \n" // store 2
+
+ "subs %w6, %w6, #16 \n" // 8 sample -> 16 sample
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(src_temp), // %2
+ "+r"(src_temp1), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_ptr1), // %5
+ "+r"(dst_width) // %6
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 1;
+ asm volatile(
+ "movi v31.8h, #3 \n"
+
+ "1: \n"
+ "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
+ "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.4s, v0.4h, #0 \n" // 0123 (32b)
+ "ushll2 v3.4s, v0.8h, #0 \n" // 4567 (32b)
+ "ushll v4.4s, v1.4h, #0 \n" // 1234 (32b)
+ "ushll2 v5.4s, v1.8h, #0 \n" // 5678 (32b)
+
+ "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd)
+ "umlal2 v3.4s, v1.8h, v31.8h \n" // 3*near+far (2, odd)
+ "umlal v4.4s, v0.4h, v31.4h \n" // 3*near+far (1, even)
+ "umlal2 v5.4s, v0.8h, v31.8h \n" // 3*near+far (2, even)
+
+ "rshrn v0.4h, v4.4s, #2 \n" // 3/4*near+1/4*far
+ "rshrn2 v0.8h, v5.4s, #2 \n" // 3/4*near+1/4*far (even)
+ "rshrn v1.4h, v2.4s, #2 \n" // 3/4*near+1/4*far
+ "rshrn2 v1.8h, v3.4s, #2 \n" // 3/4*near+1/4*far (odd)
+
+ "st2 {v0.8h, v1.8h}, [%2], #32 \n" // store
+ "subs %w3, %w3, #16 \n" // 8 sample -> 16 sample
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_temp), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v31" // Clobber List
+ );
+}
+
+void ScaleRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 1;
+ const uint16_t* src_temp1 = src_ptr1 + 1;
+
+ asm volatile(
+ "movi v31.4h, #3 \n"
+ "movi v30.4s, #3 \n"
+
+ "1: \n"
+ "ldr d0, [%0], #8 \n" // 0123 (16b)
+ "ldr d1, [%2], #8 \n" // 1234 (16b)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "ushll v2.4s, v0.4h, #0 \n" // 0123 (32b)
+ "ushll v3.4s, v1.4h, #0 \n" // 1234 (32b)
+ "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd)
+ "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (1, even)
+
+ "ldr d0, [%1], #8 \n" // 0123 (16b)
+ "ldr d1, [%3], #8 \n" // 1234 (16b)
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+ "ushll v4.4s, v0.4h, #0 \n" // 0123 (32b)
+ "ushll v5.4s, v1.4h, #0 \n" // 1234 (32b)
+ "umlal v4.4s, v1.4h, v31.4h \n" // 3*near+far (2, odd)
+ "umlal v5.4s, v0.4h, v31.4h \n" // 3*near+far (2, even)
+
+ "mov v0.16b, v4.16b \n"
+ "mov v1.16b, v5.16b \n"
+ "mla v4.4s, v2.4s, v30.4s \n" // 9 3 3 1 (1, odd)
+ "mla v5.4s, v3.4s, v30.4s \n" // 9 3 3 1 (1, even)
+ "mla v2.4s, v0.4s, v30.4s \n" // 9 3 3 1 (2, odd)
+ "mla v3.4s, v1.4s, v30.4s \n" // 9 3 3 1 (2, even)
+
+ "rshrn v1.4h, v4.4s, #4 \n" // 3/4*near+1/4*far
+ "rshrn v0.4h, v5.4s, #4 \n" // 3/4*near+1/4*far
+ "rshrn v5.4h, v2.4s, #4 \n" // 3/4*near+1/4*far
+ "rshrn v4.4h, v3.4s, #4 \n" // 3/4*near+1/4*far
+
+ "st2 {v0.4h, v1.4h}, [%4], #16 \n" // store 1
+ "st2 {v4.4h, v5.4h}, [%5], #16 \n" // store 2
+
+ "subs %w6, %w6, #8 \n" // 4 sample -> 8 sample
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(src_temp), // %2
+ "+r"(src_temp1), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_ptr1), // %5
+ "+r"(dst_width) // %6
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Linear_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_temp = src_ptr + 2;
asm volatile(
- "ld1 {v30.8h}, [%4] \n"
- "ld1 {v31.16b}, [%5] \n"
- "add %2, %2, %0 \n"
- "1: \n"
-
- // 00 40 01 41 02 42 03 43
- // 10 50 11 51 12 52 13 53
- // 20 60 21 61 22 62 23 63
- // 30 70 31 71 32 72 33 73
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n"
- "subs %w3, %w3, #12 \n"
-
- // Shuffle the input data around to get align the data
- // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
- // 00 10 01 11 02 12 03 13
- // 40 50 41 51 42 52 43 53
- "trn1 v16.8b, v0.8b, v1.8b \n"
- "trn2 v17.8b, v0.8b, v1.8b \n"
- "trn1 v18.8b, v4.8b, v5.8b \n"
- "trn2 v19.8b, v4.8b, v5.8b \n"
-
- // 20 30 21 31 22 32 23 33
- // 60 70 61 71 62 72 63 73
- "trn1 v0.8b, v2.8b, v3.8b \n"
- "trn2 v1.8b, v2.8b, v3.8b \n"
- "trn1 v4.8b, v6.8b, v7.8b \n"
- "trn2 v5.8b, v6.8b, v7.8b \n"
-
- // 00+10 01+11 02+12 03+13
- // 40+50 41+51 42+52 43+53
- "uaddlp v16.4h, v16.8b \n"
- "uaddlp v17.4h, v17.8b \n"
- "uaddlp v18.4h, v18.8b \n"
- "uaddlp v19.4h, v19.8b \n"
-
- // 60+70 61+71 62+72 63+73
- "uaddlp v1.4h, v1.8b \n"
- "uaddlp v5.4h, v5.8b \n"
-
- // combine source lines
- "add v16.4h, v16.4h, v18.4h \n"
- "add v17.4h, v17.4h, v19.4h \n"
- "add v2.4h, v1.4h, v5.4h \n"
-
- // dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4
- "uqrshrn v2.8b, v2.8h, #2 \n"
-
- // Shuffle 2,3 reg around so that 2 can be added to the
- // 0,1 reg and 3 can be added to the 4,5 reg. This
- // requires expanding from u8 to u16 as the 0,1 and 4,5
- // registers are already expanded. Then do transposes
- // to get aligned.
- // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
-
- // combine source lines
- "uaddl v0.8h, v0.8b, v4.8b \n"
-
- // xx 20 xx 21 xx 22 xx 23
- // xx 30 xx 31 xx 32 xx 33
- "trn1 v1.8h, v0.8h, v0.8h \n"
- "trn2 v4.8h, v0.8h, v0.8h \n"
- "xtn v0.4h, v1.4s \n"
- "xtn v4.4h, v4.4s \n"
-
- // 0+1+2, 3+4+5
- "add v16.8h, v16.8h, v0.8h \n"
- "add v17.8h, v17.8h, v4.8h \n"
-
- // Need to divide, but can't downshift as the the value
- // isn't a power of 2. So multiply by 65536 / n
- // and take the upper 16 bits.
- "sqrdmulh v0.8h, v16.8h, v30.8h \n"
- "sqrdmulh v1.8h, v17.8h, v30.8h \n"
-
- // Align for table lookup, vtbl requires registers to
- // be adjacent
-
- "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v31.16b \n"
-
- "st1 {v3.8b}, [%1], #8 \n"
- "st1 {v3.s}[2], [%1], #4 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(tmp_src_stride), // %2
- "+r"(dst_width) // %3
- : "r"(&kMult38_Div6), // %4
- "r"(&kShuf38_2) // %5
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v30", "v31", "memory", "cc");
+ "movi v31.8b, #3 \n"
+
+ "1: \n"
+ "ldr d0, [%0], #8 \n" // 00112233 (1u1v)
+ "ldr d1, [%1], #8 \n" // 11223344 (1u1v)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.8h, v0.8b, #0 \n" // 00112233 (1u1v, 16b)
+ "ushll v3.8h, v1.8b, #0 \n" // 11223344 (1u1v, 16b)
+
+ "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (odd)
+ "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (even)
+
+ "rshrn v2.8b, v2.8h, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn v1.8b, v3.8h, #2 \n" // 3/4*near+1/4*far (even)
+
+ "st2 {v1.4h, v2.4h}, [%2], #16 \n" // store
+ "subs %w3, %w3, #8 \n" // 4 uv -> 8 uv
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_temp), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v31" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Bilinear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ uint8_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint8_t* src_temp = src_ptr + 2;
+ const uint8_t* src_temp1 = src_ptr1 + 2;
+
+ asm volatile(
+ "movi v31.8b, #3 \n"
+ "movi v30.8h, #3 \n"
+
+ "1: \n"
+ "ldr d0, [%0], #8 \n"
+ "ldr d1, [%2], #8 \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.8h, v0.8b, #0 \n"
+ "ushll v3.8h, v1.8b, #0 \n"
+ "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (1, odd)
+ "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (1, even)
+
+ "ldr d0, [%1], #8 \n"
+ "ldr d1, [%3], #8 \n"
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v4.8h, v0.8b, #0 \n"
+ "ushll v5.8h, v1.8b, #0 \n"
+ "umlal v4.8h, v1.8b, v31.8b \n" // 3*near+far (2, odd)
+ "umlal v5.8h, v0.8b, v31.8b \n" // 3*near+far (2, even)
+
+ "mov v0.16b, v4.16b \n"
+ "mov v1.16b, v5.16b \n"
+ "mla v4.8h, v2.8h, v30.8h \n" // 9 3 3 1 (1, odd)
+ "mla v5.8h, v3.8h, v30.8h \n" // 9 3 3 1 (1, even)
+ "mla v2.8h, v0.8h, v30.8h \n" // 9 3 3 1 (2, odd)
+ "mla v3.8h, v1.8h, v30.8h \n" // 9 3 3 1 (2, even)
+
+ "rshrn v2.8b, v2.8h, #4 \n" // 2, odd
+ "rshrn v1.8b, v3.8h, #4 \n" // 2, even
+ "rshrn v4.8b, v4.8h, #4 \n" // 1, odd
+ "rshrn v3.8b, v5.8h, #4 \n" // 1, even
+
+ "st2 {v1.4h, v2.4h}, [%5], #16 \n" // store 2
+ "st2 {v3.4h, v4.4h}, [%4], #16 \n" // store 1
+ "subs %w6, %w6, #8 \n" // 4 uv -> 8 uv
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(src_temp), // %2
+ "+r"(src_temp1), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_ptr1), // %5
+ "+r"(dst_width) // %6
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const uint16_t* src_temp = src_ptr + 2;
+ asm volatile(
+ "movi v31.8h, #3 \n"
+
+ "1: \n"
+ "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
+ "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+
+ "ushll v2.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b)
+ "ushll v3.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b)
+ "ushll2 v4.4s, v0.8h, #0 \n" // 2233 (1u1v, 32b)
+ "ushll2 v5.4s, v1.8h, #0 \n" // 3344 (1u1v, 32b)
+
+ "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (odd)
+ "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (even)
+ "umlal2 v4.4s, v1.8h, v31.8h \n" // 3*near+far (odd)
+ "umlal2 v5.4s, v0.8h, v31.8h \n" // 3*near+far (even)
+
+ "rshrn v2.4h, v2.4s, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn v1.4h, v3.4s, #2 \n" // 3/4*near+1/4*far (even)
+ "rshrn v4.4h, v4.4s, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn v3.4h, v5.4s, #2 \n" // 3/4*near+1/4*far (even)
+
+ "st2 {v1.2s, v2.2s}, [%2], #16 \n" // store
+ "st2 {v3.2s, v4.2s}, [%2], #16 \n" // store
+ "subs %w3, %w3, #8 \n" // 4 uv -> 8 uv
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_temp), // %1
+ "+r"(dst_ptr), // %2
+ "+r"(dst_width) // %3
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v31" // Clobber List
+ );
+}
+
+void ScaleUVRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+ uint16_t* dst_ptr1 = dst_ptr + dst_stride;
+ const uint16_t* src_temp = src_ptr + 2;
+ const uint16_t* src_temp1 = src_ptr1 + 2;
+
+ asm volatile(
+ "movi v31.4h, #3 \n"
+ "movi v30.4s, #3 \n"
+
+ "1: \n"
+ "ldr d0, [%0], #8 \n"
+ "ldr d1, [%2], #8 \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "ushll v2.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b)
+ "ushll v3.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b)
+ "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd)
+ "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (1, even)
+
+ "ldr d0, [%1], #8 \n"
+ "ldr d1, [%3], #8 \n"
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+ "ushll v4.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b)
+ "ushll v5.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b)
+ "umlal v4.4s, v1.4h, v31.4h \n" // 3*near+far (2, odd)
+ "umlal v5.4s, v0.4h, v31.4h \n" // 3*near+far (2, even)
+
+ "mov v0.16b, v4.16b \n"
+ "mov v1.16b, v5.16b \n"
+ "mla v4.4s, v2.4s, v30.4s \n" // 9 3 3 1 (1, odd)
+ "mla v5.4s, v3.4s, v30.4s \n" // 9 3 3 1 (1, even)
+ "mla v2.4s, v0.4s, v30.4s \n" // 9 3 3 1 (2, odd)
+ "mla v3.4s, v1.4s, v30.4s \n" // 9 3 3 1 (2, even)
+
+ "rshrn v1.4h, v2.4s, #4 \n" // 2, odd
+ "rshrn v0.4h, v3.4s, #4 \n" // 2, even
+ "rshrn v3.4h, v4.4s, #4 \n" // 1, odd
+ "rshrn v2.4h, v5.4s, #4 \n" // 1, even
+
+ "st2 {v0.2s, v1.2s}, [%5], #16 \n" // store 2
+ "st2 {v2.2s, v3.2s}, [%4], #16 \n" // store 1
+ "subs %w6, %w6, #4 \n" // 2 uv -> 4 uv
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src_ptr1), // %1
+ "+r"(src_temp), // %2
+ "+r"(src_temp1), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_ptr1), // %5
+ "+r"(dst_width) // %6
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30",
+ "v31" // Clobber List
+ );
}
// Add a row of bytes to a row of shorts. Used for box filter.
@@ -521,14 +995,15 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
asm volatile(
- "1: \n"
- "ld1 {v1.8h, v2.8h}, [%1] \n" // load accumulator
- "ld1 {v0.16b}, [%0], #16 \n" // load 16 bytes
- "uaddw2 v2.8h, v2.8h, v0.16b \n" // add
- "uaddw v1.8h, v1.8h, v0.8b \n"
- "st1 {v1.8h, v2.8h}, [%1], #32 \n" // store accumulator
- "subs %w2, %w2, #16 \n" // 16 processed per loop
- "b.gt 1b \n"
+ "1: \n"
+ "ld1 {v1.8h, v2.8h}, [%1] \n" // load accumulator
+ "ld1 {v0.16b}, [%0], #16 \n" // load 16 bytes
+ "uaddw2 v2.8h, v2.8h, v0.16b \n" // add
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "uaddw v1.8h, v1.8h, v0.8b \n"
+ "st1 {v1.8h, v2.8h}, [%1], #32 \n" // store accumulator
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_width) // %2
@@ -537,15 +1012,12 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr,
);
}
-// TODO(Yang Zhang): Investigate less load instructions for
-// the x/dx stepping
-#define LOAD2_DATA8_LANE(n) \
- "lsr %5, %3, #16 \n" \
- "add %6, %1, %5 \n" \
- "add %3, %3, %4 \n" \
- "ld2 {v4.b, v5.b}[" #n "], [%6] \n"
+#define SCALE_FILTER_COLS_STEP_ADDR \
+ "sbfx %[tmp_offset], %x[x], #16, #16 \n" \
+ "add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \
+ "add %w[x], %w[x], %w[dx] \n"
-// The NEON version mimics this formula (from row_common.cc):
+// The Neon version mimics this formula (from scale_common.cc):
// #define BLENDER(a, b, f) (uint8_t)((int)(a) +
// ((((int)((f)) * ((int)(b) - (int)(a))) + 0x8000) >> 16))
@@ -555,171 +1027,89 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
int x,
int dx) {
int dx_offset[4] = {0, 1, 2, 3};
- int* tmp = dx_offset;
- const uint8_t* src_tmp = src_ptr;
- int64_t x64 = (int64_t)x; // NOLINT
- int64_t dx64 = (int64_t)dx; // NOLINT
- asm volatile (
- "dup v0.4s, %w3 \n" // x
- "dup v1.4s, %w4 \n" // dx
- "ld1 {v2.4s}, [%5] \n" // 0 1 2 3
- "shl v3.4s, v1.4s, #2 \n" // 4 * dx
- "mul v1.4s, v1.4s, v2.4s \n"
- // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
- "add v1.4s, v1.4s, v0.4s \n"
- // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
- "add v2.4s, v1.4s, v3.4s \n"
- "shl v0.4s, v3.4s, #1 \n" // 8 * dx
- "1: \n"
- LOAD2_DATA8_LANE(0)
- LOAD2_DATA8_LANE(1)
- LOAD2_DATA8_LANE(2)
- LOAD2_DATA8_LANE(3)
- LOAD2_DATA8_LANE(4)
- LOAD2_DATA8_LANE(5)
- LOAD2_DATA8_LANE(6)
- LOAD2_DATA8_LANE(7)
- "mov v6.16b, v1.16b \n"
- "mov v7.16b, v2.16b \n"
- "uzp1 v6.8h, v6.8h, v7.8h \n"
- "ushll v4.8h, v4.8b, #0 \n"
- "ushll v5.8h, v5.8b, #0 \n"
- "ssubl v16.4s, v5.4h, v4.4h \n"
- "ssubl2 v17.4s, v5.8h, v4.8h \n"
- "ushll v7.4s, v6.4h, #0 \n"
- "ushll2 v6.4s, v6.8h, #0 \n"
- "mul v16.4s, v16.4s, v7.4s \n"
- "mul v17.4s, v17.4s, v6.4s \n"
- "rshrn v6.4h, v16.4s, #16 \n"
- "rshrn2 v6.8h, v17.4s, #16 \n"
- "add v4.8h, v4.8h, v6.8h \n"
- "xtn v4.8b, v4.8h \n"
-
- "st1 {v4.8b}, [%0], #8 \n" // store pixels
- "add v1.4s, v1.4s, v0.4s \n"
- "add v2.4s, v2.4s, v0.4s \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "b.gt 1b \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(dst_width), // %2
- "+r"(x64), // %3
- "+r"(dx64), // %4
- "+r"(tmp), // %5
- "+r"(src_tmp) // %6
- :
- : "memory", "cc", "v0", "v1", "v2", "v3",
- "v4", "v5", "v6", "v7", "v16", "v17"
- );
-}
-
-#undef LOAD2_DATA8_LANE
-
-// 16x2 -> 16x1
-void ScaleFilterRows_NEON(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- int y_fraction = 256 - source_y_fraction;
+ int64_t tmp_offset;
+ uint8_t* tmp_ptr;
asm volatile(
- "cmp %w4, #0 \n"
- "b.eq 100f \n"
- "add %2, %2, %1 \n"
- "cmp %w4, #64 \n"
- "b.eq 75f \n"
- "cmp %w4, #128 \n"
- "b.eq 50f \n"
- "cmp %w4, #192 \n"
- "b.eq 25f \n"
-
- "dup v5.8b, %w4 \n"
- "dup v4.8b, %w5 \n"
- // General purpose row blend.
- "1: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "ld1 {v1.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "umull v6.8h, v0.8b, v4.8b \n"
- "umull2 v7.8h, v0.16b, v4.16b \n"
- "umlal v6.8h, v1.8b, v5.8b \n"
- "umlal2 v7.8h, v1.16b, v5.16b \n"
- "rshrn v0.8b, v6.8h, #8 \n"
- "rshrn2 v0.16b, v7.8h, #8 \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 1b \n"
- "b 99f \n"
-
- // Blend 25 / 75.
- "25: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "ld1 {v1.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 25b \n"
- "b 99f \n"
-
- // Blend 50 / 50.
- "50: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "ld1 {v1.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 50b \n"
- "b 99f \n"
-
- // Blend 75 / 25.
- "75: \n"
- "ld1 {v1.16b}, [%1], #16 \n"
- "ld1 {v0.16b}, [%2], #16 \n"
- "subs %w3, %w3, #16 \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "urhadd v0.16b, v0.16b, v1.16b \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 75b \n"
- "b 99f \n"
-
- // Blend 100 / 0 - Copy row unchanged.
- "100: \n"
- "ld1 {v0.16b}, [%1], #16 \n"
- "subs %w3, %w3, #16 \n"
- "st1 {v0.16b}, [%0], #16 \n"
- "b.gt 100b \n"
-
- "99: \n"
- "st1 {v0.b}[15], [%0] \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(src_stride), // %2
- "+r"(dst_width), // %3
- "+r"(source_y_fraction), // %4
- "+r"(y_fraction) // %5
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory", "cc");
+ "dup v0.4s, %w[x] \n"
+ "dup v1.4s, %w[dx] \n"
+ "ld1 {v2.4s}, [%[dx_offset]] \n" // 0 1 2 3
+ "shl v3.4s, v1.4s, #2 \n" // 4 * dx
+ "shl v22.4s, v1.4s, #3 \n" // 8 * dx
+
+ "mul v1.4s, v1.4s, v2.4s \n"
+ // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
+ "add v1.4s, v1.4s, v0.4s \n"
+ // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
+ "add v2.4s, v1.4s, v3.4s \n"
+
+ "movi v0.8h, #0 \n"
+
+ // truncate to uint16_t
+ "trn1 v22.8h, v22.8h, v0.8h \n"
+ "trn1 v20.8h, v1.8h, v0.8h \n"
+ "trn1 v21.8h, v2.8h, v0.8h \n"
+
+ "1: \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ldr h6, [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[1], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[2], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[3], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[4], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[5], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[6], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[7], [%[tmp_ptr]] \n"
+
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop
+ "trn1 v4.16b, v6.16b, v0.16b \n"
+ "trn2 v5.16b, v6.16b, v0.16b \n"
+
+ "ssubl v16.4s, v5.4h, v4.4h \n"
+ "ssubl2 v17.4s, v5.8h, v4.8h \n"
+ "mul v16.4s, v16.4s, v20.4s \n"
+ "mul v17.4s, v17.4s, v21.4s \n"
+ "rshrn v6.4h, v16.4s, #16 \n"
+ "rshrn2 v6.8h, v17.4s, #16 \n"
+ "add v4.8h, v4.8h, v6.8h \n"
+ "xtn v4.8b, v4.8h \n"
+
+ "add v20.8h, v20.8h, v22.8h \n"
+ "add v21.8h, v21.8h, v22.8h \n"
+
+ "st1 {v4.8b}, [%[dst_ptr]], #8 \n" // store pixels
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width), // %[width]
+ [x] "+r"(x), // %[x]
+ [dx] "+r"(dx), // %[dx]
+ [tmp_offset] "=&r"(tmp_offset), // %[tmp_offset]
+ [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr]
+ : [dx_offset] "r"(dx_offset) // %[dx_offset]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
+ "v20", "v21", "v22");
}
+#undef SCALE_FILTER_COLS_STEP_ADDR
+
void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- // load 16 ARGB pixels with even pixels into q0/q2, odd into q1/q3
- "ld4 {v0.4s,v1.4s,v2.4s,v3.4s}, [%0], #64 \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "mov v2.16b, v3.16b \n"
- "st2 {v1.4s,v2.4s}, [%1], #32 \n" // store 8 odd pixels
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst), // %1
- "+r"(dst_width) // %2
+ "1: \n"
+ "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[src]], #64 \n"
+ "subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uzp2 v0.4s, v0.4s, v1.4s \n"
+ "uzp2 v1.4s, v2.4s, v3.4s \n"
+ "st1 {v0.4s, v1.4s}, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(dst_width) // %[width]
:
- : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
- );
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
@@ -727,79 +1117,79 @@ void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
+ const uint8_t* src_argb1 = src_argb + 32;
asm volatile(
- "1: \n"
- // load 16 ARGB pixels with even pixels into q0/q2, odd into q1/q3
- "ld4 {v0.4s,v1.4s,v2.4s,v3.4s}, [%0], #64 \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
-
- "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
- "urhadd v1.16b, v2.16b, v3.16b \n"
- "st2 {v0.4s,v1.4s}, [%1], #32 \n" // store 8 pixels
- "b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
+ "1: \n"
+ "ld2 {v0.4s, v1.4s}, [%[src]] \n"
+ "add %[src], %[src], #64 \n"
+ "ld2 {v2.4s, v3.4s}, [%[src1]] \n"
+ "add %[src1], %[src1], #64 \n"
+ "urhadd v0.16b, v0.16b, v1.16b \n"
+ "urhadd v1.16b, v2.16b, v3.16b \n"
+ "subs %w[width], %w[width], #8 \n"
+ "st1 {v0.16b, v1.16b}, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [src1] "+r"(src_argb1), // %[src1]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(dst_width) // %[width]
:
- : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
- );
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
asm volatile(
- // change the stride to row 2 pointer
- "add %1, %1, %0 \n"
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 8 ARGB
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "uaddlp v3.8h, v3.16b \n" // A 16 bytes -> 8 shorts.
- "ld4 {v16.16b,v17.16b,v18.16b,v19.16b}, [%1], #64 \n" // load 8
- "uadalp v0.8h, v16.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v17.16b \n" // G 16 bytes -> 8 shorts.
- "uadalp v2.8h, v18.16b \n" // R 16 bytes -> 8 shorts.
- "uadalp v3.8h, v19.16b \n" // A 16 bytes -> 8 shorts.
- "rshrn v0.8b, v0.8h, #2 \n" // round and pack
- "rshrn v1.8b, v1.8h, #2 \n"
- "rshrn v2.8b, v2.8h, #2 \n"
- "rshrn v3.8b, v3.8h, #2 \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(src_stride), // %1
- "+r"(dst), // %2
- "+r"(dst_width) // %3
+ "1: \n"
+ "ld2 {v0.4s, v1.4s}, [%[src]], #32 \n"
+ "ld2 {v20.4s, v21.4s}, [%[src1]], #32 \n"
+ "uaddl v2.8h, v0.8b, v1.8b \n"
+ "uaddl2 v3.8h, v0.16b, v1.16b \n"
+ "uaddl v22.8h, v20.8b, v21.8b \n"
+ "uaddl2 v23.8h, v20.16b, v21.16b \n"
+ "add v0.8h, v2.8h, v22.8h \n"
+ "add v1.8h, v3.8h, v23.8h \n"
+ "rshrn v0.8b, v0.8h, #2 \n"
+ "rshrn v1.8b, v1.8h, #2 \n"
+ "subs %w[width], %w[width], #4 \n"
+ "stp d0, d1, [%[dst]], #16 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_ptr), [src1] "+r"(src_ptr1), [dst] "+r"(dst),
+ [width] "+r"(dst_width)
:
- : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v20", "v21", "v22", "v23");
}
-// Reads 4 pixels at a time.
-// Alignment requirement: src_argb 4 byte aligned.
void ScaleARGBRowDownEven_NEON(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
+ const uint8_t* src_argb1 = src_argb + src_stepx * 4;
+ const uint8_t* src_argb2 = src_argb + src_stepx * 8;
+ const uint8_t* src_argb3 = src_argb + src_stepx * 12;
+ int64_t i = 0;
(void)src_stride;
asm volatile(
- "1: \n"
- "ld1 {v0.s}[0], [%0], %3 \n"
- "ld1 {v0.s}[1], [%0], %3 \n"
- "ld1 {v0.s}[2], [%0], %3 \n"
- "ld1 {v0.s}[3], [%0], %3 \n"
- "subs %w2, %w2, #4 \n" // 4 pixels per loop.
- "st1 {v0.16b}, [%1], #16 \n"
- "b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- : "r"((int64_t)(src_stepx * 4)) // %3
- : "memory", "cc", "v0");
+ "1: \n"
+ "ldr w10, [%[src], %[i]] \n"
+ "ldr w11, [%[src1], %[i]] \n"
+ "ldr w12, [%[src2], %[i]] \n"
+ "ldr w13, [%[src3], %[i]] \n"
+ "add %[i], %[i], %[step] \n"
+ "subs %w[width], %w[width], #4 \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "stp w10, w11, [%[dst]], #8 \n"
+ "stp w12, w13, [%[dst]], #8 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), [src1] "+r"(src_argb1), [src2] "+r"(src_argb2),
+ [src3] "+r"(src_argb3), [dst] "+r"(dst_argb), [width] "+r"(dst_width),
+ [i] "+r"(i)
+ : [step] "r"((int64_t)(src_stepx * 16))
+ : "memory", "cc", "w10", "w11", "w12", "w13");
}
// Reads 4 pixels at a time.
@@ -812,39 +1202,39 @@ void ScaleARGBRowDownEvenBox_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int dst_width) {
asm volatile(
- "add %1, %1, %0 \n"
- "1: \n"
- "ld1 {v0.8b}, [%0], %4 \n" // Read 4 2x2 -> 2x1
- "ld1 {v1.8b}, [%1], %4 \n"
- "ld1 {v2.8b}, [%0], %4 \n"
- "ld1 {v3.8b}, [%1], %4 \n"
- "ld1 {v4.8b}, [%0], %4 \n"
- "ld1 {v5.8b}, [%1], %4 \n"
- "ld1 {v6.8b}, [%0], %4 \n"
- "ld1 {v7.8b}, [%1], %4 \n"
- "uaddl v0.8h, v0.8b, v1.8b \n"
- "uaddl v2.8h, v2.8b, v3.8b \n"
- "uaddl v4.8h, v4.8b, v5.8b \n"
- "uaddl v6.8h, v6.8b, v7.8b \n"
- "mov v16.d[1], v0.d[1] \n" // ab_cd -> ac_bd
- "mov v0.d[1], v2.d[0] \n"
- "mov v2.d[0], v16.d[1] \n"
- "mov v16.d[1], v4.d[1] \n" // ef_gh -> eg_fh
- "mov v4.d[1], v6.d[0] \n"
- "mov v6.d[0], v16.d[1] \n"
- "add v0.8h, v0.8h, v2.8h \n" // (a+b)_(c+d)
- "add v4.8h, v4.8h, v6.8h \n" // (e+f)_(g+h)
- "rshrn v0.8b, v0.8h, #2 \n" // first 2 pixels.
- "rshrn2 v0.16b, v4.8h, #2 \n" // next 2 pixels.
- "subs %w3, %w3, #4 \n" // 4 pixels per loop.
- "st1 {v0.16b}, [%2], #16 \n"
- "b.gt 1b \n"
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "ld1 {v0.8b}, [%0], %4 \n" // Read 4 2x2 -> 2x1
+ "ld1 {v1.8b}, [%1], %4 \n"
+ "ld1 {v2.8b}, [%0], %4 \n"
+ "ld1 {v3.8b}, [%1], %4 \n"
+ "ld1 {v4.8b}, [%0], %4 \n"
+ "ld1 {v5.8b}, [%1], %4 \n"
+ "ld1 {v6.8b}, [%0], %4 \n"
+ "ld1 {v7.8b}, [%1], %4 \n"
+ "uaddl v0.8h, v0.8b, v1.8b \n"
+ "uaddl v2.8h, v2.8b, v3.8b \n"
+ "uaddl v4.8h, v4.8b, v5.8b \n"
+ "uaddl v6.8h, v6.8b, v7.8b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "zip1 v1.2d, v0.2d, v2.2d \n"
+ "zip2 v2.2d, v0.2d, v2.2d \n"
+ "zip1 v5.2d, v4.2d, v6.2d \n"
+ "zip2 v6.2d, v4.2d, v6.2d \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "add v0.8h, v1.8h, v2.8h \n" // (a+b)_(c+d)
+ "add v4.8h, v5.8h, v6.8h \n" // (e+f)_(g+h)
+ "rshrn v0.8b, v0.8h, #2 \n" // first 2 pixels.
+ "rshrn v1.8b, v4.8h, #2 \n" // next 2 pixels.
+ "subs %w3, %w3, #4 \n" // 4 pixels per loop.
+ "stp d0, d1, [%2], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_stride), // %1
"+r"(dst_argb), // %2
"+r"(dst_width) // %3
: "r"((int64_t)(src_stepx * 4)) // %4
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
// TODO(Yang Zhang): Investigate less load instructions for
@@ -864,8 +1254,8 @@ void ScaleARGBCols_NEON(uint8_t* dst_argb,
int64_t x64 = (int64_t)x; // NOLINT
int64_t dx64 = (int64_t)dx; // NOLINT
int64_t tmp64;
- asm volatile(
- "1: \n"
+ asm volatile (
+ "1: \n"
// clang-format off
LOAD1_DATA32_LANE(v0, 0)
LOAD1_DATA32_LANE(v0, 1)
@@ -875,10 +1265,11 @@ void ScaleARGBCols_NEON(uint8_t* dst_argb,
LOAD1_DATA32_LANE(v1, 1)
LOAD1_DATA32_LANE(v1, 2)
LOAD1_DATA32_LANE(v1, 3)
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
// clang-format on
- "st1 {v0.4s, v1.4s}, [%0], #32 \n" // store pixels
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "b.gt 1b \n"
+ "st1 {v0.4s, v1.4s}, [%0], #32 \n" // store pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "b.gt 1b \n"
: "+r"(dst_argb), // %0
"+r"(src_argb), // %1
"+r"(dst_width), // %2
@@ -892,13 +1283,14 @@ void ScaleARGBCols_NEON(uint8_t* dst_argb,
#undef LOAD1_DATA32_LANE
-// TODO(Yang Zhang): Investigate less load instructions for
-// the x/dx stepping
-#define LOAD2_DATA32_LANE(vn1, vn2, n) \
- "lsr %5, %3, #16 \n" \
- "add %6, %1, %5, lsl #2 \n" \
- "add %3, %3, %4 \n" \
- "ld2 {" #vn1 ".s, " #vn2 ".s}[" #n "], [%6] \n"
+static const uvec8 kScaleARGBFilterColsShuffleIndices = {
+ 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6,
+};
+
+#define SCALE_ARGB_FILTER_COLS_STEP_ADDR \
+ "lsr %5, %3, #16 \n" \
+ "add %6, %1, %5, lsl #2 \n" \
+ "add %3, %3, %4 \n"
void ScaleARGBFilterCols_NEON(uint8_t* dst_argb,
const uint8_t* src_argb,
@@ -906,64 +1298,130 @@ void ScaleARGBFilterCols_NEON(uint8_t* dst_argb,
int x,
int dx) {
int dx_offset[4] = {0, 1, 2, 3};
- int* tmp = dx_offset;
+ int64_t tmp;
const uint8_t* src_tmp = src_argb;
- int64_t x64 = (int64_t)x; // NOLINT
- int64_t dx64 = (int64_t)dx; // NOLINT
- asm volatile (
- "dup v0.4s, %w3 \n" // x
- "dup v1.4s, %w4 \n" // dx
- "ld1 {v2.4s}, [%5] \n" // 0 1 2 3
- "shl v6.4s, v1.4s, #2 \n" // 4 * dx
- "mul v1.4s, v1.4s, v2.4s \n"
- "movi v3.16b, #0x7f \n" // 0x7F
- "movi v4.8h, #0x7f \n" // 0x7F
- // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
- "add v5.4s, v1.4s, v0.4s \n"
- "1: \n"
- // d0, d1: a
- // d2, d3: b
- LOAD2_DATA32_LANE(v0, v1, 0)
- LOAD2_DATA32_LANE(v0, v1, 1)
- LOAD2_DATA32_LANE(v0, v1, 2)
- LOAD2_DATA32_LANE(v0, v1, 3)
- "shrn v2.4h, v5.4s, #9 \n"
- "and v2.8b, v2.8b, v4.8b \n"
- "dup v16.8b, v2.b[0] \n"
- "dup v17.8b, v2.b[2] \n"
- "dup v18.8b, v2.b[4] \n"
- "dup v19.8b, v2.b[6] \n"
- "ext v2.8b, v16.8b, v17.8b, #4 \n"
- "ext v17.8b, v18.8b, v19.8b, #4 \n"
- "ins v2.d[1], v17.d[0] \n" // f
- "eor v7.16b, v2.16b, v3.16b \n" // 0x7f ^ f
- "umull v16.8h, v0.8b, v7.8b \n"
- "umull2 v17.8h, v0.16b, v7.16b \n"
- "umull v18.8h, v1.8b, v2.8b \n"
- "umull2 v19.8h, v1.16b, v2.16b \n"
- "add v16.8h, v16.8h, v18.8h \n"
- "add v17.8h, v17.8h, v19.8h \n"
- "shrn v0.8b, v16.8h, #7 \n"
- "shrn2 v0.16b, v17.8h, #7 \n"
-
- "st1 {v0.4s}, [%0], #16 \n" // store pixels
- "add v5.4s, v5.4s, v6.4s \n"
- "subs %w2, %w2, #4 \n" // 4 processed per loop
- "b.gt 1b \n"
- : "+r"(dst_argb), // %0
- "+r"(src_argb), // %1
- "+r"(dst_width), // %2
- "+r"(x64), // %3
- "+r"(dx64), // %4
- "+r"(tmp), // %5
- "+r"(src_tmp) // %6
- :
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
- "v6", "v7", "v16", "v17", "v18", "v19"
- );
+ int64_t x64 = (int64_t)x;
+ int64_t dx64 = (int64_t)dx;
+ asm volatile(
+ "dup v0.4s, %w3 \n"
+ "dup v1.4s, %w4 \n"
+ "ld1 {v2.4s}, [%[kOffsets]] \n"
+ "shl v6.4s, v1.4s, #2 \n"
+ "mul v1.4s, v1.4s, v2.4s \n"
+ "movi v3.16b, #0x7f \n"
+
+ "add v5.4s, v1.4s, v0.4s \n"
+ "ldr q18, [%[kIndices]] \n"
+
+ "1: \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ldr d1, [%6] \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ldr d2, [%6] \n"
+ "shrn v4.4h, v5.4s, #9 \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ld1 {v1.d}[1], [%6] \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ld1 {v2.d}[1], [%6] \n"
+
+ "subs %w2, %w2, #4 \n" // 4 processed per loop
+ "and v4.8b, v4.8b, v3.8b \n"
+ "trn1 v0.4s, v1.4s, v2.4s \n"
+ "tbl v4.16b, {v4.16b}, v18.16b \n" // f
+ "trn2 v1.4s, v1.4s, v2.4s \n"
+ "eor v7.16b, v4.16b, v3.16b \n" // 0x7f ^ f
+
+ "umull v16.8h, v1.8b, v4.8b \n"
+ "umull2 v17.8h, v1.16b, v4.16b \n"
+ "umlal v16.8h, v0.8b, v7.8b \n"
+ "umlal2 v17.8h, v0.16b, v7.16b \n"
+
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+ "shrn v0.8b, v16.8h, #7 \n"
+ "shrn v1.8b, v17.8h, #7 \n"
+ "add v5.4s, v5.4s, v6.4s \n"
+ "stp d0, d1, [%0], #16 \n" // store pixels
+ "b.gt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(src_argb), // %1
+ "+r"(dst_width), // %2
+ "+r"(x64), // %3
+ "+r"(dx64), // %4
+ "=&r"(tmp), // %5
+ "+r"(src_tmp) // %6
+ : [kIndices] "r"(&kScaleARGBFilterColsShuffleIndices), // %[kIndices]
+ [kOffsets] "r"(dx_offset) // %[kOffsets]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
}
-#undef LOAD2_DATA32_LANE
+#undef SCALE_ARGB_FILTER_COLS_STEP_ADDR
+
+void ScaleRowDown2_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "subs %w[dst_width], %w[dst_width], #32 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "ldp q2, q3, [%[src_ptr], #32] \n"
+ "ldp q4, q5, [%[src_ptr], #64] \n"
+ "ldp q6, q7, [%[src_ptr], #96] \n"
+ "add %[src_ptr], %[src_ptr], #128 \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
+ "uzp2 v2.8h, v4.8h, v5.8h \n"
+ "uzp2 v3.8h, v6.8h, v7.8h \n"
+ "subs %w[dst_width], %w[dst_width], #32 \n" // 32 elems per
+ // iteration.
+ "stp q0, q1, [%[dst_ptr]] \n"
+ "stp q2, q3, [%[dst_ptr], #32] \n"
+ "add %[dst_ptr], %[dst_ptr], #64 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], #32 \n"
+ "b.eq 99f \n"
+
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "ldp q2, q3, [%[src_ptr], #32] \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
+ "stp q0, q1, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+
+void ScaleRowDown2Linear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "ld2 {v0.8h, v1.8h}, [%[src_ptr]], #32 \n"
+ "ld2 {v2.8h, v3.8h}, [%[src_ptr]], #32 \n"
+ "subs %w[dst_width], %w[dst_width], #16 \n"
+ "urhadd v0.8h, v0.8h, v1.8h \n"
+ "urhadd v1.8h, v2.8h, v3.8h \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n"
+ "stp q0, q1, [%[dst_ptr]], #32 \n"
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3");
+}
// Read 16x2 average down and write 8x1.
void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
@@ -972,76 +1430,157 @@ void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
int dst_width) {
asm volatile(
// change the stride to row 2 pointer
- "add %1, %0, %1, lsl #1 \n" // ptr + stide * 2
- "1: \n"
- "ld1 {v0.8h, v1.8h}, [%0], #32 \n" // load row 1 and post inc
- "ld1 {v2.8h, v3.8h}, [%1], #32 \n" // load row 2 and post inc
- "subs %w3, %w3, #8 \n" // 8 processed per loop
- "uaddlp v0.4s, v0.8h \n" // row 1 add adjacent
- "uaddlp v1.4s, v1.8h \n"
- "uadalp v0.4s, v2.8h \n" // +row 2 add adjacent
- "uadalp v1.4s, v3.8h \n"
- "rshrn v0.4h, v0.4s, #2 \n" // round and pack
- "rshrn2 v0.8h, v1.4s, #2 \n"
- "st1 {v0.8h}, [%2], #16 \n"
- "b.gt 1b \n"
+ "add %1, %0, %1, lsl #1 \n" // ptr + stide * 2
+ "1: \n"
+ "ld1 {v0.8h, v1.8h}, [%0], #32 \n" // load row 1 and post inc
+ "ld1 {v2.8h, v3.8h}, [%1], #32 \n" // load row 2 and post inc
+ "subs %w3, %w3, #8 \n" // 8 processed per loop
+ "uaddlp v0.4s, v0.8h \n" // row 1 add adjacent
+ "uaddlp v1.4s, v1.8h \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "uadalp v0.4s, v2.8h \n" // +row 2 add adjacent
+ "uadalp v1.4s, v3.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "rshrn v0.4h, v0.4s, #2 \n" // round and pack
+ "rshrn2 v0.8h, v1.4s, #2 \n"
+ "st1 {v0.8h}, [%2], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_stride), // %1
"+r"(dst), // %2
"+r"(dst_width) // %3
:
- : "v0", "v1", "v2", "v3" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
);
}
-// Read 8x2 upsample with filtering and write 16x1.
-// Actually reads an extra pixel, so 9x2.
-void ScaleRowUp2_16_NEON(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
+void ScaleUVRowDown2_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
asm volatile(
- "add %1, %0, %1, lsl #1 \n" // ptr + stide * 2
- "movi v0.8h, #9 \n" // constants
- "movi v1.4s, #3 \n"
+ "1: \n"
+ "ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "st1 {v1.8h}, [%1], #16 \n" // store 8 UV
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "v0", "v1");
+}
- "1: \n"
- "ld1 {v3.8h}, [%0], %4 \n" // TL read first 8
- "ld1 {v4.8h}, [%0], %5 \n" // TR read 8 offset by 1
- "ld1 {v5.8h}, [%1], %4 \n" // BL read 8 from next row
- "ld1 {v6.8h}, [%1], %5 \n" // BR offset by 1
- "subs %w3, %w3, #16 \n" // 16 dst pixels per loop
- "umull v16.4s, v3.4h, v0.4h \n"
- "umull2 v7.4s, v3.8h, v0.8h \n"
- "umull v18.4s, v4.4h, v0.4h \n"
- "umull2 v17.4s, v4.8h, v0.8h \n"
- "uaddw v16.4s, v16.4s, v6.4h \n"
- "uaddl2 v19.4s, v6.8h, v3.8h \n"
- "uaddl v3.4s, v6.4h, v3.4h \n"
- "uaddw2 v6.4s, v7.4s, v6.8h \n"
- "uaddl2 v7.4s, v5.8h, v4.8h \n"
- "uaddl v4.4s, v5.4h, v4.4h \n"
- "uaddw v18.4s, v18.4s, v5.4h \n"
- "mla v16.4s, v4.4s, v1.4s \n"
- "mla v18.4s, v3.4s, v1.4s \n"
- "mla v6.4s, v7.4s, v1.4s \n"
- "uaddw2 v4.4s, v17.4s, v5.8h \n"
- "uqrshrn v16.4h, v16.4s, #4 \n"
- "mla v4.4s, v19.4s, v1.4s \n"
- "uqrshrn2 v16.8h, v6.4s, #4 \n"
- "uqrshrn v17.4h, v18.4s, #4 \n"
- "uqrshrn2 v17.8h, v4.4s, #4 \n"
- "st2 {v16.8h-v17.8h}, [%2], #32 \n"
- "b.gt 1b \n"
+void ScaleUVRowDown2Linear_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "st1 {v0.8h}, [%1], #16 \n" // store 8 UV
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "v0", "v1");
+}
+
+void ScaleUVRowDown2Box_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ asm volatile(
+ // change the stride to row 2 pointer
+ "add %1, %1, %0 \n"
+ "1: \n"
+ "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 UV
+ "subs %w3, %w3, #8 \n" // 8 processed per loop.
+ "uaddlp v0.8h, v0.16b \n" // U 16 bytes -> 8 shorts.
+ "uaddlp v1.8h, v1.16b \n" // V 16 bytes -> 8 shorts.
+ "ld2 {v16.16b,v17.16b}, [%1], #32 \n" // load 16
+ "uadalp v0.8h, v16.16b \n" // U 16 bytes -> 8 shorts.
+ "uadalp v1.8h, v17.16b \n" // V 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "rshrn v0.8b, v0.8h, #2 \n" // round and pack
+ "prfm pldl1keep, [%1, 448] \n"
+ "rshrn v1.8b, v1.8h, #2 \n"
+ "st2 {v0.8b,v1.8b}, [%2], #16 \n"
+ "b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_stride), // %1
"+r"(dst), // %2
"+r"(dst_width) // %3
- : "r"(2LL), // %4
- "r"(14LL) // %5
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19" // Clobber List
- );
+ :
+ : "memory", "cc", "v0", "v1", "v16", "v17");
+}
+
+// Reads 4 pixels at a time.
+void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int src_stepx, // pixel step
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src1_ptr = src_ptr + src_stepx * 2;
+ const uint8_t* src2_ptr = src_ptr + src_stepx * 4;
+ const uint8_t* src3_ptr = src_ptr + src_stepx * 6;
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "ld1 {v0.h}[0], [%0], %6 \n"
+ "ld1 {v1.h}[0], [%1], %6 \n"
+ "ld1 {v2.h}[0], [%2], %6 \n"
+ "ld1 {v3.h}[0], [%3], %6 \n"
+ "subs %w5, %w5, #4 \n" // 4 pixels per loop.
+ "st4 {v0.h, v1.h, v2.h, v3.h}[0], [%4], #8 \n"
+ "b.gt 1b \n"
+ : "+r"(src_ptr), // %0
+ "+r"(src1_ptr), // %1
+ "+r"(src2_ptr), // %2
+ "+r"(src3_ptr), // %3
+ "+r"(dst_ptr), // %4
+ "+r"(dst_width) // %5
+ : "r"((int64_t)(src_stepx * 8)) // %6
+ : "memory", "cc", "v0", "v1", "v2", "v3");
+}
+
+// Scale UV columns by 3 box filter.
+// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
+void ScaleUVAddCols3_NEON(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ asm volatile(
+ "dup v6.8h, %w[scaleval] \n"
+ "1: \n"
+ "ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n"
+ "ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n"
+ "subs %w[dst_width], %w[dst_width], #8 \n"
+ "add v0.8h, v0.8h, v1.8h \n"
+ "add v3.8h, v3.8h, v4.8h \n"
+ "add v0.8h, v0.8h, v2.8h \n"
+ "add v3.8h, v3.8h, v5.8h \n"
+ "umull v1.4s, v0.4h, v6.4h \n"
+ "umull2 v2.4s, v0.8h, v6.8h \n"
+ "umull v4.4s, v3.4h, v6.4h \n"
+ "umull2 v5.4s, v3.8h, v6.8h \n"
+ "uzp2 v0.8h, v1.8h, v2.8h \n"
+ "uzp2 v3.8h, v4.8h, v5.8h \n"
+ "uzp1 v0.16b, v0.16b, v3.16b \n"
+ "st1 {v0.16b}, [%[dst_uv]], #16 \n"
+ "b.gt 1b \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ : [scaleval] "r"(scaleval) // %[scaleval]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
diff --git a/libs/libyuv/source/scale_rgb.cc b/libs/libyuv/source/scale_rgb.cc
new file mode 100644
index 0000000000..0a2f2f6b9a
--- /dev/null
+++ b/libs/libyuv/source/scale_rgb.cc
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/scale.h" /* For FilterMode */
+
+#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libyuv/convert_argb.h"
+#include "libyuv/convert_from_argb.h"
+#include "libyuv/row.h"
+#include "libyuv/scale_argb.h"
+#include "libyuv/scale_rgb.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Scale a 24 bit image.
+// Converts to ARGB as intermediate step
+
+LIBYUV_API
+int RGBScale(const uint8_t* src_rgb,
+ int src_stride_rgb,
+ int src_width,
+ int src_height,
+ uint8_t* dst_rgb,
+ int dst_stride_rgb,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+ if (!src_rgb || !dst_rgb || src_width <= 0 || src_width > INT_MAX / 4 ||
+ src_height == 0 || src_height == INT_MIN || dst_width <= 0 ||
+ dst_width > INT_MAX / 4 || dst_height <= 0) {
+ return -1;
+ }
+ const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ const uint64_t src_argb_size = (uint64_t)src_width * abs_src_height * 4;
+ const uint64_t dst_argb_size = (uint64_t)dst_width * dst_height * 4;
+ if (src_argb_size > (UINT64_MAX - dst_argb_size)) {
+ return -1; // Invalid size.
+ }
+ const uint64_t argb_size = src_argb_size + dst_argb_size;
+#if UINT64_MAX > SIZE_MAX
+ if (argb_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+#endif
+ uint8_t* src_argb = (uint8_t*)malloc((size_t)argb_size);
+ if (!src_argb) {
+ return 1; // Out of memory runtime error.
+ }
+ uint8_t* dst_argb = src_argb + (size_t)src_argb_size;
+
+ r = RGB24ToARGB(src_rgb, src_stride_rgb, src_argb, src_width * 4, src_width,
+ src_height);
+ if (!r) {
+ r = ARGBScale(src_argb, src_width * 4, src_width, abs_src_height, dst_argb,
+ dst_width * 4, dst_width, dst_height, filtering);
+ if (!r) {
+ r = ARGBToRGB24(dst_argb, dst_width * 4, dst_rgb, dst_stride_rgb,
+ dst_width, dst_height);
+ }
+ }
+ free(src_argb);
+ return r;
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/scale_rvv.cc b/libs/libyuv/source/scale_rvv.cc
new file mode 100644
index 0000000000..238a9a9539
--- /dev/null
+++ b/libs/libyuv/source/scale_rvv.cc
@@ -0,0 +1,1348 @@
+/*
+ * Copyright 2023 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+/*
+ * Copyright (c) 2023 SiFive, Inc. All rights reserved.
+ *
+ * Contributed by Darren Hsieh <darren.hsieh@sifive.com>
+ * Contributed by Bruce Lai <bruce.lai@sifive.com>
+ */
+
+#include "libyuv/row.h"
+#include "libyuv/scale_row.h"
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+// This module is for RVV (RISC-V Vector extension)
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
+#include <assert.h>
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// vsetvli/vsetivli write the vl and vtype CSRs. Only "vl" is declared, and
+// only for GCC; clang historically rejected it. The "vtype" register is
+// intentionally not declared, and whether it should be is unresolved.
+// TODO(fbarchard): consider adding "vtype" (and "vl" for clang) to the
+// clobber list once confirmed with compiler maintainers.
+#ifndef RVV_VL_CLOBBER
+#ifdef __clang__
+#define RVV_VL_CLOBBER
+#else
+#define RVV_VL_CLOBBER "vl",
+#endif
+#endif
+
+#ifdef HAS_SCALEARGBFILTERCOLS_RVV
+void ScaleARGBFilterCols_RVV(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ assert(x >= 0);
+ int vl, vl_dx;
+ asm volatile(
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vmv.v.x v0, %[x] \n"
+ "vid.v v2 \n"
+ "vmacc.vx v0, %[dx], v2 \n"
+
+ "1: \n"
+ // idx is x >> 16, byte index is (x >> 14) & ~3u
+ "vsrl.vi v2, v0, 14 \n"
+ "vand.vi v2, v2, -4 \n"
+ // Read Packed ARGB w/ byte index.
+ "vluxseg2ei32.v v4, (%[src_argb]), v2 \n"
+ // xf = (x >> 9) & 0x7f;
+ "vsrl.vi v8, v0, 9 \n"
+ "vand.vx v8, v8, %[c_7f] \n"
+ "mul %[vl_dx], %[vl], %[dx] \n"
+ "vadd.vx v0, v0, %[vl_dx] \n"
+ // duplicate v_xf0_u32[i] from {0,0,0,f[i]} to {f[i],f[i],f[i],f[i]}
+ "vmul.vx v8, v8, %[c_01010101] \n"
+ // TODO(fbarchard): Replace 0x7f ^ f with 128-f. bug=607.
+ "vxor.vx v10, v8, %[c_7f7f7f7f] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "vsetvli zero, %[vl], e8, m2, ta, ma \n"
+ // ((a) * (0x7f ^ f) + (b)*f) >> 7
+ "vwmulu.vv v12, v4, v10 \n"
+ "vwmaccu.vv v12, v6, v8 \n"
+ "vnsrl.wi v4, v12, 7 \n"
+ "vse8.v v4, (%[dst_argb]) \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [vl] "=&r"(vl), // %[vl]
+ [vl_dx] "=&r"(vl_dx) // %[vl_dx]
+ : [src_argb] "r"(src_argb), // %[src_argb]
+ [x] "r"(x), // %[x]
+ [dx] "r"(dx), // %[dx]
+ [c_7f] "r"(0x7f), // %[c_7f]
+ [c_01010101] "r"(0x01010101u), // %[c_01010101]
+ [c_7f7f7f7f] "r"(0x7f7f7f7fu) // %[c_7f7f7f7f]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15");
+}
+#endif
+
+#ifdef HAS_SCALEADDROW_RVV
+void ScaleAddRow_RVV(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle16.v v8, (%[dst_ptr]) \n"
+ "vle8.v v16, (%[src_ptr]) \n"
+ // Use widening multiply-add instead of widening + add
+ "vwmaccu.vx v8, %[one], v16 \n"
+ "vse16.v v8, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(src_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [one] "r"(1) // %[one]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWN2_RVV
+void ScaleARGBRowDown2_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vlseg2e32.v v8, (%[src]) \n"
+ "vse32.v v12, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src], %[src], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWN2LINEAR_RVV
+void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+ // register) is set to round-to-nearest-up mode(0).
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vlseg2e32.v v8, (%[src]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "vsetvli zero, %[vl], e8, m4, ta, ma \n"
+ "vaaddu.vv v8, v8, v12 \n"
+ "vse8.v v8, (%[dst_argb]) \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src], %[src], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src] "+r"(src_argb), // %[src]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWN2BOX_RVV
+void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ int vl;
+ asm volatile(
+ "add %[src1], %[src0], %[src1] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vlseg2e32.v v16, (%[src0]) \n"
+ "vlseg2e32.v v20, (%[src1]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "vsetvli zero, %[vl], e8, m2, ta, ma \n"
+ "vwaddu.vx v8, v16, %[c2] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vnsrl.wi v16, v8, 2 \n"
+ "vse8.v v16, (%[dst_argb]) \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src0], %[src0], %[vl] \n"
+ "add %[src1], %[src1], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src0] "+r"(src_argb), // %[src0]
+ [src1] "+r"(src_stride), // %[src1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [vl] "=&r"(vl) // %[vl]
+ : [c2] "r"(2) // %[c2]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWNEVEN_RVV
+void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ ptrdiff_t src_step;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m4, ta, ma \n"
+ "vlse32.v v8, (%[src]), %[stride_byte] \n"
+ "vse32.v v8, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "mul %[src_step], %[vl], %[stride_byte]\n"
+ "add %[src], %[src], %[src_step] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [vl] "=&r"(vl), // %[vl]
+ [src_step] "=&r"(src_step) // %[src_step]
+ : [stride_byte] "r"((ptrdiff_t)src_stepx * 4) // %[stride_byte]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWNEVENBOX_RVV
+void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ int vl;
+ ptrdiff_t src_step;
+ asm volatile(
+ "add %[src1], %[src0], %[src1] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vlsseg2e32.v v16, (%[src0]), %[stride_byte] \n"
+ "vlsseg2e32.v v20, (%[src1]), %[stride_byte] \n"
+ "sub %[w], %[w], %[vl] \n"
+ "mul %[src_step], %[vl], %[stride_byte]\n"
+ "slli %[vl], %[vl], 2 \n"
+ "vsetvli zero, %[vl], e8, m2, ta, ma \n"
+ "vwaddu.vx v8, v16, %[c2] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vnsrl.wi v16, v8, 2 \n"
+ "vse8.v v16, (%[dst_argb]) \n"
+ "add %[src0], %[src0], %[src_step] \n"
+ "add %[src1], %[src1], %[src_step] \n"
+ "add %[dst_argb], %[dst_argb], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src0] "+r"(src_argb), // %[src0]
+ [src1] "+r"(src_stride), // %[src1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [vl] "=&r"(vl), // %[vl]
+ [src_step] "=&r"(src_step) // %[src_step]
+ : [stride_byte] "r"((ptrdiff_t)src_stepx * 4), // %[stride_byte]
+ [c2] "r"(2) // %[c2]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN2_RVV
+void ScaleRowDown2_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle16.v v8, (%[src_ptr]) \n"
+ "vnsrl.wi v16, v8, 8 \n"
+ "vse8.v v16, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst] "+r"(dst), // %[dst]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN2LINEAR_RVV
+void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+ // register) is set to round-to-nearest-up mode(0).
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vle16.v v8, (%[src_ptr]) \n"
+ "vnsrl.wi v16, v8, 0 \n"
+ "vnsrl.wi v18, v8, 8 \n"
+ "vaaddu.vv v16, v16, v18 \n"
+ "vse8.v v16, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst] "+r"(dst), // %[dst]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN2BOX_RVV
+void ScaleRowDown2Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ int vl;
+ asm volatile(
+ "add %[t], %[s], %[t] \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg2e8.v v16, (%[s]) \n"
+ "vlseg2e8.v v20, (%[t]) \n"
+ "vwaddu.vx v8, v16, %[c2] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vnsrl.wi v16, v8, 2 \n"
+ "vse8.v v16, (%[dst]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst], %[dst], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[s], %[s], %[vl] \n"
+ "add %[t], %[t], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [s] "+r"(src_ptr), // %[s]
+ [t] "+r"(src_stride), // %[t]
+ [dst] "+r"(dst), // %[dst]
+ [vl] "=&r"(vl) // %[vl]
+ : [c2] "r"(2) // %[c2]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN4_RVV
+void ScaleRowDown4_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_ptr]) \n"
+ "vse8.v v12, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN4BOX_RVV
+void ScaleRowDown4Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_ptr1;
+ const uint8_t* src_ptr2;
+ int vl;
+ asm volatile(
+ "add %[src_ptr1], %[src_ptr], %[src_ptr3]\n"
+ "add %[src_ptr2], %[src_ptr1], %[src_ptr3]\n"
+ "add %[src_ptr3], %[src_ptr2], %[src_ptr3]\n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v16, (%[src_ptr]) \n"
+ "vlseg4e8.v v24, (%[src_ptr1]) \n"
+ "vwaddu.vx v8, v16, %[c8] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vwaddu.wv v8, v8, v24 \n"
+ "vwaddu.wv v8, v8, v26 \n"
+ "vwaddu.wv v8, v8, v28 \n"
+ "vwaddu.wv v8, v8, v30 \n"
+ "vlseg4e8.v v16, (%[src_ptr2]) \n"
+ "vlseg4e8.v v24, (%[src_ptr3]) \n"
+ "vwaddu.wv v8, v8, v16 \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v20 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vwaddu.wv v8, v8, v24 \n"
+ "vwaddu.wv v8, v8, v26 \n"
+ "vwaddu.wv v8, v8, v28 \n"
+ "vwaddu.wv v8, v8, v30 \n"
+ "vnsrl.wi v16, v8, 4 \n"
+ "vse8.v v16, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[src_ptr1], %[src_ptr1], %[vl]\n"
+ "add %[src_ptr2], %[src_ptr2], %[vl]\n"
+ "add %[src_ptr3], %[src_ptr3], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "=&r"(src_ptr1), // %[src_ptr1]
+ [src_ptr2] "=&r"(src_ptr2), // %[src_ptr2]
+ [src_ptr3] "+r"(src_stride), // %[src_ptr3]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [c8] "r"(8) // %[c8]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",
+ "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN34_RVV
+void ScaleRowDown34_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ dst_width /= 3;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_ptr]) \n"
+ "vmv2r.v v12, v14 \n"
+ "vsseg3e8.v v8, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN34_0_BOX_RVV
+void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ dst_width /= 3;
+ const uint8_t* t;
+ int vl;
+ // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+ // register) is set to round-to-nearest-up mode(0).
+ asm volatile(
+ "add %[t], %[s], %[src_stride] \n"
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v24, (%[s]) \n"
+ "bnez %[src_stride], 2f \n"
+ "vwaddu.vx v20, v24, %[c2] \n"
+ "vwaddu.vx v16, v26, %[c2] \n"
+ "vwaddu.vx v12, v28, %[c2] \n"
+ "vwaddu.vx v8, v30, %[c2] \n"
+ "j 3f \n"
+
+ "2: \n"
+ "vlseg4e8.v v0, (%[t]) \n"
+ "vwaddu.vx v20, v0, %[c2] \n"
+ "vwaddu.vx v16, v2, %[c2] \n"
+ "vwaddu.vx v12, v4, %[c2] \n"
+ "vwaddu.vx v8, v6, %[c2] \n"
+
+ "3: \n"
+ "vwmaccu.vx v20, %[c3], v24 \n"
+ "vnsrl.wi v24, v20, 2 \n"
+ "vwmaccu.vx v16, %[c3], v26 \n"
+ "vnsrl.wi v20, v16, 2 \n"
+ "vwmaccu.vx v12, %[c3], v28 \n"
+ "vnsrl.wi v16, v12, 2 \n"
+ "vwmaccu.vx v8, %[c3], v30 \n"
+ "vnsrl.wi v12, v8, 2 \n"
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "vwaddu.vx v8, v20, %[c2] \n"
+ "vwmaccu.vx v8, %[c3], v24 \n"
+ "vnsrl.wi v14, v8, 2 \n"
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "vwaddu.vx v8, v16, %[c2] \n"
+ "vwmaccu.vx v8, %[c3], v12 \n"
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "vaaddu.vv v16, v20, v16 \n"
+ "vnsrl.wi v18, v8, 2 \n"
+ "vsseg3e8.v v14, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[s], %[s], %[vl] \n"
+ "add %[t], %[t], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [s] "+r"(src_ptr), // %[s]
+ [t] "=&r"(t), // %[t]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28", "v29", "v30", "v31");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN34_1_BOX_RVV
+void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ dst_width /= 3;
+ const uint8_t* t;
+ int vl;
+ // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+ // register) is set to round-to-nearest-up mode(0).
+ asm volatile(
+ "add %[t], %[s], %[src_stride] \n"
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v14, (%[s]) \n"
+ "bnez %[src_stride], 2f \n"
+ "vaaddu.vv v14, v14, v14 \n"
+ "vaaddu.vv v8, v16, v16 \n"
+ "vaaddu.vv v10, v18, v18 \n"
+ "vaaddu.vv v12, v20, v20 \n"
+ "j 3f \n"
+
+ "2: \n"
+ "vlseg4e8.v v22, (%[t]) \n"
+ "vaaddu.vv v14, v14, v22 \n"
+ "vaaddu.vv v8, v16, v24 \n"
+ "vaaddu.vv v10, v18, v26 \n"
+ "vaaddu.vv v12, v20, v28 \n"
+
+ "3: \n"
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ "vwaddu.vx v16, v8, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v14 \n"
+ "vnsrl.wi v20, v16, 2 \n"
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ "vwaddu.vx v16, v10, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v12 \n"
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ "vaaddu.vv v22, v8, v10 \n"
+ "vnsrl.wi v24, v16, 2 \n"
+ "vsseg3e8.v v20, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[s], %[s], %[vl] \n"
+ "add %[t], %[t], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [s] "+r"(src_ptr), // %[s]
+ [t] "=&r"(t), // %[t]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN38_RVV
+void ScaleRowDown38_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ assert(dst_width % 3 == 0);
+ dst_width /= 3;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ "vlseg8e8.v v8, (%[src_ptr]) \n"
+ "vmv1r.v v9, v11 \n"
+ "vmv1r.v v10, v14 \n"
+ "vsseg3e8.v v8, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN38_2_BOX_RVV
+void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ assert((dst_width % 3 == 0) && (dst_width > 0));
+ dst_width /= 3;
+ int vl;
+ asm volatile(
+ "add %[src_ptr1], %[src_ptr], %[src_ptr1]\n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ // s: e00, e10, e20, f00, f10, f20, g00, g10
+ "vlseg8e8.v v14, (%[src_ptr]) \n"
+ // t: e01, e11, e21, f01, f11, f21, g01, g11
+ "vlseg8e8.v v22, (%[src_ptr1]) \n"
+ "vwaddu.vv v8, v14, v22 \n"
+ "vwaddu.vv v10, v15, v23 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v8, v8, v10 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v10, v17, v25 \n"
+ "vwaddu.vv v12, v18, v26 \n"
+ "vwaddu.vv v14, v16, v24 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v10, v10, v12 \n"
+ "vadd.vv v8, v8, v14 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v12, v19, v27 \n"
+ "vwaddu.vv v14, v20, v28 \n"
+ "vwaddu.vv v16, v21, v29 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v10, v10, v12 \n"
+ "vadd.vv v12, v14, v16 \n"
+ // Average in 16-bit fixed-point
+ "vmulhu.vx v8, v8, %[coeff_a] \n"
+ "vmulhu.vx v10, v10, %[coeff_a] \n"
+ "vmulhu.vx v12, v12, %[coeff_b] \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v14, v8, 0 \n"
+ "vnsrl.wi v15, v10, 0 \n"
+ "vnsrl.wi v16, v12, 0 \n"
+ "vsseg3e8.v v14, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[src_ptr1], %[src_ptr1], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_stride), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [coeff_a] "r"(65536u / 6u), // %[coeff_a]
+ [coeff_b] "r"(65536u / 4u) // %[coeff_b]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29");
+}
+#endif
+
+#ifdef HAS_SCALEROWDOWN38_3_BOX_RVV
+void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ assert((dst_width % 3 == 0) && (dst_width > 0));
+ dst_width /= 3;
+ const uint8_t* src_ptr1;
+ int vl;
+ asm volatile(
+ "add %[src_ptr1], %[src_ptr], %[src_ptr2]\n"
+ "add %[src_ptr2], %[src_ptr1], %[src_ptr2]\n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m1, ta, ma \n"
+ // s: e00, e10, e20, f00, f10, f20, g00, g10
+ "vlseg8e8.v v8, (%[src_ptr]) \n"
+ // t: e01, e11, e21, f01, f11, f21, g01, g11
+ "vlseg8e8.v v19, (%[src_ptr1]) \n"
+ // u: e02, e12, e22, f02, f12, f22, g02, g12
+ "vlseg8e8.v v0, (%[src_ptr2]) \n"
+ // Calculate sum of [e00, e22]
+ "vwaddu.vv v16, v8, v19 \n"
+ "vwaddu.vv v18, v9, v20 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v8, v16, v18 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v16, v10, v21 \n"
+ "vwaddu.vv v18, v0, v1 \n"
+ "vwcvtu.x.x.v v20, v2 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v16, v16, v18 \n"
+ "vadd.vv v8, v8, v20 \n"
+ "vadd.vv v8, v8, v16 \n"
+ // Calculate sum of [f00, f22]
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v16, v11, v22 \n"
+ "vwaddu.vv v10, v12, v23 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v10, v16, v10 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v16, v13, v24 \n"
+ "vwaddu.vv v12, v3, v4 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v12, v16, v12 \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwcvtu.x.x.v v16, v5 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v10, v10, v16 \n"
+ "vadd.vv v10, v10, v12 \n"
+ // Calculate sum of [g00, g12]
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vwaddu.vv v12, v14, v25 \n"
+ "vwaddu.vv v16, v15, v26 \n"
+ "vwaddu.vv v14, v6, v7 \n"
+ "vsetvli zero, zero, e16, m2, ta, ma \n"
+ "vadd.vv v12, v12, v16 \n"
+ "vadd.vv v12, v12, v14 \n"
+ // Average in 16-bit fixed-point
+ "vmulhu.vx v8, v8, %[coeff_a] \n"
+ "vmulhu.vx v10, v10, %[coeff_a] \n"
+ "vmulhu.vx v12, v12, %[coeff_b] \n"
+ "vsetvli zero, zero, e8, m1, ta, ma \n"
+ "vnsrl.wi v14, v8, 0 \n"
+ "vnsrl.wi v15, v10, 0 \n"
+ "vnsrl.wi v16, v12, 0 \n"
+ "vsseg3e8.v v14, (%[dst_ptr]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_ptr], %[dst_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "add %[src_ptr1], %[src_ptr1], %[vl]\n"
+ "add %[src_ptr2], %[src_ptr2], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "=&r"(src_ptr1), // %[src_ptr1]
+ [src_ptr2] "+r"(src_stride), // %[src_ptr2]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [vl] "=&r"(vl) // %[vl]
+ : [coeff_a] "r"(65536u / 9u), // %[coeff_a]
+ [coeff_b] "r"(65536u / 6u) // %[coeff_b]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26");
+}
+#endif
+
+// ScaleUVRowUp2_(Bi)linear_RVV function is equal to other platforms'
+// ScaleRowUp2_(Bi)linear_Any_XXX. We process entire row in this function. Other
+// platforms only implement non-edge part of image and process edge with scalar.
+
+#ifdef HAS_SCALEROWUP2_LINEAR_RVV
+void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ int src_width = (dst_width - 1) >> 1;
+ dst_ptr[0] = src_ptr[0];
+ if (src_width > 0) {
+ int vl;
+ const uint8_t* work_src_next;
+ uint8_t* work_dst_ptr = dst_ptr + 1;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[src_width], e8, m2, ta, ma\n"
+ "vle8.v v12, (%[src_ptr]) \n"
+ "addi %[work_src_next], %[src_ptr], 1\n"
+ "vle8.v v24, (%[work_src_next]) \n"
+ "vwaddu.vx v16, v12, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v24 \n"
+ "vnsrl.wi v26, v16, 2 \n"
+ "vwaddu.vx v16, v24, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v12 \n"
+ "vnsrl.wi v24, v16, 2 \n"
+ "vsseg2e8.v v24, (%[work_dst_ptr]) \n"
+ "sub %[src_width], %[src_width], %[vl]\n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[work_dst_ptr], %[work_dst_ptr], %[vl]\n"
+ "bgtz %[src_width], 1b \n"
+ : [src_width] "+r"(src_width), // %[src_width]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [work_dst_ptr] "+r"(work_dst_ptr), // %[work_dst_ptr]
+ [vl] "=&r"(vl), // %[vl]
+ [work_src_next] "=&r"(work_src_next) // %[work_src_next]
+ : [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v12", "v13", "v16", "v17", "v18", "v19",
+ "v24", "v25", "v26", "v27");
+ }
+ dst_ptr[dst_width - 1] = src_ptr[0];
+}
+#endif
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_RVV
+void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ int src_width = (dst_width - 1) >> 1;
+ const uint8_t* t = src_ptr + src_stride;
+ uint8_t* e = dst_ptr + dst_stride;
+ dst_ptr[0] = (3 * src_ptr[0] + t[0] + 2) >> 2;
+ e[0] = (src_ptr[0] + 3 * t[0] + 2) >> 2;
+ if (src_width > 0) {
+ int vl;
+ const uint8_t* tmp_ptr;
+ uint8_t* work_d = dst_ptr + 1;
+ uint8_t* work_e = e + 1;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[src_width], e8, m2, ta, ma\n"
+ "vle8.v v0, (%[work_s]) \n"
+ "addi %[tmp_ptr], %[work_s], 1 \n"
+ "vle8.v v2, (%[tmp_ptr]) \n"
+ "vwaddu.vx v8, v0, %[c2] \n"
+ "vwaddu.vx v12, v2, %[c2] \n"
+ "vwmaccu.vx v8, %[c3], v2 \n"
+ "vwmaccu.vx v12, %[c3], v0 \n"
+ "vle8.v v4, (%[work_t]) \n"
+ "addi %[tmp_ptr], %[work_t], 1 \n"
+ "vle8.v v6, (%[tmp_ptr]) \n"
+ "vwaddu.vx v16, v4, %[c2] \n"
+ "vwaddu.vx v20, v6, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v6 \n"
+ "vwmaccu.vx v20, %[c3], v4 \n"
+ "vmv4r.v v24, v16 \n"
+ "vmv4r.v v28, v20 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmacc.vx v16, %[c3], v8 \n"
+ "vmacc.vx v20, %[c3], v12 \n"
+ "vmacc.vx v8, %[c3], v24 \n"
+ "vmacc.vx v12, %[c3], v28 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ "vnsrl.wi v0, v20, 4 \n"
+ "vnsrl.wi v2, v16, 4 \n"
+ "vsseg2e8.v v0, (%[work_d]) \n"
+ "vnsrl.wi v4, v12, 4 \n"
+ "vnsrl.wi v6, v8, 4 \n"
+ "vsseg2e8.v v4, (%[work_e]) \n"
+ "sub %[src_width], %[src_width], %[vl]\n"
+ "add %[work_s], %[work_s], %[vl] \n"
+ "add %[work_t], %[work_t], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[work_d], %[work_d], %[vl] \n"
+ "add %[work_e], %[work_e], %[vl] \n"
+ "bgtz %[src_width], 1b \n"
+ : [src_width] "+r"(src_width), // %[src_width]
+ [work_s] "+r"(src_ptr), // %[work_s]
+ [work_t] "+r"(t), // %[work_t]
+ [work_d] "+r"(work_d), // %[work_d]
+ [work_e] "+r"(work_e), // %[work_e]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr]
+ : [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+ "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27", "v28", "v29", "v30", "v31");
+ }
+ dst_ptr[dst_width - 1] = (3 * src_ptr[0] + t[0] + 2) >> 2;
+ e[dst_width - 1] = (src_ptr[0] + 3 * t[0] + 2) >> 2;
+}
+#endif
+
+#ifdef HAS_SCALEUVROWDOWN2_RVV
+void ScaleUVRowDown2_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m2, ta, ma \n"
+ "vle32.v v8, (%[src_uv]) \n"
+ "vnsrl.wi v16, v8, 16 \n"
+ "vse16.v v16, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWDOWN2LINEAR_RVV
+void ScaleUVRowDown2Linear_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
+ // register) is set to round-to-nearest-up mode(0).
+ asm volatile(
+ "csrwi vxrm, 0 \n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v8, (%[src_uv]) \n"
+ "vaaddu.vv v8, v8, v12 \n"
+ "vaaddu.vv v10, v10, v14 \n"
+ "vsseg2e8.v v8, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWDOWN2BOX_RVV
+void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ int vl;
+ asm volatile(
+ "add %[src_uv_row1], %[src_uv], %[src_uv_row1]\n"
+
+ "1: \n"
+ "vsetvli %[vl], %[w], e8, m2, ta, ma \n"
+ "vlseg4e8.v v14, (%[src_uv]) \n"
+ "vlseg4e8.v v22, (%[src_uv_row1]) \n"
+ "vwaddu.vx v8, v14, %[c2] \n"
+ "vwaddu.wv v8, v8, v18 \n"
+ "vwaddu.wv v8, v8, v22 \n"
+ "vwaddu.wv v8, v8, v26 \n"
+ "vwaddu.vx v12, v16, %[c2] \n"
+ "vwaddu.wv v12, v12, v20 \n"
+ "vwaddu.wv v12, v12, v24 \n"
+ "vwaddu.wv v12, v12, v28 \n"
+ "vnsrl.wi v16, v8, 2 \n"
+ "vnsrl.wi v18, v12, 2 \n"
+ "vsseg2e8.v v16, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "add %[src_uv_row1], %[src_uv_row1], %[vl]\n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [src_uv_row1] "+r"(src_stride), // %[src_uv_row1]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl) // %[vl]
+ : [c2] "r"(2) // %[c2]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+ "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+ "v25", "v26", "v27", "v28", "v29");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWDOWN4_RVV
+void ScaleUVRowDown4_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ (void)src_stepx;
+ int vl;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vle64.v v8, (%[src_uv]) \n"
+ "vnsrl.wi v16, v8, 0 \n"
+ "vsetvli zero, zero, e16, m1, ta, ma \n"
+ "vnsrl.wi v8, v16, 0 \n"
+ "vse16.v v8, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "slli %[vl], %[vl], 2 \n"
+ "add %[src_uv], %[src_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
+}
+#endif
+
+#ifdef HAS_SCALEUVROWDOWNEVEN_RVV
+void ScaleUVRowDownEven_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ int vl;
+ ptrdiff_t src_step;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e16, m4, ta, ma \n"
+ "vlse16.v v8, (%[src_uv]), %[stride_byte]\n"
+ "vse16.v v8, (%[dst_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "mul %[src_step], %[vl], %[stride_byte]\n"
+ "add %[src_uv], %[src_uv], %[src_step]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(dst_width), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl), // %[vl]
+ [src_step] "=&r"(src_step) // %[src_step]
+ : [stride_byte] "r"((ptrdiff_t)src_stepx * 2) // %[stride_byte]
+ : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
+}
+#endif
+
+// ScaleUVRowUp2_(Bi)linear_RVV function is equal to other platforms'
+// ScaleUVRowUp2_(Bi)linear_Any_XXX. We process entire row in this function.
+// Other platforms only implement non-edge part of image and process edge with
+// scalar.
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_RVV
+void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ int src_pairs = (dst_width - 1) >> 1;
+ dst_ptr[0] = src_ptr[0];
+ dst_ptr[1] = src_ptr[1];
+ if (src_pairs > 0) {
+ int vl;
+ const uint8_t* work_src_next;
+ uint8_t* work_dst_ptr = dst_ptr + 2;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[src_pairs], e8, m2, ta, ma\n"
+ "addi %[work_src_next], %[src_ptr], 2\n"
+ "vlseg2e8.v v16, (%[work_src_next]) \n"
+ "vlseg2e8.v v20, (%[src_ptr]) \n"
+ "vwaddu.vx v12, v16, %[c2] \n"
+ "vwmaccu.vx v12, %[c3], v20 \n"
+ "vnsrl.wi v24, v12, 2 \n"
+ "vwaddu.vx v12, v20, %[c2] \n"
+ "vwmaccu.vx v12, %[c3], v16 \n"
+ "vnsrl.wi v28, v12, 2 \n"
+ "vwaddu.vx v12, v18, %[c2] \n"
+ "vwmaccu.vx v12, %[c3], v22 \n"
+ "vnsrl.wi v26, v12, 2 \n"
+ "vwaddu.vx v12, v22, %[c2] \n"
+ "vwmaccu.vx v12, %[c3], v18 \n"
+ "vnsrl.wi v30, v12, 2 \n"
+ "vsseg4e8.v v24, (%[work_dst_ptr]) \n"
+ "sub %[src_pairs], %[src_pairs], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[src_ptr], %[src_ptr], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[work_dst_ptr], %[work_dst_ptr], %[vl]\n"
+ "bgtz %[src_pairs], 1b \n"
+ : [src_pairs] "+r"(src_pairs), // %[src_pairs]
+ [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [work_dst_ptr] "+r"(work_dst_ptr), // %[work_dst_ptr]
+ [vl] "=&r"(vl), // %[vl]
+ [work_src_next] "=&r"(work_src_next) // %[work_src_next]
+ : [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v12", "v13", "v14", "v15", "v16", "v17",
+ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
+ "v28", "v29", "v30", "v31");
+ }
+ dst_ptr[2 * dst_width - 2] = src_ptr[0];
+ dst_ptr[2 * dst_width - 1] = src_ptr[1];
+}
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_RVV
+void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ int src_pairs = (dst_width - 1) >> 1;
+ const uint8_t* t = src_ptr + src_stride;
+ uint8_t* e = dst_ptr + dst_stride;
+ dst_ptr[0] = (3 * src_ptr[0] + t[0] + 2) >> 2;
+ e[0] = (src_ptr[0] + 3 * t[0] + 2) >> 2;
+ dst_ptr[1] = (3 * src_ptr[1] + t[1] + 2) >> 2;
+ e[1] = (src_ptr[1] + 3 * t[1] + 2) >> 2;
+ if (src_pairs > 0) {
+ int vl;
+ const uint8_t* tmp_ptr;
+ uint8_t* work_d = dst_ptr + 2;
+ uint8_t* work_e = e + 2;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[src_pairs], e8, m2, ta, ma\n"
+ // Load s_u0 (v0), s_v0 (v2) and s_u1 (v4), s_v1 (v6)
+ "vlseg2e8.v v0, (%[work_s]) \n"
+ "addi %[tmp_ptr], %[work_s], 2 \n"
+ "vlseg2e8.v v4, (%[tmp_ptr]) \n"
+ // Load t_u0 (v8), t_v0 (v10) and t_u1 (v12), t_v1 (v14)
+ "vlseg2e8.v v8, (%[work_t]) \n"
+ "addi %[tmp_ptr], %[work_t], 2 \n"
+ "vlseg2e8.v v12, (%[tmp_ptr]) \n"
+ // Compute U channel horizontal interpolation into v16..v31
+ "vwaddu.vx v16, v0, %[c2] \n"
+ "vwaddu.vx v20, v4, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v4 \n"
+ "vwmaccu.vx v20, %[c3], v0 \n"
+ "vwaddu.vx v24, v8, %[c2] \n"
+ "vwaddu.vx v28, v12, %[c2] \n"
+ "vwmaccu.vx v24, %[c3], v12 \n"
+ "vwmaccu.vx v28, %[c3], v8 \n"
+ // Move t_v0 (v10) -> v0, t_v1 (v14) -> v4 so v8..v15 are free
+ "vmv2r.v v0, v10 \n"
+ "vmv2r.v v4, v14 \n"
+ "vmv4r.v v8, v24 \n"
+ "vmv4r.v v12, v28 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmacc.vx v24, %[c3], v16 \n"
+ "vmacc.vx v28, %[c3], v20 \n"
+ "vmacc.vx v16, %[c3], v8 \n"
+ "vmacc.vx v20, %[c3], v12 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ // Move s_v0 (v2) -> v8, s_v1 (v6) -> v10, t_v0 (v0) -> v12, t_v1 (v4)
+ // -> v14
+ "vmv2r.v v8, v2 \n"
+ "vmv2r.v v10, v6 \n"
+ "vmv2r.v v12, v0 \n"
+ "vmv2r.v v14, v4 \n"
+ // Narrow U results into v0 (u_d_even), v2 (u_d_odd), v4 (u_e_even), v6
+ // (u_e_odd)
+ "vnsrl.wi v0, v28, 4 \n"
+ "vnsrl.wi v2, v24, 4 \n"
+ "vnsrl.wi v4, v20, 4 \n"
+ "vnsrl.wi v6, v16, 4 \n"
+ // Compute V channel horizontal interpolation from v8, v10, v12, v14
+ // into v16..v31
+ "vwaddu.vx v16, v8, %[c2] \n"
+ "vwaddu.vx v20, v10, %[c2] \n"
+ "vwmaccu.vx v16, %[c3], v10 \n"
+ "vwmaccu.vx v20, %[c3], v8 \n"
+ "vwaddu.vx v24, v12, %[c2] \n"
+ "vwaddu.vx v28, v14, %[c2] \n"
+ "vwmaccu.vx v24, %[c3], v14 \n"
+ "vwmaccu.vx v28, %[c3], v12 \n"
+ "vmv4r.v v8, v24 \n"
+ "vmv4r.v v12, v28 \n"
+ "vsetvli zero, zero, e16, m4, ta, ma \n"
+ "vmacc.vx v24, %[c3], v16 \n"
+ "vmacc.vx v28, %[c3], v20 \n"
+ "vmacc.vx v16, %[c3], v8 \n"
+ "vmacc.vx v20, %[c3], v12 \n"
+ "vsetvli zero, zero, e8, m2, ta, ma \n"
+ // Assemble and store row 0: (u_d_even, v_d_even, u_d_odd, v_d_odd) in
+ // v8..v15
+ "vmv2r.v v8, v0 \n"
+ "vnsrl.wi v10, v28, 4 \n"
+ "vmv2r.v v12, v2 \n"
+ "vnsrl.wi v14, v24, 4 \n"
+ "vsseg4e8.v v8, (%[work_d]) \n"
+ // Assemble and store row 1: (u_e_even, v_e_even, u_e_odd, v_e_odd) in
+ // v8..v15
+ "vmv2r.v v8, v4 \n"
+ "vnsrl.wi v10, v20, 4 \n"
+ "vmv2r.v v12, v6 \n"
+ "vnsrl.wi v14, v16, 4 \n"
+ "vsseg4e8.v v8, (%[work_e]) \n"
+ "sub %[src_pairs], %[src_pairs], %[vl]\n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[work_s], %[work_s], %[vl] \n"
+ "add %[work_t], %[work_t], %[vl] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "add %[work_d], %[work_d], %[vl] \n"
+ "add %[work_e], %[work_e], %[vl] \n"
+ "bgtz %[src_pairs], 1b \n"
+ : [src_pairs] "+r"(src_pairs), // %[src_pairs]
+ [work_s] "+r"(src_ptr), // %[work_s]
+ [work_t] "+r"(t), // %[work_t]
+ [work_d] "+r"(work_d), // %[work_d]
+ [work_e] "+r"(work_e), // %[work_e]
+ [vl] "=&r"(vl), // %[vl]
+ [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr]
+ : [c2] "r"(2), // %[c2]
+ [c3] "r"(3) // %[c3]
+ : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+ "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
+ "v27", "v28", "v29", "v30", "v31");
+ }
+ dst_ptr[2 * dst_width - 2] = (3 * src_ptr[0] + t[0] + 2) >> 2;
+ e[2 * dst_width - 2] = (src_ptr[0] + 3 * t[0] + 2) >> 2;
+ dst_ptr[2 * dst_width - 1] = (3 * src_ptr[1] + t[1] + 2) >> 2;
+ e[2 * dst_width - 1] = (src_ptr[1] + 3 * t[1] + 2) >> 2;
+}
+#endif
+
+#ifdef HAS_SCALEUVADDCOLS3_RVV
+void ScaleUVAddCols3_RVV(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ size_t vl, vl2;
+ size_t w = (size_t)dst_width;
+ asm volatile(
+ "1: \n"
+ "vsetvli %[vl], %[w], e32, m2, ta, ma \n"
+ "vlseg3e32.v v8, (%[src_uv]) \n"
+ "sub %[w], %[w], %[vl] \n"
+ "slli %[vl2], %[vl], 2 \n"
+ "add %[src_uv], %[src_uv], %[vl2] \n"
+ "slli %[vl2], %[vl2], 1 \n"
+ "add %[src_uv], %[src_uv], %[vl2] \n"
+ "slli %[vl], %[vl], 1 \n"
+ "vsetvli zero, %[vl], e16, m2, ta, ma \n"
+ "vadd.vv v8, v8, v10 \n"
+ "vadd.vv v8, v8, v12 \n"
+ "vmulhu.vx v8, v8, %[scaleval] \n"
+ "vsetvli zero, %[vl], e8, m1, ta, ma \n"
+ "vnsrl.wi v4, v8, 0 \n"
+ "vse8.v v4, (%[dst_uv]) \n"
+ "add %[dst_uv], %[dst_uv], %[vl] \n"
+ "bgtz %[w], 1b \n"
+ : [w] "+r"(w), // %[w]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [vl] "=&r"(vl), // %[vl]
+ [vl2] "=&r"(vl2) // %[vl2]
+ : [scaleval] "r"(scaleval) // %[scaleval]
+ : RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13");
+}
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
diff --git a/libs/libyuv/source/scale_sme.cc b/libs/libyuv/source/scale_sme.cc
new file mode 100644
index 0000000000..1dc5c17bac
--- /dev/null
+++ b/libs/libyuv/source/scale_sme.cc
@@ -0,0 +1,628 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/scale_row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+__arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1b {z1.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "st1b {z1.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.h \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "st1h {z1.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "urhadd z0.b, p0/m, z0.b, z1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "urhadd z0.b, p0/m, z0.b, z1.b \n"
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.h \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "urhadd z0.h, p0/m, z0.h, z1.h \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "urhadd z0.h, p0/m, z0.h, z1.h \n"
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+#define SCALEROWDOWN2BOX_SVE \
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n" \
+ "ld2b {z2.b, z3.b}, p0/z, [%[src2_ptr]] \n" \
+ "incb %[src_ptr], all, mul #2 \n" \
+ "incb %[src2_ptr], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n" \
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n" \
+ "incb %[dst_ptr] \n"
+
+__arm_locally_streaming void ScaleRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_ptr = src_ptr + src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n" //
+ SCALEROWDOWN2BOX_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n" //
+ SCALEROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
+}
+
+#undef SCALEROWDOWN2BOX_SVE
+
+#define SCALEROWDOWN2BOX_16_SVE \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n" \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src2_ptr]] \n" \
+ "incb %[src_ptr], all, mul #2 \n" \
+ "incb %[src2_ptr], all, mul #2 \n" \
+ "uaddlb z4.s, z0.h, z1.h \n" \
+ "uaddlt z5.s, z0.h, z1.h \n" \
+ "uaddlb z6.s, z2.h, z3.h \n" \
+ "uaddlt z7.s, z2.h, z3.h \n" \
+ "add z4.s, z4.s, z6.s \n" \
+ "add z5.s, z5.s, z7.s \n" \
+ "rshrnb z0.h, z4.s, #2 \n" \
+ "rshrnt z0.h, z5.s, #2 \n" \
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n" \
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n" \
+ "incb %[dst_ptr] \n"
+
+__arm_locally_streaming void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint16_t* src2_ptr = src_ptr + src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n" //
+ SCALEROWDOWN2BOX_16_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n" //
+ SCALEROWDOWN2BOX_16_SVE
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
+}
+
+#undef SCALEROWDOWN2BOX_16_SVE
+
+__arm_locally_streaming void ScaleUVRowDown2_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "st1h {z1.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleUVRowDown2Linear_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #2 \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1h {z0.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #1 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1h {z0.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0", "p1");
+}
+
+#define SCALEUVROWDOWN2BOX_SVE \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n" \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src2_uv]] \n" \
+ "incb %[src_uv], all, mul #2 \n" \
+ "incb %[src2_uv], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "st1h {z0.h}, p0, [%[dst_uv]] \n" \
+ "incb %[dst_uv], all, mul #1 \n"
+
+__arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_uv = src_uv + src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n" //
+ SCALEUVROWDOWN2BOX_SVE
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n" //
+ SCALEUVROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [src2_uv] "+r"(src2_uv), // %[src2_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+#undef SCALEUVROWDOWN2BOX_SVE
+
+__arm_locally_streaming void ScaleARGBRowDown2_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "incb %[src_argb], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1w {z1.s}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "st1w {z1.s}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleARGBRowDown2Linear_SME(
+ const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "incb %[src_argb], all, mul #2 \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1w {z0.s}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #1 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1w {z0.s}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0", "p1");
+}
+
+#define SCALEARGBROWDOWN2BOX_SVE \
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" \
+ "ld2w {z2.s, z3.s}, p0/z, [%[src2_argb]] \n" \
+ "incb %[src_argb], all, mul #2 \n" \
+ "incb %[src2_argb], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "st1w {z0.s}, p0, [%[dst_argb]] \n" \
+ "incb %[dst_argb], all, mul #1 \n"
+
+__arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_argb = src_argb + src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n" //
+ SCALEARGBROWDOWN2BOX_SVE
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n" //
+ SCALEARGBROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src2_argb] "+r"(src2_argb), // %[src2_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+// Scale UV columns by 3 box filter using Streaming SVE.
+__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ int vl_w;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "cntw %x[vl_w] \n"
+ "dup z6.h, %w[scaleval] \n"
+ "ptrue p3.h \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "ptrue p1.h \n"
+ "1: \n"
+ "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #3 \n"
+ "ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #3 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "add z0.h, z0.h, z1.h \n"
+ "add z3.h, z3.h, z4.h \n"
+ "add z0.h, z0.h, z2.h \n"
+ "add z3.h, z3.h, z5.h \n"
+ "umulh z0.h, p3/m, z0.h, z6.h \n"
+ "umulh z3.h, p3/m, z3.h, z6.h \n"
+ "uzp1 z0.b, z0.b, z3.b \n"
+ "st1h {z0.h}, p1, [%[dst_uv]] \n"
+ "incb %[dst_uv] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n"
+ "whilelt p2.s, %w[vl_w], %w[dst_width] \n"
+ "whilelt p1.h, wzr, %w[dst_width] \n"
+ "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #3 \n"
+ "ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n"
+ "add z0.h, z0.h, z1.h \n"
+ "add z3.h, z3.h, z4.h \n"
+ "add z0.h, z0.h, z2.h \n"
+ "add z3.h, z3.h, z5.h \n"
+ "umulh z0.h, p3/m, z0.h, z6.h \n"
+ "umulh z3.h, p3/m, z3.h, z6.h \n"
+ "uzp1 z0.b, z0.b, z3.b \n"
+ "st1h {z0.h}, p1, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=&r"(vl), // %[vl]
+ [vl_w] "=&r"(vl_w) // %[vl_w]
+ : [scaleval] "r"(scaleval) // %[scaleval]
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1",
+ "p2", "p3");
+}
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/scale_uv.cc b/libs/libyuv/source/scale_uv.cc
new file mode 100644
index 0000000000..706f27a80a
--- /dev/null
+++ b/libs/libyuv/source/scale_uv.cc
@@ -0,0 +1,1986 @@
+/*
+ * Copyright 2020 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/scale_uv.h"
+
+#include <assert.h>
+#include <limits.h>
+#include <string.h>
+
+#include "libyuv/cpu_id.h"
+#include "libyuv/planar_functions.h" // For CopyUV
+#include "libyuv/row.h"
+#include "libyuv/scale_row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Macros to enable specialized scalers
+
+#ifndef HAS_SCALEUVDOWN2
+#define HAS_SCALEUVDOWN2 1
+#endif
+#ifndef HAS_SCALEUVDOWN4BOX
+#define HAS_SCALEUVDOWN4BOX 1
+#endif
+#ifndef HAS_SCALEUVDOWNEVEN
+#define HAS_SCALEUVDOWNEVEN 1
+#endif
+#ifndef HAS_SCALEUVBOX
+#define HAS_SCALEUVBOX 1
+#endif
+#ifndef HAS_SCALEUVBILINEARDOWN
+#define HAS_SCALEUVBILINEARDOWN 1
+#endif
+#ifndef HAS_SCALEUVBILINEARUP
+#define HAS_SCALEUVBILINEARUP 1
+#endif
+#ifndef HAS_UVCOPY
+#define HAS_UVCOPY 1
+#endif
+#ifndef HAS_SCALEPLANEVERTICAL
+#define HAS_SCALEPLANEVERTICAL 1
+#endif
+
+#define MIN1(x) ((x) < 1 ? 1 : (x))
+
+// ScaleUV, 1/2
+// This is an optimized version for scaling down a UV to 1/2 of
+// its original size.
+#if HAS_SCALEUVDOWN2
+static void ScaleUVDown2(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
+ void (*ScaleUVRowDown2)(const uint8_t* src_uv, ptrdiff_t src_stride,
+ uint8_t* dst_uv, int dst_width) =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_C
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_C
+ : ScaleUVRowDown2Box_C);
+ (void)src_width;
+ (void)src_height;
+ (void)dx;
+ assert(dx == 65536 * 2); // Test scale factor of 2.
+ assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2.
+ // Advance to odd row, even column.
+ if (filtering) {
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+ } else {
+ src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2;
+ }
+
+#if defined(HAS_SCALEUVROWDOWN2_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_Any_AVX2
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_AVX2
+ : ScaleUVRowDown2Box_Any_AVX2);
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_AVX2
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_AVX2
+ : ScaleUVRowDown2Box_AVX2);
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_Any_NEON
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_NEON
+ : ScaleUVRowDown2Box_Any_NEON);
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_NEON
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_NEON
+ : ScaleUVRowDown2Box_NEON);
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleUVRowDown2 = filtering == kFilterNone ? ScaleUVRowDown2_SME
+ : filtering == kFilterLinear ? ScaleUVRowDown2Linear_SME
+ : ScaleUVRowDown2Box_SME;
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_RVV
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_RVV
+ : ScaleUVRowDown2Box_RVV);
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleUVRowDown2 =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_HVX
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_HVX
+ : ScaleUVRowDown2Box_HVX);
+ }
+#endif
+
+ if (filtering == kFilterLinear) {
+ src_stride = 0;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+}
+
+static void ScaleUVDown2_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
+ void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride,
+ uint16_t* dst_uv, int dst_width) =
+ filtering == kFilterNone
+ ? ScaleUVRowDown2_16_C
+ : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_16_C
+ : ScaleUVRowDown2Box_16_C);
+ (void)src_width;
+ (void)src_height;
+ (void)dx;
+ assert(dx == 65536 * 2); // Test scale factor of 2.
+ assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2.
+ // Advance to odd row, even column.
+ if (filtering) {
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+ } else {
+ src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2;
+ }
+
+ if (filtering == kFilterLinear) {
+ src_stride = 0;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+}
+#endif // HAS_SCALEUVDOWN2
+
+// ScaleUV, 1/4
+// This is an optimized version for scaling down a UV to 1/4 of
+// its original size.
+#if HAS_SCALEUVDOWN4BOX
+static int ScaleUVDown4Box(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j;
+ // Allocate 2 rows of UV.
+ const int row_size = (dst_width * 2 * 2 + 15) & ~15;
+ align_buffer_64(row, row_size * 2);
+ if (!row)
+ return 1;
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
+ void (*ScaleUVRowDown2)(const uint8_t* src_uv, ptrdiff_t src_stride,
+ uint8_t* dst_uv, int dst_width) =
+ ScaleUVRowDown2Box_C;
+ // Advance to odd row, even column.
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+ (void)src_width;
+ (void)src_height;
+ (void)dx;
+ assert(dx == 65536 * 4); // Test scale factor of 4.
+ assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
+
+#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2BOX_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_NEON;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_SME;
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2BOX_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_RVV;
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWN2BOX_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_HVX;
+ }
+#endif
+
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDown2(src_uv, src_stride, row, dst_width * 2);
+ ScaleUVRowDown2(src_uv + src_stride * 2, src_stride, row + row_size,
+ dst_width * 2);
+ ScaleUVRowDown2(row, row_size, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+
+static int ScaleUVDown4Box_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j;
+ // Allocate 2 rows of 16-bit UV.
+ const int row_stride_words = (dst_width * 2 * 2 + 15) & ~15;
+ align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t));
+ if (!row)
+ return 1;
+ uint16_t* row16 = (uint16_t*)row;
+ ptrdiff_t row_stride = src_stride * (dy >> 16);
+ void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride,
+ uint16_t* dst_uv, int dst_width) =
+ ScaleUVRowDown2Box_16_C;
+ // Advance to odd row, even column.
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+ (void)src_width;
+ (void)src_height;
+ (void)dx;
+ assert(dx == 65536 * 4); // Test scale factor of 4.
+ assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
+
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDown2(src_uv, src_stride, row16, dst_width * 2);
+ ScaleUVRowDown2(src_uv + src_stride * 2, src_stride,
+ row16 + row_stride_words, dst_width * 2);
+ ScaleUVRowDown2(row16, row_stride_words, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+ free_aligned_buffer_64(row);
+ return 0;
+}
+#endif // HAS_SCALEUVDOWN4BOX
+
+// ScaleUV, any dimensions, box filter.
+// The counterpart of ScalePlaneBox in scale.cc. Rows are summed into 16 bit
+// accumulators, then each destination pixel averages its box of source pixels.
+#if HAS_SCALEUVBOX
+static void ScaleUVAddCols1_C(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ int i, j;
+ int boxwidth = MIN1(dx >> 16);
+ int scaleval = 65536 / (boxwidth * boxheight);
+ x >>= 16;
+ for (i = 0; i < dst_width; ++i) {
+ int u = 0;
+ int v = 0;
+ for (j = 0; j < boxwidth; ++j) {
+ u += src_uv[(x + j) * 2];
+ v += src_uv[(x + j) * 2 + 1];
+ }
+ x += boxwidth;
+ *dst_uv++ = (uint8_t)(u * scaleval >> 16);
+ *dst_uv++ = (uint8_t)(v * scaleval >> 16);
+ }
+}
+
+static void ScaleUVAddCols2_C(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ int i, j;
+ int minboxwidth = dx >> 16;
+ int scaletbl[2];
+ scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight);
+ scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight);
+ for (i = 0; i < dst_width; ++i) {
+ int ix = x >> 16;
+ int u = 0;
+ int v = 0;
+ int boxwidth;
+ int scaleval;
+ x += dx;
+ boxwidth = MIN1((x >> 16) - ix);
+ assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1));
+ scaleval = scaletbl[boxwidth - minboxwidth];
+ for (j = 0; j < boxwidth; ++j) {
+ u += src_uv[(ix + j) * 2];
+ v += src_uv[(ix + j) * 2 + 1];
+ }
+ *dst_uv++ = (uint8_t)(u * scaleval >> 16);
+ *dst_uv++ = (uint8_t)(v * scaleval >> 16);
+ }
+}
+
+static int ScaleUVBox(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j, k;
+ const uint32_t max_y = (uint32_t)src_height << 16;
+ // Sized on src_width to keep ScaleAddRow aligned.
+ const int row_samples = src_width * 2;
+ {
+ // Row accumulator, two uint16_t per source pixel.
+ align_buffer_64(row16, row_samples * 2);
+ if (!row16)
+ return 1;
+ {
+ void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
+ const uint16_t* src_uv, uint8_t* dst_uv) =
+ (dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
+ void (*ScaleUVAddCols3)(int dst_width, int scaleval,
+ const uint16_t* src_uv, uint8_t* dst_uv) = NULL;
+ // Summing a row is position wise, so an interleaved UV row can be fed to
+ // the plane accumulator as src_width * 2 samples.
+ void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,
+ int src_width) = ScaleAddRow_C;
+ if (dx == 0x30000) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_C;
+#if defined(HAS_SCALEUVADDCOLS3_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI)
+ if (TestCpuFlag(kCpuHasAVX512VBMI)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI;
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_SME;
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_RVV;
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_Any_LSX;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASM)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_WASMSIMD;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVADDCOLS3_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleUVAddCols3 = ScaleUVAddCols3_HVX;
+ }
+#endif
+ }
+#if defined(HAS_SCALEADDROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ ScaleAddRow = ScaleAddRow_Any_SSE2;
+ if (IS_ALIGNED(row_samples, 16)) {
+ ScaleAddRow = ScaleAddRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_SCALEADDROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleAddRow = ScaleAddRow_Any_AVX2;
+ if (IS_ALIGNED(row_samples, 32)) {
+ ScaleAddRow = ScaleAddRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SCALEADDROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ ScaleAddRow = ScaleAddRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_SCALEADDROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleAddRow = ScaleAddRow_Any_NEON;
+ if (IS_ALIGNED(row_samples, 16)) {
+ ScaleAddRow = ScaleAddRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_SCALEADDROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ScaleAddRow = ScaleAddRow_Any_LSX;
+ if (IS_ALIGNED(row_samples, 16)) {
+ ScaleAddRow = ScaleAddRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_SCALEADDROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleAddRow = ScaleAddRow_RVV;
+ }
+#endif
+#if defined(HAS_SCALEADDROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ ScaleAddRow = ScaleAddRow_HVX;
+ }
+#endif
+#if defined(HAS_SCALEADDROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASM)) {
+ ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
+ if (IS_ALIGNED(row_samples, 16)) {
+ ScaleAddRow = ScaleAddRow_WASMSIMD;
+ }
+ }
+#endif
+
+ uint32_t uy = y;
+ for (j = 0; j < dst_height; ++j) {
+ int boxheight;
+ int iy = uy >> 16;
+ const uint8_t* src = src_uv + iy * src_stride;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ boxheight = MIN1((int)(uy >> 16) - iy);
+ memset(row16, 0, row_samples * 2);
+ for (k = 0; k < boxheight; ++k) {
+ ScaleAddRow(src, (uint16_t*)(row16), row_samples);
+ src += src_stride;
+ }
+ if (ScaleUVAddCols3 && boxheight <= 85) {
+ ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
+ (const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2,
+ dst_uv);
+ } else {
+ ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16),
+ dst_uv);
+ }
+ dst_uv += dst_stride;
+ }
+ }
+ free_aligned_buffer_64(row16);
+ }
+ return 0;
+}
+
+static void ScaleUVAddCols1_16_C(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint32_t* src_uv,
+ uint16_t* dst_uv) {
+ int i, j;
+ int boxwidth = MIN1(dx >> 16);
+ uint32_t scaleval = 65536 / (boxwidth * boxheight);
+ x >>= 16;
+ for (i = 0; i < dst_width; ++i) {
+ uint32_t u = 0;
+ uint32_t v = 0;
+ for (j = 0; j < boxwidth; ++j) {
+ u += src_uv[(x + j) * 2];
+ v += src_uv[(x + j) * 2 + 1];
+ }
+ x += boxwidth;
+ *dst_uv++ = (uint16_t)((u * scaleval) >> 16);
+ *dst_uv++ = (uint16_t)((v * scaleval) >> 16);
+ }
+}
+
+static void ScaleUVAddCols2_16_C(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint32_t* src_uv,
+ uint16_t* dst_uv) {
+ int i, j;
+ int minboxwidth = dx >> 16;
+ uint32_t scaletbl[2];
+ scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight);
+ scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight);
+ for (i = 0; i < dst_width; ++i) {
+ int ix = x >> 16;
+ uint32_t u = 0;
+ uint32_t v = 0;
+ int boxwidth;
+ uint32_t scaleval;
+ x += dx;
+ boxwidth = MIN1((x >> 16) - ix);
+ assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1));
+ scaleval = scaletbl[boxwidth - minboxwidth];
+ for (j = 0; j < boxwidth; ++j) {
+ u += src_uv[(ix + j) * 2];
+ v += src_uv[(ix + j) * 2 + 1];
+ }
+ *dst_uv++ = (uint16_t)((u * scaleval) >> 16);
+ *dst_uv++ = (uint16_t)((v * scaleval) >> 16);
+ }
+}
+
+static int ScaleUVBox_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j, k;
+ const uint32_t max_y = (uint32_t)src_height << 16;
+ const int row_samples = src_width * 2;
+ {
+ // Row accumulator, two uint32_t per source UV pixel.
+ align_buffer_64(row32_buf, row_samples * (int)sizeof(uint32_t));
+ if (!row32_buf)
+ return 1;
+ {
+ uint32_t* row32 = (uint32_t*)row32_buf;
+ void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
+ const uint32_t* src_uv, uint16_t* dst_uv) =
+ (dx & 0xffff) ? ScaleUVAddCols2_16_C : ScaleUVAddCols1_16_C;
+ void (*ScaleUVAddCols3)(int dst_width, int scaleval,
+ const uint32_t* src_uv, uint16_t* dst_uv) =
+ (dx == 0x30000) ? ScaleUVAddCols3_16_C : NULL;
+ void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr,
+ int src_width) = ScaleAddRow_16_C;
+
+ uint32_t uy = y;
+ for (j = 0; j < dst_height; ++j) {
+ int boxheight;
+ int iy = uy >> 16;
+ const uint16_t* src = src_uv + iy * src_stride;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ boxheight = MIN1((int)(uy >> 16) - iy);
+ memset(row32, 0, row_samples * sizeof(uint32_t));
+ for (k = 0; k < boxheight; ++k) {
+ ScaleAddRow(src, row32, row_samples);
+ src += src_stride;
+ }
+ if (ScaleUVAddCols3 && boxheight <= 21845) {
+ ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
+ row32 + (ptrdiff_t)(x >> 16) * 2, dst_uv);
+ } else {
+ ScaleUVAddCols(dst_width, boxheight, x, dx, row32, dst_uv);
+ }
+ dst_uv += dst_stride;
+ }
+ }
+ free_aligned_buffer_64(row32_buf);
+ }
+ return 0;
+}
+#endif // HAS_SCALEUVBOX
+
+// ScaleUV Even
+// This is an optimized version for scaling down a UV to even
+// multiple of its original size.
+#if HAS_SCALEUVDOWNEVEN
+static void ScaleUVDownEven(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ int col_step = dx >> 16;
+ ptrdiff_t row_stride = (dy >> 16) * src_stride;
+ void (*ScaleUVRowDownEven)(const uint8_t* src_uv, ptrdiff_t src_stride,
+ int src_step, uint8_t* dst_uv, int dst_width) =
+ filtering ? ScaleUVRowDownEvenBox_C : ScaleUVRowDownEven_C;
+ (void)src_width;
+ (void)src_height;
+ assert(IS_ALIGNED(src_width, 2));
+ assert(IS_ALIGNED(src_height, 2));
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+#if defined(HAS_SCALEUVROWDOWNEVEN_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2) && !filtering) {
+ ScaleUVRowDownEven = ScaleUVRowDownEven_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
+ ScaleUVRowDownEven = ScaleUVRowDownEven_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_SCALEUVROWDOWNEVEN_NEON)
+ if (TestCpuFlag(kCpuHasNEON) && !filtering) {
+ ScaleUVRowDownEven = ScaleUVRowDownEven_Any_NEON;
+ if (IS_ALIGNED(dst_width, 4)) {
+ ScaleUVRowDownEven = ScaleUVRowDownEven_NEON;
+ }
+ }
+#endif // TODO(fbarchard): Enable Box filter
+#if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV)
+ if (TestCpuFlag(kCpuHasRVV) && !filtering) {
+#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
+ ScaleUVRowDownEven = ScaleUVRowDownEven_RVV;
+#endif
+#if defined(HAS_SCALEUVROWDOWN4_RVV)
+ if (col_step == 4) {
+ ScaleUVRowDownEven = ScaleUVRowDown4_RVV;
+ }
+#endif
+ }
+#endif
+
+ if (filtering == kFilterLinear) {
+ src_stride = 0;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+}
+
+static void ScaleUVDownEven_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ int col_step = dx >> 16;
+ ptrdiff_t row_stride = (dy >> 16) * src_stride;
+ void (*ScaleUVRowDownEven)(const uint16_t* src_uv, ptrdiff_t src_stride,
+ int src_step, uint16_t* dst_uv, int dst_width) =
+ filtering ? ScaleUVRowDownEvenBox_16_C : ScaleUVRowDownEven_16_C;
+ (void)src_width;
+ (void)src_height;
+ assert(IS_ALIGNED(src_width, 2));
+ assert(IS_ALIGNED(src_height, 2));
+ src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
+
+ if (filtering == kFilterLinear) {
+ src_stride = 0;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width);
+ src_uv += row_stride;
+ dst_uv += dst_stride;
+ }
+}
+#endif
+
+// Scale UV down with bilinear interpolation.
+#if HAS_SCALEUVBILINEARDOWN
+static int ScaleUVBilinearDown(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ void (*InterpolateRow)(uint8_t* dst_uv, const uint8_t* src_uv,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_C;
+ void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv,
+ int dst_width, int x, int dx) =
+ (src_width >= 32768) ? ScaleUVFilterCols64_C : ScaleUVFilterCols_C;
+ int64_t xlast = x + (int64_t)(dst_width - 1) * dx;
+ int64_t xl = (dx >= 0) ? x : xlast;
+ int64_t xr = (dx >= 0) ? xlast : x;
+ int clip_src_width;
+ xl = (xl >> 16) & ~3; // Left edge aligned.
+ xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels.
+ xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel.
+ if (xr > src_width) {
+ xr = src_width;
+ }
+ clip_src_width = (int)(xr - xl) * 2; // Width aligned to 2.
+ src_uv += xl * 2;
+ x -= (int)(xl << 16);
+#if defined(HAS_INTERPOLATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_Any_AVX2;
+ if (IS_ALIGNED(clip_src_width, 32)) {
+ InterpolateRow = InterpolateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_Any_NEON;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(clip_src_width, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
+ }
+ }
+#endif
+ // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
+ // Allocate a row of UV.
+ {
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ uint32_t uy = y;
+ // clip_src_width is already in bytes (2 per UV pixel).
+ align_buffer_64(row, clip_src_width);
+ if (!row)
+ return 1;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ int yi = uy >> 16;
+ const uint8_t* src = src_uv + yi * src_stride;
+ if (filtering == kFilterLinear) {
+ ScaleUVFilterCols(dst_uv, src, dst_width, x, dx);
+ } else {
+ int yf = (uy >> 8) & 255;
+ InterpolateRow(row, src, src_stride, clip_src_width, yf);
+ ScaleUVFilterCols(dst_uv, row, dst_width, x, dx);
+ }
+ dst_uv += dst_stride;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+
+static int ScaleUVBilinearDown_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ int j;
+ void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_16_C;
+ void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv,
+ int dst_width, int x, int dx) =
+ (src_width >= 32768) ? ScaleUVFilterCols64_16_C : ScaleUVFilterCols_16_C;
+ int64_t xlast = x + (int64_t)(dst_width - 1) * dx;
+ int64_t xl = (dx >= 0) ? x : xlast;
+ int64_t xr = (dx >= 0) ? xlast : x;
+ int clip_src_width;
+ xl = (xl >> 16) & ~3; // Left edge aligned.
+ xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels.
+ xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel.
+ if (xr > src_width) {
+ xr = src_width;
+ }
+ clip_src_width = (int)(xr - xl) * 2; // Width in uint16_t words.
+ src_uv += xl * 2;
+ x -= (int)(xl << 16);
+#if defined(HAS_INTERPOLATEROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_16_Any_AVX2;
+ if (IS_ALIGNED(clip_src_width, 16)) {
+ InterpolateRow = InterpolateRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_16_Any_NEON;
+ if (IS_ALIGNED(clip_src_width, 8)) {
+ InterpolateRow = InterpolateRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_16_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(clip_src_width, 8)) {
+ InterpolateRow = InterpolateRow_16_WASMSIMD;
+ }
+ }
+#endif
+ // Allocate a row of 16-bit UV.
+ {
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ uint32_t uy = y;
+ align_buffer_64(row, clip_src_width * (int)sizeof(uint16_t));
+ if (!row)
+ return 1;
+ uint16_t* row16 = (uint16_t*)row;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ for (j = 0; j < dst_height; ++j) {
+ int yi = uy >> 16;
+ const uint16_t* src = src_uv + yi * src_stride;
+ if (filtering == kFilterLinear) {
+ ScaleUVFilterCols(dst_uv, src, dst_width, x, dx);
+ } else {
+ int yf = (uy >> 8) & 255;
+ InterpolateRow(row16, src, src_stride, clip_src_width, yf);
+ ScaleUVFilterCols(dst_uv, row16, dst_width, x, dx);
+ }
+ dst_uv += dst_stride;
+ uy += dy;
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+#endif
+
+// Scale UV up with bilinear interpolation.
+#if HAS_SCALEUVBILINEARUP
+static int ScaleUVBilinearUp(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ assert(src_width > 0);
+ assert(src_height > 0);
+ assert(dst_width > 0);
+ assert(dst_height > 0);
+ assert(dy <= 65536);
+
+ int j;
+ void (*InterpolateRow)(uint8_t* dst_uv, const uint8_t* src_uv,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_C;
+ void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv,
+ int dst_width, int x, int dx) =
+ filtering ? ScaleUVFilterCols_C : ScaleUVCols_C;
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ uint32_t uy = y;
+#if defined(HAS_INTERPOLATEROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 16)) {
+ InterpolateRow = InterpolateRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_Any_NEON;
+ if (IS_ALIGNED(dst_width, 8)) {
+ InterpolateRow = InterpolateRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ InterpolateRow = InterpolateRow_SVE2;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ InterpolateRow = InterpolateRow_Any_LSX;
+ if (IS_ALIGNED(dst_width, 16)) {
+ InterpolateRow = InterpolateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ InterpolateRow = InterpolateRow_Any_LASX;
+ if (IS_ALIGNED(dst_width, 32)) {
+ InterpolateRow = InterpolateRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ InterpolateRow = InterpolateRow_RVV;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 8)) {
+ InterpolateRow = InterpolateRow_WASMSIMD;
+ }
+ }
+#endif
+ if (src_width >= 32768) {
+ ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C;
+ }
+ if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
+ ScaleUVFilterCols = ScaleUVColsUp2_C;
+ }
+
+ if (uy > max_y) {
+ uy = max_y;
+ }
+
+ {
+ int yi = uy >> 16;
+ const uint8_t* src = src_uv + yi * src_stride;
+
+ // Allocate 2 rows of UV.
+ const int row_size = (dst_width * 2 + 15) & ~15;
+ align_buffer_64(row, row_size * 2);
+ if (!row)
+ return 1;
+
+ uint8_t* rowptr = row;
+ ptrdiff_t rowstride = row_size;
+ int lasty = yi;
+
+ ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
+ if (src_height > 1) {
+ src += src_stride;
+ }
+ ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx);
+ if (src_height > 2) {
+ src += src_stride;
+ }
+
+ // 2-row rolling buffer:
+ // rowptr and (rowptr + rowstride) hold the scaled rows for yi and yi + 1.
+ // Because dy <= 65536 (dy <= 1.0 in 16.16), yi advances in unit steps.
+ // When yi != lasty:
+ // 1. Scale the next source row into the older buffer (rowptr).
+ // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
+ // so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
+ // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
+ // otherwise clamp src at (src_height - 1) to avoid reading out of
+ // bounds.
+ for (j = 0; j < dst_height; ++j) {
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ yi = uy >> 16;
+ if (yi != lasty) {
+ ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
+ rowptr += rowstride;
+ rowstride = -rowstride;
+ lasty = yi;
+ if ((uy + 65536) < max_y) {
+ src += src_stride;
+ }
+ }
+ if (filtering == kFilterLinear) {
+ InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0);
+ } else {
+ int yf = (uy >> 8) & 255;
+ InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf);
+ }
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+
+static int ScaleUVBilinearUp_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy,
+ enum FilterMode filtering) {
+ assert(src_width > 0);
+ assert(src_height > 0);
+ assert(dst_width > 0);
+ assert(dst_height > 0);
+ assert(dy <= 65536);
+
+ int j;
+ void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv,
+ ptrdiff_t src_stride, int dst_width,
+ int source_y_fraction) = InterpolateRow_16_C;
+ void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv,
+ int dst_width, int x, int dx) =
+ filtering ? ScaleUVFilterCols_16_C : ScaleUVCols_16_C;
+ const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
+ uint32_t uy = y;
+#if defined(HAS_INTERPOLATEROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ InterpolateRow = InterpolateRow_16_Any_AVX2;
+ if (IS_ALIGNED(dst_width, 8)) {
+ InterpolateRow = InterpolateRow_16_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ InterpolateRow = InterpolateRow_16_AVX512BW;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ InterpolateRow = InterpolateRow_16_Any_NEON;
+ if (IS_ALIGNED(dst_width, 4)) {
+ InterpolateRow = InterpolateRow_16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ InterpolateRow = InterpolateRow_16_HVX;
+ }
+#endif
+#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
+ if (TestCpuFlag(kCpuHasWASMSIMD)) {
+ InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
+ if (IS_ALIGNED(dst_width, 4)) {
+ InterpolateRow = InterpolateRow_16_WASMSIMD;
+ }
+ }
+#endif
+ if (src_width >= 32768) {
+ ScaleUVFilterCols =
+ filtering ? ScaleUVFilterCols64_16_C : ScaleUVCols64_16_C;
+ }
+ if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
+ ScaleUVFilterCols = ScaleUVColsUp2_16_C;
+ }
+
+ if (uy > max_y) {
+ uy = max_y;
+ }
+
+ {
+ int yi = uy >> 16;
+ const uint16_t* src = src_uv + yi * src_stride;
+
+ // Allocate 2 rows of 16-bit UV.
+ const int row_stride_words = (dst_width * 2 + 15) & ~15;
+ align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t));
+ if (!row)
+ return 1;
+
+ uint16_t* rowptr = (uint16_t*)row;
+ ptrdiff_t rowstride = row_stride_words;
+ int lasty = yi;
+
+ ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
+ if (src_height > 1) {
+ src += src_stride;
+ }
+ ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx);
+ if (src_height > 2) {
+ src += src_stride;
+ }
+
+ for (j = 0; j < dst_height; ++j) {
+ if (uy > max_y) {
+ uy = max_y;
+ }
+ yi = uy >> 16;
+ if (yi != lasty) {
+ ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
+ rowptr += rowstride;
+ rowstride = -rowstride;
+ lasty = yi;
+ if ((uy + 65536) < max_y) {
+ src += src_stride;
+ }
+ }
+ if (filtering == kFilterLinear) {
+ InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0);
+ } else {
+ int yf = (uy >> 8) & 255;
+ InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf);
+ }
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+#endif // HAS_SCALEUVBILINEARUP
+
+// Scale UV, horizontally up by 2 times.
+// Uses linear filter horizontally, nearest vertically.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original width, using linear interpolation.
+// This is used to scale U and V planes of NV16 to NV24.
+static void ScaleUVLinearUp2(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv) {
+ void (*ScaleRowUp)(const uint8_t* src_uv, uint8_t* dst_uv, int dst_width) =
+ ScaleUVRowUp2_Linear_Any_C;
+ int i;
+
+ // This function can only scale up by 2 times horizontally.
+ (void)src_width;
+ assert(src_width == ((dst_width + 1) / 2));
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_SSSE3
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_Any_SSSE3;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_Any_NEON;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_RVV
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_RVV;
+ }
+#endif
+
+ if (dst_height == 1) {
+ ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
+ } else {
+ uint32_t uy = (1 << 15) - 1;
+ int dy = FixedDiv(src_height - 1, dst_height - 1);
+ for (i = 0; i < dst_height; ++i) {
+ int yi = uy >> 16;
+ ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+ }
+}
+
+// Scale plane, up by 2 times.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original size, using bilinear interpolation.
+// This is used to scale U and V planes of NV12 to NV24.
+static void ScaleUVBilinearUp2(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_ptr,
+ uint8_t* dst_ptr) {
+ void (*Scale2RowUp)(const uint8_t* src_ptr, ptrdiff_t src_stride,
+ uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
+ ScaleUVRowUp2_Bilinear_Any_C;
+ int x;
+
+ // This function can only scale up by 2 times.
+ (void)src_width;
+ assert(src_width == ((dst_width + 1) / 2));
+ assert(src_height == ((dst_height + 1) / 2));
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_SSSE3
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_SSSE3;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_NEON;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_RVV
+ if (TestCpuFlag(kCpuHasRVV)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_RVV;
+ }
+#endif
+
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ dst_ptr += dst_stride;
+ for (x = 0; x < src_height - 1; ++x) {
+ Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
+ src_ptr += src_stride;
+ // TODO(fbarchard): Test performance of writing one row of destination at a
+ // time.
+ dst_ptr += 2 * dst_stride;
+ }
+ if (!(dst_height & 1)) {
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ }
+}
+
+// Scale 16 bit UV, horizontally up by 2 times.
+// Uses linear filter horizontally, nearest vertically.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original width, using linear interpolation.
+// This is used to scale U and V planes of P210 to P410.
+static void ScaleUVLinearUp2_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv) {
+ void (*ScaleRowUp)(const uint16_t* src_uv, uint16_t* dst_uv, int dst_width) =
+ ScaleUVRowUp2_Linear_16_Any_C;
+ int i;
+
+ // This function can only scale up by 2 times horizontally.
+ (void)src_width;
+ assert(src_width == ((dst_width + 1) / 2));
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_NEON;
+ }
+#endif
+
+ if (dst_height == 1) {
+ ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
+ } else {
+ uint32_t uy = (1 << 15) - 1;
+ int dy = FixedDiv(src_height - 1, dst_height - 1);
+ for (i = 0; i < dst_height; ++i) {
+ int yi = uy >> 16;
+ ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+ }
+}
+
+// Scale 16 bit UV, up by 2 times.
+// This is an optimized version for scaling up a plane to 2 times of
+// its original size, using bilinear interpolation.
+// This is used to scale U and V planes of P010 to P410.
+static void ScaleUVBilinearUp2_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_ptr,
+ uint16_t* dst_ptr) {
+ void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride,
+ uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
+ ScaleUVRowUp2_Bilinear_16_Any_C;
+ int x;
+
+ // This function can only scale up by 2 times.
+ (void)src_width;
+ assert(src_width == ((dst_width + 1) / 2));
+ assert(src_height == ((dst_height + 1) / 2));
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_SSE41;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_AVX2;
+ }
+#endif
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_NEON
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_NEON;
+ }
+#endif
+
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ dst_ptr += dst_stride;
+ for (x = 0; x < src_height - 1; ++x) {
+ Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
+ src_ptr += src_stride;
+ // TODO(fbarchard): Test performance of writing one row of destination at a
+ // time.
+ dst_ptr += 2 * dst_stride;
+ }
+ if (!(dst_height & 1)) {
+ Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
+ }
+}
+
+// Scale UV to/from any dimensions, without interpolation.
+// Fixed point math is used for performance: The upper 16 bits
+// of x and dx is the integer part of the source position and
+// the lower 16 bits are the fixed decimal part.
+
+static void ScaleUVSimple(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint8_t* src_uv,
+ uint8_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j;
+ void (*ScaleUVCols)(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width,
+ int x, int dx) =
+ (src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C;
+ (void)src_height;
+ if (src_width * 2 == dst_width && x < 0x8000) {
+ ScaleUVCols = ScaleUVColsUp2_C;
+ }
+
+ uint32_t uy = y;
+ for (j = 0; j < dst_height; ++j) {
+ int yi = uy >> 16;
+ ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx);
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+}
+
+static void ScaleUVSimple_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ ptrdiff_t src_stride,
+ ptrdiff_t dst_stride,
+ const uint16_t* src_uv,
+ uint16_t* dst_uv,
+ int x,
+ int dx,
+ int y,
+ int dy) {
+ int j;
+ void (*ScaleUVCols)(uint16_t* dst_uv, const uint16_t* src_uv, int dst_width,
+ int x, int dx) =
+ (src_width >= 32768) ? ScaleUVCols64_16_C : ScaleUVCols_16_C;
+ (void)src_height;
+ if (src_width * 2 == dst_width && x < 0x8000) {
+ ScaleUVCols = ScaleUVColsUp2_16_C;
+ }
+
+ uint32_t uy = y;
+ for (j = 0; j < dst_height; ++j) {
+ int yi = uy >> 16;
+ ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx);
+ dst_uv += dst_stride;
+ uy += dy;
+ }
+}
+
+// Copy UV with optional flipping
+#if HAS_UVCOPY
+static int UVCopy(const uint8_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ if (!src_uv || !dst_uv || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uv = src_uv + (height - 1) * (ptrdiff_t)src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ }
+
+ CopyPlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, width * 2, height);
+ return 0;
+}
+
+static int UVCopy_16(const uint16_t* src_uv,
+ int src_stride_uv,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ if (!src_uv || !dst_uv || width <= 0 || height == 0 || height == INT_MIN) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ src_uv = src_uv + (height - 1) * (ptrdiff_t)src_stride_uv;
+ src_stride_uv = -src_stride_uv;
+ }
+
+ CopyPlane_16(src_uv, src_stride_uv, dst_uv, dst_stride_uv, width * 2, height);
+ return 0;
+}
+#endif // HAS_UVCOPY
+
+// Scale a UV plane (from NV12)
+// This function in turn calls a scaling function
+// suitable for handling the desired resolutions.
+static int ScaleUV(const uint8_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint8_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ int clip_x,
+ int clip_y,
+ int clip_width,
+ int clip_height,
+ enum FilterMode filtering) {
+ // Initial source x/y coordinate and step values as 16.16 fixed point.
+ int x = 0;
+ int y = 0;
+ int dx = 0;
+ int dy = 0;
+ // Simplify filtering when possible.
+ filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
+ filtering);
+
+ // Negative src_height means invert the image.
+ if (src_height < 0) {
+ src_height = -src_height;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
+ src_stride = -src_stride;
+ }
+ ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
+ &dx, &dy);
+ if (clip_x) {
+ int64_t clipf = (int64_t)(clip_x)*dx;
+ x += (clipf & 0xffff);
+ src += (clipf >> 16) * 2;
+ dst += clip_x * 2;
+ }
+ if (clip_y) {
+ int64_t clipf = (int64_t)(clip_y)*dy;
+ y += (clipf & 0xffff);
+ src += (clipf >> 16) * (ptrdiff_t)src_stride;
+ dst += clip_y * (ptrdiff_t)dst_stride;
+ }
+
+#if HAS_SCALEUVBOX
+ // Exactly 4 goes to the faster ScaleUVDown4Box below. Clipping is excluded
+ // because whole source rows are accumulated.
+ if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) &&
+ clip_x == 0 && clip_y == 0) {
+ return ScaleUVBox(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy);
+ }
+#endif
+
+ // Special case for integer step values.
+ if (((dx | dy) & 0xffff) == 0) {
+ if (!dx || !dy) { // 1 pixel wide and/or tall.
+ filtering = kFilterNone;
+ } else {
+ // Optimized even scale down. ie 2, 4, 6, 8, 10x.
+ if (!(dx & 0x10000) && !(dy & 0x10000)) {
+#if HAS_SCALEUVDOWN2
+ if (dx == 0x20000 && dy == 0x20000) {
+ // Optimized 1/2 downsample.
+ ScaleUVDown2(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ return 0;
+ }
+#endif
+#if HAS_SCALEUVDOWN4BOX
+ if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
+ // Optimized 1/4 box downsample.
+ return ScaleUVDown4Box(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y,
+ dy);
+ }
+#endif
+#if HAS_SCALEUVDOWNEVEN
+ ScaleUVDownEven(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ return 0;
+#endif
+ }
+ // Optimized odd scale down. ie 3, 5, 7, 9x.
+ if ((dx & 0x10000) && (dy & 0x10000)) {
+ filtering = kFilterNone;
+#ifdef HAS_UVCOPY
+ if (dx == 0x10000 && dy == 0x10000) {
+ // Straight copy.
+ return UVCopy(src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2,
+ src_stride, dst, dst_stride, clip_width, clip_height);
+ }
+#endif
+ }
+ }
+ }
+ // HAS_SCALEPLANEVERTICAL
+ if (dx == 0x10000 && (x & 0xffff) == 0) {
+ // Arbitrary scale vertically, but unscaled horizontally.
+ ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
+ dst_stride, src, dst, x, y, dy, /*bpp=*/2, filtering);
+ return 0;
+ }
+ if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
+ ScaleUVLinearUp2(src_width, src_height, clip_width, clip_height, src_stride,
+ dst_stride, src, dst);
+ return 0;
+ }
+ if ((clip_height + 1) / 2 == src_height &&
+ (clip_width + 1) / 2 == src_width &&
+ (filtering == kFilterBilinear || filtering == kFilterBox)) {
+ ScaleUVBilinearUp2(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+#if HAS_SCALEUVBILINEARUP
+ if (filtering && dy < 65536) {
+ return ScaleUVBilinearUp(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ }
+#endif
+#if HAS_SCALEUVBILINEARDOWN
+ if (filtering) {
+ return ScaleUVBilinearDown(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ }
+#endif
+ ScaleUVSimple(src_width, src_height, clip_width, clip_height, src_stride,
+ dst_stride, src, dst, x, dx, y, dy);
+ return 0;
+}
+
+// Scale an UV image.
+LIBYUV_API
+int UVScale(const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 ||
+ src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ return ScaleUV(src_uv, src_stride_uv, src_width, src_height, dst_uv,
+ dst_stride_uv, dst_width, dst_height, 0, 0, dst_width,
+ dst_height, filtering);
+}
+
+// Scale a 16-bit UV plane (from P010/P210)
+static int ScaleUV_16(const uint16_t* src,
+ int src_stride,
+ int src_width,
+ int src_height,
+ uint16_t* dst,
+ int dst_stride,
+ int dst_width,
+ int dst_height,
+ int clip_x,
+ int clip_y,
+ int clip_width,
+ int clip_height,
+ enum FilterMode filtering) {
+ // Initial source x/y coordinate and step values as 16.16 fixed point.
+ int x = 0;
+ int y = 0;
+ int dx = 0;
+ int dy = 0;
+ const enum FilterMode orig_filtering = filtering;
+ // Simplify filtering when possible.
+ filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
+ filtering);
+
+ // Negative src_height means invert the image.
+ if (src_height < 0) {
+ src_height = -src_height;
+ src = src + (src_height - 1) * (ptrdiff_t)src_stride;
+ src_stride = -src_stride;
+ }
+ ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
+ &dx, &dy);
+ if (clip_x) {
+ int64_t clipf = (int64_t)(clip_x)*dx;
+ x += (clipf & 0xffff);
+ src += (clipf >> 16) * 2;
+ dst += clip_x * 2;
+ }
+ if (clip_y) {
+ int64_t clipf = (int64_t)(clip_y)*dy;
+ y += (clipf & 0xffff);
+ src += (clipf >> 16) * (ptrdiff_t)src_stride;
+ dst += clip_y * (ptrdiff_t)dst_stride;
+ }
+
+#if HAS_SCALEUVBOX
+ // Exactly 4 goes to the faster ScaleUVDown4Box_16 below. Clipping is excluded
+ // because whole source rows are accumulated.
+ if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) &&
+ clip_x == 0 && clip_y == 0) {
+ return ScaleUVBox_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy);
+ }
+#endif
+
+ // Special case for integer step values.
+ if (((dx | dy) & 0xffff) == 0) {
+ if (!dx || !dy) { // 1 pixel wide and/or tall.
+ filtering = kFilterNone;
+ } else {
+ // Optimized even scale down. ie 2, 4, 6, 8, 10x.
+ if (!(dx & 0x10000) && !(dy & 0x10000)) {
+#if HAS_SCALEUVDOWN2
+ if (dx == 0x20000 && dy == 0x20000) {
+ // Optimized 1/2 downsample.
+ ScaleUVDown2_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ return 0;
+ }
+#endif
+#if HAS_SCALEUVDOWN4BOX
+ if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
+ // Optimized 1/4 box downsample.
+ return ScaleUVDown4Box_16(src_width, src_height, clip_width,
+ clip_height, src_stride, dst_stride, src,
+ dst, x, dx, y, dy);
+ }
+#endif
+#if HAS_SCALEUVDOWNEVEN
+ ScaleUVDownEven_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ return 0;
+#endif
+ }
+ // Optimized odd scale down. ie 3, 5, 7, 9x.
+ if ((dx & 0x10000) && (dy & 0x10000)) {
+ filtering = kFilterNone;
+#ifdef HAS_UVCOPY
+ if (dx == 0x10000 && dy == 0x10000) {
+ // Straight copy.
+ return UVCopy_16(
+ src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2,
+ src_stride, dst, dst_stride, clip_width, clip_height);
+ }
+#endif
+ }
+ }
+ }
+
+ // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone to avoid
+ // reading a second source column. The Up2 kernels clamp their last pixel and
+ // only read the 1 source column when dst_width <= 2, so restore Linear or
+ // Bilinear for 2x upscales (e.g., P010ToP410 and P210ToP410 on 1-2 pixel wide
+ // images) when the caller requested filtering.
+ if (orig_filtering != kFilterNone && filtering == kFilterNone &&
+ src_width == 1 && dst_width <= 2 && clip_x == 0 && clip_y == 0) {
+ if (clip_height == src_height && (dst_width + 1) / 2 == src_width) {
+ ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ if ((clip_height + 1) / 2 == src_height &&
+ (clip_width + 1) / 2 == src_width && orig_filtering != kFilterLinear) {
+ ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ }
+
+ // HAS_SCALEPLANEVERTICAL
+ if (dx == 0x10000 && (x & 0xffff) == 0) {
+ // Arbitrary scale vertically, but unscaled horizontally.
+ ScalePlaneVertical_16(src_height, clip_width, clip_height, src_stride,
+ dst_stride, src, dst, x, y, dy, /*wpp=*/2, filtering);
+ return 0;
+ }
+ if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
+ ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+ if ((clip_height + 1) / 2 == src_height &&
+ (clip_width + 1) / 2 == src_width &&
+ (filtering == kFilterBilinear || filtering == kFilterBox)) {
+ ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst);
+ return 0;
+ }
+#if HAS_SCALEUVBILINEARUP
+ if (filtering && dy < 65536) {
+ return ScaleUVBilinearUp_16(src_width, src_height, clip_width, clip_height,
+ src_stride, dst_stride, src, dst, x, dx, y, dy,
+ filtering);
+ }
+#endif
+#if HAS_SCALEUVBILINEARDOWN
+ if (filtering) {
+ return ScaleUVBilinearDown_16(src_width, src_height, clip_width,
+ clip_height, src_stride, dst_stride, src, dst,
+ x, dx, y, dy, filtering);
+ }
+#endif
+ ScaleUVSimple_16(src_width, src_height, clip_width, clip_height, src_stride,
+ dst_stride, src, dst, x, dx, y, dy);
+ return 0;
+}
+
+// Scale a 16 bit UV image.
+LIBYUV_API
+int UVScale_16(const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 ||
+ src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
+ dst_height <= 0) {
+ return -1;
+ }
+ return ScaleUV_16(src_uv, src_stride_uv, src_width, src_height, dst_uv,
+ dst_stride_uv, dst_width, dst_height, 0, 0, dst_width,
+ dst_height, filtering);
+}
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/libs/libyuv/source/scale_wasm.cc b/libs/libyuv/source/scale_wasm.cc
new file mode 100644
index 0000000000..62e613ea3b
--- /dev/null
+++ b/libs/libyuv/source/scale_wasm.cc
@@ -0,0 +1,713 @@
+/*
+ * Copyright 2026 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/row.h"
+#include "libyuv/scale_row.h"
+
+#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
+#include <wasm_simd128.h>
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+// Arbitrary i8x16.shuffle masks are rematerialized on every use by V8, so
+// shuffles that do not map to a single instruction use swizzle with masks
+// loaded before the loop. Mask indices of 128 produce 0.
+#ifdef __wasm_relaxed_simd__
+#define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle
+#else
+#define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle
+#endif
+
+// Load a constant vector. The empty asm hides the address so the compiler
+// keeps the vector in a local instead of sinking a v128.const into the loop.
+static inline v128_t LoadConst(const uint8_t* p) {
+ asm("" : "+r"(p));
+ return wasm_v128_load(p);
+}
+
+#ifdef HAS_SCALEROWDOWN2_WASMSIMD
+// Point sample 32 pixels to 16, keeping the odd pixels.
+void ScaleRowDown2_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_u16x8_shr(wasm_v128_load(src_ptr), 8);
+ v128_t b = wasm_u16x8_shr(wasm_v128_load(src_ptr + 16), 8);
+ wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b));
+ src_ptr += 32;
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+
+// Average pairs of pixels: (s0 + s1 + 1) >> 1.
+void ScaleRowDown2Linear_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t zero = wasm_i16x8_const_splat(0);
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr));
+ v128_t b = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16));
+ a = wasm_u16x8_avgr(a, zero);
+ b = wasm_u16x8_avgr(b, zero);
+ wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b));
+ src_ptr += 32;
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+
+// Average 2x2 boxes: (s0 + s1 + t0 + t1 + 2) >> 2.
+void ScaleRowDown2Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* t = src_ptr + src_stride;
+ const v128_t two = wasm_i16x8_const_splat(2);
+ while (dst_width > 0) {
+ v128_t a = wasm_i16x8_add(
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr)),
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t)));
+ v128_t b = wasm_i16x8_add(
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16)),
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t + 16)));
+ a = wasm_u16x8_shr(wasm_i16x8_add(a, two), 2);
+ b = wasm_u16x8_shr(wasm_i16x8_add(b, two), 2);
+ wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b));
+ src_ptr += 32;
+ t += 32;
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+#endif // HAS_SCALEROWDOWN2_WASMSIMD
+
+#ifdef HAS_SCALEROWDOWN4_WASMSIMD
+// Point sample 64 pixels to 16, keeping pixel 2 of every 4. Odd 16 bit
+// lanes then even bytes, which are unzips (uzp on Arm, pack on x86).
+void ScaleRowDown4_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t mask = wasm_i16x8_const_splat(0xff);
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_i16x8_shuffle(wasm_v128_load(src_ptr),
+ wasm_v128_load(src_ptr + 16), 1, 3, 5, 7, 9,
+ 11, 13, 15);
+ v128_t b = wasm_i16x8_shuffle(wasm_v128_load(src_ptr + 32),
+ wasm_v128_load(src_ptr + 48), 1, 3, 5, 7, 9,
+ 11, 13, 15);
+ wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(wasm_v128_and(a, mask),
+ wasm_v128_and(b, mask)));
+ src_ptr += 64;
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+
+// Average 4x4 boxes: (sum + 8) >> 4. 8 pixels per loop.
+void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t eight = wasm_i16x8_const_splat(8);
+ while (dst_width > 0) {
+ v128_t a = wasm_i16x8_const_splat(0);
+ v128_t b = wasm_i16x8_const_splat(0);
+ const uint8_t* s = src_ptr;
+ for (int i = 0; i < 4; ++i) {
+ a = wasm_i16x8_add(a,
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s)));
+ b = wasm_i16x8_add(
+ b, wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s + 16)));
+ s += src_stride;
+ }
+ a = wasm_u16x8_narrow_i32x4(wasm_u32x4_extadd_pairwise_u16x8(a),
+ wasm_u32x4_extadd_pairwise_u16x8(b));
+ a = wasm_u16x8_shr(wasm_i16x8_add(a, eight), 4);
+ wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(a, a), 0);
+ src_ptr += 32;
+ dst_ptr += 8;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEROWDOWN4_WASMSIMD
+
+#ifdef HAS_SCALEROWDOWN34_WASMSIMD
+static const uint8_t kShuf34[8][16] = {
+ // ScaleRowDown34: 16 bytes from a, then b, then 8 bytes from b.
+ {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128},
+ {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 3, 4},
+ {5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128, 128, 128, 128, 128},
+ // Filter34: s0 s1 s3 and s0 s2 s2 of every 4.
+ {0, 1, 3, 3, 4, 5, 7, 7, 8, 9, 11, 11, 12, 13, 15, 15},
+ {0, 2, 2, 3, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, 14, 15},
+ // Store34: bytes 0, 1, 2 of every 4 from a, then b, then 8 from b.
+ {0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128},
+ {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 2, 4},
+ {5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128, 128, 128, 128, 128},
+};
+
+// Point sample 32 pixels to 24, keeping pixels 0, 1 and 3 of every 4.
+void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t m0 = LoadConst(kShuf34[0]);
+ const v128_t m1 = LoadConst(kShuf34[1]);
+ const v128_t m2 = LoadConst(kShuf34[2]);
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_v128_load(src_ptr);
+ v128_t b = wasm_v128_load(src_ptr + 16);
+ wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0),
+ LIBYUV_WASM_SWIZZLE(b, m1)));
+ wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0);
+ src_ptr += 32;
+ dst_ptr += 24;
+ dst_width -= 24;
+ }
+}
+
+// Rounding-down average: (a + b) >> 1.
+static inline v128_t AvgFloor(v128_t a, v128_t b) {
+ return wasm_i8x16_sub(
+ wasm_u8x16_avgr(a, b),
+ wasm_v128_and(wasm_v128_xor(a, b), wasm_i8x16_const_splat(1)));
+}
+
+// Horizontal 4 to 3 filter of 16 pixels. Result bytes 4n+0..2 are
+// (3 * s0 + s1 + 2) >> 2, (s1 + s2 + 1) >> 1, (s2 + 3 * s3 + 2) >> 2
+// using (3 * a + b + 2) >> 2 == avgr(a, (a + b) >> 1). Byte 4n+3 is unused.
+static inline v128_t Filter34(v128_t s, v128_t mp, v128_t ma) {
+ v128_t p = LIBYUV_WASM_SWIZZLE(s, mp); // s0 s1 s3
+ v128_t a = LIBYUV_WASM_SWIZZLE(s, ma); // s0 s2 s2
+ v128_t b = wasm_u32x4_shr(s, 8); // s1 s2 s3
+ return wasm_u8x16_avgr(p, AvgFloor(a, b));
+}
+
+// Filter rows 1 and 2 together, 1 : 1
+void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* t = src_ptr + src_stride;
+ const v128_t mp = LoadConst(kShuf34[3]);
+ const v128_t ma = LoadConst(kShuf34[4]);
+ const v128_t m0 = LoadConst(kShuf34[5]);
+ const v128_t m1 = LoadConst(kShuf34[6]);
+ const v128_t m2 = LoadConst(kShuf34[7]);
+ while (dst_width > 0) {
+ v128_t a = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr), mp, ma),
+ Filter34(wasm_v128_load(t), mp, ma));
+ v128_t b = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr + 16), mp, ma),
+ Filter34(wasm_v128_load(t + 16), mp, ma));
+ wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0),
+ LIBYUV_WASM_SWIZZLE(b, m1)));
+ wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0);
+ src_ptr += 32;
+ t += 32;
+ dst_ptr += 24;
+ dst_width -= 24;
+ }
+}
+
+// Filter rows 0 and 1 together, 3 : 1
+void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* t = src_ptr + src_stride;
+ const v128_t mp = LoadConst(kShuf34[3]);
+ const v128_t ma = LoadConst(kShuf34[4]);
+ const v128_t m0 = LoadConst(kShuf34[5]);
+ const v128_t m1 = LoadConst(kShuf34[6]);
+ const v128_t m2 = LoadConst(kShuf34[7]);
+ while (dst_width > 0) {
+ v128_t a0 = Filter34(wasm_v128_load(src_ptr), mp, ma);
+ v128_t b0 = Filter34(wasm_v128_load(src_ptr + 16), mp, ma);
+ v128_t a1 = Filter34(wasm_v128_load(t), mp, ma);
+ v128_t b1 = Filter34(wasm_v128_load(t + 16), mp, ma);
+ v128_t a = wasm_u8x16_avgr(a0, AvgFloor(a0, a1));
+ v128_t b = wasm_u8x16_avgr(b0, AvgFloor(b0, b1));
+ wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0),
+ LIBYUV_WASM_SWIZZLE(b, m1)));
+ wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0);
+ src_ptr += 32;
+ t += 32;
+ dst_ptr += 24;
+ dst_width -= 24;
+ }
+}
+#endif // HAS_SCALEROWDOWN34_WASMSIMD
+
+#ifdef HAS_SCALEROWDOWN38_WASMSIMD
+static const uint8_t kShuf38[3][16] = {
+ // ScaleRowDown38: 6 bytes from a, then 6 bytes from b.
+ {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
+ {128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128},
+ // Store38: bytes 0, 1, 3 of every 4.
+ {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128},
+};
+
+// Point sample 32 pixels to 12, keeping pixels 0, 3 and 6 of every 8.
+void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t m0 = LoadConst(kShuf38[0]);
+ const v128_t m1 = LoadConst(kShuf38[1]);
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t d =
+ wasm_v128_or(LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr), m0),
+ LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr + 16), m1));
+ wasm_v128_store64_lane(dst_ptr, d, 0);
+ wasm_v128_store32_lane(dst_ptr + 8, d, 2);
+ src_ptr += 32;
+ dst_ptr += 12;
+ dst_width -= 12;
+ }
+}
+
+// Sums of columns 0-2, 3-5 and 6-7 of 8 columns of u16 are scaled by the
+// i16 factors in lanes 0, 3 and 6 of k. Returns the high 16 bits of the
+// products in 32 bit lanes 0, 1 and 3.
+static inline v128_t Sum38(v128_t s, v128_t k) {
+ const v128_t zero = wasm_i16x8_const_splat(0);
+ v128_t s1 = wasm_i8x16_shuffle(s, zero, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
+ 13, 14, 15, 16, 17);
+ v128_t s2 = wasm_i8x16_shuffle(s, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+ 15, 16, 17, 18, 19);
+ s = wasm_i16x8_add(wasm_i16x8_add(s, s1), s2);
+ return wasm_u32x4_shr(wasm_i32x4_dot_i16x8(s, k), 16);
+}
+
+// Pack 12 results from Sum38 and store them.
+static inline void Store38(uint8_t* dst_ptr,
+ v128_t m,
+ v128_t a,
+ v128_t b,
+ v128_t c,
+ v128_t d) {
+ v128_t r = wasm_u8x16_narrow_i16x8(wasm_u16x8_narrow_i32x4(a, b),
+ wasm_u16x8_narrow_i32x4(c, d));
+ r = LIBYUV_WASM_SWIZZLE(r, m);
+ wasm_v128_store64_lane(dst_ptr, r, 0);
+ wasm_v128_store32_lane(dst_ptr + 8, r, 2);
+}
+
+// 8x3 -> 3x1
+void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t k =
+ wasm_i16x8_const(65536 / 9, 0, 0, 65536 / 9, 0, 0, 65536 / 6, 0);
+ const v128_t m = LoadConst(kShuf38[2]);
+ while (dst_width > 0) {
+ v128_t s[4];
+ for (int i = 0; i < 4; ++i) {
+ const uint8_t* p = src_ptr + i * 8;
+ s[i] = wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_load8x8(p),
+ wasm_u16x8_load8x8(p + src_stride)),
+ wasm_u16x8_load8x8(p + src_stride * 2));
+ }
+ Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k),
+ Sum38(s[3], k));
+ src_ptr += 32;
+ dst_ptr += 12;
+ dst_width -= 12;
+ }
+}
+
+// 8x2 -> 3x1
+void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const v128_t k =
+ wasm_i16x8_const(65536 / 6, 0, 0, 65536 / 6, 0, 0, 65536 / 4, 0);
+ const v128_t m = LoadConst(kShuf38[2]);
+ while (dst_width > 0) {
+ v128_t s[4];
+ for (int i = 0; i < 4; ++i) {
+ const uint8_t* p = src_ptr + i * 8;
+ s[i] = wasm_i16x8_add(wasm_u16x8_load8x8(p),
+ wasm_u16x8_load8x8(p + src_stride));
+ }
+ Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k),
+ Sum38(s[3], k));
+ src_ptr += 32;
+ dst_ptr += 12;
+ dst_width -= 12;
+ }
+}
+#endif // HAS_SCALEROWDOWN38_WASMSIMD
+
+#ifdef HAS_SCALEFILTERCOLS_WASMSIMD
+// Bilinear column filtering with 7 bit fractions, matching ScaleFilterCols_C:
+// a + ((f * (b - a) + 64) >> 7). 8 pixels per loop. The fraction
+// (x >> 9) & 0x7f only uses the low 16 bits of x, so x is tracked in 16 bit
+// lanes.
+void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ uint32_t ux = (uint32_t)x;
+ const uint32_t udx = (uint32_t)dx;
+ v128_t xv = wasm_i16x8_make((int16_t)ux, (int16_t)(ux + udx),
+ (int16_t)(ux + udx * 2), (int16_t)(ux + udx * 3),
+ (int16_t)(ux + udx * 4), (int16_t)(ux + udx * 5),
+ (int16_t)(ux + udx * 6), (int16_t)(ux + udx * 7));
+ const v128_t dx8 = wasm_i16x8_splat((int16_t)(udx * 8));
+ const v128_t maskff = wasm_i16x8_const_splat(0xff);
+ const v128_t round = wasm_i16x8_const_splat(64);
+ while (dst_width > 0) {
+ v128_t p = wasm_i16x8_const_splat(0);
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 0);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 1);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 2);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 3);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 4);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 5);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 6);
+ ux += udx;
+ p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 7);
+ ux += udx;
+ v128_t f = wasm_u16x8_shr(xv, 9);
+ xv = wasm_i16x8_add(xv, dx8);
+ v128_t a = wasm_v128_and(p, maskff);
+ v128_t d = wasm_i16x8_sub(wasm_u16x8_shr(p, 8), a);
+ d = wasm_i16x8_shr(wasm_i16x8_add(wasm_i16x8_mul(d, f), round), 7);
+ d = wasm_i16x8_add(d, a);
+ wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(d, d), 0);
+ dst_ptr += 8;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEFILTERCOLS_WASMSIMD
+
+#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD) || \
+ defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD)
+// Average 2x2 boxes of ARGB pixels. s0 s1 hold 4 pairs of pixels from row
+// 0 and t0 t1 hold the same pairs from row 1. Returns 4 pixels.
+static inline v128_t BoxARGB(v128_t s0, v128_t s1, v128_t t0, v128_t t1) {
+ v128_t se = wasm_i32x4_shuffle(s0, s1, 0, 2, 4, 6);
+ v128_t so = wasm_i32x4_shuffle(s0, s1, 1, 3, 5, 7);
+ v128_t te = wasm_i32x4_shuffle(t0, t1, 0, 2, 4, 6);
+ v128_t to = wasm_i32x4_shuffle(t0, t1, 1, 3, 5, 7);
+ // Interleave even and odd pixels so pairwise adds sum the same channel.
+ v128_t a = wasm_i16x8_add(
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle(
+ se, so, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)),
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle(
+ te, to, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)));
+ v128_t b = wasm_i16x8_add(wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle(
+ se, so, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28,
+ 13, 29, 14, 30, 15, 31)),
+ wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle(
+ te, to, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28,
+ 13, 29, 14, 30, 15, 31)));
+ a = wasm_u16x8_shr(wasm_i16x8_add(a, wasm_i16x8_const_splat(2)), 2);
+ b = wasm_u16x8_shr(wasm_i16x8_add(b, wasm_i16x8_const_splat(2)), 2);
+ return wasm_u8x16_narrow_i16x8(a, b);
+}
+#endif
+
+#ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD
+// Point sample 8 ARGB pixels to 4, keeping the odd pixels.
+void ScaleARGBRowDown2_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_v128_load(src_argb);
+ v128_t b = wasm_v128_load(src_argb + 16);
+ wasm_v128_store(dst_argb, wasm_i32x4_shuffle(a, b, 1, 3, 5, 7));
+ src_argb += 32;
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+
+// Average pairs of ARGB pixels.
+void ScaleARGBRowDown2Linear_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t a = wasm_v128_load(src_argb);
+ v128_t b = wasm_v128_load(src_argb + 16);
+ wasm_v128_store(dst_argb,
+ wasm_u8x16_avgr(wasm_i32x4_shuffle(a, b, 0, 2, 4, 6),
+ wasm_i32x4_shuffle(a, b, 1, 3, 5, 7)));
+ src_argb += 32;
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+
+// Average 2x2 boxes of ARGB pixels.
+void ScaleARGBRowDown2Box_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const uint8_t* t = src_argb + src_stride;
+ while (dst_width > 0) {
+ wasm_v128_store(
+ dst_argb,
+ BoxARGB(wasm_v128_load(src_argb), wasm_v128_load(src_argb + 16),
+ wasm_v128_load(t), wasm_v128_load(t + 16)));
+ src_argb += 32;
+ t += 32;
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+#endif // HAS_SCALEARGBROWDOWN2_WASMSIMD
+
+#ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD
+// Point sample ARGB pixels with an even step. 4 pixels per loop.
+void ScaleARGBRowDownEven_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ (void)src_stride;
+ while (dst_width > 0) {
+ v128_t v = wasm_v128_load32_zero(src_argb);
+ v = wasm_v128_load32_lane(src_argb + step, v, 1);
+ v = wasm_v128_load32_lane(src_argb + step * 2, v, 2);
+ v = wasm_v128_load32_lane(src_argb + step * 3, v, 3);
+ wasm_v128_store(dst_argb, v);
+ src_argb += step * 4;
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+
+// Average 2x2 boxes of ARGB pixels with an even step. 4 pixels per loop.
+void ScaleARGBRowDownEvenBox_WASMSIMD(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ while (dst_width > 0) {
+ const uint8_t* t = src_argb + src_stride;
+ v128_t s0 = wasm_v128_load64_zero(src_argb);
+ v128_t t0 = wasm_v128_load64_zero(t);
+ s0 = wasm_v128_load64_lane(src_argb + step, s0, 1);
+ t0 = wasm_v128_load64_lane(t + step, t0, 1);
+ v128_t s1 = wasm_v128_load64_zero(src_argb + step * 2);
+ v128_t t1 = wasm_v128_load64_zero(t + step * 2);
+ s1 = wasm_v128_load64_lane(src_argb + step * 3, s1, 1);
+ t1 = wasm_v128_load64_lane(t + step * 3, t1, 1);
+ wasm_v128_store(dst_argb, BoxARGB(s0, s1, t0, t1));
+ src_argb += step * 4;
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+#endif // HAS_SCALEARGBROWDOWNEVEN_WASMSIMD
+
+#ifdef HAS_SCALEARGBCOLS_WASMSIMD
+// Point sample ARGB columns. 4 pixels per loop.
+void ScaleARGBCols_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ uint32_t ux = (uint32_t)x;
+ const uint32_t udx = (uint32_t)dx;
+ while (dst_width > 0) {
+ v128_t v = wasm_v128_load32_zero(src_argb + ((int)ux >> 16) * 4);
+ ux += udx;
+ v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 1);
+ ux += udx;
+ v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 2);
+ ux += udx;
+ v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 3);
+ ux += udx;
+ wasm_v128_store(dst_argb, v);
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+#endif // HAS_SCALEARGBCOLS_WASMSIMD
+
+#ifdef HAS_SCALEARGBFILTERCOLS_WASMSIMD
+// Bilinear ARGB column filtering with 7 bit fractions, matching
+// ScaleARGBFilterCols_C: (a * (127 - f) + b * f) >> 7. 4 pixels per loop.
+// The fraction (x >> 9) & 0x7f only uses the low 16 bits of x, so x is
+// tracked in 16 bit lanes, 4 per pixel to match the channels.
+void ScaleARGBFilterCols_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ uint32_t ux = (uint32_t)x;
+ const uint32_t udx = (uint32_t)dx;
+ int16_t x0 = (int16_t)ux;
+ int16_t x1 = (int16_t)(ux + udx);
+ int16_t x2 = (int16_t)(ux + udx * 2);
+ int16_t x3 = (int16_t)(ux + udx * 3);
+ v128_t x01 = wasm_i16x8_make(x0, x0, x0, x0, x1, x1, x1, x1);
+ v128_t x23 = wasm_i16x8_make(x2, x2, x2, x2, x3, x3, x3, x3);
+ const v128_t dx4 = wasm_i16x8_splat((int16_t)(udx * 4));
+ const v128_t k7f = wasm_i16x8_const_splat(0x7f);
+ while (dst_width > 0) {
+ // Load pixel pairs (a, b) for 4 pixels.
+ v128_t p0 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4);
+ ux += udx;
+ p0 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p0, 1);
+ ux += udx;
+ v128_t p1 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4);
+ ux += udx;
+ p1 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p1, 1);
+ ux += udx;
+ v128_t a = wasm_i32x4_shuffle(p0, p1, 0, 2, 4, 6); // a0 a1 a2 a3
+ v128_t b = wasm_i32x4_shuffle(p0, p1, 1, 3, 5, 7); // b0 b1 b2 b3
+ v128_t f01 = wasm_u16x8_shr(x01, 9);
+ v128_t f23 = wasm_u16x8_shr(x23, 9);
+ x01 = wasm_i16x8_add(x01, dx4);
+ x23 = wasm_i16x8_add(x23, dx4);
+ v128_t r0 = wasm_i16x8_add(
+ wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(a), wasm_v128_xor(f01, k7f)),
+ wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(b), f01));
+ v128_t r1 =
+ wasm_i16x8_add(wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(a),
+ wasm_v128_xor(f23, k7f)),
+ wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(b), f23));
+ wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(r0, 7),
+ wasm_u16x8_shr(r1, 7)));
+ dst_argb += 16;
+ dst_width -= 4;
+ }
+}
+#endif // HAS_SCALEARGBFILTERCOLS_WASMSIMD
+
+#ifdef HAS_SCALEARGBCOLSUP2_WASMSIMD
+// Point sample ARGB up by 2x. 8 pixels per loop.
+void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ (void)x;
+ (void)dx;
+ while (dst_width > 0) {
+ // V8 lowers an Arm zip of a vector with itself to several instructions,
+ // so zip with copies of the pixels loaded by load64_splat instead.
+ v128_t s = wasm_v128_load(src_argb);
+ v128_t t = wasm_v128_load64_splat(src_argb);
+ v128_t u = wasm_v128_load64_splat(src_argb + 8);
+ wasm_v128_store(dst_argb, wasm_i32x4_shuffle(s, t, 0, 4, 1, 5));
+ wasm_v128_store(dst_argb + 16, wasm_i32x4_shuffle(s, u, 2, 6, 3, 7));
+ src_argb += 16;
+ dst_argb += 32;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEARGBCOLSUP2_WASMSIMD
+
+#ifdef HAS_SCALEADDROW_WASMSIMD
+// Reads 16 bytes and accumulates to 16 shorts at a time.
+void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ while (src_width > 0) {
+ v128_t s = wasm_v128_load(src_ptr);
+ v128_t d0 = wasm_v128_load(dst_ptr);
+ v128_t d1 = wasm_v128_load(dst_ptr + 8);
+ d0 = wasm_i16x8_add(d0, wasm_u16x8_extend_low_u8x16(s));
+ d1 = wasm_i16x8_add(d1, wasm_u16x8_extend_high_u8x16(s));
+ wasm_v128_store(dst_ptr, d0);
+ wasm_v128_store(dst_ptr + 8, d1);
+ src_ptr += 16;
+ dst_ptr += 16;
+ src_width -= 16;
+ }
+}
+#endif // HAS_SCALEADDROW_WASMSIMD
+
+#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
+// Reads 24 UV pixels (48 uint16_t) and writes 8 UV pixels (16 bytes).
+void ScaleUVAddCols3_WASMSIMD(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ const v128_t scale = wasm_i16x8_splat((int16_t)scaleval);
+ while (dst_width > 0) {
+ v128_t s0 = wasm_v128_load(src_uv);
+ v128_t s1 = wasm_v128_load(src_uv + 8);
+ v128_t s2 = wasm_v128_load(src_uv + 16);
+ v128_t s3 = wasm_v128_load(src_uv + 24);
+ v128_t s4 = wasm_v128_load(src_uv + 32);
+ v128_t s5 = wasm_v128_load(src_uv + 40);
+ src_uv += 48;
+
+ // Group 0..3 of 3 UV pairs from s0, s1, s2 (dwords 0..11)
+ v128_t a0 = wasm_i32x4_shuffle(s0, s1, 0, 3, 6, 1);
+ v128_t b0 = wasm_i32x4_shuffle(s0, s1, 1, 4, 7, 2);
+ v128_t c0 = wasm_i32x4_shuffle(s0, s1, 2, 5, 0, 3);
+ a0 = wasm_i32x4_shuffle(a0, s2, 0, 1, 2, 5);
+ b0 = wasm_i32x4_shuffle(b0, s2, 0, 1, 2, 6);
+ c0 = wasm_i32x4_shuffle(c0, s2, 0, 1, 4, 7);
+ v128_t sum0 = wasm_i16x8_add(wasm_i16x8_add(a0, b0), c0);
+
+ // Group 4..7 of 3 UV pairs from s3, s4, s5 (dwords 0..11)
+ v128_t a1 = wasm_i32x4_shuffle(s3, s4, 0, 3, 6, 1);
+ v128_t b1 = wasm_i32x4_shuffle(s3, s4, 1, 4, 7, 2);
+ v128_t c1 = wasm_i32x4_shuffle(s3, s4, 2, 5, 0, 3);
+ a1 = wasm_i32x4_shuffle(a1, s5, 0, 1, 2, 5);
+ b1 = wasm_i32x4_shuffle(b1, s5, 0, 1, 2, 6);
+ c1 = wasm_i32x4_shuffle(c1, s5, 0, 1, 4, 7);
+ v128_t sum1 = wasm_i16x8_add(wasm_i16x8_add(a1, b1), c1);
+
+ // Multiply unsigned 16-bit sums by scaleval and shift right by 16.
+ v128_t lo0 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum0, scale), 16);
+ v128_t hi0 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum0, scale), 16);
+ v128_t lo1 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum1, scale), 16);
+ v128_t hi1 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum1, scale), 16);
+ v128_t p0 = wasm_i16x8_narrow_i32x4(lo0, hi0);
+ v128_t p1 = wasm_i16x8_narrow_i32x4(lo1, hi1);
+ wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(p0, p1));
+ dst_uv += 16;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEUVADDCOLS3_WASMSIMD
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
diff --git a/libs/libyuv/source/scale_win.cc b/libs/libyuv/source/scale_win.cc
index c5fc86f3e9..93921c794a 100644
--- a/libs/libyuv/source/scale_win.cc
+++ b/libs/libyuv/source/scale_win.cc
@@ -11,83 +11,30 @@
#include "libyuv/row.h"
#include "libyuv/scale_row.h"
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+#include <emmintrin.h>
+#include <immintrin.h>
+#include <string.h>
+#include <tmmintrin.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+#include <intrin.h>
+#endif
+#endif
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-// This module is for 32 bit Visual C x86 and clangcl
-#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER)
-
-// Offsets for source bytes 0 to 9
-static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
- 128, 128, 128, 128, 128, 128, 128, 128};
-
-// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12.
-static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12,
- 128, 128, 128, 128, 128, 128, 128, 128};
-
-// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.
-static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15,
- 128, 128, 128, 128, 128, 128, 128, 128};
-
-// Offsets for source bytes 0 to 10
-static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
-
-// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13.
-static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7,
- 8, 9, 9, 10, 10, 11, 12, 13};
-
-// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.
-static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10,
- 10, 11, 12, 13, 13, 14, 14, 15};
-
-// Coefficients for source bytes 0 to 10
-static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2};
-
-// Coefficients for source bytes 10 to 21
-static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1};
-
-// Coefficients for source bytes 21 to 31
-static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3};
-
-// Coefficients for source bytes 21 to 31
-static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2};
-
-static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128,
- 128, 128, 128, 128, 128, 128, 128, 128};
-
-static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3,
- 6, 8, 11, 14, 128, 128, 128, 128};
-
-// Arrange words 0,3,6 into 0,1,2
-static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128,
- 128, 128, 128, 128, 128, 128, 128, 128};
-
-// Arrange words 0,3,6 into 3,4,5
-static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1,
- 6, 7, 12, 13, 128, 128, 128, 128};
-
-// Scaling values for boxes of 3x3 and 2x3
-static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9,
- 65536 / 9, 65536 / 6, 0, 0};
-
-// Arrange first value for pixels 0,1,2,3,4,5
-static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128,
- 11, 128, 14, 128, 128, 128, 128, 128};
-
-// Arrange second value for pixels 0,1,2,3,4,5
-static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128,
- 12, 128, 15, 128, 128, 128, 128, 128};
-
-// Arrange third value for pixels 0,1,2,3,4,5
-static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128,
- 13, 128, 128, 128, 128, 128, 128, 128};
-
-// Scaling values for boxes of 3x2 and 2x2
-static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,
- 65536 / 3, 65536 / 2, 0, 0};
+// This module is for 32 bit Visual C x86
+#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
+ !defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN)
+#ifdef HAS_SCALEROWDOWN2_SSSE3
// Reads 32 pixels, throws half away and writes 16 pixels.
__declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -103,7 +50,7 @@ __declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
movdqu xmm0, [eax]
movdqu xmm1, [eax + 16]
lea eax, [eax + 32]
- psrlw xmm0, 8 // isolate odd pixels.
+ psrlw xmm0, 8 // isolate odd pixels.
psrlw xmm1, 8
packuswb xmm0, xmm1
movdqu [edx], xmm0
@@ -137,7 +84,7 @@ __declspec(naked) void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
lea eax, [eax + 32]
pmaddubsw xmm0, xmm4 // horizontal add
pmaddubsw xmm1, xmm4
- pavgw xmm0, xmm5 // (x + 1) / 2
+ pavgw xmm0, xmm5 // (x + 1) / 2
pavgw xmm1, xmm5
packuswb xmm0, xmm1
movdqu [edx], xmm0
@@ -192,122 +139,9 @@ __declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
ret
}
}
+#endif // HAS_SCALEROWDOWN2_SSSE3
-#ifdef HAS_SCALEROWDOWN2_AVX2
-// Reads 64 pixels, throws half away and writes 32 pixels.
-__declspec(naked) void ScaleRowDown2_AVX2(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- // src_stride ignored
- mov edx, [esp + 12] // dst_ptr
- mov ecx, [esp + 16] // dst_width
-
- wloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpsrlw ymm0, ymm0, 8 // isolate odd pixels.
- vpsrlw ymm1, ymm1, 8
- vpackuswb ymm0, ymm0, ymm1
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg wloop
-
- vzeroupper
- ret
- }
-}
-
-// Blends 64x1 rectangle to 32x1.
-__declspec(naked) void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- // src_stride
- mov edx, [esp + 12] // dst_ptr
- mov ecx, [esp + 16] // dst_width
-
- vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b
- vpsrlw ymm4, ymm4, 15
- vpackuswb ymm4, ymm4, ymm4
- vpxor ymm5, ymm5, ymm5 // constant 0
-
- wloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpmaddubsw ymm0, ymm0, ymm4 // horizontal add
- vpmaddubsw ymm1, ymm1, ymm4
- vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2
- vpavgw ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg wloop
-
- vzeroupper
- ret
- }
-}
-
-// For rounding, average = (sum + 2) / 4
-// becomes average((sum >> 1), 0)
-// Blends 64x2 rectangle to 32x1.
-__declspec(naked) void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_ptr
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_ptr
- mov ecx, [esp + 4 + 16] // dst_width
-
- vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b
- vpsrlw ymm4, ymm4, 15
- vpackuswb ymm4, ymm4, ymm4
- vpxor ymm5, ymm5, ymm5 // constant 0
-
- wloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + esi]
- vmovdqu ymm3, [eax + esi + 32]
- lea eax, [eax + 64]
- vpmaddubsw ymm0, ymm0, ymm4 // horizontal add
- vpmaddubsw ymm1, ymm1, ymm4
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- vpaddw ymm0, ymm0, ymm2 // vertical add
- vpaddw ymm1, ymm1, ymm3
- vpsrlw ymm0, ymm0, 1 // (x + 2) / 4 = (x / 2 + 1) / 2
- vpsrlw ymm1, ymm1, 1
- vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2
- vpavgw ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vmovdqu [edx], ymm0
- lea edx, [edx + 32]
- sub ecx, 32
- jg wloop
-
- pop esi
- vzeroupper
- ret
- }
-}
-#endif // HAS_SCALEROWDOWN2_AVX2
-
+#ifdef HAS_SCALEROWDOWN4_SSSE3
// Point samples 32 pixels to 8 pixels.
__declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -318,7 +152,7 @@ __declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,
// src_stride ignored
mov edx, [esp + 12] // dst_ptr
mov ecx, [esp + 16] // dst_width
- pcmpeqb xmm5, xmm5 // generate mask 0x00ff0000
+ pcmpeqb xmm5, xmm5 // generate mask 0x00ff0000
psrld xmm5, 24
pslld xmm5, 16
@@ -397,993 +231,2389 @@ __declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,
ret
}
}
+#endif // HAS_SCALEROWDOWN4_SSSE3
-#ifdef HAS_SCALEROWDOWN4_AVX2
-// Point samples 64 pixels to 16 pixels.
-__declspec(naked) void ScaleRowDown4_AVX2(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
+#ifdef HAS_FIXEDDIV_X86
+// Divide num by div and return as 16.16 fixed point result.
+__declspec(naked) int FixedDiv_X86(int num, int div) {
__asm {
- mov eax, [esp + 4] // src_ptr
- // src_stride ignored
- mov edx, [esp + 12] // dst_ptr
- mov ecx, [esp + 16] // dst_width
- vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff0000
- vpsrld ymm5, ymm5, 24
- vpslld ymm5, ymm5, 16
-
- wloop:
- vmovdqu ymm0, [eax]
- vmovdqu ymm1, [eax + 32]
- lea eax, [eax + 64]
- vpand ymm0, ymm0, ymm5
- vpand ymm1, ymm1, ymm5
- vpackuswb ymm0, ymm0, ymm1
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vpsrlw ymm0, ymm0, 8
- vpackuswb ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vmovdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg wloop
-
- vzeroupper
+ mov eax, [esp + 4] // num
+ cdq // extend num to 64 bits
+ shld edx, eax, 16 // 32.16
+ shl eax, 16
+ idiv dword ptr [esp + 8]
ret
}
}
-// Blends 64x4 rectangle to 16x1.
-__declspec(naked) void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
+// Divide num by div and return as 16.16 fixed point result.
+__declspec(naked) int FixedDiv1_X86(int num, int div) {
__asm {
- push esi
- push edi
- mov eax, [esp + 8 + 4] // src_ptr
- mov esi, [esp + 8 + 8] // src_stride
- mov edx, [esp + 8 + 12] // dst_ptr
- mov ecx, [esp + 8 + 16] // dst_width
- lea edi, [esi + esi * 2] // src_stride * 3
- vpcmpeqb ymm4, ymm4, ymm4 // constant 0x0101
- vpsrlw ymm4, ymm4, 15
- vpsllw ymm5, ymm4, 3 // constant 0x0008
- vpackuswb ymm4, ymm4, ymm4
-
- wloop:
- vmovdqu ymm0, [eax] // average rows
- vmovdqu ymm1, [eax + 32]
- vmovdqu ymm2, [eax + esi]
- vmovdqu ymm3, [eax + esi + 32]
- vpmaddubsw ymm0, ymm0, ymm4 // horizontal add
- vpmaddubsw ymm1, ymm1, ymm4
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- vpaddw ymm0, ymm0, ymm2 // vertical add rows 0, 1
- vpaddw ymm1, ymm1, ymm3
- vmovdqu ymm2, [eax + esi * 2]
- vmovdqu ymm3, [eax + esi * 2 + 32]
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- vpaddw ymm0, ymm0, ymm2 // add row 2
- vpaddw ymm1, ymm1, ymm3
- vmovdqu ymm2, [eax + edi]
- vmovdqu ymm3, [eax + edi + 32]
- lea eax, [eax + 64]
- vpmaddubsw ymm2, ymm2, ymm4
- vpmaddubsw ymm3, ymm3, ymm4
- vpaddw ymm0, ymm0, ymm2 // add row 3
- vpaddw ymm1, ymm1, ymm3
- vphaddw ymm0, ymm0, ymm1 // mutates
- vpermq ymm0, ymm0, 0xd8 // unmutate vphaddw
- vpaddw ymm0, ymm0, ymm5 // + 8 for round
- vpsrlw ymm0, ymm0, 4 // /32 for average of 4 * 4
- vpackuswb ymm0, ymm0, ymm0
- vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb
- vmovdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 16
- jg wloop
-
- pop edi
- pop esi
- vzeroupper
+ mov eax, [esp + 4] // num
+ mov ecx, [esp + 8] // denom
+ cdq // extend num to 64 bits
+ shld edx, eax, 16 // 32.16
+ shl eax, 16
+ sub eax, 0x00010001
+ sbb edx, 0
+ sub ecx, 1
+ idiv ecx
ret
}
}
-#endif // HAS_SCALEROWDOWN4_AVX2
-
-// Point samples 32 pixels to 24 pixels.
-// Produces three 8 byte values. For each 8 bytes, 16 bytes are read.
-// Then shuffled to do the scaling.
+#endif // HAS_FIXEDDIV_X86
+#endif // !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86)
+
+// The following are available for Visual C 32/64 bit, and for clang with
+// LIBYUV_ENABLE_ROWWIN:
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
+ defined(_M_IX86)) && \
+ ((defined(_MSC_VER) && !defined(__clang__)) || \
+ defined(LIBYUV_ENABLE_ROWWIN))
+
+#if defined(__clang__) || defined(__GNUC__)
+#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
+#define LIBYUV_TARGET_AVX512BW \
+ __attribute__((target("avx512bw,avx512vl,avx512f")))
+#define LIBYUV_TARGET_AVX512VBMI \
+ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
+#else
+#define LIBYUV_TARGET_AVX2
+#define LIBYUV_TARGET_AVX512BW
+#define LIBYUV_TARGET_AVX512VBMI
+#endif // defined(__clang__) || defined(__GNUC__)
+
+static inline uint16_t LoadU16(const void* p) {
+ uint16_t v;
+ memcpy(&v, p, sizeof(v));
+ return v;
+}
-__declspec(naked) void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- // src_stride ignored
- mov edx, [esp + 12] // dst_ptr
- mov ecx, [esp + 16] // dst_width
- movdqa xmm3, xmmword ptr kShuf0
- movdqa xmm4, xmmword ptr kShuf1
- movdqa xmm5, xmmword ptr kShuf2
+static inline uint32_t LoadU32(const void* p) {
+ uint32_t v;
+ memcpy(&v, p, sizeof(v));
+ return v;
+}
- wloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- movdqa xmm2, xmm1
- palignr xmm1, xmm0, 8
- pshufb xmm0, xmm3
- pshufb xmm1, xmm4
- pshufb xmm2, xmm5
- movq qword ptr [edx], xmm0
- movq qword ptr [edx + 8], xmm1
- movq qword ptr [edx + 16], xmm2
- lea edx, [edx + 24]
- sub ecx, 24
- jg wloop
+static inline uint64_t LoadU64(const void* p) {
+ uint64_t v;
+ memcpy(&v, p, sizeof(v));
+ return v;
+}
- ret
- }
+static inline void StoreU32(void* p, uint32_t v) {
+ memcpy(p, &v, sizeof(v));
}
-// Blends 32x2 rectangle to 24x1
-// Produces three 8 byte values. For each 8 bytes, 16 bytes are read.
-// Then shuffled to do the scaling.
-
-// Register usage:
-// xmm0 src_row 0
-// xmm1 src_row 1
-// xmm2 shuf 0
-// xmm3 shuf 1
-// xmm4 shuf 2
-// xmm5 madd 0
-// xmm6 madd 1
-// xmm7 kRound34
-
-// Note that movdqa+palign may be better than movdqu.
-__declspec(naked) void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_ptr
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_ptr
- mov ecx, [esp + 4 + 16] // dst_width
- movdqa xmm2, xmmword ptr kShuf01
- movdqa xmm3, xmmword ptr kShuf11
- movdqa xmm4, xmmword ptr kShuf21
- movdqa xmm5, xmmword ptr kMadd01
- movdqa xmm6, xmmword ptr kMadd11
- movdqa xmm7, xmmword ptr kRound34
+#if defined(HAS_FIXEDDIV_X86) && \
+ !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \
+ !defined(LIBYUV_ENABLE_ROWWIN))
+int FixedDiv_X86(int num, int div) {
+ return (int)(((int64_t)num * 65536) / div);
+}
+#endif
- wloop:
- movdqu xmm0, [eax] // pixels 0..7
- movdqu xmm1, [eax + esi]
- pavgb xmm0, xmm1
- pshufb xmm0, xmm2
- pmaddubsw xmm0, xmm5
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx], xmm0
- movdqu xmm0, [eax + 8] // pixels 8..15
- movdqu xmm1, [eax + esi + 8]
- pavgb xmm0, xmm1
- pshufb xmm0, xmm3
- pmaddubsw xmm0, xmm6
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx + 8], xmm0
- movdqu xmm0, [eax + 16] // pixels 16..23
- movdqu xmm1, [eax + esi + 16]
- lea eax, [eax + 32]
- pavgb xmm0, xmm1
- pshufb xmm0, xmm4
- movdqa xmm1, xmmword ptr kMadd21
- pmaddubsw xmm0, xmm1
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx + 16], xmm0
- lea edx, [edx + 24]
- sub ecx, 24
- jg wloop
+#if defined(HAS_FIXEDDIV1_X86) && \
+ !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \
+ !defined(LIBYUV_ENABLE_ROWWIN))
+int FixedDiv1_X86(int num, int div) {
+ return (int)((((int64_t)num * 65536) - 0x00010001) / (div - 1));
+}
+#endif
- pop esi
- ret
+#ifdef HAS_SCALEROWDOWN2_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_srli_epi16(
+ _mm256_loadu_si256((const __m256i*)src_ptr), 8);
+ __m256i s1 = _mm256_srli_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), 8);
+ src_ptr += 64;
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 32;
+ dst_width -= 32;
}
}
-// Note that movdqa+palign may be better than movdqu.
-__declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_ptr
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_ptr
- mov ecx, [esp + 4 + 16] // dst_width
- movdqa xmm2, xmmword ptr kShuf01
- movdqa xmm3, xmmword ptr kShuf11
- movdqa xmm4, xmmword ptr kShuf21
- movdqa xmm5, xmmword ptr kMadd01
- movdqa xmm6, xmmword ptr kMadd11
- movdqa xmm7, xmmword ptr kRound34
-
- wloop:
- movdqu xmm0, [eax] // pixels 0..7
- movdqu xmm1, [eax + esi]
- pavgb xmm1, xmm0
- pavgb xmm0, xmm1
- pshufb xmm0, xmm2
- pmaddubsw xmm0, xmm5
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx], xmm0
- movdqu xmm0, [eax + 8] // pixels 8..15
- movdqu xmm1, [eax + esi + 8]
- pavgb xmm1, xmm0
- pavgb xmm0, xmm1
- pshufb xmm0, xmm3
- pmaddubsw xmm0, xmm6
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx + 8], xmm0
- movdqu xmm0, [eax + 16] // pixels 16..23
- movdqu xmm1, [eax + esi + 16]
- lea eax, [eax + 32]
- pavgb xmm1, xmm0
- pavgb xmm0, xmm1
- pshufb xmm0, xmm4
- movdqa xmm1, xmmword ptr kMadd21
- pmaddubsw xmm0, xmm1
- paddsw xmm0, xmm7
- psrlw xmm0, 2
- packuswb xmm0, xmm0
- movq qword ptr [edx + 16], xmm0
- lea edx, [edx+24]
- sub ecx, 24
- jg wloop
-
- pop esi
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i zero = _mm256_setzero_si256();
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)src_ptr), ones);
+ __m256i s1 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
+ src_ptr += 64;
+ s0 = _mm256_avg_epu16(s0, zero);
+ s1 = _mm256_avg_epu16(s1, zero);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 32;
+ dst_width -= 32;
}
}
-// 3/8 point sampler
-
-// Scale 32 pixels to 12
-__declspec(naked) void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- // src_stride ignored
- mov edx, [esp + 12] // dst_ptr
- mov ecx, [esp + 16] // dst_width
- movdqa xmm4, xmmword ptr kShuf38a
- movdqa xmm5, xmmword ptr kShuf38b
-
- xloop:
- movdqu xmm0, [eax] // 16 pixels -> 0,1,2,3,4,5
- movdqu xmm1, [eax + 16] // 16 pixels -> 6,7,8,9,10,11
- lea eax, [eax + 32]
- pshufb xmm0, xmm4
- pshufb xmm1, xmm5
- paddusb xmm0, xmm1
-
- movq qword ptr [edx], xmm0 // write 12 pixels
- movhlps xmm1, xmm0
- movd [edx + 8], xmm1
- lea edx, [edx + 12]
- sub ecx, 12
- jg xloop
-
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i zero = _mm256_setzero_si256();
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)src_ptr), ones);
+ __m256i s1 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
+ __m256i s2 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), ones);
+ __m256i s3 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), ones);
+ src_ptr += 64;
+ s0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s0, s2), 1), zero);
+ s1 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s1, s3), 1), zero);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 32;
+ dst_width -= 32;
}
}
+#endif // HAS_SCALEROWDOWN2_AVX2
-// Scale 16x3 pixels to 6x1 with interpolation
-__declspec(naked) void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_ptr
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_ptr
- mov ecx, [esp + 4 + 16] // dst_width
- movdqa xmm2, xmmword ptr kShufAc
- movdqa xmm3, xmmword ptr kShufAc3
- movdqa xmm4, xmmword ptr kScaleAc33
- pxor xmm5, xmm5
-
- xloop:
- movdqu xmm0, [eax] // sum up 3 rows into xmm0/1
- movdqu xmm6, [eax + esi]
- movhlps xmm1, xmm0
- movhlps xmm7, xmm6
- punpcklbw xmm0, xmm5
- punpcklbw xmm1, xmm5
- punpcklbw xmm6, xmm5
- punpcklbw xmm7, xmm5
- paddusw xmm0, xmm6
- paddusw xmm1, xmm7
- movdqu xmm6, [eax + esi * 2]
- lea eax, [eax + 16]
- movhlps xmm7, xmm6
- punpcklbw xmm6, xmm5
- punpcklbw xmm7, xmm5
- paddusw xmm0, xmm6
- paddusw xmm1, xmm7
-
- movdqa xmm6, xmm0 // 8 pixels -> 0,1,2 of xmm6
- psrldq xmm0, 2
- paddusw xmm6, xmm0
- psrldq xmm0, 2
- paddusw xmm6, xmm0
- pshufb xmm6, xmm2
-
- movdqa xmm7, xmm1 // 8 pixels -> 3,4,5 of xmm6
- psrldq xmm1, 2
- paddusw xmm7, xmm1
- psrldq xmm1, 2
- paddusw xmm7, xmm1
- pshufb xmm7, xmm3
- paddusw xmm6, xmm7
-
- pmulhuw xmm6, xmm4 // divide by 9,9,6, 9,9,6
- packuswb xmm6, xmm6
-
- movd [edx], xmm6 // write 6 pixels
- psrlq xmm6, 16
- movd [edx + 2], xmm6
- lea edx, [edx + 6]
- sub ecx, 6
- jg xloop
+#ifdef HAS_SCALEROWDOWN2_16_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_srli_epi32(
+ _mm256_loadu_si256((const __m256i*)src_ptr), 16);
+ __m256i s1 = _mm256_srli_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 16)), 16);
+ src_ptr += 32;
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
- pop esi
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i mask = _mm256_set1_epi32(0x0000ffff);
+ while (dst_width > 0) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16));
+ src_ptr += 32;
+ __m256i e = _mm256_packus_epi32(_mm256_and_si256(r0, mask),
+ _mm256_and_si256(r1, mask));
+ __m256i o = _mm256_packus_epi32(_mm256_srli_epi32(r0, 16),
+ _mm256_srli_epi32(r1, 16));
+ __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu16(e, o), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 16;
+ dst_width -= 16;
}
}
-// Scale 16x2 pixels to 6x1 with interpolation
-__declspec(naked) void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_ptr
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_ptr
- mov ecx, [esp + 4 + 16] // dst_width
- movdqa xmm2, xmmword ptr kShufAb0
- movdqa xmm3, xmmword ptr kShufAb1
- movdqa xmm4, xmmword ptr kShufAb2
- movdqa xmm5, xmmword ptr kScaleAb2
-
- xloop:
- movdqu xmm0, [eax] // average 2 rows into xmm0
- movdqu xmm1, [eax + esi]
- lea eax, [eax + 16]
- pavgb xmm0, xmm1
-
- movdqa xmm1, xmm0 // 16 pixels -> 0,1,2,3,4,5 of xmm1
- pshufb xmm1, xmm2
- movdqa xmm6, xmm0
- pshufb xmm6, xmm3
- paddusw xmm1, xmm6
- pshufb xmm0, xmm4
- paddusw xmm1, xmm0
-
- pmulhuw xmm1, xmm5 // divide by 3,3,2, 3,3,2
- packuswb xmm1, xmm1
-
- movd [edx], xmm1 // write 6 pixels
- psrlq xmm1, 16
- movd [edx + 2], xmm1
- lea edx, [edx + 6]
- sub ecx, 6
- jg xloop
+LIBYUV_TARGET_AVX2
+void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const __m256i mask = _mm256_set1_epi32(0x0000ffff);
+ const __m256i round2 = _mm256_set1_epi32(2);
+ while (dst_width > 0) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16));
+ __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
+ __m256i r3 =
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 16));
+ src_ptr += 32;
+ __m256i s0 = _mm256_add_epi32(
+ _mm256_add_epi32(_mm256_and_si256(r0, mask), _mm256_srli_epi32(r0, 16)),
+ _mm256_add_epi32(_mm256_and_si256(r2, mask),
+ _mm256_srli_epi32(r2, 16)));
+ __m256i s1 = _mm256_add_epi32(
+ _mm256_add_epi32(_mm256_and_si256(r1, mask), _mm256_srli_epi32(r1, 16)),
+ _mm256_add_epi32(_mm256_and_si256(r3, mask),
+ _mm256_srli_epi32(r3, 16)));
+ s0 = _mm256_srli_epi32(_mm256_add_epi32(s0, round2), 2);
+ s1 = _mm256_srli_epi32(_mm256_add_epi32(s1, round2), 2);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+#endif // HAS_SCALEROWDOWN2_16_AVX2
+
+#ifdef HAS_SCALEROWDOWN2_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width >= 32) {
+ __m512i s =
+ _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), 8);
+ src_ptr += 64;
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+ if (dst_width > 0) {
+ __mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
+ __m512i s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), 8);
+ _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
+ }
+}
- pop esi
- ret
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m512i ones = _mm512_set1_epi8(1);
+ const __m512i zero = _mm512_setzero_si512();
+ while (dst_width >= 32) {
+ __m512i s = _mm512_avg_epu16(
+ _mm512_maddubs_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), ones),
+ zero);
+ src_ptr += 64;
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+ if (dst_width > 0) {
+ __mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
+ __m512i s = _mm512_avg_epu16(
+ _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones),
+ zero);
+ _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
}
}
-// Reads 16 bytes and accumulates to 16 shorts at a time.
-__declspec(naked) void ScaleAddRow_SSE2(const uint8_t* src_ptr,
- uint16_t* dst_ptr,
- int src_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- mov edx, [esp + 8] // dst_ptr
- mov ecx, [esp + 12] // src_width
- pxor xmm5, xmm5
-
- // sum rows
- xloop:
- movdqu xmm3, [eax] // read 16 bytes
- lea eax, [eax + 16]
- movdqu xmm0, [edx] // read 16 words from destination
- movdqu xmm1, [edx + 16]
- movdqa xmm2, xmm3
- punpcklbw xmm2, xmm5
- punpckhbw xmm3, xmm5
- paddusw xmm0, xmm2 // sum 16 words
- paddusw xmm1, xmm3
- movdqu [edx], xmm0 // write 16 words to destination
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 16
- jg xloop
- ret
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m512i ones = _mm512_set1_epi8(1);
+ const __m512i zero = _mm512_setzero_si512();
+ while (dst_width >= 32) {
+ __m512i s0 = _mm512_maddubs_epi16(
+ _mm512_loadu_si512((const __m512i*)src_ptr), ones);
+ __m512i s1 = _mm512_maddubs_epi16(
+ _mm512_loadu_si512((const __m512i*)(src_ptr + src_stride)), ones);
+ src_ptr += 64;
+ __m512i s =
+ _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero);
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+ if (dst_width > 0) {
+ __mmask32 k = (__mmask32)((1ULL << dst_width) - 1);
+ __m512i s0 =
+ _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones);
+ __m512i s1 = _mm512_maddubs_epi16(
+ _mm512_maskz_loadu_epi16(k, src_ptr + src_stride), ones);
+ __m512i s =
+ _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero);
+ _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s));
}
}
-#ifdef HAS_SCALEADDROW_AVX2
-// Reads 32 bytes and accumulates to 32 shorts at a time.
-__declspec(naked) void ScaleAddRow_AVX2(const uint8_t* src_ptr,
- uint16_t* dst_ptr,
- int src_width) {
- __asm {
- mov eax, [esp + 4] // src_ptr
- mov edx, [esp + 8] // dst_ptr
- mov ecx, [esp + 12] // src_width
- vpxor ymm5, ymm5, ymm5
-
- // sum rows
- xloop:
- vmovdqu ymm3, [eax] // read 32 bytes
- lea eax, [eax + 32]
- vpermq ymm3, ymm3, 0xd8 // unmutate for vpunpck
- vpunpcklbw ymm2, ymm3, ymm5
- vpunpckhbw ymm3, ymm3, ymm5
- vpaddusw ymm0, ymm2, [edx] // sum 16 words
- vpaddusw ymm1, ymm3, [edx + 32]
- vmovdqu [edx], ymm0 // write 32 words to destination
- vmovdqu [edx + 32], ymm1
- lea edx, [edx + 64]
- sub ecx, 32
- jg xloop
-
- vzeroupper
- ret
+LIBYUV_TARGET_AVX512BW
+void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ dst_width -= 1;
+ if (dst_width > 0) {
+ ScaleRowDown2Box_AVX512BW(src_ptr, src_stride, dst_ptr, dst_width);
+ src_ptr += dst_width * 2;
+ dst_ptr += dst_width;
}
+ dst_ptr[0] = (uint8_t)((src_ptr[0] + src_ptr[src_stride] + 1) >> 1);
}
-#endif // HAS_SCALEADDROW_AVX2
+#endif // HAS_SCALEROWDOWN2_AVX512BW
-// Constant for making pixels signed to avoid pmaddubsw
-// saturation.
-static const uvec8 kFsub80 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
- 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
-
-// Constant for making pixels unsigned and adding .5 for rounding.
-static const uvec16 kFadd40 = {0x4040, 0x4040, 0x4040, 0x4040,
- 0x4040, 0x4040, 0x4040, 0x4040};
-
-// Bilinear column filtering. SSSE3 version.
-__declspec(naked) void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- __asm {
- push ebx
- push esi
- push edi
- mov edi, [esp + 12 + 4] // dst_ptr
- mov esi, [esp + 12 + 8] // src_ptr
- mov ecx, [esp + 12 + 12] // dst_width
- movd xmm2, [esp + 12 + 16] // x
- movd xmm3, [esp + 12 + 20] // dx
- mov eax, 0x04040000 // shuffle to line up fractions with pixel.
- movd xmm5, eax
- pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction.
- psrlw xmm6, 9
- pcmpeqb xmm7, xmm7 // generate 0x0001
- psrlw xmm7, 15
- pextrw eax, xmm2, 1 // get x0 integer. preroll
- sub ecx, 2
- jl xloop29
-
- movdqa xmm0, xmm2 // x1 = x0 + dx
- paddd xmm0, xmm3
- punpckldq xmm2, xmm0 // x0 x1
- punpckldq xmm3, xmm3 // dx dx
- paddd xmm3, xmm3 // dx * 2, dx * 2
- pextrw edx, xmm2, 3 // get x1 integer. preroll
-
- // 2 Pixel loop.
- xloop2:
- movdqa xmm1, xmm2 // x0, x1 fractions.
- paddd xmm2, xmm3 // x += dx
- movzx ebx, word ptr [esi + eax] // 2 source x0 pixels
- movd xmm0, ebx
- psrlw xmm1, 9 // 7 bit fractions.
- movzx ebx, word ptr [esi + edx] // 2 source x1 pixels
- movd xmm4, ebx
- pshufb xmm1, xmm5 // 0011
- punpcklwd xmm0, xmm4
- psubb xmm0, xmmword ptr kFsub80 // make pixels signed.
- pxor xmm1, xmm6 // 0..7f and 7f..0
- paddusb xmm1, xmm7 // +1 so 0..7f and 80..1
- pmaddubsw xmm1, xmm0 // 16 bit, 2 pixels.
- pextrw eax, xmm2, 1 // get x0 integer. next iteration.
- pextrw edx, xmm2, 3 // get x1 integer. next iteration.
- paddw xmm1, xmmword ptr kFadd40 // make pixels unsigned and round.
- psrlw xmm1, 7 // 8.7 fixed point to low 8 bits.
- packuswb xmm1, xmm1 // 8 bits, 2 pixels.
- movd ebx, xmm1
- mov [edi], bx
- lea edi, [edi + 2]
- sub ecx, 2 // 2 pixels
- jge xloop2
-
- xloop29:
- add ecx, 2 - 1
- jl xloop99
-
- // 1 pixel remainder
- movzx ebx, word ptr [esi + eax] // 2 source x0 pixels
- movd xmm0, ebx
- psrlw xmm2, 9 // 7 bit fractions.
- pshufb xmm2, xmm5 // 0011
- psubb xmm0, xmmword ptr kFsub80 // make pixels signed.
- pxor xmm2, xmm6 // 0..7f and 7f..0
- paddusb xmm2, xmm7 // +1 so 0..7f and 80..1
- pmaddubsw xmm2, xmm0 // 16 bit
- paddw xmm2, xmmword ptr kFadd40 // make pixels unsigned and round.
- psrlw xmm2, 7 // 8.7 fixed point to low 8 bits.
- packuswb xmm2, xmm2 // 8 bits
- movd ebx, xmm2
- mov [edi], bl
-
- xloop99:
+#ifdef HAS_SCALEROWDOWN4_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowDown4_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i mask = _mm256_set1_epi32(0x00ff0000);
+ while (dst_width > 0) {
+ __m256i s0 =
+ _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_ptr), mask);
+ __m256i s1 = _mm256_and_si256(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), mask);
+ src_ptr += 64;
+ __m256i t = _mm256_srli_epi16(
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8), 8);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
- pop edi
- pop esi
- pop ebx
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i round8 = _mm256_set1_epi16(8);
+ const ptrdiff_t stridex3 = src_stride * 3;
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)src_ptr), ones);
+ __m256i s1 = _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones);
+ s0 = _mm256_add_epi16(
+ s0, _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)),
+ ones));
+ s1 = _mm256_add_epi16(
+ s1, _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)),
+ ones));
+ s0 = _mm256_add_epi16(
+ s0, _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)),
+ ones));
+ s1 = _mm256_add_epi16(
+ s1,
+ _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2 + 32)),
+ ones));
+ s0 = _mm256_add_epi16(
+ s0,
+ _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3)), ones));
+ s1 = _mm256_add_epi16(
+ s1, _mm256_maddubs_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3 + 32)),
+ ones));
+ src_ptr += 64;
+ __m256i t = _mm256_permute4x64_epi64(_mm256_hadd_epi16(s0, s1), 0xd8);
+ t = _mm256_srli_epi16(_mm256_add_epi16(t, round8), 4);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
+ dst_ptr += 16;
+ dst_width -= 16;
}
}
+#endif // HAS_SCALEROWDOWN4_AVX2
-// Reads 16 pixels, duplicates them and writes 32 pixels.
-__declspec(naked) void ScaleColsUp2_SSE2(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- int dst_width,
- int x,
- int dx) {
- __asm {
- mov edx, [esp + 4] // dst_ptr
- mov eax, [esp + 8] // src_ptr
- mov ecx, [esp + 12] // dst_width
+#ifdef HAS_SCALEROWDOWN34_AVX2
+static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10};
+static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7,
+ 8, 9, 9, 10, 10, 11, 12, 13};
+static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10,
+ 10, 11, 12, 13, 13, 14, 14, 15};
+static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2};
+static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1};
+static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3};
- wloop:
- movdqu xmm0, [eax]
- lea eax, [eax + 16]
- movdqa xmm1, xmm0
- punpcklbw xmm0, xmm0
- punpckhbw xmm1, xmm1
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 32
- jg wloop
+LIBYUV_TARGET_AVX2
+void ScaleRowDown34_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i shuf0 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf0));
+ const __m256i shuf1 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf1));
+ const __m256i shuf2 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf2));
+ while (dst_width >= 48) {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32));
+ src_ptr += 64;
+ __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
+ __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
+ __m256i mid = _mm256_alignr_epi8(hi, lo, 8);
+ __m256i d0 = _mm256_shuffle_epi8(lo, shuf0);
+ __m256i d1 = _mm256_shuffle_epi8(mid, shuf1);
+ __m256i d2 = _mm256_shuffle_epi8(hi, shuf2);
+ __m256i d01 = _mm256_unpacklo_epi64(d0, d1);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(d2));
+ _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(d2, 1));
+ dst_ptr += 48;
+ dst_width -= 48;
+ }
+ if (dst_width > 0) {
+ __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m128i lo = _mm256_castsi256_si128(s0);
+ __m128i hi = _mm256_extracti128_si256(s0, 1);
+ __m128i mid = _mm_alignr_epi8(hi, lo, 8);
+ __m128i d0 = _mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf0));
+ __m128i d1 = _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf1));
+ __m128i d2 = _mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf2));
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(d0, d1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), d2);
+ }
+}
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i shuf01 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01));
+ const __m256i shuf11 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11));
+ const __m256i shuf21 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21));
+ const __m256i madd01 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01));
+ const __m256i madd11 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11));
+ const __m256i madd21 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21));
+ const __m256i round2 = _mm256_set1_epi16(2);
+ while (dst_width >= 48) {
+ __m256i s0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_ptr),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
+ __m256i s1 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)));
+ src_ptr += 64;
+ __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
+ __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
+ __m256i mid = _mm256_alignr_epi8(hi, lo, 8);
+ __m256i p0 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01),
+ round2),
+ 2);
+ __m256i p1 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11),
+ round2),
+ 2);
+ __m256i p2 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21),
+ round2),
+ 2);
+ p0 = _mm256_packus_epi16(p0, p0);
+ p1 = _mm256_packus_epi16(p1, p1);
+ p2 = _mm256_packus_epi16(p2, p2);
+ __m256i d01 = _mm256_unpacklo_epi64(p0, p1);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2));
+ _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1));
+ dst_ptr += 48;
+ dst_width -= 48;
+ }
+ if (dst_width > 0) {
+ __m256i s0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_ptr),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
+ __m128i lo = _mm256_castsi256_si128(s0);
+ __m128i hi = _mm256_extracti128_si256(s0, 1);
+ __m128i mid = _mm_alignr_epi8(hi, lo, 8);
+ __m128i r2 = _mm256_castsi256_si128(round2);
+ __m128i p0 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)),
+ _mm256_castsi256_si128(madd01)),
+ r2),
+ 2);
+ __m128i p1 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(
+ _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)),
+ _mm256_castsi256_si128(madd11)),
+ r2),
+ 2);
+ __m128i p2 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)),
+ _mm256_castsi256_si128(madd21)),
+ r2),
+ 2);
+ p0 = _mm_packus_epi16(p0, p0);
+ p1 = _mm_packus_epi16(p1, p1);
+ p2 = _mm_packus_epi16(p2, p2);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2);
}
}
-// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6)
-__declspec(naked) void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- // src_stride ignored
- mov edx, [esp + 12] // dst_argb
- mov ecx, [esp + 16] // dst_width
+LIBYUV_TARGET_AVX2
+void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i shuf01 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01));
+ const __m256i shuf11 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11));
+ const __m256i shuf21 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21));
+ const __m256i madd01 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01));
+ const __m256i madd11 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11));
+ const __m256i madd21 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21));
+ const __m256i round2 = _mm256_set1_epi16(2);
+ while (dst_width >= 48) {
+ __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
+ __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0));
+ __m256i a1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32));
+ __m256i b1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32));
+ __m256i s1 = _mm256_avg_epu8(a1, _mm256_avg_epu8(a1, b1));
+ src_ptr += 64;
+ __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20);
+ __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31);
+ __m256i mid = _mm256_alignr_epi8(hi, lo, 8);
+ __m256i p0 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01),
+ round2),
+ 2);
+ __m256i p1 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11),
+ round2),
+ 2);
+ __m256i p2 = _mm256_srli_epi16(
+ _mm256_adds_epi16(
+ _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21),
+ round2),
+ 2);
+ p0 = _mm256_packus_epi16(p0, p0);
+ p1 = _mm256_packus_epi16(p1, p1);
+ p2 = _mm256_packus_epi16(p2, p2);
+ __m256i d01 = _mm256_unpacklo_epi64(p0, p1);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2));
+ _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1));
+ dst_ptr += 48;
+ dst_width -= 48;
+ }
+ if (dst_width > 0) {
+ __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
+ __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0));
+ __m128i lo = _mm256_castsi256_si128(s0);
+ __m128i hi = _mm256_extracti128_si256(s0, 1);
+ __m128i mid = _mm_alignr_epi8(hi, lo, 8);
+ __m128i r2 = _mm256_castsi256_si128(round2);
+ __m128i p0 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)),
+ _mm256_castsi256_si128(madd01)),
+ r2),
+ 2);
+ __m128i p1 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(
+ _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)),
+ _mm256_castsi256_si128(madd11)),
+ r2),
+ 2);
+ __m128i p2 = _mm_srli_epi16(
+ _mm_adds_epi16(
+ _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)),
+ _mm256_castsi256_si128(madd21)),
+ r2),
+ 2);
+ p0 = _mm_packus_epi16(p0, p0);
+ p1 = _mm_packus_epi16(p1, p1);
+ p2 = _mm_packus_epi16(p2, p2);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2);
+ }
+}
+#endif // HAS_SCALEROWDOWN34_AVX2
- wloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- shufps xmm0, xmm1, 0xdd
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg wloop
+#ifdef HAS_SCALEROWDOWN38_AVX2
+static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3,
+ 6, 8, 11, 14, 128, 128, 128, 128};
+static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1,
+ 6, 7, 12, 13, 128, 128, 128, 128};
+static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9,
+ 65536 / 9, 65536 / 6, 0, 0};
+static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128,
+ 11, 128, 14, 128, 128, 128, 128, 128};
+static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128,
+ 12, 128, 15, 128, 128, 128, 128, 128};
+static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128,
+ 13, 128, 128, 128, 128, 128, 128, 128};
+static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,
+ 65536 / 3, 65536 / 2, 0, 0};
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown38_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i shuf38 = _mm256_inserti128_si256(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf38a)),
+ _mm_loadu_si128((const __m128i*)&kShuf38b), 1);
+ while (dst_width >= 24) {
+ __m256i s0 =
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38);
+ __m256i s1 = _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), shuf38);
+ src_ptr += 64;
+ __m128i d0 =
+ _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1));
+ __m128i d1 =
+ _mm_or_si128(_mm256_castsi256_si128(s1), _mm256_extracti128_si256(s1, 1));
+ _mm_storel_epi64((__m128i*)dst_ptr, d0);
+ StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8)));
+ _mm_storel_epi64((__m128i*)(dst_ptr + 12), d1);
+ StoreU32(dst_ptr + 20, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d1, 8)));
+ dst_ptr += 24;
+ dst_width -= 24;
+ }
+ if (dst_width > 0) {
+ __m256i s0 =
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38);
+ __m128i d0 =
+ _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1));
+ _mm_storel_epi64((__m128i*)dst_ptr, d0);
+ StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8)));
}
}
-// Blends 8x1 rectangle to 4x1.
-__declspec(naked) void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- __asm {
- mov eax, [esp + 4] // src_argb
- // src_stride ignored
- mov edx, [esp + 12] // dst_argb
- mov ecx, [esp + 16] // dst_width
-
- wloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- lea eax, [eax + 32]
- movdqa xmm2, xmm0
- shufps xmm0, xmm1, 0x88 // even pixels
- shufps xmm2, xmm1, 0xdd // odd pixels
- pavgb xmm0, xmm2
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg wloop
+LIBYUV_TARGET_AVX2
+void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i shuf_ab0 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb0));
+ const __m256i shuf_ab1 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb1));
+ const __m256i shuf_ab2 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb2));
+ const __m256i scale_ab2 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAb2));
+ const __m256i zero = _mm256_setzero_si256();
+ while (dst_width > 0) {
+ __m256i r = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_ptr),
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)));
+ src_ptr += 32;
+ __m256i t = _mm256_adds_epu16(
+ _mm256_adds_epu16(_mm256_shuffle_epi8(r, shuf_ab0),
+ _mm256_shuffle_epi8(r, shuf_ab1)),
+ _mm256_shuffle_epi8(r, shuf_ab2));
+ t = _mm256_packus_epi16(_mm256_mulhi_epu16(t, scale_ab2), zero);
+ __m128i d = _mm_or_si128(_mm256_castsi256_si128(t),
+ _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6));
+ _mm_storel_epi64((__m128i*)dst_ptr, d);
+ StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8)));
+ dst_ptr += 12;
+ dst_width -= 12;
+ }
+}
- ret
+LIBYUV_TARGET_AVX2
+void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i shuf_ac =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc));
+ const __m256i shuf_ac3 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc3));
+ const __m256i scale_ac33 =
+ _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAc33));
+ const __m256i zero = _mm256_setzero_si256();
+ while (dst_width > 0) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride));
+ __m256i r2 =
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2));
+ src_ptr += 32;
+ __m256i s0 =
+ _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpacklo_epi8(r0, zero),
+ _mm256_unpacklo_epi8(r1, zero)),
+ _mm256_unpacklo_epi8(r2, zero));
+ __m256i s1 =
+ _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpackhi_epi8(r0, zero),
+ _mm256_unpackhi_epi8(r1, zero)),
+ _mm256_unpackhi_epi8(r2, zero));
+ __m256i t0 = _mm256_shuffle_epi8(
+ _mm256_adds_epu16(_mm256_adds_epu16(s0, _mm256_srli_si256(s0, 2)),
+ _mm256_srli_si256(s0, 4)),
+ shuf_ac);
+ __m256i t1 = _mm256_shuffle_epi8(
+ _mm256_adds_epu16(_mm256_adds_epu16(s1, _mm256_srli_si256(s1, 2)),
+ _mm256_srli_si256(s1, 4)),
+ shuf_ac3);
+ __m256i t = _mm256_packus_epi16(
+ _mm256_mulhi_epu16(_mm256_adds_epu16(t0, t1), scale_ac33), zero);
+ __m128i d = _mm_or_si128(_mm256_castsi256_si128(t),
+ _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6));
+ _mm_storel_epi64((__m128i*)dst_ptr, d);
+ StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8)));
+ dst_ptr += 12;
+ dst_width -= 12;
}
}
+#endif // HAS_SCALEROWDOWN38_AVX2
-// Blends 8x2 rectangle to 4x1.
-__declspec(naked) void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- uint8_t* dst_argb,
- int dst_width) {
- __asm {
- push esi
- mov eax, [esp + 4 + 4] // src_argb
- mov esi, [esp + 4 + 8] // src_stride
- mov edx, [esp + 4 + 12] // dst_argb
- mov ecx, [esp + 4 + 16] // dst_width
+#ifdef HAS_SCALEADDROW_SSE2
+// Reads 16 bytes and accumulates to 16 shorts at a time.
+void ScaleAddRow_SSE2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ const __m128i zero = _mm_setzero_si128();
+ while (src_width > 0) {
+ __m128i s = _mm_loadu_si128((const __m128i*)src_ptr);
+ src_ptr += 16;
+ __m128i d0 = _mm_loadu_si128((const __m128i*)dst_ptr);
+ __m128i d1 = _mm_loadu_si128((const __m128i*)(dst_ptr + 8));
+ d0 = _mm_adds_epu16(d0, _mm_unpacklo_epi8(s, zero));
+ d1 = _mm_adds_epu16(d1, _mm_unpackhi_epi8(s, zero));
+ _mm_storeu_si128((__m128i*)dst_ptr, d0);
+ _mm_storeu_si128((__m128i*)(dst_ptr + 8), d1);
+ dst_ptr += 16;
+ src_width -= 16;
+ }
+}
+#endif // HAS_SCALEADDROW_SSE2
- wloop:
- movdqu xmm0, [eax]
- movdqu xmm1, [eax + 16]
- movdqu xmm2, [eax + esi]
- movdqu xmm3, [eax + esi + 16]
- lea eax, [eax + 32]
- pavgb xmm0, xmm2 // average rows
- pavgb xmm1, xmm3
- movdqa xmm2, xmm0 // average columns (8 to 4 pixels)
- shufps xmm0, xmm1, 0x88 // even pixels
- shufps xmm2, xmm1, 0xdd // odd pixels
- pavgb xmm0, xmm2
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg wloop
+#ifdef HAS_SCALEADDROW_AVX2
+// Reads 32 bytes and accumulates to 32 shorts at a time.
+LIBYUV_TARGET_AVX2
+void ScaleAddRow_AVX2(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ while (src_width > 0) {
+ __m256i s0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_ptr));
+ __m256i s1 =
+ _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_ptr + 16)));
+ src_ptr += 32;
+ __m256i d0 = _mm256_loadu_si256((const __m256i*)dst_ptr);
+ __m256i d1 = _mm256_loadu_si256((const __m256i*)(dst_ptr + 16));
+ d0 = _mm256_adds_epu16(d0, s0);
+ d1 = _mm256_adds_epu16(d1, s1);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d0);
+ _mm256_storeu_si256((__m256i*)(dst_ptr + 16), d1);
+ dst_ptr += 32;
+ src_width -= 32;
+ }
+}
+#endif // HAS_SCALEADDROW_AVX2
- pop esi
- ret
+#ifdef HAS_SCALEADDROW_AVX512BW
+// Reads 64 bytes and accumulates to 64 shorts at a time, with masked tail.
+LIBYUV_TARGET_AVX512BW
+void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
+ uint16_t* dst_ptr,
+ int src_width) {
+ while (src_width >= 64) {
+ __m512i s0 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr));
+ __m512i s1 = _mm512_cvtepu8_epi16(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 32)));
+ src_ptr += 64;
+ __m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr);
+ __m512i d1 = _mm512_loadu_si512((const __m512i*)(dst_ptr + 32));
+ d0 = _mm512_adds_epu16(d0, s0);
+ d1 = _mm512_adds_epu16(d1, s1);
+ _mm512_storeu_si512((__m512i*)dst_ptr, d0);
+ _mm512_storeu_si512((__m512i*)(dst_ptr + 32), d1);
+ dst_ptr += 64;
+ src_width -= 64;
+ }
+ if (src_width >= 32) {
+ __m512i s0 =
+ _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr));
+ src_ptr += 32;
+ __m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr);
+ d0 = _mm512_adds_epu16(d0, s0);
+ _mm512_storeu_si512((__m512i*)dst_ptr, d0);
+ dst_ptr += 32;
+ src_width -= 32;
+ }
+ if (src_width > 0) {
+ __mmask32 k = (__mmask32)((1ULL << src_width) - 1);
+ __m512i s0 = _mm512_maskz_cvtepu8_epi16(
+ k, _mm256_maskz_loadu_epi8(k, (const void*)src_ptr));
+ __m512i d0 = _mm512_maskz_loadu_epi16(k, (const void*)dst_ptr);
+ d0 = _mm512_maskz_adds_epu16(k, d0, s0);
+ _mm512_mask_storeu_epi16((void*)dst_ptr, k, d0);
}
}
+#endif // HAS_SCALEADDROW_AVX512BW
-// Reads 4 pixels at a time.
-__declspec(naked) void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- __asm {
- push ebx
- push edi
- mov eax, [esp + 8 + 4] // src_argb
- // src_stride ignored
- mov ebx, [esp + 8 + 12] // src_stepx
- mov edx, [esp + 8 + 16] // dst_argb
- mov ecx, [esp + 8 + 20] // dst_width
- lea ebx, [ebx * 4]
- lea edi, [ebx + ebx * 2]
+#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW)
+static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15};
+#endif
- wloop:
- movd xmm0, [eax]
- movd xmm1, [eax + ebx]
- punpckldq xmm0, xmm1
- movd xmm2, [eax + ebx * 2]
- movd xmm3, [eax + edi]
- lea eax, [eax + ebx * 4]
- punpckldq xmm2, xmm3
- punpcklqdq xmm0, xmm2
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg wloop
+#ifdef HAS_SCALEADDCOLS1_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleAddCols1_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr) {
+ int boxwidth = dx >> 16;
+ if (boxwidth < 1) {
+ boxwidth = 1;
+ }
+ int scaleval = 65536 / (boxwidth * boxheight);
+ src_ptr += (x >> 16);
+ const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff);
+ const __m512i vscale = _mm512_set1_epi32(scaleval);
+ if (boxwidth <= 4) {
+ __m512i seq =
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15));
+ seq = _mm512_or_si512(seq, _mm512_slli_epi32(seq, 16));
+ __m512i vbw = _mm512_set1_epi16((int16_t)boxwidth);
+ __m512i base = _mm512_mullo_epi16(seq, vbw);
+ __m512i c01 = _mm512_set1_epi32(0x00010000);
+ __m512i c23 = _mm512_set1_epi32(0x00030002);
+ __m512i idx01 = _mm512_add_epi16(base, c01);
+ __m512i idx23 = _mm512_add_epi16(base, c23);
+ __mmask32 k2 = _mm512_cmpgt_epu16_mask(vbw, c01);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c23);
+ int span = boxwidth * 16;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __mmask32 k1 = (__mmask32)m64;
+ __mmask32 k4 = (__mmask32)(m64 >> 32);
+ while (dst_width >= 16) {
+ __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr);
+ __m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32);
+ src_ptr += span;
+ __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
+ __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
+ __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
+ _mm512_srli_epi32(p01, 16));
+ __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
+ _mm512_srli_epi32(p23, 16));
+ __m512i sum = _mm512_add_epi32(sum01, sum23);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum));
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+ if (dst_width > 0) {
+ int rem_span = dst_width * boxwidth;
+ uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr);
+ __m512i s1 =
+ _mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32);
+ __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
+ __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
+ __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
+ _mm512_srli_epi32(p01, 16));
+ __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
+ _mm512_srli_epi32(p23, 16));
+ __m512i sum = _mm512_add_epi32(sum01, sum23);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
+ __mmask16 km = (__mmask16)((1U << dst_width) - 1);
+ // Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
+ // because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
+ // to unpoison destination shadow memory.
+ _mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi32_epi8(sum));
+ }
+ } else if (boxwidth <= 8) {
+ __m512i seq =
+ _mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15));
+ seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 16));
+ seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 32));
+ __m512i vbw = _mm512_set1_epi16((int16_t)boxwidth);
+ __m512i base = _mm512_mullo_epi16(seq, vbw);
+ __m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL);
+ __m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL);
+ __m512i idx03 = _mm512_add_epi16(base, c03);
+ __m512i idx47 = _mm512_add_epi16(base, c47);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c47);
+ int span = boxwidth * 8;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __mmask32 k1 = (__mmask32)m64;
+ __mmask32 k4 = (__mmask32)(m64 >> 32);
+ while (dst_width >= 8) {
+ __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr);
+ __m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32);
+ src_ptr += span;
+ __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
+ __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
+ __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
+ _mm512_srli_epi32(p03, 16));
+ __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
+ _mm512_srli_epi32(p47, 16));
+ __m512i sum = _mm512_add_epi32(sum03, sum47);
+ sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
+ _mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum));
+ dst_ptr += 8;
+ dst_width -= 8;
+ }
+ if (dst_width > 0) {
+ int rem_span = dst_width * boxwidth;
+ uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr);
+ __m512i s1 =
+ _mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32);
+ __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
+ __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
+ __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
+ _mm512_srli_epi32(p03, 16));
+ __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
+ _mm512_srli_epi32(p47, 16));
+ __m512i sum = _mm512_add_epi32(sum03, sum47);
+ sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16);
+ __mmask8 km = (__mmask8)((1U << dst_width) - 1);
+ // Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi64_storeu_epi8
+ // because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.qb.mem
+ // to unpoison destination shadow memory.
+ _mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi64_epi8(sum));
+ }
+ } else {
+ int rem = boxwidth & 31;
+ __mmask32 k1 = (__mmask32)((1ULL << rem) - 1);
+ while (dst_width > 0) {
+ __m512i acc = _mm512_setzero_si512();
+ const uint16_t* p = src_ptr;
+ src_ptr += boxwidth;
+ int n = boxwidth;
+ while (n >= 32) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)p);
+ p += 32;
+ acc = _mm512_add_epi32(acc,
+ _mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
+ _mm512_srli_epi32(s, 16)));
+ n -= 32;
+ }
+ if (n > 0) {
+ __m512i s = _mm512_maskz_loadu_epi16(k1, p);
+ acc = _mm512_add_epi32(acc,
+ _mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
+ _mm512_srli_epi32(s, 16)));
+ }
+ uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc);
+ *dst_ptr++ = (uint8_t)((total * (uint32_t)scaleval) >> 16);
+ --dst_width;
+ }
+ }
+}
+#endif // HAS_SCALEADDCOLS1_AVX512BW
+
+#ifdef HAS_SCALEADDCOLS2_AVX512BW
+LIBYUV_TARGET_AVX512BW
+void ScaleAddCols2_AVX512BW(int dst_width,
+ int boxheight,
+ int x,
+ int dx,
+ const uint16_t* src_ptr,
+ uint8_t* dst_ptr) {
+ int minboxwidth = dx >> 16;
+ int scale0 = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight);
+ int scale1 =
+ 65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight);
+ const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff);
+ const __m512i vscale0 = _mm512_set1_epi32(scale0);
+ const __m512i vscale1 = _mm512_set1_epi32(scale1);
+ if (minboxwidth >= 1 && minboxwidth <= 3) {
+ __m512i seq =
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15));
+ __m512i vdx = _mm512_set1_epi32(dx);
+ __m512i step0 = _mm512_mullo_epi32(seq, vdx);
+ __m512i step1 = _mm512_add_epi32(step0, vdx);
+ __m512i c01 = _mm512_set1_epi32(0x00010000);
+ __m512i c23 = _mm512_set1_epi32(0x00030002);
+ __m512i vminbw = _mm512_set1_epi32(minboxwidth);
+ while (dst_width >= 16) {
+ int base_ix = x >> 16;
+ __m512i xfrac = _mm512_set1_epi32(x & 0xffff);
+ x += dx * 16;
+ int span = (x >> 16) - base_ix;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
+ __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
+ src_ptr + base_ix + 32);
+ __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
+ __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
+ __m512i bw = _mm512_sub_epi32(ix1, ix0);
+ __mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw);
+ __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16));
+ __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16));
+ __m512i idx01 = _mm512_add_epi16(ix0_16, c01);
+ __m512i idx23 = _mm512_add_epi16(ix0_16, c23);
+ __mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23);
+ __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
+ __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
+ __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
+ _mm512_srli_epi32(p01, 16));
+ __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
+ _mm512_srli_epi32(p23, 16));
+ __m512i sum = _mm512_add_epi32(sum01, sum23);
+ __m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum));
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+ if (dst_width > 0) {
+ int base_ix = x >> 16;
+ __m512i xfrac = _mm512_set1_epi32(x & 0xffff);
+ int end_x = x + dx * dst_width;
+ int span = (end_x >> 16) - base_ix;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
+ __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
+ src_ptr + base_ix + 32);
+ __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
+ __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
+ __m512i bw = _mm512_sub_epi32(ix1, ix0);
+ __mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw);
+ __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16));
+ __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16));
+ __m512i idx01 = _mm512_add_epi16(ix0_16, c01);
+ __m512i idx23 = _mm512_add_epi16(ix0_16, c23);
+ __mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23);
+ __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1);
+ __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1);
+ __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff),
+ _mm512_srli_epi32(p01, 16));
+ __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff),
+ _mm512_srli_epi32(p23, 16));
+ __m512i sum = _mm512_add_epi32(sum01, sum23);
+ __m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
+ __mmask16 km = (__mmask16)((1U << dst_width) - 1);
+ // Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi32_storeu_epi8
+ // because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.db.mem
+ // to unpoison destination shadow memory.
+ _mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi32_epi8(sum));
+ }
+ } else if (minboxwidth >= 4 && minboxwidth <= 7) {
+ __m512i seq =
+ _mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15));
+ __m512i vdx = _mm512_set1_epi64(dx);
+ __m512i step0 = _mm512_mullo_epi32(seq, vdx);
+ __m512i step1 = _mm512_add_epi64(step0, vdx);
+ __m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL);
+ __m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL);
+ __m512i vminbw = _mm512_set1_epi64(minboxwidth);
+ while (dst_width >= 8) {
+ int base_ix = x >> 16;
+ __m512i xfrac = _mm512_set1_epi64(x & 0xffff);
+ x += dx * 8;
+ int span = (x >> 16) - base_ix;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
+ __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
+ src_ptr + base_ix + 32);
+ __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
+ __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
+ __m512i bw = _mm512_sub_epi32(ix1, ix0);
+ __mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw);
+ __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16));
+ ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32));
+ __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16));
+ bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32));
+ __m512i idx03 = _mm512_add_epi16(ix0_16, c03);
+ __m512i idx47 = _mm512_add_epi16(ix0_16, c47);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47);
+ __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
+ __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
+ __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
+ _mm512_srli_epi32(p03, 16));
+ __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
+ _mm512_srli_epi32(p47, 16));
+ __m512i sum = _mm512_add_epi32(sum03, sum47);
+ sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
+ __m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
+ _mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum));
+ dst_ptr += 8;
+ dst_width -= 8;
+ }
+ if (dst_width > 0) {
+ int base_ix = x >> 16;
+ __m512i xfrac = _mm512_set1_epi64(x & 0xffff);
+ int end_x = x + dx * dst_width;
+ int span = (end_x >> 16) - base_ix;
+ uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix);
+ __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32),
+ src_ptr + base_ix + 32);
+ __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16);
+ __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16);
+ __m512i bw = _mm512_sub_epi32(ix1, ix0);
+ __mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw);
+ __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16));
+ ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32));
+ __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16));
+ bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32));
+ __m512i idx03 = _mm512_add_epi16(ix0_16, c03);
+ __m512i idx47 = _mm512_add_epi16(ix0_16, c47);
+ __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47);
+ __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1);
+ __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1);
+ __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff),
+ _mm512_srli_epi32(p03, 16));
+ __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff),
+ _mm512_srli_epi32(p47, 16));
+ __m512i sum = _mm512_add_epi32(sum03, sum47);
+ sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32));
+ __m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1);
+ sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16);
+ __mmask8 km = (__mmask8)((1U << dst_width) - 1);
+ // Use _mm_mask_storeu_epi8 instead of _mm512_mask_cvtepi64_storeu_epi8
+ // because LLVM MSan does not instrument @llvm.x86.avx512.mask.pmov.qb.mem
+ // to unpoison destination shadow memory.
+ _mm_mask_storeu_epi8(dst_ptr, km, _mm512_cvtepi64_epi8(sum));
+ }
+ } else {
+ while (dst_width > 0) {
+ int ix = x >> 16;
+ x += dx;
+ int bw = (x >> 16) - ix;
+ if (bw < 1) {
+ bw = 1;
+ }
+ int sc = (bw > minboxwidth) ? scale1 : scale0;
+ __m512i acc = _mm512_setzero_si512();
+ const uint16_t* p = src_ptr + ix;
+ int n = bw;
+ while (n >= 32) {
+ __m512i s = _mm512_loadu_si512((const __m512i*)p);
+ p += 32;
+ acc = _mm512_add_epi32(acc,
+ _mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
+ _mm512_srli_epi32(s, 16)));
+ n -= 32;
+ }
+ if (n > 0) {
+ __mmask32 k1 = (__mmask32)((1ULL << n) - 1);
+ __m512i s = _mm512_maskz_loadu_epi16(k1, p);
+ acc = _mm512_add_epi32(acc,
+ _mm512_add_epi32(_mm512_and_si512(s, mask_ffff),
+ _mm512_srli_epi32(s, 16)));
+ }
+ uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc);
+ *dst_ptr++ = (uint8_t)((total * (uint32_t)sc) >> 16);
+ --dst_width;
+ }
+ }
+}
+#endif // HAS_SCALEADDCOLS2_AVX512BW
+
+#ifdef HAS_SCALEUVADDCOLS3_AVX2
+static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
+static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
+static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
+
+// Scale UV columns by 3 box filter.
+// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
+LIBYUV_TARGET_AVX2
+void ScaleUVAddCols3_AVX2(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ const __m256i vscale = _mm256_set1_epi16((int16_t)scaleval);
+ const __m256i perm0 = _mm256_loadu_si256((const __m256i*)&kPermUV3_0);
+ const __m256i perm1 = _mm256_loadu_si256((const __m256i*)&kPermUV3_1);
+ const __m256i perm2 = _mm256_loadu_si256((const __m256i*)&kPermUV3_2);
+ while (dst_width >= 16) {
+ __m256i a0 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)src_uv), perm0);
+ __m256i a1 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0);
+ __m256i a2 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0);
+ __m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0);
+ __m256i c1 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1);
+ __m256i c2 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2);
+ __m256i sum0 = _mm256_mulhi_epu16(
+ _mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale);
+
+ __m256i b0 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 48)), perm0);
+ __m256i b1 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 64)), perm0);
+ __m256i b2 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 80)), perm0);
+ src_uv += 96;
+ __m256i d0 = _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x38), b2, 0xc0);
+ __m256i d1 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0xc0), b2, 0x07), perm1);
+ __m256i d2 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x07), b2, 0x38), perm2);
+ __m256i sum1 = _mm256_mulhi_epu16(
+ _mm256_add_epi16(_mm256_add_epi16(d0, d1), d2), vscale);
+
+ __m256i packed =
+ _mm256_permute4x64_epi64(_mm256_packus_epi16(sum0, sum1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_uv, packed);
+ dst_uv += 32;
+ dst_width -= 16;
+ }
+ while (dst_width > 0) {
+ __m256i a0 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)src_uv), perm0);
+ __m256i a1 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0);
+ __m256i a2 = _mm256_permutevar8x32_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0);
+ src_uv += 48;
+ __m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0);
+ __m256i c1 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1);
+ __m256i c2 = _mm256_permutevar8x32_epi32(
+ _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2);
+ __m256i sum = _mm256_mulhi_epu16(
+ _mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale);
+ __m128i lo = _mm256_castsi256_si128(sum);
+ __m128i hi = _mm256_extracti128_si256(sum, 1);
+ _mm_storeu_si128((__m128i*)dst_uv, _mm_packus_epi16(lo, hi));
+ dst_uv += 16;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEUVADDCOLS3_AVX2
+
+#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
+static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
+ 24, 27, 30, 1, 4, 7, 10, 13};
+static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
+ 25, 28, 31, 2, 5, 8, 11, 14};
+static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
+ 26, 29, 0, 3, 6, 9, 12, 15};
+static const uint8_t kPackEvenBytes[64] = {
+ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
+ 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
+ 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
+ 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
+ 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
+
+// Under MemorySanitizer, use a portable lane-by-lane C loop so shadow
+// propagation relies only on standard C memory/array semantics rather than
+// target-specific LLVM intrinsic modeling. In non-MSan builds, call the native
+// AVX-512 intrinsic directly.
+LIBYUV_TARGET_AVX512VBMI
+static inline __m512i MaskPermutexvarEpi32(__m512i src,
+ __mmask16 k,
+ __m512i idx,
+ __m512i v) {
+#if defined(LIBYUV_MSAN)
+ uint32_t src_arr[16];
+ uint32_t idx_arr[16];
+ uint32_t v_arr[16];
+ uint32_t out_arr[16];
+ _mm512_storeu_si512((__m512i*)src_arr, src);
+ _mm512_storeu_si512((__m512i*)idx_arr, idx);
+ _mm512_storeu_si512((__m512i*)v_arr, v);
+ for (int i = 0; i < 16; ++i) {
+ out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 15] : src_arr[i];
+ }
+ return _mm512_loadu_si512((const __m512i*)out_arr);
+#else
+ return _mm512_mask_permutexvar_epi32(src, k, idx, v);
+#endif
+}
- pop edi
- pop ebx
- ret
+// Scale UV columns by 3 box filter using AVX512VBMI.
+// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
+// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
+LIBYUV_TARGET_AVX512VBMI
+void ScaleUVAddCols3_AVX512VBMI(int dst_width,
+ int scaleval,
+ const uint16_t* src_uv,
+ uint8_t* dst_uv) {
+ const __m512i vscale = _mm512_set1_epi16((int16_t)scaleval);
+ const __m512i col0 =
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col0));
+ const __m512i col1 =
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col1));
+ const __m512i col2 =
+ _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col2));
+ const __m512i pack_even = _mm512_loadu_si512((const __m512i*)kPackEvenBytes);
+ const __mmask16 k5 = 0xf800;
+ const __mmask16 k6 = 0xfc00;
+
+ while (dst_width >= 32) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
+ __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
+ __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
+ k5, col0, s2);
+ __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
+ k5, col1, s2);
+ __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
+ k6, col2, s2);
+ __m512i sum0 = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
+
+ __m512i s3 = _mm512_loadu_si512((const __m512i*)(src_uv + 96));
+ __m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128));
+ __m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160));
+ src_uv += 192;
+ __m512i b0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col0, s4),
+ k5, col0, s5);
+ __m512i b1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col1, s4),
+ k5, col1, s5);
+ __m512i b2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col2, s4),
+ k6, col2, s5);
+ __m512i sum1 = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale);
+
+ __m512i out = _mm512_permutex2var_epi8(sum0, pack_even, sum1);
+ _mm512_storeu_si512((__m512i*)dst_uv, out);
+ dst_uv += 64;
+ dst_width -= 32;
+ }
+ if (dst_width >= 16) {
+ __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
+ __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
+ __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
+ src_uv += 96;
+ __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
+ k5, col0, s2);
+ __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
+ k5, col1, s2);
+ __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
+ k6, col2, s2);
+ __m512i sum0 = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
+ _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0));
+ dst_uv += 32;
+ dst_width -= 16;
+ }
+ if (dst_width > 0) {
+ uint64_t m64 = (1ULL << (dst_width * 3)) - 1;
+ __mmask16 k1 = (__mmask16)m64;
+ __mmask16 k2 = (__mmask16)(m64 >> 16);
+ __mmask16 k3 = (__mmask16)(m64 >> 32);
+ __mmask16 k4 = (__mmask16)((1U << dst_width) - 1);
+ __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv);
+ __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32);
+ __m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64);
+ __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
+ k5, col0, s2);
+ __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
+ k5, col1, s2);
+ __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
+ k6, col2, s2);
+ __m512i sum0 = _mm512_mulhi_epu16(
+ _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
+ _mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0));
}
}
+#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
-// Blends four 2x2 to 4x1.
-__declspec(naked) void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb,
- ptrdiff_t src_stride,
- int src_stepx,
- uint8_t* dst_argb,
- int dst_width) {
- __asm {
- push ebx
- push esi
- push edi
- mov eax, [esp + 12 + 4] // src_argb
- mov esi, [esp + 12 + 8] // src_stride
- mov ebx, [esp + 12 + 12] // src_stepx
- mov edx, [esp + 12 + 16] // dst_argb
- mov ecx, [esp + 12 + 20] // dst_width
- lea esi, [eax + esi] // row1 pointer
- lea ebx, [ebx * 4]
- lea edi, [ebx + ebx * 2]
+#if defined(HAS_SCALEROWUP2_LINEAR_AVX2) || \
+ defined(HAS_SCALEROWUP2_BILINEAR_AVX2)
+static const uvec8 kLinearMadd31 = {3, 1, 1, 3, 3, 1, 1, 3,
+ 3, 1, 1, 3, 3, 1, 1, 3};
+#endif
- wloop:
- movq xmm0, qword ptr [eax] // row0 4 pairs
- movhps xmm0, qword ptr [eax + ebx]
- movq xmm1, qword ptr [eax + ebx * 2]
- movhps xmm1, qword ptr [eax + edi]
- lea eax, [eax + ebx * 4]
- movq xmm2, qword ptr [esi] // row1 4 pairs
- movhps xmm2, qword ptr [esi + ebx]
- movq xmm3, qword ptr [esi + ebx * 2]
- movhps xmm3, qword ptr [esi + edi]
- lea esi, [esi + ebx * 4]
- pavgb xmm0, xmm2 // average rows
- pavgb xmm1, xmm3
- movdqa xmm2, xmm0 // average columns (8 to 4 pixels)
- shufps xmm0, xmm1, 0x88 // even pixels
- shufps xmm2, xmm1, 0xdd // odd pixels
- pavgb xmm0, xmm2
- movdqu [edx], xmm0
- lea edx, [edx + 16]
- sub ecx, 4
- jg wloop
+#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2) || \
+ defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2)
+static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5,
+ 10, 11, 8, 9, 14, 15, 12, 13};
+#endif
- pop edi
- pop esi
- pop ebx
- ret
+#ifdef HAS_SCALEROWUP2_LINEAR_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i madd31 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kLinearMadd31));
+ const __m256i round2 = _mm256_set1_epi16(2);
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ __m256i s1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
+ 0xd8);
+ src_ptr += 16;
+ s0 = _mm256_unpacklo_epi16(s0, s0);
+ s1 = _mm256_unpacklo_epi16(s1, s1);
+ __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(s0, s1), madd31);
+ __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(s0, s1), madd31);
+ lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2);
+ hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2);
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+}
+#endif // HAS_SCALEROWUP2_LINEAR_AVX2
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const __m256i madd31 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kLinearMadd31));
+ const __m256i round8 = _mm256_set1_epi16(8);
+ while (dst_width > 0) {
+ __m256i a0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ __m256i a1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
+ 0xd8);
+ a0 = _mm256_unpacklo_epi16(a0, a0);
+ a1 = _mm256_unpacklo_epi16(a1, a1);
+ __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(a0, a1), madd31);
+ __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(a0, a1), madd31);
+
+ __m256i b0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
+ 0xd8);
+ __m256i b1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))),
+ 0xd8);
+ src_ptr += 16;
+ b0 = _mm256_unpacklo_epi16(b0, b0);
+ b1 = _mm256_unpacklo_epi16(b1, b1);
+ __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(b0, b1), madd31);
+ __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(b0, b1), madd31);
+
+ __m256i d0_lo = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo),
+ _mm256_add_epi16(r1_lo, round8)),
+ 4);
+ __m256i d1_lo = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo),
+ _mm256_add_epi16(r0_lo, round8)),
+ 4);
+ __m256i d0_hi = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi),
+ _mm256_add_epi16(r1_hi, round8)),
+ 4);
+ __m256i d1_hi = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi),
+ _mm256_add_epi16(r0_hi, round8)),
+ 4);
+
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi));
+ _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
+ _mm256_packus_epi16(d1_lo, d1_hi));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+}
+#endif // HAS_SCALEROWUP2_BILINEAR_AVX2
+
+#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const __m256i shuf_far = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kLinearShuffleFar));
+ const __m256i round2 = _mm256_set1_epi16(2);
+ while (dst_width > 0) {
+ __m256i s0 =
+ _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)src_ptr),
+ 0xd8);
+ __m256i s1 = _mm256_permute4x64_epi64(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + 1)), 0xd8);
+ src_ptr += 16;
+ __m256i near_lo = _mm256_unpacklo_epi16(s0, s1);
+ __m256i near_hi = _mm256_unpackhi_epi16(s0, s1);
+ __m256i far_lo = _mm256_shuffle_epi8(near_lo, shuf_far);
+ __m256i far_hi = _mm256_shuffle_epi8(near_hi, shuf_far);
+ __m256i d_lo = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(near_lo, near_lo),
+ _mm256_add_epi16(near_lo, _mm256_add_epi16(far_lo, round2))),
+ 2);
+ __m256i d_hi = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(near_hi, near_hi),
+ _mm256_add_epi16(near_hi, _mm256_add_epi16(far_hi, round2))),
+ 2);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d_lo);
+ _mm256_storeu_si256((__m256i*)(dst_ptr + 16), d_hi);
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+}
+#endif // HAS_SCALEROWUP2_LINEAR_12_AVX2
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const __m256i shuf_far = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kLinearShuffleFar));
+ const __m256i round8 = _mm256_set1_epi16(8);
+ while (dst_width > 0) {
+ __m256i a0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ __m256i a1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))),
+ 0xd8);
+ __m256i near0 = _mm256_unpacklo_epi16(a0, a1);
+ __m256i far0 = _mm256_shuffle_epi8(near0, shuf_far);
+ __m256i r0 = _mm256_add_epi16(_mm256_add_epi16(near0, near0),
+ _mm256_add_epi16(near0, far0));
+
+ __m256i b0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
+ 0xd8);
+ __m256i b1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))),
+ 0xd8);
+ src_ptr += 8;
+ __m256i near1 = _mm256_unpacklo_epi16(b0, b1);
+ __m256i far1 = _mm256_shuffle_epi8(near1, shuf_far);
+ __m256i r1 = _mm256_add_epi16(_mm256_add_epi16(near1, near1),
+ _mm256_add_epi16(near1, far1));
+
+ __m256i d0 = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0, r0), r0),
+ _mm256_add_epi16(r1, round8)),
+ 4);
+ __m256i d1 = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1, r1), r1),
+ _mm256_add_epi16(r0, round8)),
+ 4);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d0);
+ _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), d1);
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+#endif // HAS_SCALEROWUP2_BILINEAR_12_AVX2
+
+#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const __m256i round2 = _mm256_set1_epi32(2);
+ while (dst_width > 0) {
+ __m256i near_lo =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
+ __m256i near_hi =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1)));
+ src_ptr += 8;
+ __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0xb1);
+ __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0xb1);
+ __m256i d_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo),
+ _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))),
+ 2);
+ __m256i d_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi),
+ _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))),
+ 2);
+ __m256i d = _mm256_shuffle_epi32(_mm256_packus_epi32(d_lo, d_hi), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_ptr, d);
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+#endif // HAS_SCALEROWUP2_LINEAR_16_AVX2
+
+#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const __m256i round8 = _mm256_set1_epi32(8);
+ while (dst_width > 0) {
+ __m256i a0 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
+ __m256i a1 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1)));
+ __m256i r0_lo = _mm256_add_epi32(
+ _mm256_add_epi32(a0, a0),
+ _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0xb1)));
+ __m256i r0_hi = _mm256_add_epi32(
+ _mm256_add_epi32(a1, a1),
+ _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0xb1)));
+
+ __m256i b0 = _mm256_cvtepu16_epi32(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)));
+ __m256i b1 = _mm256_cvtepu16_epi32(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1)));
+ src_ptr += 8;
+ __m256i r1_lo = _mm256_add_epi32(
+ _mm256_add_epi32(b0, b0),
+ _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0xb1)));
+ __m256i r1_hi = _mm256_add_epi32(
+ _mm256_add_epi32(b1, b1),
+ _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0xb1)));
+
+ __m256i d0_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo),
+ _mm256_add_epi32(r1_lo, round8)),
+ 4);
+ __m256i d1_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo),
+ _mm256_add_epi32(r0_lo, round8)),
+ 4);
+ __m256i d0_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi),
+ _mm256_add_epi32(r1_hi, round8)),
+ 4);
+ __m256i d1_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi),
+ _mm256_add_epi32(r0_hi, round8)),
+ 4);
+
+ _mm256_storeu_si256(
+ (__m256i*)dst_ptr,
+ _mm256_shuffle_epi32(_mm256_packus_epi32(d0_lo, d0_hi), 0xd8));
+ _mm256_storeu_si256(
+ (__m256i*)(dst_ptr + dst_stride),
+ _mm256_shuffle_epi32(_mm256_packus_epi32(d1_lo, d1_hi), 0xd8));
+ dst_ptr += 16;
+ dst_width -= 16;
+ }
+}
+#endif // HAS_SCALEROWUP2_BILINEAR_16_AVX2
+
+#ifdef HAS_SCALEFILTERCOLS_AVX2
+static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12,
+ 128, 128, 128, 128, 128, 128, 128, 128};
+
+LIBYUV_TARGET_AVX2
+void ScaleFilterCols_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ int n = dst_width & ~7;
+ int r = dst_width & 7;
+ if (n > 0) {
+ SIMD_ALIGNED(int32_t xi_buf[8]);
+ SIMD_ALIGNED(uint16_t pix_buf[8]);
+ __m256i vx = _mm256_add_epi32(
+ _mm256_set1_epi32(x),
+ _mm256_mullo_epi32(
+ _mm256_set1_epi32(dx),
+ _mm256_loadu_si256((const __m256i*)kStep0_7_Filter)));
+ const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3);
+ const __m256i shuf_frac = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kShufFracCols));
+ const __m128i mask_7f = _mm_set1_epi16(0x007f);
+ const __m128i mask_ff = _mm_set1_epi16(0x00ff);
+ const __m128i round_40 = _mm_set1_epi16(0x0040);
+ for (int i = 0; i < n; i += 8) {
+ _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16));
+ __m256i f256 = _mm256_permute4x64_epi64(
+ _mm256_shuffle_epi8(_mm256_srli_epi16(vx, 9), shuf_frac), 0xd8);
+ vx = _mm256_add_epi32(vx, vdx8);
+ __m128i weights = _mm_xor_si128(_mm256_castsi256_si128(f256), mask_7f);
+ pix_buf[0] = LoadU16(src_ptr + xi_buf[0]);
+ pix_buf[1] = LoadU16(src_ptr + xi_buf[1]);
+ pix_buf[2] = LoadU16(src_ptr + xi_buf[2]);
+ pix_buf[3] = LoadU16(src_ptr + xi_buf[3]);
+ pix_buf[4] = LoadU16(src_ptr + xi_buf[4]);
+ pix_buf[5] = LoadU16(src_ptr + xi_buf[5]);
+ pix_buf[6] = LoadU16(src_ptr + xi_buf[6]);
+ pix_buf[7] = LoadU16(src_ptr + xi_buf[7]);
+ __m128i pix = _mm_load_si128((const __m128i*)pix_buf);
+ __m128i a = _mm_and_si128(pix, mask_ff);
+ __m128i res = _mm_srli_epi16(
+ _mm_add_epi16(_mm_add_epi16(_mm_maddubs_epi16(pix, weights), a),
+ round_40),
+ 7);
+ _mm_storel_epi64((__m128i*)dst_ptr, _mm_packus_epi16(res, res));
+ dst_ptr += 8;
+ }
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ int xi = x >> 16;
+ int a = src_ptr[xi];
+ int b = src_ptr[xi + 1];
+ dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7));
+ x = (int)((uint32_t)x + (uint32_t)dx);
+ }
+}
+#endif // HAS_SCALEFILTERCOLS_AVX2
+
+#ifdef HAS_SCALECOLSUP2_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleColsUp2_AVX2(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ int dst_width,
+ int x,
+ int dx) {
+ (void)x;
+ (void)dx;
+ while (dst_width >= 32) {
+ __m256i s = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ src_ptr += 16;
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_unpacklo_epi8(s, s));
+ dst_ptr += 32;
+ dst_width -= 32;
+ }
+ if (dst_width & 16) {
+ __m128i s = _mm_loadl_epi64((const __m128i*)src_ptr);
+ src_ptr += 8;
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s));
+ dst_ptr += 16;
+ }
+ if (dst_width & 8) {
+ __m128i s = _mm_cvtsi32_si128((int)LoadU32(src_ptr));
+ _mm_storel_epi64((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s));
+ }
+}
+#endif // HAS_SCALECOLSUP2_AVX2
+
+#ifdef HAS_SCALEARGBROWDOWN2_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ __m256 s0 = _mm256_loadu_ps((const float*)src_argb);
+ __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32));
+ src_argb += 64;
+ __m256i d = _mm256_permute4x64_epi64(
+ _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, d);
+ dst_argb += 32;
+ dst_width -= 8;
}
}
-// Column scaling unfiltered. SSE2 version.
-__declspec(naked) void ScaleARGBCols_SSE2(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- __asm {
- push edi
- push esi
- mov edi, [esp + 8 + 4] // dst_argb
- mov esi, [esp + 8 + 8] // src_argb
- mov ecx, [esp + 8 + 12] // dst_width
- movd xmm2, [esp + 8 + 16] // x
- movd xmm3, [esp + 8 + 20] // dx
-
- pshufd xmm2, xmm2, 0 // x0 x0 x0 x0
- pshufd xmm0, xmm3, 0x11 // dx 0 dx 0
- paddd xmm2, xmm0
- paddd xmm3, xmm3 // 0, 0, 0, dx * 2
- pshufd xmm0, xmm3, 0x05 // dx * 2, dx * 2, 0, 0
- paddd xmm2, xmm0 // x3 x2 x1 x0
- paddd xmm3, xmm3 // 0, 0, 0, dx * 4
- pshufd xmm3, xmm3, 0 // dx * 4, dx * 4, dx * 4, dx * 4
-
- pextrw eax, xmm2, 1 // get x0 integer.
- pextrw edx, xmm2, 3 // get x1 integer.
-
- cmp ecx, 0
- jle xloop99
- sub ecx, 4
- jl xloop49
-
- // 4 Pixel loop.
- xloop4:
- movd xmm0, [esi + eax * 4] // 1 source x0 pixels
- movd xmm1, [esi + edx * 4] // 1 source x1 pixels
- pextrw eax, xmm2, 5 // get x2 integer.
- pextrw edx, xmm2, 7 // get x3 integer.
- paddd xmm2, xmm3 // x += dx
- punpckldq xmm0, xmm1 // x0 x1
-
- movd xmm1, [esi + eax * 4] // 1 source x2 pixels
- movd xmm4, [esi + edx * 4] // 1 source x3 pixels
- pextrw eax, xmm2, 1 // get x0 integer. next iteration.
- pextrw edx, xmm2, 3 // get x1 integer. next iteration.
- punpckldq xmm1, xmm4 // x2 x3
- punpcklqdq xmm0, xmm1 // x0 x1 x2 x3
- movdqu [edi], xmm0
- lea edi, [edi + 16]
- sub ecx, 4 // 4 pixels
- jge xloop4
-
- xloop49:
- test ecx, 2
- je xloop29
-
- // 2 Pixels.
- movd xmm0, [esi + eax * 4] // 1 source x0 pixels
- movd xmm1, [esi + edx * 4] // 1 source x1 pixels
- pextrw eax, xmm2, 5 // get x2 integer.
- punpckldq xmm0, xmm1 // x0 x1
-
- movq qword ptr [edi], xmm0
- lea edi, [edi + 8]
-
- xloop29:
- test ecx, 1
- je xloop99
-
- // 1 Pixels.
- movd xmm0, [esi + eax * 4] // 1 source x2 pixels
- movd dword ptr [edi], xmm0
- xloop99:
+LIBYUV_TARGET_AVX2
+void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width > 0) {
+ __m256 s0 = _mm256_loadu_ps((const float*)src_argb);
+ __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32));
+ src_argb += 64;
+ __m256i e = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0x88));
+ __m256i o = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd));
+ __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, d);
+ dst_argb += 32;
+ dst_width -= 8;
+ }
+}
- pop esi
- pop edi
- ret
+LIBYUV_TARGET_AVX2
+void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ while (dst_width > 0) {
+ __m256i r0 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)src_argb),
+ _mm256_loadu_si256((const __m256i*)(src_argb + src_stride)));
+ __m256i r1 = _mm256_avg_epu8(
+ _mm256_loadu_si256((const __m256i*)(src_argb + 32)),
+ _mm256_loadu_si256((const __m256i*)(src_argb + src_stride + 32)));
+ src_argb += 64;
+ __m256i e = _mm256_castps_si256(
+ _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1),
+ 0x88));
+ __m256i o = _mm256_castps_si256(
+ _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1),
+ 0xdd));
+ __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, d);
+ dst_argb += 32;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEARGBROWDOWN2_AVX2
+
+#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ SIMD_ALIGNED(uint32_t pix_buf[8]);
+ while (dst_width > 0) {
+ pix_buf[0] = LoadU32(src_argb);
+ pix_buf[1] = LoadU32(src_argb + step);
+ pix_buf[2] = LoadU32(src_argb + step * 2);
+ pix_buf[3] = LoadU32(src_argb + step * 3);
+ pix_buf[4] = LoadU32(src_argb + step * 4);
+ pix_buf[5] = LoadU32(src_argb + step * 5);
+ pix_buf[6] = LoadU32(src_argb + step * 6);
+ pix_buf[7] = LoadU32(src_argb + step * 7);
+ src_argb += step * 8;
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_load_si256((const __m256i*)pix_buf));
+ dst_argb += 32;
+ dst_width -= 8;
}
}
-// Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version.
-// TODO(fbarchard): Port to Neon
+LIBYUV_TARGET_AVX2
+void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ const uint8_t* row1 = src_argb + src_stride;
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 4;
+ SIMD_ALIGNED(uint64_t r0_lo[4]);
+ SIMD_ALIGNED(uint64_t r0_hi[4]);
+ SIMD_ALIGNED(uint64_t r1_lo[4]);
+ SIMD_ALIGNED(uint64_t r1_hi[4]);
+ while (dst_width > 0) {
+ r0_lo[0] = LoadU64(src_argb);
+ r0_lo[1] = LoadU64(src_argb + step);
+ r0_lo[2] = LoadU64(src_argb + step * 2);
+ r0_lo[3] = LoadU64(src_argb + step * 3);
+ r0_hi[0] = LoadU64(src_argb + step * 4);
+ r0_hi[1] = LoadU64(src_argb + step * 5);
+ r0_hi[2] = LoadU64(src_argb + step * 6);
+ r0_hi[3] = LoadU64(src_argb + step * 7);
+ r1_lo[0] = LoadU64(row1);
+ r1_lo[1] = LoadU64(row1 + step);
+ r1_lo[2] = LoadU64(row1 + step * 2);
+ r1_lo[3] = LoadU64(row1 + step * 3);
+ r1_hi[0] = LoadU64(row1 + step * 4);
+ r1_hi[1] = LoadU64(row1 + step * 5);
+ r1_hi[2] = LoadU64(row1 + step * 6);
+ r1_hi[3] = LoadU64(row1 + step * 7);
+ src_argb += step * 8;
+ row1 += step * 8;
+ __m256i a0 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_lo),
+ _mm256_load_si256((const __m256i*)r1_lo));
+ __m256i a1 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_hi),
+ _mm256_load_si256((const __m256i*)r1_hi));
+ __m256i e = _mm256_castps_si256(
+ _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1),
+ 0x88));
+ __m256i o = _mm256_castps_si256(
+ _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1),
+ 0xdd));
+ __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_argb, d);
+ dst_argb += 32;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2
+
+#ifdef HAS_SCALEARGBCOLS_AVX2
+static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+
+LIBYUV_TARGET_AVX2
+void ScaleARGBCols_AVX2(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ int n = dst_width & ~7;
+ int r = dst_width & 7;
+ if (n > 0) {
+ SIMD_ALIGNED(int32_t xi_buf[8]);
+ SIMD_ALIGNED(uint32_t pix_buf[8]);
+ __m256i vx = _mm256_add_epi32(
+ _mm256_set1_epi32(x),
+ _mm256_mullo_epi32(
+ _mm256_set1_epi32(dx),
+ _mm256_loadu_si256((const __m256i*)kStep0_7_Cols)));
+ const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3);
+ for (int i = 0; i < n; i += 8) {
+ _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16));
+ vx = _mm256_add_epi32(vx, vdx8);
+ pix_buf[0] = LoadU32(src_argb + (ptrdiff_t)xi_buf[0] * 4);
+ pix_buf[1] = LoadU32(src_argb + (ptrdiff_t)xi_buf[1] * 4);
+ pix_buf[2] = LoadU32(src_argb + (ptrdiff_t)xi_buf[2] * 4);
+ pix_buf[3] = LoadU32(src_argb + (ptrdiff_t)xi_buf[3] * 4);
+ pix_buf[4] = LoadU32(src_argb + (ptrdiff_t)xi_buf[4] * 4);
+ pix_buf[5] = LoadU32(src_argb + (ptrdiff_t)xi_buf[5] * 4);
+ pix_buf[6] = LoadU32(src_argb + (ptrdiff_t)xi_buf[6] * 4);
+ pix_buf[7] = LoadU32(src_argb + (ptrdiff_t)xi_buf[7] * 4);
+ _mm256_storeu_si256((__m256i*)dst_argb,
+ _mm256_load_si256((const __m256i*)pix_buf));
+ dst_argb += 32;
+ }
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4;
+ dst_argb[0] = src[0];
+ dst_argb[1] = src[1];
+ dst_argb[2] = src[2];
+ dst_argb[3] = src[3];
+ dst_argb += 4;
+ x = (int)((uint32_t)x + (uint32_t)dx);
+ }
+}
+#endif // HAS_SCALEARGBCOLS_AVX2
+
+#ifdef HAS_SCALEARGBCOLSUP2_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ (void)x;
+ (void)dx;
+ while (dst_width > 0) {
+ __m256i s = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_argb)),
+ 0xd8);
+ src_argb += 16;
+ _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi32(s, s));
+ dst_argb += 32;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEARGBCOLSUP2_AVX2
-// Shuffle table for arranging 2 pixels into pairs for pmaddubsw
+#ifdef HAS_SCALEARGBFILTERCOLS_AVX2
+static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3};
static const uvec8 kShuffleColARGB = {
- 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel
- 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel
+ 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u,
+ 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u};
+static const uint8_t kShuffleFractionsAVX2[32] = {
+ 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
+ 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u,
};
-// Shuffle table for duplicating 2 fractions into 8 bytes each
-static const uvec8 kShuffleFractions = {
- 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u,
-};
-
-__declspec(naked) void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- __asm {
- push esi
- push edi
- mov edi, [esp + 8 + 4] // dst_argb
- mov esi, [esp + 8 + 8] // src_argb
- mov ecx, [esp + 8 + 12] // dst_width
- movd xmm2, [esp + 8 + 16] // x
- movd xmm3, [esp + 8 + 20] // dx
- movdqa xmm4, xmmword ptr kShuffleColARGB
- movdqa xmm5, xmmword ptr kShuffleFractions
- pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction.
- psrlw xmm6, 9
- pextrw eax, xmm2, 1 // get x0 integer. preroll
- sub ecx, 2
- jl xloop29
-
- movdqa xmm0, xmm2 // x1 = x0 + dx
- paddd xmm0, xmm3
- punpckldq xmm2, xmm0 // x0 x1
- punpckldq xmm3, xmm3 // dx dx
- paddd xmm3, xmm3 // dx * 2, dx * 2
- pextrw edx, xmm2, 3 // get x1 integer. preroll
-
- // 2 Pixel loop.
- xloop2:
- movdqa xmm1, xmm2 // x0, x1 fractions.
- paddd xmm2, xmm3 // x += dx
- movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels
- psrlw xmm1, 9 // 7 bit fractions.
- movhps xmm0, qword ptr [esi + edx * 4] // 2 source x1 pixels
- pshufb xmm1, xmm5 // 0000000011111111
- pshufb xmm0, xmm4 // arrange pixels into pairs
- pxor xmm1, xmm6 // 0..7f and 7f..0
- pmaddubsw xmm0, xmm1 // argb_argb 16 bit, 2 pixels.
- pextrw eax, xmm2, 1 // get x0 integer. next iteration.
- pextrw edx, xmm2, 3 // get x1 integer. next iteration.
- psrlw xmm0, 7 // argb 8.7 fixed point to low 8 bits.
- packuswb xmm0, xmm0 // argb_argb 8 bits, 2 pixels.
- movq qword ptr [edi], xmm0
- lea edi, [edi + 8]
- sub ecx, 2 // 2 pixels
- jge xloop2
-
- xloop29:
-
- add ecx, 2 - 1
- jl xloop99
-
- // 1 pixel remainder
- psrlw xmm2, 9 // 7 bit fractions.
- movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels
- pshufb xmm2, xmm5 // 00000000
- pshufb xmm0, xmm4 // arrange pixels into pairs
- pxor xmm2, xmm6 // 0..7f and 7f..0
- pmaddubsw xmm0, xmm2 // argb 16 bit, 1 pixel.
- psrlw xmm0, 7
- packuswb xmm0, xmm0 // argb 8 bits, 1 pixel.
- movd [edi], xmm0
-
- xloop99:
-
- pop edi
- pop esi
- ret
+LIBYUV_TARGET_AVX2
+void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ int n = dst_width & ~3;
+ int r = dst_width & 3;
+ if (n > 0) {
+ SIMD_ALIGNED(int32_t xi_buf[4]);
+ SIMD_ALIGNED(uint64_t pix_buf[4]);
+ const __m256i shuf_argb = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kShuffleColARGB));
+ const __m256i shuf_frac =
+ _mm256_loadu_si256((const __m256i*)kShuffleFractionsAVX2);
+ const __m256i mask_7f = _mm256_set1_epi16(0x007f);
+ __m128i vx = _mm_add_epi32(
+ _mm_set1_epi32(x),
+ _mm_mullo_epi32(_mm_set1_epi32(dx),
+ _mm_loadu_si128((const __m128i*)kStep0_3_ARGB)));
+ const __m128i vdx4 = _mm_slli_epi32(_mm_set1_epi32(dx), 2);
+ for (int i = 0; i < n; i += 4) {
+ _mm_store_si128((__m128i*)xi_buf, _mm_srai_epi32(vx, 16));
+ __m128i f128 = _mm_srli_epi16(vx, 9);
+ vx = _mm_add_epi32(vx, vdx4);
+ pix_buf[0] = LoadU64(src_argb + (ptrdiff_t)xi_buf[0] * 4);
+ pix_buf[1] = LoadU64(src_argb + (ptrdiff_t)xi_buf[1] * 4);
+ pix_buf[2] = LoadU64(src_argb + (ptrdiff_t)xi_buf[2] * 4);
+ pix_buf[3] = LoadU64(src_argb + (ptrdiff_t)xi_buf[3] * 4);
+ __m256i pix = _mm256_shuffle_epi8(
+ _mm256_load_si256((const __m256i*)pix_buf), shuf_argb);
+ __m256i weights = _mm256_xor_si256(
+ _mm256_shuffle_epi8(
+ _mm256_inserti128_si256(_mm256_zextsi128_si256(f128), f128, 1),
+ shuf_frac),
+ mask_7f);
+ __m256i res = _mm256_srli_epi16(_mm256_maddubs_epi16(pix, weights), 7);
+ res = _mm256_permute4x64_epi64(_mm256_packus_epi16(res, res), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_argb, _mm256_castsi256_si128(res));
+ dst_argb += 16;
+ }
+ x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx);
+ }
+ for (int j = 0; j < r; ++j) {
+ ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4;
+ int xf = (x >> 9) & 0x7f;
+ const uint8_t* a = src_argb + xi;
+ const uint8_t* b = a + 4;
+ dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7);
+ dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7);
+ dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7);
+ dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7);
+ dst_argb += 4;
+ x = (int)((uint32_t)x + (uint32_t)dx);
}
}
+#endif // HAS_SCALEARGBFILTERCOLS_AVX2
+
+#ifdef HAS_SCALEUVROWDOWN2_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowDown2_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ while (dst_width >= 16) {
+ __m256i s0 =
+ _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16);
+ __m256i s1 = _mm256_srli_epi32(
+ _mm256_loadu_si256((const __m256i*)(src_uv + 32)), 16);
+ src_uv += 64;
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_uv, d);
+ dst_uv += 32;
+ dst_width -= 16;
+ }
+ if (dst_width > 0) {
+ __m256i s0 =
+ _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d));
+ }
+}
+#endif // HAS_SCALEUVROWDOWN2_AVX2
+
+#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ const __m256i mask = _mm256_set1_epi32(0x0000ffff);
+ while (dst_width >= 16) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
+ src_uv += 64;
+ __m256i s0 =
+ _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask);
+ __m256i s1 =
+ _mm256_and_si256(_mm256_avg_epu8(r1, _mm256_srli_epi32(r1, 16)), mask);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8);
+ _mm256_storeu_si256((__m256i*)dst_uv, d);
+ dst_uv += 32;
+ dst_width -= 16;
+ }
+ if (dst_width > 0) {
+ __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv);
+ __m256i s0 =
+ _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d));
+ }
+}
+#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2
+
+#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2
+static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u,
+ 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u};
+static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u,
+ 6u, 14u, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80};
+
+LIBYUV_TARGET_AVX2
+void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i split = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kShuffleSplitUV));
+ const __m256i merge = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kShuffleMergeUV));
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i zero = _mm256_setzero_si256();
+ while (dst_width > 0) {
+ __m256i r0 = _mm256_maddubs_epi16(
+ _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), split),
+ ones);
+ __m256i r1 = _mm256_maddubs_epi16(
+ _mm256_shuffle_epi8(
+ _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), split),
+ ones);
+ src_ptr += 32;
+ __m256i s =
+ _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(r0, r1), 1), zero);
+ __m256i d = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(s, merge), 0xd8);
+ _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d));
+ dst_ptr += 16;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEUVROWDOWN2BOX_AVX2
+
+#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_uv,
+ int dst_width) {
+ (void)src_stride;
+ const ptrdiff_t step = (ptrdiff_t)src_stepx * 2;
+ SIMD_ALIGNED(uint16_t uv_buf[8]);
+ while (dst_width > 0) {
+ uv_buf[0] = LoadU16(src_uv);
+ uv_buf[1] = LoadU16(src_uv + step);
+ uv_buf[2] = LoadU16(src_uv + step * 2);
+ uv_buf[3] = LoadU16(src_uv + step * 3);
+ uv_buf[4] = LoadU16(src_uv + step * 4);
+ uv_buf[5] = LoadU16(src_uv + step * 5);
+ uv_buf[6] = LoadU16(src_uv + step * 6);
+ uv_buf[7] = LoadU16(src_uv + step * 7);
+ src_uv += step * 8;
+ _mm_storeu_si128((__m128i*)dst_uv, _mm_load_si128((const __m128i*)uv_buf));
+ dst_uv += 16;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEUVROWDOWNEVEN_AVX2
-// Reads 4 pixels, duplicates them and writes 8 pixels.
-__declspec(naked) void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
- const uint8_t* src_argb,
- int dst_width,
- int x,
- int dx) {
- __asm {
- mov edx, [esp + 4] // dst_argb
- mov eax, [esp + 8] // src_argb
- mov ecx, [esp + 12] // dst_width
-
- wloop:
- movdqu xmm0, [eax]
- lea eax, [eax + 16]
- movdqa xmm1, xmm0
- punpckldq xmm0, xmm0
- punpckhdq xmm1, xmm1
- movdqu [edx], xmm0
- movdqu [edx + 16], xmm1
- lea edx, [edx + 32]
- sub ecx, 8
- jg wloop
+#if defined(HAS_SCALEUVROWUP2_LINEAR_AVX2) || \
+ defined(HAS_SCALEUVROWUP2_BILINEAR_AVX2)
+static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3,
+ 3, 1, 3, 1, 1, 3, 1, 3};
+#endif
- ret
+#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const __m256i madd31 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kUVLinearMadd31));
+ const __m256i round2 = _mm256_set1_epi16(2);
+ while (dst_width > 0) {
+ __m256i s0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ __m256i s1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))),
+ 0xd8);
+ src_ptr += 16;
+ __m256i p = _mm256_unpacklo_epi8(s0, s1);
+ __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(p, p), madd31);
+ __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(p, p), madd31);
+ lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2);
+ hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2);
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi));
+ dst_ptr += 32;
+ dst_width -= 16;
}
}
-
-// Divide num by div and return as 16.16 fixed point result.
-__declspec(naked) int FixedDiv_X86(int num, int div) {
- __asm {
- mov eax, [esp + 4] // num
- cdq // extend num to 64 bits
- shld edx, eax, 16 // 32.16
- shl eax, 16
- idiv dword ptr [esp + 8]
- ret
+#endif // HAS_SCALEUVROWUP2_LINEAR_AVX2
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const __m256i madd31 = _mm256_broadcastsi128_si256(
+ _mm_loadu_si128((const __m128i*)&kUVLinearMadd31));
+ const __m256i round8 = _mm256_set1_epi16(8);
+ while (dst_width > 0) {
+ __m256i a0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8);
+ __m256i a1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))),
+ 0xd8);
+ __m256i pa = _mm256_unpacklo_epi8(a0, a1);
+ __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pa, pa), madd31);
+ __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pa, pa), madd31);
+
+ __m256i b0 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))),
+ 0xd8);
+ __m256i b1 = _mm256_permute4x64_epi64(
+ _mm256_zextsi128_si256(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))),
+ 0xd8);
+ src_ptr += 16;
+ __m256i pb = _mm256_unpacklo_epi8(b0, b1);
+ __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pb, pb), madd31);
+ __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pb, pb), madd31);
+
+ __m256i d0_lo = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo),
+ _mm256_add_epi16(r1_lo, round8)),
+ 4);
+ __m256i d1_lo = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo),
+ _mm256_add_epi16(r0_lo, round8)),
+ 4);
+ __m256i d0_hi = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi),
+ _mm256_add_epi16(r1_hi, round8)),
+ 4);
+ __m256i d1_hi = _mm256_srli_epi16(
+ _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi),
+ _mm256_add_epi16(r0_hi, round8)),
+ 4);
+
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi));
+ _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
+ _mm256_packus_epi16(d1_lo, d1_hi));
+ dst_ptr += 32;
+ dst_width -= 16;
}
}
-
-// Divide num by div and return as 16.16 fixed point result.
-__declspec(naked) int FixedDiv1_X86(int num, int div) {
- __asm {
- mov eax, [esp + 4] // num
- mov ecx, [esp + 8] // denom
- cdq // extend num to 64 bits
- shld edx, eax, 16 // 32.16
- shl eax, 16
- sub eax, 0x00010001
- sbb edx, 0
- sub ecx, 1
- idiv ecx
- ret
+#endif // HAS_SCALEUVROWUP2_BILINEAR_AVX2
+
+#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr,
+ uint16_t* dst_ptr,
+ int dst_width) {
+ const __m256i round2 = _mm256_set1_epi32(2);
+ while (dst_width > 0) {
+ __m256i near_lo =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
+ __m256i near_hi =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2)));
+ src_ptr += 8;
+ __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0x4e);
+ __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0x4e);
+ __m256i d_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo),
+ _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))),
+ 2);
+ __m256i d_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi),
+ _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))),
+ 2);
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d_lo, d_hi));
+ dst_ptr += 16;
+ dst_width -= 8;
}
}
-#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
+#endif // HAS_SCALEUVROWUP2_LINEAR_16_AVX2
+
+#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+LIBYUV_TARGET_AVX2
+void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ const __m256i round8 = _mm256_set1_epi32(8);
+ while (dst_width > 0) {
+ __m256i a0 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr));
+ __m256i a1 =
+ _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2)));
+ __m256i r0_lo = _mm256_add_epi32(
+ _mm256_add_epi32(a0, a0),
+ _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0x4e)));
+ __m256i r0_hi = _mm256_add_epi32(
+ _mm256_add_epi32(a1, a1),
+ _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0x4e)));
+
+ __m256i b0 = _mm256_cvtepu16_epi32(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)));
+ __m256i b1 = _mm256_cvtepu16_epi32(
+ _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2)));
+ src_ptr += 8;
+ __m256i r1_lo = _mm256_add_epi32(
+ _mm256_add_epi32(b0, b0),
+ _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0x4e)));
+ __m256i r1_hi = _mm256_add_epi32(
+ _mm256_add_epi32(b1, b1),
+ _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0x4e)));
+
+ __m256i d0_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo),
+ _mm256_add_epi32(r1_lo, round8)),
+ 4);
+ __m256i d1_lo = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo),
+ _mm256_add_epi32(r0_lo, round8)),
+ 4);
+ __m256i d0_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi),
+ _mm256_add_epi32(r1_hi, round8)),
+ 4);
+ __m256i d1_hi = _mm256_srli_epi32(
+ _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi),
+ _mm256_add_epi32(r0_hi, round8)),
+ 4);
+
+ _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d0_lo, d0_hi));
+ _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride),
+ _mm256_packus_epi32(d1_lo, d1_hi));
+ dst_ptr += 16;
+ dst_width -= 8;
+ }
+}
+#endif // HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
+
+#undef LIBYUV_TARGET_AVX2
+#undef LIBYUV_TARGET_AVX512BW
+#undef LIBYUV_TARGET_AVX512VBMI
+
+#endif // !defined(LIBYUV_DISABLE_X86) && ...
#ifdef __cplusplus
} // extern "C"
diff --git a/libs/libyuv/source/test.sh b/libs/libyuv/source/test.sh
new file mode 100755
index 0000000000..7f12c3c156
--- /dev/null
+++ b/libs/libyuv/source/test.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -x
+
+function runbenchmark1 {
+ perf record /google/src/cloud/fbarchard/clean/google3/blaze-bin/third_party/libyuv/libyuv_test --gunit_filter=*$1 --libyuv_width=1280 --libyuv_height=720 --libyuv_repeat=1000 --libyuv_flags=-1 --libyuv_cpu_info=-1
+ perf report | grep AVX
+}
+
+runbenchmark1 ABGRToI420
+runbenchmark1 Android420ToI420
+runbenchmark1 ARGBToI420
+runbenchmark1 Convert16To8Plane
+runbenchmark1 ConvertToARGB
+runbenchmark1 ConvertToI420
+runbenchmark1 CopyPlane
+runbenchmark1 H010ToAB30
+runbenchmark1 H010ToAR30
+runbenchmark1 HalfFloatPlane
+runbenchmark1 I010ToAB30
+runbenchmark1 I010ToAR30
+runbenchmark1 I420Copy
+runbenchmark1 I420Psnr
+runbenchmark1 I420Scale
+runbenchmark1 I420Ssim
+runbenchmark1 I420ToARGB
+runbenchmark1 I420ToNV12
+runbenchmark1 I420ToUYVY
+runbenchmark1 I422ToI420
+runbenchmark1 InitCpuFlags
+runbenchmark1 J420ToARGB
+runbenchmark1 NV12ToARGB
+runbenchmark1 NV12ToI420
+runbenchmark1 NV12ToI420Rotate
+runbenchmark1 SetCpuFlags
+runbenchmark1 YUY2ToI420
diff --git a/libs/libyuv/tools_libyuv/OWNERS b/libs/libyuv/tools_libyuv/OWNERS
index 2cb971d2b7..aae4fb6e02 100644
--- a/libs/libyuv/tools_libyuv/OWNERS
+++ b/libs/libyuv/tools_libyuv/OWNERS
@@ -1 +1,4 @@
-phoglund@chromium.org
+mbonadei@chromium.org
+fbarchard@chromium.org
+pbos@chromium.org
+
diff --git a/libs/libyuv/tools_libyuv/autoroller/roll_deps.py b/libs/libyuv/tools_libyuv/autoroller/roll_deps.py
index 2b95e302cd..472cbe6cbe 100755
--- a/libs/libyuv/tools_libyuv/autoroller/roll_deps.py
+++ b/libs/libyuv/tools_libyuv/autoroller/roll_deps.py
@@ -1,18 +1,14 @@
-#!/usr/bin/env python
-# Copyright 2017 The LibYuv Project Authors. All rights reserved.
+#!/usr/bin/env vpython3
+
+# Copyright (c) 2017 The LibYUV project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
+# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
+"""Script to automatically roll dependencies in the LibYUV DEPS file."""
-# This is a modified copy of the script in
-# https://webrtc.googlesource.com/src/+/master/tools_webrtc/autoroller/roll_deps.py
-# customized for libyuv.
-
-
-"""Script to automatically roll dependencies in the libyuv DEPS file."""
import argparse
import base64
@@ -22,12 +18,54 @@ import os
import re
import subprocess
import sys
-import urllib2
+import urllib.request
+
+def FindRootPath():
+ """Returns the absolute path to the highest level repo root.
+
+ If this repo is checked out as a submodule of the chromium/src
+ superproject, this returns the superproect root. Otherwise, it returns the
+ webrtc/src repo root.
+ """
+ root_dir = os.path.dirname(os.path.abspath(__file__))
+ while os.path.basename(root_dir) not in ('src', 'chromium'):
+ par_dir = os.path.normpath(os.path.join(root_dir, os.pardir))
+ if par_dir == root_dir:
+ raise RuntimeError('Could not find the repo root.')
+ root_dir = par_dir
+ return root_dir
# Skip these dependencies (list without solution name prefix).
DONT_AUTOROLL_THESE = [
- 'src/third_party/gflags/src',
+ 'third_party/fuchsia-gn-sdk',
+ 'src/third_party/gflags/src',
+ 'src/third_party/mockito/src',
+]
+
+# These dependencies are missing in chromium/src/DEPS, either unused or already
+# in-tree. For instance, src/base is a part of the Chromium source git repo,
+# but we pull it through a subtree mirror, so therefore it isn't listed in
+# Chromium's deps but it is in ours.
+LIBYUV_ONLY_DEPS = [
+ 'src/build',
+ 'src/buildtools',
+ 'src/ios',
+ 'src/testing',
+ 'src/third_party',
+ 'src/third_party/android_support_test_runner',
+ 'src/third_party/errorprone/lib',
+ 'src/third_party/findbugs',
+ 'src/third_party/gson',
+ 'src/third_party/gtest-parallel',
+ 'src/third_party/guava',
+ 'src/third_party/intellij',
+ 'src/third_party/jsr-305/src',
+ 'src/third_party/ow2_asm',
+ 'src/third_party/proguard',
+ 'src/third_party/ub-uiautomator/lib',
+ 'src/tools',
+ 'src/tools/clang/dsymutil',
]
LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv'
@@ -37,199 +75,381 @@ CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
-CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([0-9a-z]+)\'$')
+CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$')
ROLL_BRANCH_NAME = 'roll_chromium_revision'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
- os.pardir))
-CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
+CHECKOUT_ROOT_DIR = FindRootPath()
+GCLIENT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_ROOT_DIR, os.pardir))
+
+# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy.
+ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ==='
+ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ==='
+# Location of automically gathered android deps.
+ANDROID_DEPS_PATH = 'src/third_party/android_deps/'
+
+sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
+import find_depot_tools # pylint: disable=wrong-import-position
-sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
-import find_depot_tools
find_depot_tools.add_depot_tools_to_path()
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
-CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
- 'clang', 'scripts', 'update.py')
+CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(
+ CHECKOUT_ROOT_DIR, 'tools', 'clang', 'scripts', 'update.py'
+)
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
-ChangedDep = collections.namedtuple('ChangedDep',
- 'path url current_rev new_rev')
+ChangedDep = collections.namedtuple(
+ 'ChangedDep', 'path url current_rev new_rev'
+)
+CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages')
+GcsDepsEntry = collections.namedtuple('GcsDepsEntry', 'path bucket objects')
+VersionEntry = collections.namedtuple('VersionEntry', 'version')
+ChangedCipdPackage = collections.namedtuple(
+ 'ChangedCipdPackage', 'path package current_version new_version'
+)
+ChangedVersionEntry = collections.namedtuple(
+ 'ChangedVersionEntry', 'path current_version new_version'
+)
+
+ChromiumRevisionUpdate = collections.namedtuple(
+ 'ChromiumRevisionUpdate', 'current_chromium_rev new_chromium_rev '
+)
+
class RollError(Exception):
- pass
+ pass
+
+
+def StrExpansion():
+ return lambda str_value: str_value
def VarLookup(local_scope):
- return lambda var_name: local_scope['vars'][var_name]
+ return lambda var_name: local_scope['vars'][var_name]
def ParseDepsDict(deps_content):
- local_scope = {}
- global_scope = {
- 'Var': VarLookup(local_scope),
- 'deps_os': {},
- }
- exec(deps_content, global_scope, local_scope)
- return local_scope
+ local_scope = {}
+ global_scope = {
+ 'Str': StrExpansion(),
+ 'Var': VarLookup(local_scope),
+ 'deps_os': {},
+ }
+ exec(deps_content, global_scope, local_scope)
+ return local_scope
def ParseLocalDepsFile(filename):
- with open(filename, 'rb') as f:
- deps_content = f.read()
- return ParseDepsDict(deps_content)
-
-
-def ParseRemoteCrDepsFile(revision):
- deps_content = ReadRemoteCrFile('DEPS', revision)
- return ParseDepsDict(deps_content)
+ with open(filename, 'rb') as f:
+ deps_content = f.read().decode('utf-8')
+ return ParseDepsDict(deps_content)
def ParseCommitPosition(commit_message):
- for line in reversed(commit_message.splitlines()):
- m = COMMIT_POSITION_RE.match(line.strip())
- if m:
- return int(m.group(1))
- logging.error('Failed to parse commit position id from:\n%s\n',
- commit_message)
- sys.exit(-1)
-
-
-def _RunCommand(command, working_dir=None, ignore_exit_code=False,
- extra_env=None):
- """Runs a command and returns the output from that command.
-
- If the command fails (exit code != 0), the function will exit the process.
-
- Returns:
- A tuple containing the stdout and stderr outputs as strings.
- """
- working_dir = working_dir or CHECKOUT_SRC_DIR
- logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
- env = os.environ.copy()
- if extra_env:
- assert all(isinstance(value, str) for value in extra_env.values())
- logging.debug('extra env: %s', extra_env)
- env.update(extra_env)
- p = subprocess.Popen(command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=env,
- cwd=working_dir, universal_newlines=True)
- std_output = p.stdout.read()
- err_output = p.stderr.read()
- p.wait()
- p.stdout.close()
- p.stderr.close()
- if not ignore_exit_code and p.returncode != 0:
- logging.error('Command failed: %s\n'
- 'stdout:\n%s\n'
- 'stderr:\n%s\n', ' '.join(command), std_output, err_output)
- sys.exit(p.returncode)
- return std_output, err_output
+ for line in reversed(commit_message.splitlines()):
+ m = COMMIT_POSITION_RE.match(line.strip())
+ if m:
+ return int(m.group(1))
+ logging.error(
+ 'Failed to parse commit position id from:\n%s\n', commit_message
+ )
+ sys.exit(-1)
+
+
+def _RunCommand(
+ command,
+ working_dir=None,
+ ignore_exit_code=False,
+ extra_env=None,
+ input_data=None,
+):
+ """Runs a command and returns the output from that command.
+
+ If the command fails (exit code != 0), the function will exit the process.
+
+ Returns:
+ A tuple containing the stdout and stderr outputs as strings.
+ """
+ working_dir = working_dir or CHECKOUT_ROOT_DIR
+ logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
+ env = os.environ.copy()
+ if extra_env:
+ assert all(isinstance(value, str) for value in extra_env.values())
+ logging.debug('extra env: %s', extra_env)
+ env.update(extra_env)
+ p = subprocess.Popen(
+ command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ cwd=working_dir,
+ universal_newlines=True,
+ )
+ std_output, err_output = p.communicate(input_data)
+ p.stdout.close()
+ p.stderr.close()
+ if not ignore_exit_code and p.returncode != 0:
+ logging.error(
+ 'Command failed: %s\nstdout:\n%s\nstderr:\n%s\n',
+ ' '.join(command),
+ std_output,
+ err_output,
+ )
+ sys.exit(p.returncode)
+ return std_output, err_output
def _GetBranches():
- """Returns a tuple of active,branches.
-
- The 'active' is the name of the currently active branch and 'branches' is a
- list of all branches.
- """
- lines = _RunCommand(['git', 'branch'])[0].split('\n')
- branches = []
- active = ''
- for line in lines:
- if '*' in line:
- # The assumption is that the first char will always be the '*'.
- active = line[1:].strip()
- branches.append(active)
- else:
- branch = line.strip()
- if branch:
- branches.append(branch)
- return active, branches
+ """Returns a tuple of active,branches.
+
+ The 'active' is the name of the currently active branch and 'branches' is a
+ list of all branches.
+ """
+ lines = _RunCommand(['git', 'branch'])[0].split('\n')
+ branches = []
+ active = ''
+ for line in lines:
+ if '*' in line:
+ # The assumption is that the first char will always be the '*'.
+ active = line[1:].strip()
+ branches.append(active)
+ else:
+ branch = line.strip()
+ if branch:
+ branches.append(branch)
+ return active, branches
def _ReadGitilesContent(url):
- # Download and decode BASE64 content until
- # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
- base64_content = ReadUrlContent(url + '?format=TEXT')
- return base64.b64decode(base64_content[0])
+ # Download and decode BASE64 content until
+ # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
+ base64_content = ReadUrlContent(url + '?format=TEXT')
+ return base64.b64decode(base64_content[0]).decode('utf-8')
def ReadRemoteCrFile(path_below_src, revision):
- """Reads a remote Chromium file of a specific revision. Returns a string."""
- return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
- path_below_src))
+ """Reads a remote Chromium file of a specific revision.
+
+ Args:
+ path_below_src: A path to the target file relative to src dir.
+ revision: Revision to read.
+
+ Returns:
+ A string with file content.
+ """
+ return _ReadGitilesContent(
+ CHROMIUM_FILE_TEMPLATE % (revision, path_below_src)
+ )
def ReadRemoteCrCommit(revision):
- """Reads a remote Chromium commit message. Returns a string."""
- return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
+ """Reads a remote Chromium commit message. Returns a string."""
+ return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
def ReadUrlContent(url):
- """Connect to a remote host and read the contents. Returns a list of lines."""
- conn = urllib2.urlopen(url)
- try:
- return conn.readlines()
- except IOError as e:
- logging.exception('Error connecting to %s. Error: %s', url, e)
- raise
- finally:
- conn.close()
+ """Connect to a remote host and read the contents.
+
+ Args:
+ url: URL to connect to.
+
+ Returns:
+ A list of lines.
+ """
+ conn = urllib.request.urlopen(url)
+ try:
+ return conn.readlines()
+ except IOError as e:
+ logging.exception('Error connecting to %s. Error: %s', url, e)
+ raise
+ finally:
+ conn.close()
def GetMatchingDepsEntries(depsentry_dict, dir_path):
- """Gets all deps entries matching the provided path.
-
- This list may contain more than one DepsEntry object.
- Example: dir_path='src/testing' would give results containing both
- 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
- Example 2: dir_path='src/build' should return 'src/build' but not
- 'src/buildtools'.
-
- Returns:
- A list of DepsEntry objects.
- """
- result = []
- for path, depsentry in depsentry_dict.iteritems():
- if path == dir_path:
- result.append(depsentry)
- else:
- parts = path.split('/')
- if all(part == parts[i]
- for i, part in enumerate(dir_path.split('/'))):
- result.append(depsentry)
- return result
+ """Gets all deps entries matching the provided path.
+
+ This list may contain more than one DepsEntry object.
+ Example: dir_path='src/testing' would give results containing both
+ 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's
+ DEPS.
+ Example 2: dir_path='src/build' should return 'src/build' but not
+ 'src/buildtools'.
+
+ Returns:
+ A list of DepsEntry objects.
+ """
+ result = []
+ for path, depsentry in depsentry_dict.items():
+ if path == dir_path:
+ result.append(depsentry)
+ else:
+ parts = path.split('/')
+ if all(
+ part == parts[i] for i, part in enumerate(dir_path.split('/'))
+ ):
+ result.append(depsentry)
+ return result
def BuildDepsentryDict(deps_dict):
- """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
- result = {}
- def AddDepsEntries(deps_subdict):
- for path, deps_url_spec in deps_subdict.iteritems():
- # The deps url is either an URL and a condition, or just the URL.
- if isinstance(deps_url_spec, dict):
- if deps_url_spec.get('dep_type') == 'cipd':
- continue
- deps_url = deps_url_spec['url']
- else:
- deps_url = deps_url_spec
-
- if not result.has_key(path):
- url, revision = deps_url.split('@') if deps_url else (None, None)
- result[path] = DepsEntry(path, url, revision)
-
- AddDepsEntries(deps_dict['deps'])
- for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
- AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
- return result
+ """Builds a dict of paths to DepsEntry objects from a raw deps dict."""
+ result = {}
+
+ def AddDepsEntries(deps_subdict):
+ for path, dep in deps_subdict.items():
+ if path in result:
+ continue
+ if not isinstance(dep, dict):
+ dep = {'url': dep}
+ if dep.get('dep_type') == 'cipd':
+ result[path] = CipdDepsEntry(path, dep['packages'])
+ elif dep.get('dep_type') == 'gcs':
+ result[path] = GcsDepsEntry(path, dep['bucket'],
+ dep['objects'])
+ else:
+ if '@' not in dep['url']:
+ url, revision = dep['url'], 'HEAD'
+ else:
+ url, revision = dep['url'].split('@')
+ result[path] = DepsEntry(path, url, revision)
+
+ def AddVersionEntry(vars_subdict):
+ for key, value in vars_subdict.items():
+ if key in result:
+ continue
+ if not key.endswith('_version'):
+ continue
+ key = re.sub('_version$', '', key)
+ result[key] = VersionEntry(value)
+
+ AddDepsEntries(deps_dict['deps'])
+ for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
+ AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
+ AddVersionEntry(deps_dict.get('vars', {}))
+ return result
+
+
+def _FindChangedCipdPackages(path, old_pkgs, new_pkgs):
+ old_pkgs_names = {p['package'] for p in old_pkgs}
+ new_pkgs_names = {p['package'] for p in new_pkgs}
+ pkgs_equal = old_pkgs_names == new_pkgs_names
+ added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names]
+ removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names]
+
+ assert pkgs_equal, (
+ 'Old: %s\n New: %s.\nYou need to do a manual roll '
+ 'and remove/add entries in DEPS so the old and new '
+ 'list match.\nMost likely, you should add "%s" and '
+ 'remove "%s"' % (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)
+ )
+
+ for old_pkg in old_pkgs:
+ for new_pkg in new_pkgs:
+ old_version = old_pkg['version']
+ new_version = new_pkg['version']
+ if (
+ old_pkg['package'] == new_pkg['package']
+ and old_version != new_version
+ ):
+ logging.debug('Roll dependency %s to %s', path, new_version)
+ yield ChangedCipdPackage(
+ path, old_pkg['package'], old_version, new_version
+ )
+
+
+def _FindChangedVars(name, old_version, new_version):
+ if old_version != new_version:
+ logging.debug('Roll dependency %s to %s', name, new_version)
+ yield ChangedVersionEntry(name, old_version, new_version)
+
+
+def _FindNewDeps(old, new):
+ """Gather dependencies only in `new` and return corresponding paths."""
+ old_entries = set(BuildDepsentryDict(old))
+ new_entries = set(BuildDepsentryDict(new))
+ return [
+ path
+ for path in new_entries - old_entries
+ if path not in DONT_AUTOROLL_THESE
+ ]
+
+
+def FindAddedDeps(libyuv_deps, new_cr_deps):
+ """Calculate new deps entries of interest.
+
+ Ideally, that would mean: only appearing in chromium DEPS
+ but transitively used in LibYUV.
+
+ Since it's hard to compute, we restrict ourselves to a well defined subset:
+ deps sitting in `ANDROID_DEPS_PATH`.
+ Otherwise, assumes that's a Chromium-only dependency.
+
+ Args:
+ libyuv_deps: dict of deps as defined in the LibYUV DEPS file.
+ new_cr_deps: dict of deps as defined in the chromium DEPS file.
+
+ Caveat: Doesn't detect a new package in existing dep.
+
+ Returns:
+ A tuple consisting of:
+ A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`.
+ A list of paths for other added dependencies.
+ """
+ all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps)
+ generated_android_deps = [
+ path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH)
+ ]
+ other_deps = [
+ path for path in all_added_deps if path not in generated_android_deps
+ ]
+ return generated_android_deps, other_deps
+
+
+def FindRemovedDeps(libyuv_deps, new_cr_deps):
+ """Calculate obsolete deps entries.
+
+ Ideally, that would mean: no more appearing in chromium DEPS
+ and not used in LibYUV.
+
+ Since it's hard to compute:
+ 1/ We restrict ourselves to a well defined subset:
+ deps sitting in `ANDROID_DEPS_PATH`.
+ 2/ We rely on existing behavior of CalculateChangeDeps.
+ I.e. Assumes non-CIPD dependencies are LibYUV-only, don't remove them.
+
+ Args:
+ libyuv_deps: dict of deps as defined in the LibYUV DEPS file.
+ new_cr_deps: dict of deps as defined in the chromium DEPS file.
+
+ Caveat: Doesn't detect a deleted package in existing dep.
+
+ Returns:
+ A tuple consisting of:
+ A list of paths of dependencies removed from `ANDROID_DEPS_PATH`.
+ A list of paths of unexpected disappearing dependencies.
+ """
+ all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps)
+ generated_android_deps = sorted([
+ path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)
+ ])
+ # Webrtc-only dependencies are handled in CalculateChangedDeps.
+ other_deps = sorted([
+ path
+ for path in all_removed_deps
+ if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS
+ ])
+ return generated_android_deps, other_deps
def CalculateChangedDeps(libyuv_deps, new_cr_deps):
- """
- Calculate changed deps entries based on entries defined in the libyuv DEPS
- file:
+ """Calculate changed deps entries based on entries defined in the LibYUV
+
+ DEPS
+
+ file:
- If a shared dependency with the Chromium DEPS file: roll it to the same
revision as Chromium (i.e. entry in the new_cr_deps dict)
- If it's a Chromium sub-directory, roll it to the HEAD revision (notice
@@ -238,270 +458,474 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps):
- If it's another DEPS entry (not shared with Chromium), roll it to HEAD
unless it's configured to be skipped.
- Returns:
- A list of ChangedDep objects representing the changed deps.
- """
- result = []
- libyuv_entries = BuildDepsentryDict(libyuv_deps)
- new_cr_entries = BuildDepsentryDict(new_cr_deps)
- for path, libyuv_deps_entry in libyuv_entries.iteritems():
- if path in DONT_AUTOROLL_THESE:
- continue
- cr_deps_entry = new_cr_entries.get(path)
- if cr_deps_entry:
- # Use the revision from Chromium's DEPS file.
- new_rev = cr_deps_entry.revision
- assert libyuv_deps_entry.url == cr_deps_entry.url, (
- 'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' %
- (path, libyuv_deps_entry.url, cr_deps_entry.url))
- else:
- # Use the HEAD of the deps repo.
- stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url,
- 'HEAD'])
- new_rev = stdout.strip().split('\t')[0]
-
- # Check if an update is necessary.
- if libyuv_deps_entry.revision != new_rev:
- logging.debug('Roll dependency %s to %s', path, new_rev)
- result.append(ChangedDep(path, libyuv_deps_entry.url,
- libyuv_deps_entry.revision, new_rev))
- return sorted(result)
+ Returns:
+ A list of ChangedDep objects representing the changed deps.
+ """
+ result = []
+ libyuv_entries = BuildDepsentryDict(libyuv_deps)
+ new_cr_entries = BuildDepsentryDict(new_cr_deps)
+ for path, libyuv_deps_entry in libyuv_entries.items():
+ if path in DONT_AUTOROLL_THESE:
+ continue
+ cr_deps_entry = new_cr_entries.get(path)
+ if cr_deps_entry:
+ assert type(cr_deps_entry) is type(libyuv_deps_entry)
+
+ if isinstance(cr_deps_entry, CipdDepsEntry):
+ result.extend(
+ _FindChangedCipdPackages(path, libyuv_deps_entry.packages,
+ cr_deps_entry.packages))
+ continue
+
+ if isinstance(cr_deps_entry, GcsDepsEntry):
+ result.extend(
+ _FindChangedVars(
+ path, ','.join(x['object_name']
+ for x in libyuv_deps_entry.objects),
+ ','.join(x['object_name']
+ for x in cr_deps_entry.objects)))
+ continue
+
+ if isinstance(cr_deps_entry, VersionEntry):
+ result.extend(
+ _FindChangedVars(
+ path, libyuv_deps_entry.version, cr_deps_entry.version
+ )
+ )
+ continue
+
+ # Use the revision from Chromium's DEPS file.
+ new_rev = cr_deps_entry.revision
+ assert libyuv_deps_entry.url == cr_deps_entry.url, (
+ 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.'
+ % (
+ path,
+ libyuv_deps_entry.url,
+ cr_deps_entry.url,
+ )
+ )
+ else:
+ if isinstance(libyuv_deps_entry, DepsEntry):
+ # Use the HEAD of the deps repo.
+ stdout, _ = _RunCommand(
+ ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']
+ )
+ new_rev = stdout.strip().split('\t')[0]
+ else:
+ # The dependency has been removed from chromium.
+ # This is handled by FindRemovedDeps.
+ continue
+
+ # Check if an update is necessary.
+ if libyuv_deps_entry.revision != new_rev:
+ logging.debug('Roll dependency %s to %s', path, new_rev)
+ result.append(
+ ChangedDep(
+ path,
+ libyuv_deps_entry.url,
+ libyuv_deps_entry.revision,
+ new_rev,
+ )
+ )
+ return sorted(result)
def CalculateChangedClang(new_cr_rev):
- def GetClangRev(lines):
- for line in lines:
- match = CLANG_REVISION_RE.match(line)
- if match:
- return match.group(1)
- raise RollError('Could not parse Clang revision!')
-
- with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
- current_lines = f.readlines()
- current_rev = GetClangRev(current_lines)
-
- new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
- new_cr_rev).splitlines()
- new_rev = GetClangRev(new_clang_update_py)
- return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
-
-
-def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
- new_commit_pos, changed_deps_list, clang_change):
- current_cr_rev = current_cr_rev[0:10]
- new_cr_rev = new_cr_rev[0:10]
- rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
- git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
-
- commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
- git_number_interval)]
- commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
- commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
- rev_interval))
- if changed_deps_list:
- commit_msg.append('Changed dependencies:')
-
- for c in changed_deps_list:
- commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
- c.current_rev[0:10],
- c.new_rev[0:10]))
- change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
- commit_msg.append('DEPS diff: %s\n' % change_url)
- else:
- commit_msg.append('No dependencies changed.')
-
- if clang_change.current_rev != clang_change.new_rev:
- commit_msg.append('Clang version changed %s:%s' %
- (clang_change.current_rev, clang_change.new_rev))
- change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
- CLANG_UPDATE_SCRIPT_URL_PATH)
- commit_msg.append('Details: %s\n' % change_url)
- else:
- commit_msg.append('No update to Clang.\n')
-
- # TBR needs to be non-empty for Gerrit to process it.
- git_author = _RunCommand(['git', 'config', 'user.email'],
- working_dir=CHECKOUT_SRC_DIR)[0].strip()
- commit_msg.append('TBR=%s' % git_author)
-
- commit_msg.append('BUG=None')
- return '\n'.join(commit_msg)
-
-
-def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
- changed_deps):
- """Update the DEPS file with the new revision."""
-
- # Update the chromium_revision variable.
- with open(deps_filename, 'rb') as deps_file:
- deps_content = deps_file.read()
- deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
- with open(deps_filename, 'wb') as deps_file:
- deps_file.write(deps_content)
-
- # Update each individual DEPS entry.
- for dep in changed_deps:
- local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
- if not os.path.isdir(local_dep_dir):
- raise RollError(
- 'Cannot find local directory %s. Make sure the .gclient file\n'
- 'contains all platforms in the target_os list, i.e.\n'
- 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
- 'Then run "gclient sync" again.' % local_dep_dir)
- _RunCommand(
- ['gclient', 'setdep', '--revision', '%s@%s' % (dep.path, dep.new_rev)],
- working_dir=CHECKOUT_SRC_DIR)
+
+ def GetClangRev(lines):
+ for line in lines:
+ match = CLANG_REVISION_RE.match(line)
+ if match:
+ return match.group(1)
+ raise RollError('Could not parse Clang revision!')
+
+ with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f:
+ current_lines = f.readlines()
+ current_rev = GetClangRev(current_lines)
+
+ new_clang_update_py = ReadRemoteCrFile(
+ CLANG_UPDATE_SCRIPT_URL_PATH, new_cr_rev
+ ).splitlines()
+ new_rev = GetClangRev(new_clang_update_py)
+ return ChangedDep(
+ CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev
+ )
+
+
+def GenerateCommitMessage(
+ rev_update,
+ current_commit_pos,
+ new_commit_pos,
+ changed_deps_list,
+ added_deps_paths=None,
+ removed_deps_paths=None,
+ clang_change=None,
+):
+ current_cr_rev = rev_update.current_chromium_rev[0:10]
+ new_cr_rev = rev_update.new_chromium_rev[0:10]
+ rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
+ git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
+
+ commit_msg = [
+ 'Roll chromium_revision %s (%s)\n'
+ % (rev_interval, git_number_interval),
+ 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
+ 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval),
+ ]
+
+ def Section(adjective, deps):
+ noun = 'dependency' if len(deps) == 1 else 'dependencies'
+ commit_msg.append('%s %s' % (adjective, noun))
+
+ if changed_deps_list:
+ Section('Changed', changed_deps_list)
+
+ for c in changed_deps_list:
+ if isinstance(c, ChangedCipdPackage):
+ commit_msg.append(
+ '* %s: %s..%s' % (c.path, c.current_version, c.new_version)
+ )
+ elif isinstance(c, ChangedVersionEntry):
+ commit_msg.append(
+ '* %s_vesion: %s..%s'
+ % (c.path, c.current_version, c.new_version)
+ )
+ else:
+ commit_msg.append(
+ '* %s: %s/+log/%s..%s'
+ % (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])
+ )
+
+ if added_deps_paths:
+ Section('Added', added_deps_paths)
+ commit_msg.extend('* %s' % p for p in added_deps_paths)
+
+ if removed_deps_paths:
+ Section('Removed', removed_deps_paths)
+ commit_msg.extend('* %s' % p for p in removed_deps_paths)
+
+ if any([changed_deps_list, added_deps_paths, removed_deps_paths]):
+ change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
+ commit_msg.append('DEPS diff: %s\n' % change_url)
+ else:
+ commit_msg.append('No dependencies changed.')
+
+ if clang_change and clang_change.current_rev != clang_change.new_rev:
+ commit_msg.append(
+ 'Clang version changed %s:%s'
+ % (clang_change.current_rev, clang_change.new_rev)
+ )
+ change_url = CHROMIUM_FILE_TEMPLATE % (
+ rev_interval,
+ CLANG_UPDATE_SCRIPT_URL_PATH,
+ )
+ commit_msg.append('Details: %s\n' % change_url)
+ else:
+ commit_msg.append('No update to Clang.\n')
+
+ commit_msg.append('BUG=None')
+ return '\n'.join(commit_msg)
+
+
+def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content):
+ """Update the DEPS file with the new revision."""
+
+ with open(deps_filename, 'rb') as deps_file:
+ deps_content = deps_file.read().decode('utf-8')
+
+ # Update the chromium_revision variable.
+ deps_content = deps_content.replace(
+ rev_update.current_chromium_rev, rev_update.new_chromium_rev
+ )
+
+ # Add and remove dependencies. For now: only generated android deps.
+ # Since gclient cannot add or remove deps, we on the fact that
+ # these android deps are located in one place we can copy/paste.
+ deps_re = re.compile(
+ ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL
+ )
+ new_deps = deps_re.search(new_cr_content)
+ old_deps = deps_re.search(deps_content)
+ if not new_deps or not old_deps:
+ faulty = 'Chromium' if not new_deps else 'LibYUV'
+ raise RollError(
+ 'Was expecting to find "%s" and "%s"\nin %s DEPS'
+ % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)
+ )
+ deps_content = deps_re.sub(new_deps.group(0), deps_content)
+
+ for dep in changed_deps:
+ if isinstance(dep, ChangedVersionEntry):
+ deps_content = deps_content.replace(
+ dep.current_version, dep.new_version
+ )
+
+ with open(deps_filename, 'wb') as deps_file:
+ deps_file.write(deps_content.encode('utf-8'))
+
+ # Update each individual DEPS entry.
+ for dep in changed_deps:
+ # ChangedVersionEntry types are already been processed.
+ if isinstance(dep, ChangedVersionEntry):
+ continue
+ local_dep_dir = os.path.join(GCLIENT_ROOT_DIR, dep.path)
+ if not os.path.isdir(local_dep_dir):
+ raise RollError(
+ 'Cannot find local directory %s. Either run\n'
+ 'gclient sync --deps=all\n'
+ 'or make sure the .gclient file for your solution contains '
+ 'all platforms in the target_os list, i.e.\n'
+ 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
+ 'Then run "gclient sync" again.' % local_dep_dir
+ )
+ if isinstance(dep, ChangedCipdPackage):
+ package = dep.package.format() # Eliminate double curly brackets
+ update = '%s:%s@%s' % (dep.path, package, dep.new_version)
+ else:
+ update = '%s@%s' % (dep.path, dep.new_rev)
+ _RunCommand(
+ ['gclient', 'setdep', '--revision', update],
+ working_dir=CHECKOUT_ROOT_DIR,
+ )
def _IsTreeClean():
- stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
- if len(stdout) == 0:
- return True
+ stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
+ if len(stdout) == 0:
+ return True
- logging.error('Dirty/unversioned files:\n%s', stdout)
- return False
+ logging.error('Dirty/unversioned files:\n%s', stdout)
+ return False
-def _EnsureUpdatedMasterBranch(dry_run):
- current_branch = _RunCommand(
- ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
- if current_branch != 'master':
- logging.error('Please checkout the master branch and re-run this script.')
- if not dry_run:
- sys.exit(-1)
+def _EnsureUpdatedMainBranch(dry_run):
+ current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[
+ 0
+ ].splitlines()[0]
+ if current_branch != 'main':
+ logging.error('Please checkout the main branch and re-run this script.') # pylint: disable=line-too-long
+ if not dry_run:
+ sys.exit(-1)
- logging.info('Updating master branch...')
- _RunCommand(['git', 'pull'])
+ logging.info('Updating main branch...')
+ _RunCommand(['git', 'pull'])
def _CreateRollBranch(dry_run):
- logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
- if not dry_run:
- _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
+ logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
+ if not dry_run:
+ _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
def _RemovePreviousRollBranch(dry_run):
- active_branch, branches = _GetBranches()
- if active_branch == ROLL_BRANCH_NAME:
- active_branch = 'master'
- if ROLL_BRANCH_NAME in branches:
- logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
- if not dry_run:
- _RunCommand(['git', 'checkout', active_branch])
- _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
+ active_branch, branches = _GetBranches()
+ if active_branch == ROLL_BRANCH_NAME:
+ active_branch = 'main'
+ if ROLL_BRANCH_NAME in branches:
+ logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
+ if not dry_run:
+ _RunCommand(['git', 'checkout', active_branch])
+ _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
def _LocalCommit(commit_msg, dry_run):
- logging.info('Committing changes locally.')
- if not dry_run:
- _RunCommand(['git', 'add', '--update', '.'])
- _RunCommand(['git', 'commit', '-m', commit_msg])
+ logging.info('Committing changes locally.')
+ if not dry_run:
+ _RunCommand(['git', 'add', '--update', '.'])
+ _RunCommand(['git', 'commit', '-m', commit_msg])
def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos):
- if skip_cq:
- return 0
- if (new_commit_pos - current_commit_pos) < cq_over:
- return 1
- return 2
-
-
-def _UploadCL(commit_queue_mode):
- """Upload the committed changes as a changelist to Gerrit.
-
- commit_queue_mode:
- - 2: Submit to commit queue.
- - 1: Run trybots but do not submit to CQ.
- - 0: Skip CQ, upload only.
- """
- cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks', '--send-mail']
- if commit_queue_mode >= 2:
- logging.info('Sending the CL to the CQ...')
- cmd.extend(['--use-commit-queue'])
- elif commit_queue_mode >= 1:
- logging.info('Starting CQ dry run...')
- cmd.extend(['--cq-dry-run'])
- extra_env = {
- 'EDITOR': 'true',
- 'SKIP_GCE_AUTH_FOR_GIT': '1',
- }
- stdout, stderr = _RunCommand(cmd, extra_env=extra_env)
- logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s',
- stdout, stderr)
+ if skip_cq:
+ return 0
+ if (new_commit_pos - current_commit_pos) < cq_over:
+ return 1
+ return 2
+
+
+def _GetCcRecipients(changed_deps_list):
+ """Returns a list of emails to notify based on the changed deps list."""
+ cc_recipients = []
+ for _ in changed_deps_list:
+ pass
+ return cc_recipients
+
+
+def _UploadCL(commit_queue_mode, add_cc=None):
+ """Upload the committed changes as a changelist to Gerrit.
+
+ commit_queue_mode:
+ - 2: Submit to commit queue.
+ - 1: Run trybots but do not submit to CQ.
+ - 0: Skip CQ, upload only.
+
+ add_cc: A list of email addresses to add as CC recipients.
+ """
+ cc_recipients = []
+ if add_cc:
+ cc_recipients.extend(add_cc)
+ cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks']
+ if commit_queue_mode >= 2:
+ logging.info('Sending the CL to the CQ...')
+ cmd.extend(['-o', 'label=Bot-Commit+1'])
+ cmd.extend(['-o', 'label=Commit-Queue+2'])
+ cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)])
+ elif commit_queue_mode >= 1:
+ logging.info('Starting CQ dry run...')
+ cmd.extend(['-o', 'label=Commit-Queue+1'])
+ extra_env = {
+ 'EDITOR': 'true',
+ 'SKIP_GCE_AUTH_FOR_GIT': '1',
+ }
+ stdout, stderr = _RunCommand(cmd, extra_env=extra_env)
+ logging.debug(
+ 'Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s',
+ stdout,
+ stderr,
+ )
+
+
+def GetRollRevisionRanges(opts, libyuv_deps):
+ current_cr_rev = libyuv_deps['vars']['chromium_revision']
+ new_cr_rev = opts.revision
+ if not new_cr_rev:
+ stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
+ head_rev = stdout.strip().split('\t')[0]
+ logging.info('No revision specified. Using HEAD: %s', head_rev)
+ new_cr_rev = head_rev
+
+ return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev)
def main():
- p = argparse.ArgumentParser()
- p.add_argument('--clean', action='store_true', default=False,
- help='Removes any previous local roll branch.')
- p.add_argument('-r', '--revision',
- help=('Chromium Git revision to roll to. Defaults to the '
- 'Chromium HEAD revision if omitted.'))
- p.add_argument('--dry-run', action='store_true', default=False,
- help=('Calculate changes and modify DEPS, but don\'t create '
- 'any local branch, commit, upload CL or send any '
- 'tryjobs.'))
- p.add_argument('-i', '--ignore-unclean-workdir', action='store_true',
- default=False,
- help=('Ignore if the current branch is not master or if there '
- 'are uncommitted changes (default: %(default)s).'))
- grp = p.add_mutually_exclusive_group()
- grp.add_argument('--skip-cq', action='store_true', default=False,
- help='Skip sending the CL to the CQ (default: %(default)s)')
- grp.add_argument('--cq-over', type=int, default=1,
- help=('Commit queue dry run if the revision difference '
- 'is below this number (default: %(default)s)'))
- p.add_argument('-v', '--verbose', action='store_true', default=False,
- help='Be extra verbose in printing of log messages.')
- opts = p.parse_args()
-
- if opts.verbose:
- logging.basicConfig(level=logging.DEBUG)
- else:
- logging.basicConfig(level=logging.INFO)
-
- if not opts.ignore_unclean_workdir and not _IsTreeClean():
- logging.error('Please clean your local checkout first.')
- return 1
-
- if opts.clean:
- _RemovePreviousRollBranch(opts.dry_run)
-
- if not opts.ignore_unclean_workdir:
- _EnsureUpdatedMasterBranch(opts.dry_run)
-
- new_cr_rev = opts.revision
- if not new_cr_rev:
- stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
- head_rev = stdout.strip().split('\t')[0]
- logging.info('No revision specified. Using HEAD: %s', head_rev)
- new_cr_rev = head_rev
-
- deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
- libyuv_deps = ParseLocalDepsFile(deps_filename)
- current_cr_rev = libyuv_deps['vars']['chromium_revision']
-
- current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
- new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
-
- new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
- changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
- clang_change = CalculateChangedClang(new_cr_rev)
- commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
- current_commit_pos, new_commit_pos,
- changed_deps, clang_change)
- logging.debug('Commit message:\n%s', commit_msg)
-
- _CreateRollBranch(opts.dry_run)
- UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
- _LocalCommit(commit_msg, opts.dry_run)
- commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over,
- current_commit_pos, new_commit_pos)
- logging.info('Uploading CL...')
- if not opts.dry_run:
- _UploadCL(commit_queue_mode)
- return 0
+ p = argparse.ArgumentParser()
+ p.add_argument(
+ '--clean',
+ action='store_true',
+ default=False,
+ help='Removes any previous local roll branch.',
+ )
+ p.add_argument(
+ '-r',
+ '--revision',
+ help=(
+ 'Chromium Git revision to roll to. Defaults to the '
+ 'Chromium HEAD revision if omitted.'
+ ),
+ )
+ p.add_argument(
+ '--dry-run',
+ action='store_true',
+ default=False,
+ help=(
+ "Calculate changes and modify DEPS, but don't create "
+ 'any local branch, commit, upload CL or send any '
+ 'tryjobs.'
+ ),
+ )
+ p.add_argument(
+ '-i',
+ '--ignore-unclean-workdir',
+ action='store_true',
+ default=False,
+ help=(
+ 'Ignore if the current branch is not main or if there '
+ 'are uncommitted changes (default: %(default)s).'
+ ),
+ )
+ grp = p.add_mutually_exclusive_group()
+ grp.add_argument(
+ '--skip-cq',
+ action='store_true',
+ default=False,
+ help='Skip sending the CL to the CQ (default: %(default)s)',
+ )
+ grp.add_argument(
+ '--cq-over',
+ type=int,
+ default=1,
+ help=(
+ 'Commit queue dry run if the revision difference '
+ 'is below this number (default: %(default)s)'
+ ),
+ )
+ p.add_argument(
+ '-v',
+ '--verbose',
+ action='store_true',
+ default=False,
+ help='Be extra verbose in printing of log messages.',
+ )
+ opts = p.parse_args()
+
+ if opts.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+
+ if not opts.ignore_unclean_workdir and not _IsTreeClean():
+ logging.error('Please clean your local checkout first.')
+ return 1
+
+ if opts.clean:
+ _RemovePreviousRollBranch(opts.dry_run)
+
+ if not opts.ignore_unclean_workdir:
+ _EnsureUpdatedMainBranch(opts.dry_run)
+
+ deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
+ libyuv_deps = ParseLocalDepsFile(deps_filename)
+
+ rev_update = GetRollRevisionRanges(opts, libyuv_deps)
+
+ current_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.current_chromium_rev)
+ )
+ new_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.new_chromium_rev)
+ )
+
+ new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev)
+ new_cr_deps = ParseDepsDict(new_cr_content)
+ changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
+ # Discard other deps, assumed to be chromium-only dependencies.
+ new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps)
+ removed_generated_android_deps, other_deps = FindRemovedDeps(
+ libyuv_deps, new_cr_deps
+ )
+ if other_deps:
+ raise RollError(
+ 'LibYUV DEPS entries are missing from Chromium: %s.\n'
+ 'Remove them or add them to either '
+ 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps
+ )
+ clang_change = CalculateChangedClang(rev_update.new_chromium_rev)
+ commit_msg = GenerateCommitMessage(
+ rev_update,
+ current_commit_pos,
+ new_commit_pos,
+ changed_deps,
+ added_deps_paths=new_generated_android_deps,
+ removed_deps_paths=removed_generated_android_deps,
+ clang_change=clang_change,
+ )
+ logging.debug('Commit message:\n%s', commit_msg)
+
+ _CreateRollBranch(opts.dry_run)
+ if not opts.dry_run:
+ UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content)
+ if _IsTreeClean():
+ logging.info('No DEPS changes detected, skipping CL creation.')
+ else:
+ _LocalCommit(commit_msg, opts.dry_run)
+ commit_queue_mode = ChooseCQMode(
+ opts.skip_cq, opts.cq_over, current_commit_pos, new_commit_pos
+ )
+ logging.info('Uploading CL...')
+ if not opts.dry_run:
+ _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps))
+ return 0
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/libs/libyuv/tools_libyuv/autoroller/unittests/roll_deps_test.py b/libs/libyuv/tools_libyuv/autoroller/unittests/roll_deps_test.py
index 477b6e402a..bd38c94094 100755
--- a/libs/libyuv/tools_libyuv/autoroller/unittests/roll_deps_test.py
+++ b/libs/libyuv/tools_libyuv/autoroller/unittests/roll_deps_test.py
@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env vpython3
+
# Copyright 2017 The LibYuv Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style license
@@ -14,25 +15,23 @@ import sys
import tempfile
import unittest
+import roll_deps
+from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile # pylint: disable=line-too-long
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
-import roll_deps
-from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \
- ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
-
TEST_DATA_VARS = {
- 'chromium_git': 'https://chromium.googlesource.com',
- 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
+ 'chromium_git': 'https://chromium.googlesource.com',
+ 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
}
DEPS_ENTRIES = {
- 'src/build': 'https://build.com',
- 'src/buildtools': 'https://buildtools.com',
- 'src/testing/gtest': 'https://gtest.com',
- 'src/testing/gmock': 'https://gmock.com',
+ 'src/build': 'https://build.com',
+ 'src/buildtools': 'https://buildtools.com',
+ 'src/testing/gtest': 'https://gtest.com',
+ 'src/testing/gmock': 'https://gmock.com',
}
BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f'
@@ -42,107 +41,124 @@ BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b'
class TestError(Exception):
- pass
+ pass
+
+class FakeCmd:
-class FakeCmd(object):
- def __init__(self):
- self.expectations = []
+ def __init__(self):
+ self.expectations = []
- def add_expectation(self, *args, **kwargs):
- returns = kwargs.pop('_returns', None)
- self.expectations.append((args, kwargs, returns))
+ def add_expectation(self, *args, **kwargs):
+ returns = kwargs.pop('_returns', None)
+ self.expectations.append((args, kwargs, returns))
- def __call__(self, *args, **kwargs):
- if not self.expectations:
- raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
- exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
- if args != exp_args or kwargs != exp_kwargs:
- message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs)
- message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
- raise TestError(message)
- return exp_returns
+ def __call__(self, *args, **kwargs):
+ if not self.expectations:
+ raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
+ exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
+ if args != exp_args or kwargs != exp_kwargs:
+ message = 'Expected:\n args: %s\n kwargs: %s\n' % (
+ exp_args,
+ exp_kwargs,
+ )
+ message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
+ raise TestError(message)
+ return exp_returns
class TestRollChromiumRevision(unittest.TestCase):
- def setUp(self):
- self._output_dir = tempfile.mkdtemp()
- for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
- shutil.copy(test_file, self._output_dir)
- self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS')
- self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old')
- self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
-
- self.fake = FakeCmd()
- self.old_RunCommand = getattr(roll_deps, '_RunCommand')
- setattr(roll_deps, '_RunCommand', self.fake)
-
- def tearDown(self):
- shutil.rmtree(self._output_dir, ignore_errors=True)
- self.assertEqual(self.fake.expectations, [])
- setattr(roll_deps, '_RunCommand', self.old_RunCommand)
-
- def testVarLookup(self):
- local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}}
- lookup = roll_deps.VarLookup(local_scope)
- self.assertEquals(lookup('foo'), 'bar')
-
- def testUpdateDepsFile(self):
- new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
-
- current_rev = TEST_DATA_VARS['chromium_revision']
- UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
- with open(self._libyuv_depsfile) as deps_file:
- deps_contents = deps_file.read()
- self.assertTrue(new_rev in deps_contents,
- 'Failed to find %s in\n%s' % (new_rev, deps_contents))
-
- def testParseDepsDict(self):
- with open(self._libyuv_depsfile) as deps_file:
- deps_contents = deps_file.read()
- local_scope = ParseDepsDict(deps_contents)
- vars_dict = local_scope['vars']
-
- def assertVar(variable_name):
- self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
- assertVar('chromium_git')
- assertVar('chromium_revision')
- self.assertEquals(len(local_scope['deps']), 3)
-
- def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
- self.assertEquals(len(entries), 1)
- self.assertEquals(entries[0], DEPS_ENTRIES['src/testing/gtest'])
-
- def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing')
- self.assertEquals(len(entries), 2)
-
- def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build')
- self.assertEquals(len(entries), 1)
- self.assertEquals(entries[0], DEPS_ENTRIES['src/build'])
-
- def testCalculateChangedDeps(self):
- _SetupGitLsRemoteCall(self.fake,
- 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV)
- libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
- new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
- changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
- self.assertEquals(len(changed_deps), 2)
- self.assertEquals(changed_deps[0].path, 'src/build')
- self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
- self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV)
-
- self.assertEquals(changed_deps[1].path, 'src/buildtools')
- self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
- self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
+
+ def setUp(self):
+ self._output_dir = tempfile.mkdtemp()
+ for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
+ shutil.copy(test_file, self._output_dir)
+ self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS')
+ self._old_cr_depsfile = os.path.join(
+ self._output_dir, 'DEPS.chromium.old'
+ )
+ self._new_cr_depsfile = os.path.join(
+ self._output_dir, 'DEPS.chromium.new'
+ )
+
+ self.fake = FakeCmd()
+ self.old_RunCommand = getattr(roll_deps, '_RunCommand')
+ setattr(roll_deps, '_RunCommand', self.fake)
+
+ def tearDown(self):
+ shutil.rmtree(self._output_dir, ignore_errors=True)
+ self.assertEqual(self.fake.expectations, [])
+ setattr(roll_deps, '_RunCommand', self.old_RunCommand)
+
+ def testVarLookup(self):
+ local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}}
+ lookup = roll_deps.VarLookup(local_scope)
+ self.assertEqual(lookup('foo'), 'bar')
+
+ def testUpdateDepsFile(self):
+ new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
+
+ current_rev = TEST_DATA_VARS['chromium_revision']
+ UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
+ with open(self._libyuv_depsfile, 'r') as deps_file:
+ deps_contents = deps_file.read()
+ self.assertTrue(
+ new_rev in deps_contents,
+ 'Failed to find %s in\n%s' % (new_rev, deps_contents),
+ )
+
+ def testParseDepsDict(self):
+ with open(self._libyuv_depsfile, 'r') as deps_file:
+ deps_contents = deps_file.read()
+ local_scope = ParseDepsDict(deps_contents)
+ vars_dict = local_scope['vars']
+
+ def assertVar(variable_name):
+ self.assertEqual(
+ vars_dict[variable_name], TEST_DATA_VARS[variable_name]
+ )
+
+ assertVar('chromium_git')
+ assertVar('chromium_revision')
+ self.assertEqual(len(local_scope['deps']), 3)
+
+ def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
+ self.assertEqual(len(entries), 1)
+ self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest'])
+
+ def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing')
+ self.assertEqual(len(entries), 2)
+
+ def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build')
+ self.assertEqual(len(entries), 1)
+ self.assertEqual(entries[0], DEPS_ENTRIES['src/build'])
+
+ def testCalculateChangedDeps(self):
+ _SetupGitLsRemoteCall(
+ self.fake,
+ 'https://chromium.googlesource.com/chromium/src/build',
+ BUILD_NEW_REV,
+ )
+ libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
+ new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
+ changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
+ self.assertEqual(len(changed_deps), 2)
+ self.assertEqual(changed_deps[0].path, 'src/build')
+ self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV)
+ self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV)
+
+ self.assertEqual(changed_deps[1].path, 'src/buildtools')
+ self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
+ self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
def _SetupGitLsRemoteCall(cmd_fake, url, revision):
- cmd = ['git', 'ls-remote', url, revision]
- cmd_fake.add_expectation(cmd, _returns=(revision, None))
+ cmd = ['git', 'ls-remote', url, revision]
+ cmd_fake.add_expectation(cmd, _returns=(revision, None))
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/libs/libyuv/tools_libyuv/autoroller/unittests/testdata/DEPS b/libs/libyuv/tools_libyuv/autoroller/unittests/testdata/DEPS
index 9fbb48a734..4f45860cda 100644
--- a/libs/libyuv/tools_libyuv/autoroller/unittests/testdata/DEPS
+++ b/libs/libyuv/tools_libyuv/autoroller/unittests/testdata/DEPS
@@ -3,6 +3,7 @@
vars = {
'chromium_git': 'https://chromium.googlesource.com',
'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
+ 'ignored_str': Str(''),
}
deps = {
diff --git a/libs/libyuv/tools_libyuv/get_landmines.py b/libs/libyuv/tools_libyuv/get_landmines.py
index c554f04a39..ee0dbecd7e 100755
--- a/libs/libyuv/tools_libyuv/get_landmines.py
+++ b/libs/libyuv/tools_libyuv/get_landmines.py
@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+
# Copyright 2016 The LibYuv Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style license
@@ -8,31 +9,30 @@
# be found in the AUTHORS file in the root of the source tree.
"""
-This file emits the list of reasons why a particular build needs to be clobbered
-(or a list of 'landmines').
+This file emits the list of reasons why a particular build needs to be
+clobbered (or a list of 'landmines').
"""
import sys
def print_landmines():
- """
- ALL LANDMINES ARE EMITTED FROM HERE.
- """
- # DO NOT add landmines as part of a regular CL. Landmines are a last-effort
- # bandaid fix if a CL that got landed has a build dependency bug and all bots
- # need to be cleaned up. If you're writing a new CL that causes build
- # dependency problems, fix the dependency problems instead of adding a
- # landmine.
- # See the Chromium version in src/build/get_landmines.py for usage examples.
- print 'Clobber to remove GYP artifacts after switching bots to GN.'
- print 'Another try to remove GYP artifacts after switching bots to GN.'
+ """ALL LANDMINES ARE EMITTED FROM HERE."""
+ # DO NOT add landmines as part of a regular CL. Landmines are a last-effort
+ # bandaid fix if a CL that got landed has a build dependency bug and all
+ # bots need to be cleaned up. If you're writing a new CL that causes build
+ # dependency problems, fix the dependency problems instead of adding a
+ # landmine.
+ # See the Chromium version in src/build/get_landmines.py for usage
+ # examples.
+ print('Clobber to remove GYP artifacts after switching bots to GN.')
+ print('Another try to remove GYP artifacts after switching bots to GN.')
def main():
- print_landmines()
- return 0
+ print_landmines()
+ return 0
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/libs/libyuv/tools_libyuv/msan/OWNERS b/libs/libyuv/tools_libyuv/msan/OWNERS
index 0a919805c2..9b67a8f678 100644
--- a/libs/libyuv/tools_libyuv/msan/OWNERS
+++ b/libs/libyuv/tools_libyuv/msan/OWNERS
@@ -1,3 +1,3 @@
+mbonadei@chromium.org
+fbarchard@chromium.org
pbos@chromium.org
-phoglund@chromium.org
-
diff --git a/libs/libyuv/tools_libyuv/ubsan/OWNERS b/libs/libyuv/tools_libyuv/ubsan/OWNERS
index da77b4ef23..9b67a8f678 100644
--- a/libs/libyuv/tools_libyuv/ubsan/OWNERS
+++ b/libs/libyuv/tools_libyuv/ubsan/OWNERS
@@ -1,4 +1,3 @@
-pbos@webrtc.org
-phoglund@webrtc.org
+mbonadei@chromium.org
fbarchard@chromium.org
-
+pbos@chromium.org
diff --git a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.bat b/libs/libyuv/tools_libyuv/valgrind/chrome_tests.bat
deleted file mode 100755
index 9d4c8ca8d3..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.bat
+++ /dev/null
@@ -1,53 +0,0 @@
-@echo off
-:: Copyright (c) 2011 The Chromium Authors. All rights reserved.
-:: Use of this source code is governed by a BSD-style license that can be
-:: found in the LICENSE file.
-
-setlocal
-
-set THISDIR=%~dp0
-set TOOL_NAME="unknown"
-
-:: Get the tool name and put it into TOOL_NAME {{{1
-:: NB: SHIFT command doesn't modify %*
-:PARSE_ARGS_LOOP
- if %1 == () GOTO:TOOLNAME_NOT_FOUND
- if %1 == --tool GOTO:TOOLNAME_FOUND
- SHIFT
- goto :PARSE_ARGS_LOOP
-
-:TOOLNAME_NOT_FOUND
-echo "Please specify a tool (e.g. drmemory) by using --tool flag"
-exit /B 1
-
-:TOOLNAME_FOUND
-SHIFT
-set TOOL_NAME=%1
-:: }}}
-if "%TOOL_NAME%" == "drmemory" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_light" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_full" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_pattern" GOTO :SETUP_DRMEMORY
-echo "Unknown tool: `%TOOL_NAME%`! Only drmemory is supported right now"
-exit /B 1
-
-:SETUP_DRMEMORY
-:: Set up DRMEMORY_COMMAND to invoke Dr. Memory {{{1
-set DRMEMORY_PATH=%THISDIR%..\..\third_party\drmemory
-set DRMEMORY_SFX=%DRMEMORY_PATH%\drmemory-windows-sfx.exe
-if EXIST %DRMEMORY_SFX% GOTO DRMEMORY_BINARY_OK
-echo "Can't find Dr. Memory executables."
-echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
-echo "for the instructions on how to get them."
-exit /B 1
-
-:DRMEMORY_BINARY_OK
-%DRMEMORY_SFX% -o%DRMEMORY_PATH%\unpacked -y
-set DRMEMORY_COMMAND=%DRMEMORY_PATH%\unpacked\bin\drmemory.exe
-:: }}}
-goto :RUN_TESTS
-
-:RUN_TESTS
-set PYTHONPATH=%THISDIR%../python/google
-set RUNNING_ON_VALGRIND=yes
-python %THISDIR%/chrome_tests.py %*
diff --git a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.py b/libs/libyuv/tools_libyuv/valgrind/chrome_tests.py
deleted file mode 100755
index fe899bce14..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.py
+++ /dev/null
@@ -1,869 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-''' Runs various chrome tests through valgrind_test.py.'''
-
-import glob
-import logging
-import multiprocessing
-import optparse
-import os
-import stat
-import subprocess
-import sys
-
-import logging_utils
-import path_utils
-
-import common
-import valgrind_test
-
-class TestNotFound(Exception): pass
-
-class MultipleGTestFiltersSpecified(Exception): pass
-
-class BuildDirNotFound(Exception): pass
-
-class BuildDirAmbiguous(Exception): pass
-
-class ExecutableNotFound(Exception): pass
-
-class BadBinary(Exception): pass
-
-class ChromeTests:
- SLOW_TOOLS = ["memcheck", "drmemory"]
- LAYOUT_TESTS_DEFAULT_CHUNK_SIZE = 300
-
- def __init__(self, options, args, test):
- if ':' in test:
- (self._test, self._gtest_filter) = test.split(':', 1)
- else:
- self._test = test
- self._gtest_filter = options.gtest_filter
-
- if self._test not in self._test_list:
- raise TestNotFound("Unknown test: %s" % test)
-
- if options.gtest_filter and options.gtest_filter != self._gtest_filter:
- raise MultipleGTestFiltersSpecified("Can not specify both --gtest_filter "
- "and --test %s" % test)
-
- self._options = options
- self._args = args
-
- script_dir = path_utils.ScriptDir()
- # Compute the top of the tree (the "source dir") from the script dir (where
- # this script lives). We assume that the script dir is in tools/valgrind/
- # relative to the top of the tree.
- self._source_dir = os.path.dirname(os.path.dirname(script_dir))
- # since this path is used for string matching, make sure it's always
- # an absolute Unix-style path
- self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/')
- valgrind_test_script = os.path.join(script_dir, "valgrind_test.py")
- self._command_preamble = ["--source-dir=%s" % (self._source_dir)]
-
- if not self._options.build_dir:
- dirs = [
- os.path.join(self._source_dir, "xcodebuild", "Debug"),
- os.path.join(self._source_dir, "out", "Debug"),
- os.path.join(self._source_dir, "build", "Debug"),
- ]
- build_dir = [d for d in dirs if os.path.isdir(d)]
- if len(build_dir) > 1:
- raise BuildDirAmbiguous("Found more than one suitable build dir:\n"
- "%s\nPlease specify just one "
- "using --build-dir" % ", ".join(build_dir))
- elif build_dir:
- self._options.build_dir = build_dir[0]
- else:
- self._options.build_dir = None
-
- if self._options.build_dir:
- build_dir = os.path.abspath(self._options.build_dir)
- self._command_preamble += ["--build-dir=%s" % (self._options.build_dir)]
-
- def _EnsureBuildDirFound(self):
- if not self._options.build_dir:
- raise BuildDirNotFound("Oops, couldn't find a build dir, please "
- "specify it manually using --build-dir")
-
- def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None):
- '''Generates the default command array that most tests will use.'''
- if exe and common.IsWindows():
- exe += '.exe'
-
- cmd = list(self._command_preamble)
-
- # Find all suppressions matching the following pattern:
- # tools/valgrind/TOOL/suppressions[_PLATFORM].txt
- # and list them with --suppressions= prefix.
- script_dir = path_utils.ScriptDir()
- tool_name = tool.ToolName();
- suppression_file = os.path.join(script_dir, tool_name, "suppressions.txt")
- if os.path.exists(suppression_file):
- cmd.append("--suppressions=%s" % suppression_file)
- # Platform-specific suppression
- for platform in common.PlatformNames():
- platform_suppression_file = \
- os.path.join(script_dir, tool_name, 'suppressions_%s.txt' % platform)
- if os.path.exists(platform_suppression_file):
- cmd.append("--suppressions=%s" % platform_suppression_file)
-
- if tool_name == "drmemory":
- if self._options.drmemory_ops:
- # prepending " " to avoid Dr. Memory's option confusing optparse
- cmd += ["--drmemory_ops", " " + self._options.drmemory_ops]
-
- if self._options.valgrind_tool_flags:
- cmd += self._options.valgrind_tool_flags.split(" ")
- if self._options.keep_logs:
- cmd += ["--keep_logs"]
- if valgrind_test_args != None:
- for arg in valgrind_test_args:
- cmd.append(arg)
- if exe:
- self._EnsureBuildDirFound()
- exe_path = os.path.join(self._options.build_dir, exe)
- if not os.path.exists(exe_path):
- raise ExecutableNotFound("Couldn't find '%s'" % exe_path)
-
- # Make sure we don't try to test ASan-built binaries
- # with other dynamic instrumentation-based tools.
- # TODO(timurrrr): also check TSan and MSan?
- # `nm` might not be available, so use try-except.
- try:
- # Do not perform this check on OS X, as 'nm' on 10.6 can't handle
- # binaries built with Clang 3.5+.
- if not common.IsMac():
- nm_output = subprocess.check_output(["nm", exe_path])
- if nm_output.find("__asan_init") != -1:
- raise BadBinary("You're trying to run an executable instrumented "
- "with AddressSanitizer under %s. Please provide "
- "an uninstrumented executable." % tool_name)
- except OSError:
- pass
-
- cmd.append(exe_path)
- # Valgrind runs tests slowly, so slow tests hurt more; show elapased time
- # so we can find the slowpokes.
- cmd.append("--gtest_print_time")
- # Built-in test launcher for gtest-based executables runs tests using
- # multiple process by default. Force the single-process mode back.
- cmd.append("--single-process-tests")
- if self._options.gtest_repeat:
- cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat)
- if self._options.gtest_shuffle:
- cmd.append("--gtest_shuffle")
- if self._options.gtest_break_on_failure:
- cmd.append("--gtest_break_on_failure")
- if self._options.test_launcher_bot_mode:
- cmd.append("--test-launcher-bot-mode")
- if self._options.test_launcher_total_shards is not None:
- cmd.append("--test-launcher-total-shards=%d"
- % self._options.test_launcher_total_shards)
- if self._options.test_launcher_shard_index is not None:
- cmd.append("--test-launcher-shard-index=%d"
- % self._options.test_launcher_shard_index)
- return cmd
-
- def Run(self):
- ''' Runs the test specified by command-line argument --test '''
- logging.info("running test %s" % (self._test))
- return self._test_list[self._test](self)
-
- def _AppendGtestFilter(self, tool, name, cmd):
- '''Append an appropriate --gtest_filter flag to the googletest binary
- invocation.
- If the user passed their own filter mentioning only one test, just use
- it. Otherwise, filter out tests listed in the appropriate gtest_exclude
- files.
- '''
- if (self._gtest_filter and
- ":" not in self._gtest_filter and
- "?" not in self._gtest_filter and
- "*" not in self._gtest_filter):
- cmd.append("--gtest_filter=%s" % self._gtest_filter)
- return
-
- filters = []
- gtest_files_dir = os.path.join(path_utils.ScriptDir(), "gtest_exclude")
-
- gtest_filter_files = [
- os.path.join(gtest_files_dir, name + ".gtest-%s.txt" % tool.ToolName())]
- # Use ".gtest.txt" files only for slow tools, as they now contain
- # Valgrind- and Dr.Memory-specific filters.
- # TODO(glider): rename the files to ".gtest_slow.txt"
- if tool.ToolName() in ChromeTests.SLOW_TOOLS:
- gtest_filter_files += [os.path.join(gtest_files_dir, name + ".gtest.txt")]
- for platform_suffix in common.PlatformNames():
- gtest_filter_files += [
- os.path.join(gtest_files_dir, name + ".gtest_%s.txt" % platform_suffix),
- os.path.join(gtest_files_dir, name + ".gtest-%s_%s.txt" % \
- (tool.ToolName(), platform_suffix))]
- logging.info("Reading gtest exclude filter files:")
- for filename in gtest_filter_files:
- # strip the leading absolute path (may be very long on the bot)
- # and the following / or \.
- readable_filename = filename.replace("\\", "/") # '\' on Windows
- readable_filename = readable_filename.replace(self._source_dir, "")[1:]
- if not os.path.exists(filename):
- logging.info(" \"%s\" - not found" % readable_filename)
- continue
- logging.info(" \"%s\" - OK" % readable_filename)
- f = open(filename, 'r')
- for line in f.readlines():
- if line.startswith("#") or line.startswith("//") or line.isspace():
- continue
- line = line.rstrip()
- test_prefixes = ["FLAKY", "FAILS"]
- for p in test_prefixes:
- # Strip prefixes from the test names.
- line = line.replace(".%s_" % p, ".")
- # Exclude the original test name.
- filters.append(line)
- if line[-2:] != ".*":
- # List all possible prefixes if line doesn't end with ".*".
- for p in test_prefixes:
- filters.append(line.replace(".", ".%s_" % p))
- # Get rid of duplicates.
- filters = set(filters)
- gtest_filter = self._gtest_filter
- if len(filters):
- if gtest_filter:
- gtest_filter += ":"
- if gtest_filter.find("-") < 0:
- gtest_filter += "-"
- else:
- gtest_filter = "-"
- gtest_filter += ":".join(filters)
- if gtest_filter:
- cmd.append("--gtest_filter=%s" % gtest_filter)
-
- @staticmethod
- def ShowTests():
- test_to_names = {}
- for name, test_function in ChromeTests._test_list.iteritems():
- test_to_names.setdefault(test_function, []).append(name)
-
- name_to_aliases = {}
- for names in test_to_names.itervalues():
- names.sort(key=lambda name: len(name))
- name_to_aliases[names[0]] = names[1:]
-
- print
- print "Available tests:"
- print "----------------"
- for name, aliases in sorted(name_to_aliases.iteritems()):
- if aliases:
- print " {} (aka {})".format(name, ', '.join(aliases))
- else:
- print " {}".format(name)
-
- def SetupLdPath(self, requires_build_dir):
- if requires_build_dir:
- self._EnsureBuildDirFound()
- elif not self._options.build_dir:
- return
-
- # Append build_dir to LD_LIBRARY_PATH so external libraries can be loaded.
- if (os.getenv("LD_LIBRARY_PATH")):
- os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"),
- self._options.build_dir))
- else:
- os.putenv("LD_LIBRARY_PATH", self._options.build_dir)
-
- def SimpleTest(self, module, name, valgrind_test_args=None, cmd_args=None):
- tool = valgrind_test.CreateTool(self._options.valgrind_tool)
- cmd = self._DefaultCommand(tool, name, valgrind_test_args)
- self._AppendGtestFilter(tool, name, cmd)
- cmd.extend(['--test-tiny-timeout=1000'])
- if cmd_args:
- cmd.extend(cmd_args)
-
- self.SetupLdPath(True)
- return tool.Run(cmd, module)
-
- def RunCmdLine(self):
- tool = valgrind_test.CreateTool(self._options.valgrind_tool)
- cmd = self._DefaultCommand(tool, None, self._args)
- self.SetupLdPath(False)
- return tool.Run(cmd, None)
-
- def TestAccessibility(self):
- return self.SimpleTest("accessibility", "accessibility_unittests")
-
- def TestAddressInput(self):
- return self.SimpleTest("addressinput", "libaddressinput_unittests")
-
- def TestAngle(self):
- return self.SimpleTest("angle", "angle_unittests")
-
- def TestAppList(self):
- return self.SimpleTest("app_list", "app_list_unittests")
-
- def TestAsh(self):
- return self.SimpleTest("ash", "ash_unittests")
-
- def TestAura(self):
- return self.SimpleTest("aura", "aura_unittests")
-
- def TestBase(self):
- return self.SimpleTest("base", "base_unittests")
-
- def TestBlinkHeap(self):
- return self.SimpleTest("blink_heap", "blink_heap_unittests")
-
- def TestBlinkPlatform(self):
- return self.SimpleTest("blink_platform", "blink_platform_unittests")
-
- def TestCacheInvalidation(self):
- return self.SimpleTest("cacheinvalidation", "cacheinvalidation_unittests")
-
- def TestCast(self):
- return self.SimpleTest("chrome", "cast_unittests")
-
- def TestCC(self):
- return self.SimpleTest("cc", "cc_unittests",
- cmd_args=[
- "--cc-layer-tree-test-long-timeout"])
-
- def TestChromeApp(self):
- return self.SimpleTest("chrome_app", "chrome_app_unittests")
-
- def TestChromeElf(self):
- return self.SimpleTest("chrome_elf", "chrome_elf_unittests")
-
- def TestChromeDriver(self):
- return self.SimpleTest("chromedriver", "chromedriver_unittests")
-
- def TestChromeOS(self):
- return self.SimpleTest("chromeos", "chromeos_unittests")
-
- def TestComponents(self):
- return self.SimpleTest("components", "components_unittests")
-
- def TestCompositor(self):
- return self.SimpleTest("compositor", "compositor_unittests")
-
- def TestContent(self):
- return self.SimpleTest("content", "content_unittests")
-
- def TestCourgette(self):
- return self.SimpleTest("courgette", "courgette_unittests")
-
- def TestCrypto(self):
- return self.SimpleTest("crypto", "crypto_unittests")
-
- def TestDevice(self):
- return self.SimpleTest("device", "device_unittests")
-
- def TestDisplay(self):
- return self.SimpleTest("display", "display_unittests")
-
- def TestEvents(self):
- return self.SimpleTest("events", "events_unittests")
-
- def TestExtensions(self):
- return self.SimpleTest("extensions", "extensions_unittests")
-
- def TestFFmpegRegressions(self):
- return self.SimpleTest("chrome", "ffmpeg_regression_tests")
-
- def TestGCM(self):
- return self.SimpleTest("gcm", "gcm_unit_tests")
-
- def TestGfx(self):
- return self.SimpleTest("gfx", "gfx_unittests")
-
- def TestGin(self):
- return self.SimpleTest("gin", "gin_unittests")
-
- def TestGoogleApis(self):
- return self.SimpleTest("google_apis", "google_apis_unittests")
-
- def TestGPU(self):
- return self.SimpleTest("gpu", "gpu_unittests")
-
- def TestIpc(self):
- return self.SimpleTest("ipc", "ipc_tests",
- valgrind_test_args=["--trace_children"])
-
- def TestInstallerUtil(self):
- return self.SimpleTest("installer_util", "installer_util_unittests")
-
- def TestInstallStatic(self):
- return self.SimpleTest("install_static", "install_static_unittests")
-
- def TestJingle(self):
- return self.SimpleTest("chrome", "jingle_unittests")
-
- def TestKeyboard(self):
- return self.SimpleTest("keyboard", "keyboard_unittests")
-
- def TestLatency(self):
- return self.SimpleTest("latency", "latency_unittests")
-
- def TestMedia(self):
- return self.SimpleTest("chrome", "media_unittests")
-
- def TestMessageCenter(self):
- return self.SimpleTest("message_center", "message_center_unittests")
-
- def TestMidi(self):
- return self.SimpleTest("chrome", "midi_unittests")
-
- def TestMojoCommon(self):
- return self.SimpleTest("mojo_common", "mojo_common_unittests")
-
- def TestMojoPublicBindings(self):
- return self.SimpleTest("mojo_public_bindings",
- "mojo_public_bindings_unittests")
-
- def TestMojoPublicSystem(self):
- return self.SimpleTest("mojo_public_system",
- "mojo_public_system_unittests")
-
- def TestMojoPublicSysPerf(self):
- return self.SimpleTest("mojo_public_sysperf",
- "mojo_public_system_perftests")
-
- def TestMojoSystem(self):
- return self.SimpleTest("mojo_system", "mojo_system_unittests")
-
- def TestNet(self):
- return self.SimpleTest("net", "net_unittests")
-
- def TestNetPerf(self):
- return self.SimpleTest("net", "net_perftests")
-
- def TestPhoneNumber(self):
- return self.SimpleTest("phonenumber", "libphonenumber_unittests")
-
- def TestPPAPI(self):
- return self.SimpleTest("chrome", "ppapi_unittests")
-
- def TestPrinting(self):
- return self.SimpleTest("chrome", "printing_unittests")
-
- def TestRemoting(self):
- return self.SimpleTest("chrome", "remoting_unittests",
- cmd_args=[
- "--ui-test-action-timeout=60000",
- "--ui-test-action-max-timeout=150000"])
-
- def TestSkia(self):
- return self.SimpleTest("skia", "skia_unittests")
-
- def TestSql(self):
- return self.SimpleTest("chrome", "sql_unittests")
-
- def TestStorage(self):
- return self.SimpleTest("storage", "storage_unittests")
-
- def TestLinuxSandbox(self):
- return self.SimpleTest("sandbox", "sandbox_linux_unittests")
-
- def TestUnit(self):
- # http://crbug.com/51716
- # Disabling all unit tests
- # Problems reappeared after r119922
- if common.IsMac() and (self._options.valgrind_tool == "memcheck"):
- logging.warning("unit_tests are disabled for memcheck on MacOS.")
- return 0;
- return self.SimpleTest("chrome", "unit_tests")
-
- def TestUIBaseUnit(self):
- return self.SimpleTest("chrome", "ui_base_unittests")
-
- def TestUIChromeOS(self):
- return self.SimpleTest("chrome", "ui_chromeos_unittests")
-
- def TestURL(self):
- return self.SimpleTest("chrome", "url_unittests")
-
- def TestViews(self):
- return self.SimpleTest("views", "views_unittests")
-
-
- # Valgrind timeouts are in seconds.
- UI_VALGRIND_ARGS = ["--timeout=14400", "--trace_children", "--indirect"]
- # UI test timeouts are in milliseconds.
- UI_TEST_ARGS = ["--ui-test-action-timeout=60000",
- "--ui-test-action-max-timeout=150000",
- "--no-sandbox"]
-
- # TODO(thestig) fine-tune these values.
- # Valgrind timeouts are in seconds.
- BROWSER_VALGRIND_ARGS = ["--timeout=50000", "--trace_children", "--indirect"]
- # Browser test timeouts are in milliseconds.
- BROWSER_TEST_ARGS = ["--ui-test-action-timeout=400000",
- "--ui-test-action-max-timeout=800000",
- "--no-sandbox"]
-
- def TestBrowser(self):
- return self.SimpleTest("chrome", "browser_tests",
- valgrind_test_args=self.BROWSER_VALGRIND_ARGS,
- cmd_args=self.BROWSER_TEST_ARGS)
-
- def TestContentBrowser(self):
- return self.SimpleTest("content", "content_browsertests",
- valgrind_test_args=self.BROWSER_VALGRIND_ARGS,
- cmd_args=self.BROWSER_TEST_ARGS)
-
- def TestInteractiveUI(self):
- return self.SimpleTest("chrome", "interactive_ui_tests",
- valgrind_test_args=self.UI_VALGRIND_ARGS,
- cmd_args=self.UI_TEST_ARGS)
-
- def TestSyncIntegration(self):
- return self.SimpleTest("chrome", "sync_integration_tests",
- valgrind_test_args=self.UI_VALGRIND_ARGS,
- cmd_args=(["--ui-test-action-max-timeout=450000"]))
-
- def TestLayoutChunk(self, chunk_num, chunk_size):
- # Run tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size) from the
- # list of tests. Wrap around to beginning of list at end.
- # If chunk_size is zero, run all tests in the list once.
- # If a text file is given as argument, it is used as the list of tests.
- assert((chunk_size == 0) != (len(self._args) == 0))
- # Build the ginormous commandline in 'cmd'.
- # It's going to be roughly
- # python valgrind_test.py ...
- # but we'll use the --indirect flag to valgrind_test.py
- # to avoid valgrinding python.
- # Start by building the valgrind_test.py commandline.
- tool = valgrind_test.CreateTool(self._options.valgrind_tool)
- cmd = self._DefaultCommand(tool)
- cmd.append("--trace_children")
- cmd.append("--indirect_webkit_layout")
- cmd.append("--ignore_exit_code")
- # Now build script_cmd, the run-webkits-tests commandline.
- # Store each chunk in its own directory so that we can find the data later
- chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num)
- out_dir = os.path.join(path_utils.ScriptDir(), "latest")
- out_dir = os.path.join(out_dir, chunk_dir)
- if os.path.exists(out_dir):
- old_files = glob.glob(os.path.join(out_dir, "*.txt"))
- for f in old_files:
- os.remove(f)
- else:
- os.makedirs(out_dir)
- script = os.path.join(self._source_dir, "third_party", "WebKit", "Tools",
- "Scripts", "run-webkit-tests")
- # http://crbug.com/260627: After the switch to content_shell from DRT, each
- # test now brings up 3 processes. Under Valgrind, they become memory bound
- # and can eventually OOM if we don't reduce the total count.
- # It'd be nice if content_shell automatically throttled the startup of new
- # tests if we're low on memory.
- jobs = max(1, int(multiprocessing.cpu_count() * 0.3))
- script_cmd = ["python", script, "-v",
- # run a separate DumpRenderTree for each test
- "--batch-size=1",
- "--fully-parallel",
- "--child-processes=%d" % jobs,
- "--time-out-ms=800000",
- "--no-retry-failures", # retrying takes too much time
- # http://crbug.com/176908: Don't launch a browser when done.
- "--no-show-results",
- "--nocheck-sys-deps",
- "--additional-driver-flag=--no-sandbox"]
- # Pass build mode to run-webkit-tests. We aren't passed it directly,
- # so parse it out of build_dir. run-webkit-tests can only handle
- # the two values "Release" and "Debug".
- # TODO(Hercules): unify how all our scripts pass around build mode
- # (--mode / --target / --build-dir / --debug)
- if self._options.build_dir:
- build_root, mode = os.path.split(self._options.build_dir)
- script_cmd.extend(["--build-directory", build_root, "--target", mode])
- if (chunk_size > 0):
- script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size))
- if len(self._args):
- # if the arg is a txt file, then treat it as a list of tests
- if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt":
- script_cmd.append("--test-list=%s" % self._args[0])
- else:
- script_cmd.extend(self._args)
- self._AppendGtestFilter(tool, "layout", script_cmd)
- # Now run script_cmd with the wrapper in cmd
- cmd.extend(["--"])
- cmd.extend(script_cmd)
-
- # Layout tests often times fail quickly, but the buildbot remains green.
- # Detect this situation when running with the default chunk size.
- if chunk_size == self.LAYOUT_TESTS_DEFAULT_CHUNK_SIZE:
- min_runtime_in_seconds=120
- else:
- min_runtime_in_seconds=0
- ret = tool.Run(cmd, "layout", min_runtime_in_seconds=min_runtime_in_seconds)
- return ret
-
-
- def TestLayout(self):
- # A "chunk file" is maintained in the local directory so that each test
- # runs a slice of the layout tests of size chunk_size that increments with
- # each run. Since tests can be added and removed from the layout tests at
- # any time, this is not going to give exact coverage, but it will allow us
- # to continuously run small slices of the layout tests under valgrind rather
- # than having to run all of them in one shot.
- chunk_size = self._options.num_tests
- if chunk_size == 0 or len(self._args):
- return self.TestLayoutChunk(0, 0)
- chunk_num = 0
- chunk_file = os.path.join("valgrind_layout_chunk.txt")
- logging.info("Reading state from " + chunk_file)
- try:
- f = open(chunk_file)
- if f:
- chunk_str = f.read()
- if len(chunk_str):
- chunk_num = int(chunk_str)
- # This should be enough so that we have a couple of complete runs
- # of test data stored in the archive (although note that when we loop
- # that we almost guaranteed won't be at the end of the test list)
- if chunk_num > 10000:
- chunk_num = 0
- f.close()
- except IOError, (errno, strerror):
- logging.error("error reading from file %s (%d, %s)" % (chunk_file,
- errno, strerror))
- # Save the new chunk size before running the tests. Otherwise if a
- # particular chunk hangs the bot, the chunk number will never get
- # incremented and the bot will be wedged.
- logging.info("Saving state to " + chunk_file)
- try:
- f = open(chunk_file, "w")
- chunk_num += 1
- f.write("%d" % chunk_num)
- f.close()
- except IOError, (errno, strerror):
- logging.error("error writing to file %s (%d, %s)" % (chunk_file, errno,
- strerror))
- # Since we're running small chunks of the layout tests, it's important to
- # mark the ones that have errors in them. These won't be visible in the
- # summary list for long, but will be useful for someone reviewing this bot.
- return self.TestLayoutChunk(chunk_num, chunk_size)
-
- # The known list of tests.
- # Recognise the original abbreviations as well as full executable names.
- _test_list = {
- "cmdline" : RunCmdLine,
- "addressinput": TestAddressInput,
- "libaddressinput_unittests": TestAddressInput,
- "accessibility": TestAccessibility,
- "angle": TestAngle, "angle_unittests": TestAngle,
- "app_list": TestAppList, "app_list_unittests": TestAppList,
- "ash": TestAsh, "ash_unittests": TestAsh,
- "aura": TestAura, "aura_unittests": TestAura,
- "base": TestBase, "base_unittests": TestBase,
- "blink_heap": TestBlinkHeap,
- "blink_platform": TestBlinkPlatform,
- "browser": TestBrowser, "browser_tests": TestBrowser,
- "cacheinvalidation": TestCacheInvalidation,
- "cacheinvalidation_unittests": TestCacheInvalidation,
- "cast": TestCast, "cast_unittests": TestCast,
- "cc": TestCC, "cc_unittests": TestCC,
- "chrome_app": TestChromeApp,
- "chrome_elf": TestChromeElf,
- "chromedriver": TestChromeDriver,
- "chromeos": TestChromeOS, "chromeos_unittests": TestChromeOS,
- "components": TestComponents,"components_unittests": TestComponents,
- "compositor": TestCompositor,"compositor_unittests": TestCompositor,
- "content": TestContent, "content_unittests": TestContent,
- "content_browsertests": TestContentBrowser,
- "courgette": TestCourgette, "courgette_unittests": TestCourgette,
- "crypto": TestCrypto, "crypto_unittests": TestCrypto,
- "device": TestDevice, "device_unittests": TestDevice,
- "display": TestDisplay, "display_unittests": TestDisplay,
- "events": TestEvents, "events_unittests": TestEvents,
- "extensions": TestExtensions, "extensions_unittests": TestExtensions,
- "ffmpeg_regression_tests": TestFFmpegRegressions,
- "gcm": TestGCM, "gcm_unit_tests": TestGCM,
- "gin": TestGin, "gin_unittests": TestGin,
- "gfx": TestGfx, "gfx_unittests": TestGfx,
- "google_apis": TestGoogleApis,
- "gpu": TestGPU, "gpu_unittests": TestGPU,
- "ipc": TestIpc, "ipc_tests": TestIpc,
- "installer_util": TestInstallerUtil,
- "installer_util_unittests": TestInstallerUtil,
- "install_static_unittests": TestInstallStatic,
- "interactive_ui": TestInteractiveUI,
- "jingle": TestJingle, "jingle_unittests": TestJingle,
- "keyboard": TestKeyboard, "keyboard_unittests": TestKeyboard,
- "latency": TestLatency, "latency_unittests": TestLatency,
- "layout": TestLayout, "layout_tests": TestLayout,
- "media": TestMedia, "media_unittests": TestMedia,
- "message_center": TestMessageCenter,
- "message_center_unittests" : TestMessageCenter,
- "midi": TestMidi, "midi_unittests": TestMidi,
- "mojo_common": TestMojoCommon,
- "mojo_common_unittests": TestMojoCommon,
- "mojo_system": TestMojoSystem,
- "mojo_system_unittests": TestMojoSystem,
- "mojo_public_system": TestMojoPublicSystem,
- "mojo_public_system_unittests": TestMojoPublicSystem,
- "mojo_public_bindings": TestMojoPublicBindings,
- "mojo_public_bindings_unittests": TestMojoPublicBindings,
- "mojo_public_sysperf": TestMojoPublicSysPerf,
- "net": TestNet, "net_unittests": TestNet,
- "net_perf": TestNetPerf, "net_perftests": TestNetPerf,
- "phonenumber": TestPhoneNumber,
- "libphonenumber_unittests": TestPhoneNumber,
- "ppapi": TestPPAPI, "ppapi_unittests": TestPPAPI,
- "printing": TestPrinting, "printing_unittests": TestPrinting,
- "remoting": TestRemoting, "remoting_unittests": TestRemoting,
- "sandbox": TestLinuxSandbox, "sandbox_linux_unittests": TestLinuxSandbox,
- "skia": TestSkia, "skia_unittests": TestSkia,
- "sql": TestSql, "sql_unittests": TestSql,
- "storage": TestStorage, "storage_unittests": TestStorage,
- "sync_integration_tests": TestSyncIntegration,
- "sync_integration": TestSyncIntegration,
- "ui_base_unit": TestUIBaseUnit, "ui_base_unittests": TestUIBaseUnit,
- "ui_chromeos": TestUIChromeOS, "ui_chromeos_unittests": TestUIChromeOS,
- "unit": TestUnit, "unit_tests": TestUnit,
- "url": TestURL, "url_unittests": TestURL,
- "views": TestViews, "views_unittests": TestViews,
- "webkit": TestLayout,
- }
-
-
-def _main():
- parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> "
- "[-t <test> ...]")
-
- parser.add_option("--help-tests", dest="help_tests", action="store_true",
- default=False, help="List all available tests")
- parser.add_option("-b", "--build-dir",
- help="the location of the compiler output")
- parser.add_option("--target", help="Debug or Release")
- parser.add_option("-t", "--test", action="append", default=[],
- help="which test to run, supports test:gtest_filter format "
- "as well.")
- parser.add_option("--baseline", action="store_true", default=False,
- help="generate baseline data instead of validating")
- parser.add_option("-f", "--force", action="store_true", default=False,
- help="run a broken test anyway")
- parser.add_option("--gtest_filter",
- help="additional arguments to --gtest_filter")
- parser.add_option("--gtest_repeat", help="argument for --gtest_repeat")
- parser.add_option("--gtest_shuffle", action="store_true", default=False,
- help="Randomize tests' orders on every iteration.")
- parser.add_option("--gtest_break_on_failure", action="store_true",
- default=False,
- help="Drop in to debugger on assertion failure. Also "
- "useful for forcing tests to exit with a stack dump "
- "on the first assertion failure when running with "
- "--gtest_repeat=-1")
- parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="verbose output - enable debug log messages")
- parser.add_option("--tool", dest="valgrind_tool", default="memcheck",
- help="specify a valgrind tool to run the tests under")
- parser.add_option("--tool_flags", dest="valgrind_tool_flags", default="",
- help="specify custom flags for the selected valgrind tool")
- parser.add_option("--keep_logs", action="store_true", default=False,
- help="store memory tool logs in the <tool>.logs directory "
- "instead of /tmp.\nThis can be useful for tool "
- "developers/maintainers.\nPlease note that the <tool>"
- ".logs directory will be clobbered on tool startup.")
- parser.add_option("-n", "--num_tests", type="int",
- default=ChromeTests.LAYOUT_TESTS_DEFAULT_CHUNK_SIZE,
- help="for layout tests: # of subtests per run. 0 for all.")
- parser.add_option("--test-launcher-bot-mode", action="store_true",
- help="run the tests with --test-launcher-bot-mode")
- parser.add_option("--test-launcher-total-shards", type=int,
- help="run the tests with --test-launcher-total-shards")
- parser.add_option("--test-launcher-shard-index", type=int,
- help="run the tests with --test-launcher-shard-index")
- parser.add_option("--drmemory_ops",
- help="extra options passed to Dr. Memory")
-
- options, args = parser.parse_args()
-
- # Bake target into build_dir.
- if options.target and options.build_dir:
- assert (options.target !=
- os.path.basename(os.path.dirname(options.build_dir)))
- options.build_dir = os.path.join(os.path.abspath(options.build_dir),
- options.target)
-
- if options.verbose:
- logging_utils.config_root(logging.DEBUG)
- else:
- logging_utils.config_root()
-
- if options.help_tests:
- ChromeTests.ShowTests()
- return 0
-
- if not options.test:
- parser.error("--test not specified")
-
- if len(options.test) != 1 and options.gtest_filter:
- parser.error("--gtest_filter and multiple tests don't make sense together")
-
- BROKEN_TESTS = {
- 'drmemory_light': [
- 'addressinput',
- 'aura',
- 'base_unittests',
- 'cc',
- 'components', # x64 only?
- 'content',
- 'gfx',
- 'mojo_public_bindings',
- ],
- 'drmemory_full': [
- 'addressinput',
- 'aura',
- 'base_unittests',
- 'blink_heap',
- 'blink_platform',
- 'browser_tests',
- 'cast',
- 'cc',
- 'chromedriver',
- 'compositor',
- 'content',
- 'content_browsertests',
- 'device',
- 'events',
- 'extensions',
- 'gfx',
- 'google_apis',
- 'gpu',
- 'ipc_tests',
- 'jingle',
- 'keyboard',
- 'media',
- 'midi',
- 'mojo_common',
- 'mojo_public_bindings',
- 'mojo_public_sysperf',
- 'mojo_public_system',
- 'mojo_system',
- 'net',
- 'remoting',
- 'unit',
- 'url',
- ],
- }
-
- for t in options.test:
- if t in BROKEN_TESTS[options.valgrind_tool] and not options.force:
- logging.info("Skipping broken %s test %s -- see crbug.com/633693" %
- (options.valgrind_tool, t))
- return 0
-
- tests = ChromeTests(options, args, t)
- ret = tests.Run()
- if ret: return ret
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(_main())
diff --git a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.sh b/libs/libyuv/tools_libyuv/valgrind/chrome_tests.sh
deleted file mode 100755
index dc17684fec..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/chrome_tests.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# Set up some paths and re-direct the arguments to chrome_tests.py
-
-export THISDIR=`dirname $0`
-ARGV_COPY="$@"
-
-# We need to set CHROME_VALGRIND iff using Memcheck:
-# tools/valgrind/chrome_tests.sh --tool memcheck
-# or
-# tools/valgrind/chrome_tests.sh --tool=memcheck
-tool="memcheck" # Default to memcheck.
-while (( "$#" ))
-do
- if [[ "$1" == "--tool" ]]
- then
- tool="$2"
- shift
- elif [[ "$1" =~ --tool=(.*) ]]
- then
- tool="${BASH_REMATCH[1]}"
- fi
- shift
-done
-
-NEEDS_VALGRIND=0
-NEEDS_DRMEMORY=0
-
-case "$tool" in
- "memcheck")
- NEEDS_VALGRIND=1
- ;;
- "drmemory" | "drmemory_light" | "drmemory_full" | "drmemory_pattern")
- NEEDS_DRMEMORY=1
- ;;
-esac
-
-if [ "$NEEDS_VALGRIND" == "1" ]
-then
- export CHROME_VALGRIND=`sh $THISDIR/locate_valgrind.sh`
- if [ "$CHROME_VALGRIND" = "" ]
- then
- # locate_valgrind.sh failed
- exit 1
- fi
- echo "Using valgrind binaries from ${CHROME_VALGRIND}"
-
- PATH="${CHROME_VALGRIND}/bin:$PATH"
- # We need to set these variables to override default lib paths hard-coded into
- # Valgrind binary.
- export VALGRIND_LIB="$CHROME_VALGRIND/lib/valgrind"
- export VALGRIND_LIB_INNER="$CHROME_VALGRIND/lib/valgrind"
-
- # Clean up some /tmp directories that might be stale due to interrupted
- # chrome_tests.py execution.
- # FYI:
- # -mtime +1 <- only print files modified more than 24h ago,
- # -print0/-0 are needed to handle possible newlines in the filenames.
- echo "Cleanup /tmp from Valgrind stuff"
- find /tmp -maxdepth 1 \(\
- -name "vgdb-pipe-*" -or -name "vg_logs_*" -or -name "valgrind.*" \
- \) -mtime +1 -print0 | xargs -0 rm -rf
-fi
-
-if [ "$NEEDS_DRMEMORY" == "1" ]
-then
- if [ -z "$DRMEMORY_COMMAND" ]
- then
- DRMEMORY_PATH="$THISDIR/../../third_party/drmemory"
- DRMEMORY_SFX="$DRMEMORY_PATH/drmemory-windows-sfx.exe"
- if [ ! -f "$DRMEMORY_SFX" ]
- then
- echo "Can't find Dr. Memory executables."
- echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
- echo "for the instructions on how to get them."
- exit 1
- fi
-
- chmod +x "$DRMEMORY_SFX" # Cygwin won't run it without +x.
- "$DRMEMORY_SFX" -o"$DRMEMORY_PATH/unpacked" -y
- export DRMEMORY_COMMAND="$DRMEMORY_PATH/unpacked/bin/drmemory.exe"
- fi
-fi
-
-PYTHONPATH=$THISDIR/../python/google python \
- "$THISDIR/chrome_tests.py" $ARGV_COPY
diff --git a/libs/libyuv/tools_libyuv/valgrind/common.py b/libs/libyuv/tools_libyuv/valgrind/common.py
deleted file mode 100644
index e9ee51e48e..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/common.py
+++ /dev/null
@@ -1,256 +0,0 @@
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-import logging
-import platform
-import os
-import signal
-import subprocess
-import sys
-import time
-
-
-class NotImplementedError(Exception):
- pass
-
-
-class TimeoutError(Exception):
- pass
-
-
-def RunSubprocessInBackground(proc):
- """Runs a subprocess in the background. Returns a handle to the process."""
- logging.info("running %s in the background" % " ".join(proc))
- return subprocess.Popen(proc)
-
-
-def RunSubprocess(proc, timeout=0):
- """ Runs a subprocess, until it finishes or |timeout| is exceeded and the
- process is killed with taskkill. A |timeout| <= 0 means no timeout.
-
- Args:
- proc: list of process components (exe + args)
- timeout: how long to wait before killing, <= 0 means wait forever
- """
-
- logging.info("running %s, timeout %d sec" % (" ".join(proc), timeout))
- sys.stdout.flush()
- sys.stderr.flush()
-
- # Manually read and print out stdout and stderr.
- # By default, the subprocess is supposed to inherit these from its parent,
- # however when run under buildbot, it seems unable to read data from a
- # grandchild process, so we have to read the child and print the data as if
- # it came from us for buildbot to read it. We're not sure why this is
- # necessary.
- # TODO(erikkay): should we buffer stderr and stdout separately?
- p = subprocess.Popen(proc, universal_newlines=True,
- bufsize=0, # unbuffered
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
- logging.info("started subprocess")
-
- did_timeout = False
- if timeout > 0:
- wait_until = time.time() + timeout
- while p.poll() is None and not did_timeout:
- # Have to use readline rather than readlines() or "for line in p.stdout:",
- # otherwise we get buffered even with bufsize=0.
- line = p.stdout.readline()
- while line and not did_timeout:
- sys.stdout.write(line)
- sys.stdout.flush()
- line = p.stdout.readline()
- if timeout > 0:
- did_timeout = time.time() > wait_until
-
- if did_timeout:
- logging.info("process timed out")
- else:
- logging.info("process ended, did not time out")
-
- if did_timeout:
- if IsWindows():
- subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
- else:
- # Does this kill all children, too?
- os.kill(p.pid, signal.SIGINT)
- logging.error("KILLED %d" % p.pid)
- # Give the process a chance to actually die before continuing
- # so that cleanup can happen safely.
- time.sleep(1.0)
- logging.error("TIMEOUT waiting for %s" % proc[0])
- raise TimeoutError(proc[0])
- else:
- for line in p.stdout:
- sys.stdout.write(line)
- if not IsMac(): # stdout flush fails on Mac
- logging.info("flushing stdout")
- sys.stdout.flush()
-
- logging.info("collecting result code")
- result = p.poll()
- if result:
- logging.error("%s exited with non-zero result code %d" % (proc[0], result))
- return result
-
-
-def IsLinux():
- return sys.platform.startswith('linux')
-
-
-def IsMac():
- return sys.platform.startswith('darwin')
-
-
-def IsWindows():
- return sys.platform == 'cygwin' or sys.platform.startswith('win')
-
-
-def WindowsVersionName():
- """Returns the name of the Windows version if it is known, or None.
-
- Possible return values are: xp, vista, 7, 8, or None
- """
- if sys.platform == 'cygwin':
- # Windows version number is hiding in system name. Looks like:
- # CYGWIN_NT-6.1-WOW64
- try:
- version_str = platform.uname()[0].split('-')[1]
- except:
- return None
- elif sys.platform.startswith('win'):
- # Normal Windows version string. Mine: 6.1.7601
- version_str = platform.version()
- else:
- return None
-
- parts = version_str.split('.')
- try:
- major = int(parts[0])
- minor = int(parts[1])
- except:
- return None # Can't parse, unknown version.
-
- if major == 5:
- return 'xp'
- elif major == 6 and minor == 0:
- return 'vista'
- elif major == 6 and minor == 1:
- return '7'
- elif major == 6 and minor == 2:
- return '8' # Future proof. ;)
- return None
-
-
-def PlatformNames():
- """Return an array of string to be used in paths for the platform
- (e.g. suppressions, gtest filters, ignore files etc.)
- The first element of the array describes the 'main' platform
- """
- if IsLinux():
- return ['linux']
- if IsMac():
- return ['mac']
- if IsWindows():
- names = ['win32']
- version_name = WindowsVersionName()
- if version_name is not None:
- names.append('win-%s' % version_name)
- return names
- raise NotImplementedError('Unknown platform "%s".' % sys.platform)
-
-
-def PutEnvAndLog(env_name, env_value):
- os.putenv(env_name, env_value)
- logging.info('export %s=%s', env_name, env_value)
-
-def BoringCallers(mangled, use_re_wildcards):
- """Return a list of 'boring' function names (optinally mangled)
- with */? wildcards (optionally .*/.).
- Boring = we drop off the bottom of stack traces below such functions.
- """
-
- need_mangling = [
- # Don't show our testing framework:
- ("testing::Test::Run", "_ZN7testing4Test3RunEv"),
- ("testing::TestInfo::Run", "_ZN7testing8TestInfo3RunEv"),
- ("testing::internal::Handle*ExceptionsInMethodIfSupported*",
- "_ZN7testing8internal3?Handle*ExceptionsInMethodIfSupported*"),
-
- # Depend on scheduling:
- ("MessageLoop::Run", "_ZN11MessageLoop3RunEv"),
- ("MessageLoop::RunTask", "_ZN11MessageLoop7RunTask*"),
- ("RunnableMethod*", "_ZN14RunnableMethod*"),
- ("DispatchToMethod*", "_Z*16DispatchToMethod*"),
- ("base::internal::Invoker*::DoInvoke*",
- "_ZN4base8internal8Invoker*DoInvoke*"), # Invoker{1,2,3}
- ("base::internal::RunnableAdapter*::Run*",
- "_ZN4base8internal15RunnableAdapter*Run*"),
- ]
-
- ret = []
- for pair in need_mangling:
- ret.append(pair[1 if mangled else 0])
-
- ret += [
- # Also don't show the internals of libc/pthread.
- "start_thread",
- "main",
- "BaseThreadInitThunk",
- ]
-
- if use_re_wildcards:
- for i in range(0, len(ret)):
- ret[i] = ret[i].replace('*', '.*').replace('?', '.')
-
- return ret
-
-def NormalizeWindowsPath(path):
- """If we're using Cygwin Python, turn the path into a Windows path.
-
- Don't turn forward slashes into backslashes for easier copy-pasting and
- escaping.
-
- TODO(rnk): If we ever want to cut out the subprocess invocation, we can use
- _winreg to get the root Cygwin directory from the registry key:
- HKEY_LOCAL_MACHINE\SOFTWARE\Cygwin\setup\rootdir.
- """
- if sys.platform.startswith("cygwin"):
- p = subprocess.Popen(["cygpath", "-m", path],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (out, err) = p.communicate()
- if err:
- logging.warning("WARNING: cygpath error: %s", err)
- return out.strip()
- else:
- return path
-
-############################
-# Common output format code
-
-def PrintUsedSuppressionsList(suppcounts):
- """ Prints out the list of used suppressions in a format common to all the
- memory tools. If the list is empty, prints nothing and returns False,
- otherwise True.
-
- suppcounts: a dictionary of used suppression counts,
- Key -> name, Value -> count.
- """
- if not suppcounts:
- return False
-
- print "-----------------------------------------------------"
- print "Suppressions used:"
- print " count name"
- for (name, count) in sorted(suppcounts.items(), key=lambda (k,v): (v,k)):
- print "%7d %s" % (count, name)
- print "-----------------------------------------------------"
- sys.stdout.flush()
- return True
diff --git a/libs/libyuv/tools_libyuv/valgrind/gdb_helper.py b/libs/libyuv/tools_libyuv/valgrind/gdb_helper.py
deleted file mode 100644
index d127f76008..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/gdb_helper.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-''' A bunch of helper functions for querying gdb.'''
-
-import logging
-import os
-import re
-import tempfile
-
-GDB_LINE_RE = re.compile(r'Line ([0-9]*) of "([^"]*)".*')
-
-def _GdbOutputToFileLine(output_line):
- ''' Parse the gdb output line, return a pair (file, line num) '''
- match = GDB_LINE_RE.match(output_line)
- if match:
- return match.groups()[1], match.groups()[0]
- else:
- return None
-
-def ResolveAddressesWithinABinary(binary_name, load_address, address_list):
- ''' For each address, return a pair (file, line num) '''
- commands = tempfile.NamedTemporaryFile()
- commands.write('add-symbol-file "%s" %s\n' % (binary_name, load_address))
- for addr in address_list:
- commands.write('info line *%s\n' % addr)
- commands.write('quit\n')
- commands.flush()
- gdb_commandline = 'gdb -batch -x %s 2>/dev/null' % commands.name
- gdb_pipe = os.popen(gdb_commandline)
- result = gdb_pipe.readlines()
-
- address_count = 0
- ret = {}
- for line in result:
- if line.startswith('Line'):
- ret[address_list[address_count]] = _GdbOutputToFileLine(line)
- address_count += 1
- if line.startswith('No line'):
- ret[address_list[address_count]] = (None, None)
- address_count += 1
- gdb_pipe.close()
- commands.close()
- return ret
-
-class AddressTable(object):
- ''' Object to do batched line number lookup. '''
- def __init__(self):
- self._load_addresses = {}
- self._binaries = {}
- self._all_resolved = False
-
- def AddBinaryAt(self, binary, load_address):
- ''' Register a new shared library or executable. '''
- self._load_addresses[binary] = load_address
-
- def Add(self, binary, address):
- ''' Register a lookup request. '''
- if binary == '':
- logging.warn('adding address %s in empty binary?' % address)
- if binary in self._binaries:
- self._binaries[binary].append(address)
- else:
- self._binaries[binary] = [address]
- self._all_resolved = False
-
- def ResolveAll(self):
- ''' Carry out all lookup requests. '''
- self._translation = {}
- for binary in self._binaries.keys():
- if binary != '' and binary in self._load_addresses:
- load_address = self._load_addresses[binary]
- addr = ResolveAddressesWithinABinary(
- binary, load_address, self._binaries[binary])
- self._translation[binary] = addr
- self._all_resolved = True
-
- def GetFileLine(self, binary, addr):
- ''' Get the (filename, linenum) result of a previously-registered lookup
- request.
- '''
- if self._all_resolved:
- if binary in self._translation:
- if addr in self._translation[binary]:
- return self._translation[binary][addr]
- return (None, None)
diff --git a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.bat b/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.bat
deleted file mode 100644
index 5fceca6762..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.bat
+++ /dev/null
@@ -1,79 +0,0 @@
-@echo off
-:: Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
-::
-:: Use of this source code is governed by a BSD-style license
-:: that can be found in the LICENSE file in the root of the source
-:: tree. An additional intellectual property rights grant can be found
-:: in the file PATENTS. All contributing project authors may
-:: be found in the AUTHORS file in the root of the source tree.
-
-:: This script is a copy of chrome_tests.bat with the following changes:
-:: - Invokes libyuv_tests.py instead of chrome_tests.py
-:: - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make
-:: it possible to execute the Python scripts properly.
-
-:: TODO(timurrrr): batch files 'export' all the variables to the parent shell
-set THISDIR=%~dp0
-set TOOL_NAME="unknown"
-
-:: Get the tool name and put it into TOOL_NAME {{{1
-:: NB: SHIFT command doesn't modify %*
-:PARSE_ARGS_LOOP
- if %1 == () GOTO:TOOLNAME_NOT_FOUND
- if %1 == --tool GOTO:TOOLNAME_FOUND
- SHIFT
- goto :PARSE_ARGS_LOOP
-
-:TOOLNAME_NOT_FOUND
-echo "Please specify a tool (tsan or drmemory) by using --tool flag"
-exit /B 1
-
-:TOOLNAME_FOUND
-SHIFT
-set TOOL_NAME=%1
-:: }}}
-if "%TOOL_NAME%" == "drmemory" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_light" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_full" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "drmemory_pattern" GOTO :SETUP_DRMEMORY
-if "%TOOL_NAME%" == "tsan" GOTO :SETUP_TSAN
-echo "Unknown tool: `%TOOL_NAME%`! Only tsan and drmemory are supported."
-exit /B 1
-
-:SETUP_DRMEMORY
-if NOT "%DRMEMORY_COMMAND%"=="" GOTO :RUN_TESTS
-:: Set up DRMEMORY_COMMAND to invoke Dr. Memory {{{1
-set DRMEMORY_PATH=%THISDIR%..\..\third_party\drmemory
-set DRMEMORY_SFX=%DRMEMORY_PATH%\drmemory-windows-sfx.exe
-if EXIST %DRMEMORY_SFX% GOTO DRMEMORY_BINARY_OK
-echo "Can't find Dr. Memory executables."
-echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
-echo "for the instructions on how to get them."
-exit /B 1
-
-:DRMEMORY_BINARY_OK
-%DRMEMORY_SFX% -o%DRMEMORY_PATH%\unpacked -y
-set DRMEMORY_COMMAND=%DRMEMORY_PATH%\unpacked\bin\drmemory.exe
-:: }}}
-goto :RUN_TESTS
-
-:SETUP_TSAN
-:: Set up PIN_COMMAND to invoke TSan {{{1
-set TSAN_PATH=%THISDIR%..\..\third_party\tsan
-set TSAN_SFX=%TSAN_PATH%\tsan-x86-windows-sfx.exe
-if EXIST %TSAN_SFX% GOTO TSAN_BINARY_OK
-echo "Can't find ThreadSanitizer executables."
-echo "See http://www.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/threadsanitizer-on-windows"
-echo "for the instructions on how to get them."
-exit /B 1
-
-:TSAN_BINARY_OK
-%TSAN_SFX% -o%TSAN_PATH%\unpacked -y
-set PIN_COMMAND=%TSAN_PATH%\unpacked\tsan-x86-windows\tsan.bat
-:: }}}
-goto :RUN_TESTS
-
-:RUN_TESTS
-set PYTHONPATH=%THISDIR%..\python\google;%THISDIR%..\valgrind
-set RUNNING_ON_VALGRIND=yes
-python %THISDIR%libyuv_tests.py %*
diff --git a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.py b/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.py
deleted file mode 100755
index e780bd95eb..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-"""Runs various libyuv tests through valgrind_test.py.
-
-This script inherits the chrome_tests.py in Chrome, but allows running any test
-instead of only the hard-coded ones. It uses the -t cmdline flag to do this, and
-only supports specifying a single test for each run.
-
-Suppression files:
-The Chrome valgrind directory we use as a DEPS dependency contains the following
-suppression files:
- valgrind/memcheck/suppressions.txt
- valgrind/memcheck/suppressions_mac.txt
- valgrind/tsan/suppressions.txt
- valgrind/tsan/suppressions_mac.txt
- valgrind/tsan/suppressions_win32.txt
-Since they're referenced from the chrome_tests.py script, we have similar files
-below the directory of this script. When executing, this script will setup both
-Chrome's suppression files and our own, so we can easily maintain libyuv
-specific suppressions in our own files.
-"""
-
-import logging
-import optparse
-import os
-import sys
-
-import logging_utils
-import path_utils
-
-import chrome_tests
-
-
-class LibyuvTest(chrome_tests.ChromeTests):
- """Class that handles setup of suppressions for libyuv.
-
- Everything else is inherited from chrome_tests.ChromeTests.
- """
-
- def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None):
- """Override command-building method so we can add more suppressions."""
- cmd = chrome_tests.ChromeTests._DefaultCommand(self, tool, exe,
- valgrind_test_args)
- # When ChromeTests._DefaultCommand has executed, it has setup suppression
- # files based on what's found in the memcheck/ or tsan/ subdirectories of
- # this script's location. If Mac or Windows is executing, additional
- # platform specific files have also been added.
- # Since only the ones located below this directory is added, we must also
- # add the ones maintained by Chrome, located in ../../tools/valgrind.
-
- # The idea is to look for --suppression arguments in the cmd list and add a
- # modified copy of each suppression file, for the corresponding file in
- # ../../tools/valgrind.
- script_dir = path_utils.ScriptDir()
- old_base, _ = os.path.split(script_dir)
-
- checkout_src = os.path.abspath(os.path.join(script_dir, os.pardir,
- os.pardir))
- new_dir = os.path.join(checkout_src, 'tools', 'valgrind')
- add_suppressions = []
- for token in cmd:
- if '--suppressions' in token:
- add_suppressions.append(token.replace(script_dir, new_dir))
- return add_suppressions + cmd
-
-
-def main(_):
- parser = optparse.OptionParser('usage: %prog -b <dir> -t <test> <test args>')
- parser.disable_interspersed_args()
- parser.add_option('-b', '--build-dir',
- help=('Location of the compiler output. Can only be used '
- 'when the test argument does not contain this path.'))
- parser.add_option("--target", help="Debug or Release")
- parser.add_option('-t', '--test', help='Test to run.')
- parser.add_option('', '--baseline', action='store_true', default=False,
- help='Generate baseline data instead of validating')
- parser.add_option('', '--gtest_filter',
- help='Additional arguments to --gtest_filter')
- parser.add_option('', '--gtest_repeat',
- help='Argument for --gtest_repeat')
- parser.add_option("--gtest_shuffle", action="store_true", default=False,
- help="Randomize tests' orders on every iteration.")
- parser.add_option("--gtest_break_on_failure", action="store_true",
- default=False,
- help="Drop in to debugger on assertion failure. Also "
- "useful for forcing tests to exit with a stack dump "
- "on the first assertion failure when running with "
- "--gtest_repeat=-1")
- parser.add_option('-v', '--verbose', action='store_true', default=False,
- help='Verbose output - enable debug log messages')
- parser.add_option('', '--tool', dest='valgrind_tool', default='memcheck',
- help='Specify a valgrind tool to run the tests under')
- parser.add_option('', '--tool_flags', dest='valgrind_tool_flags', default='',
- help='Specify custom flags for the selected valgrind tool')
- parser.add_option('', '--keep_logs', action='store_true', default=False,
- help=('Store memory tool logs in the <tool>.logs directory '
- 'instead of /tmp.\nThis can be useful for tool '
- 'developers/maintainers.\nPlease note that the <tool>'
- '.logs directory will be clobbered on tool startup.'))
- parser.add_option("--test-launcher-bot-mode", action="store_true",
- help="run the tests with --test-launcher-bot-mode")
- parser.add_option("--test-launcher-total-shards", type=int,
- help="run the tests with --test-launcher-total-shards")
- parser.add_option("--test-launcher-shard-index", type=int,
- help="run the tests with --test-launcher-shard-index")
- options, args = parser.parse_args()
-
- if options.verbose:
- logging_utils.config_root(logging.DEBUG)
- else:
- logging_utils.config_root()
-
- if not options.test:
- parser.error('--test not specified')
-
- # Support build dir both with and without the target.
- if (options.target and options.build_dir and
- not options.build_dir.endswith(options.target)):
- options.build_dir = os.path.join(options.build_dir, options.target)
-
- # If --build_dir is provided, prepend it to the test executable if needed.
- test_executable = options.test
- if options.build_dir and not test_executable.startswith(options.build_dir):
- test_executable = os.path.join(options.build_dir, test_executable)
- args = [test_executable] + args
-
- test = LibyuvTest(options, args, 'cmdline')
- return test.Run()
-
-if __name__ == '__main__':
- return_code = main(sys.argv)
- sys.exit(return_code)
diff --git a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.sh b/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.sh
deleted file mode 100755
index 249032ca96..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/libyuv_tests.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# Set up some paths and re-direct the arguments to libyuv_tests.py
-
-# This script is a copy of the chrome_tests.sh wrapper script with the following
-# changes:
-# - The locate_valgrind.sh of Chromium's Valgrind scripts dir is used to locate
-# the Valgrind framework install. If it fails a fallback path is used instead
-# (../../chromium/src/third_party/valgrind/linux_x64) and a warning message
-# is showed by |show_locate_valgrind_failed_warning|.
-# - libyuv_tests.py is invoked instead of chrome_tests.py.
-# - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make it
-# possible to execute the Python scripts properly.
-
-export THISDIR=`dirname $0`
-ARGV_COPY="$@"
-
-# We need to set CHROME_VALGRIND iff using Memcheck:
-# tools_libyuv/valgrind/libyuv_tests.sh --tool memcheck
-# or
-# tools_libyuv/valgrind/libyuv_tests.sh --tool=memcheck
-tool="memcheck" # Default to memcheck.
-while (( "$#" ))
-do
- if [[ "$1" == "--tool" ]]
- then
- tool="$2"
- shift
- elif [[ "$1" =~ --tool=(.*) ]]
- then
- tool="${BASH_REMATCH[1]}"
- fi
- shift
-done
-
-NEEDS_VALGRIND=0
-
-case "$tool" in
- "memcheck")
- NEEDS_VALGRIND=1
- ;;
-esac
-
-# For libyuv, we'll use the locate_valgrind.sh script in Chromium's Valgrind
-# scripts dir to locate the Valgrind framework install
-CHROME_VALGRIND_SCRIPTS=$THISDIR/../../tools/valgrind
-
-if [ "$NEEDS_VALGRIND" == "1" ]
-then
- CHROME_VALGRIND=`sh $THISDIR/locate_valgrind.sh`
- if [ "$CHROME_VALGRIND" = "" ]
- then
- CHROME_VALGRIND=../../src/third_party/valgrind/linux_x64
- echo
- echo "-------------------- WARNING ------------------------"
- echo "locate_valgrind.sh failed."
- echo "Using $CHROME_VALGRIND as a fallback location."
- echo "This might be because:"
- echo "1) This is a swarming bot"
- echo "2) You haven't set up the valgrind binaries correctly."
- echo "In this case, please make sure you have followed the instructions at"
- echo "http://www.chromium.org/developers/how-tos/using-valgrind/get-valgrind"
- echo "Notice: In the .gclient file, you need to add this for the 'libyuv'"
- echo "solution since our directory structure is different from Chromium's:"
- echo "\"custom_deps\": {"
- echo " \"libyuv/third_party/valgrind\":"
- echo " \"https://chromium.googlesource.com/chromium/deps/valgrind/binaries\","
- echo "},"
- echo "-----------------------------------------------------"
- echo
- fi
- echo "Using valgrind binaries from ${CHROME_VALGRIND}"
-
- PATH="${CHROME_VALGRIND}/bin:$PATH"
- # We need to set these variables to override default lib paths hard-coded into
- # Valgrind binary.
- export VALGRIND_LIB="$CHROME_VALGRIND/lib/valgrind"
- export VALGRIND_LIB_INNER="$CHROME_VALGRIND/lib/valgrind"
-
- # Clean up some /tmp directories that might be stale due to interrupted
- # chrome_tests.py execution.
- # FYI:
- # -mtime +1 <- only print files modified more than 24h ago,
- # -print0/-0 are needed to handle possible newlines in the filenames.
- echo "Cleanup /tmp from Valgrind stuff"
- find /tmp -maxdepth 1 \(\
- -name "vgdb-pipe-*" -or -name "vg_logs_*" -or -name "valgrind.*" \
- \) -mtime +1 -print0 | xargs -0 rm -rf
-fi
-
-# Add Chrome's Valgrind scripts dir to the PYTHON_PATH since it contains
-# the scripts that are needed for this script to run
-PYTHONPATH=$THISDIR/../../tools/python/google:$CHROME_VALGRIND_SCRIPTS python \
- "$THISDIR/libyuv_tests.py" $ARGV_COPY
diff --git a/libs/libyuv/tools_libyuv/valgrind/locate_valgrind.sh b/libs/libyuv/tools_libyuv/valgrind/locate_valgrind.sh
deleted file mode 100755
index d9594f4848..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/locate_valgrind.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# Prints a path to Valgrind binaries to be used for Chromium.
-# Select the valgrind from third_party/valgrind by default,
-# but allow users to override this default without editing scripts and
-# without specifying a commandline option
-
-export THISDIR=`dirname $0`
-
-# User may use their own valgrind by giving its path with CHROME_VALGRIND env.
-if [ "$CHROME_VALGRIND" = "" ]
-then
- # Guess which binaries we should use by uname
- case "$(uname -a)" in
- *Linux*x86_64*)
- PLATFORM="linux_x64"
- ;;
- *Linux*86*)
- PLATFORM="linux_x86"
- ;;
- *Darwin*9.[678].[01]*i386*)
- # Didn't test other kernels.
- PLATFORM="mac"
- ;;
- *Darwin*10.[0-9].[0-9]*i386*)
- PLATFORM="mac_10.6"
- ;;
- *Darwin*10.[0-9].[0-9]*x86_64*)
- PLATFORM="mac_10.6"
- ;;
- *Darwin*11.[0-9].[0-9]*x86_64*)
- PLATFORM="mac_10.7"
- ;;
- *)
- (echo "Sorry, your platform is not supported:" &&
- uname -a
- echo
- echo "If you're on Mac OS X, please see http://crbug.com/441425") >&2
- exit 42
- esac
-
- # The binaries should be in third_party/valgrind
- # (checked out from deps/third_party/valgrind/binaries).
- CHROME_VALGRIND="$THISDIR/../../third_party/valgrind/$PLATFORM"
-
- # TODO(timurrrr): readlink -f is not present on Mac...
- if [ "$PLATFORM" != "mac" ] && \
- [ "$PLATFORM" != "mac_10.6" ] && \
- [ "$PLATFORM" != "mac_10.7" ]
- then
- # Get rid of all "../" dirs
- CHROME_VALGRIND=$(readlink -f $CHROME_VALGRIND)
- fi
-fi
-
-if ! test -x $CHROME_VALGRIND/bin/valgrind
-then
- echo "Oops, could not find Valgrind binaries in your checkout." >&2
- echo "Please see" >&2
- echo " http://dev.chromium.org/developers/how-tos/using-valgrind/get-valgrind" >&2
- echo "for the instructions on how to download pre-built binaries." >&2
- exit 1
-fi
-
-echo $CHROME_VALGRIND
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck/OWNERS b/libs/libyuv/tools_libyuv/valgrind/memcheck/OWNERS
deleted file mode 100644
index 72e8ffc0db..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck/OWNERS
+++ /dev/null
@@ -1 +0,0 @@
-*
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck/PRESUBMIT.py b/libs/libyuv/tools_libyuv/valgrind/memcheck/PRESUBMIT.py
deleted file mode 100644
index 033292148d..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck/PRESUBMIT.py
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-"""
-Copied from Chrome's src/tools/valgrind/memcheck/PRESUBMIT.py
-
-See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
-for more details on the presubmit API built into gcl.
-"""
-
-import os
-import re
-import sys
-
-def CheckChange(input_api, output_api):
- """Checks the memcheck suppressions files for bad data."""
-
- # Add the path to the Chrome valgrind dir to the import path:
- tools_vg_path = os.path.join(input_api.PresubmitLocalPath(), '..', '..', '..',
- 'tools', 'valgrind')
- sys.path.append(tools_vg_path)
- import suppressions
-
- sup_regex = re.compile('suppressions.*\.txt$')
- suppressions = {}
- errors = []
- check_for_memcheck = False
- # skip_next_line has 3 possible values:
- # - False: don't skip the next line.
- # - 'skip_suppression_name': the next line is a suppression name, skip.
- # - 'skip_param': the next line is a system call parameter error, skip.
- skip_next_line = False
- for f in filter(lambda x: sup_regex.search(x.LocalPath()),
- input_api.AffectedFiles()):
- for line, line_num in zip(f.NewContents(),
- xrange(1, len(f.NewContents()) + 1)):
- line = line.lstrip()
- if line.startswith('#') or not line:
- continue
-
- if skip_next_line:
- if skip_next_line == 'skip_suppression_name':
- if 'insert_a_suppression_name_here' in line:
- errors.append('"insert_a_suppression_name_here" is not a valid '
- 'suppression name')
- if suppressions.has_key(line):
- if f.LocalPath() == suppressions[line][1]:
- errors.append('suppression with name "%s" at %s line %s '
- 'has already been defined at line %s' %
- (line, f.LocalPath(), line_num,
- suppressions[line][1]))
- else:
- errors.append('suppression with name "%s" at %s line %s '
- 'has already been defined at %s line %s' %
- (line, f.LocalPath(), line_num,
- suppressions[line][0], suppressions[line][1]))
- else:
- suppressions[line] = (f, line_num)
- check_for_memcheck = True;
- skip_next_line = False
- continue
- if check_for_memcheck:
- if not line.startswith('Memcheck:'):
- errors.append('"%s" should be "Memcheck:..." in %s line %s' %
- (line, f.LocalPath(), line_num))
- check_for_memcheck = False;
- if line == '{':
- skip_next_line = 'skip_suppression_name'
- continue
- if line == "Memcheck:Param":
- skip_next_line = 'skip_param'
- continue
-
- if (line.startswith('fun:') or line.startswith('obj:') or
- line.startswith('Memcheck:') or line == '}' or
- line == '...'):
- continue
- errors.append('"%s" is probably wrong: %s line %s' % (line, f.LocalPath(),
- line_num))
- if errors:
- return [output_api.PresubmitError('\n'.join(errors))]
- return []
-
-def CheckChangeOnUpload(input_api, output_api):
- return CheckChange(input_api, output_api)
-
-def CheckChangeOnCommit(input_api, output_api):
- return CheckChange(input_api, output_api)
-
-def GetPreferredTrySlaves():
- # We don't have any memcheck slaves yet, so there's no use for this method.
- # When we have, the slave name(s) should be put into this list.
- return []
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions.txt b/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions.txt
deleted file mode 100644
index 3f0f6d44f1..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-# This file is used in addition to the one already maintained in Chrome.
-# It acts as a place holder for future additions for this project.
-# It must exist for the Python wrapper script to work properly.
-
-# There are two of suppressions in this file.
-# 1. third_party libraries
-# 2. libyuv stuff
-# 3. libjingle stuff (talk folder)
-#-----------------------------------------------------------------------
-
-# third_party libraries
-{
- bug_729
- Memcheck:Free
- fun:_ZdaPv
- ...
- fun:_ZN7testing8internal12UnitTestImplD1Ev
- ...
-}
-
-# libyuv (empty so far)
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_mac.txt b/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_mac.txt
deleted file mode 100644
index 3ad0c8ccc5..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_mac.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# This file is used in addition to the one already maintained in Chrome.
-# It acts as a place holder for future additions for this project.
-# It must exist for the Python wrapper script to work properly.
-
-
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_win32.txt b/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_win32.txt
deleted file mode 100644
index 3ad0c8ccc5..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck/suppressions_win32.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# This file is used in addition to the one already maintained in Chrome.
-# It acts as a place holder for future additions for this project.
-# It must exist for the Python wrapper script to work properly.
-
-
diff --git a/libs/libyuv/tools_libyuv/valgrind/memcheck_analyze.py b/libs/libyuv/tools_libyuv/valgrind/memcheck_analyze.py
deleted file mode 100755
index 80e85eb4ab..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/memcheck_analyze.py
+++ /dev/null
@@ -1,644 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# memcheck_analyze.py
-
-''' Given a valgrind XML file, parses errors and uniques them.'''
-
-import gdb_helper
-
-from collections import defaultdict
-import hashlib
-import logging
-import optparse
-import os
-import re
-import subprocess
-import sys
-import time
-from xml.dom.minidom import parse
-from xml.parsers.expat import ExpatError
-
-import common
-
-# Global symbol table (yuck)
-TheAddressTable = None
-
-# These are regexps that define functions (using C++ mangled names)
-# we don't want to see in stack traces while pretty printing
-# or generating suppressions.
-# Just stop printing the stack/suppression frames when the current one
-# matches any of these.
-_BORING_CALLERS = common.BoringCallers(mangled=True, use_re_wildcards=True)
-
-def getTextOf(top_node, name):
- ''' Returns all text in all DOM nodes with a certain |name| that are children
- of |top_node|.
- '''
-
- text = ""
- for nodes_named in top_node.getElementsByTagName(name):
- text += "".join([node.data for node in nodes_named.childNodes
- if node.nodeType == node.TEXT_NODE])
- return text
-
-def getCDATAOf(top_node, name):
- ''' Returns all CDATA in all DOM nodes with a certain |name| that are children
- of |top_node|.
- '''
-
- text = ""
- for nodes_named in top_node.getElementsByTagName(name):
- text += "".join([node.data for node in nodes_named.childNodes
- if node.nodeType == node.CDATA_SECTION_NODE])
- if (text == ""):
- return None
- return text
-
-def shortenFilePath(source_dir, directory):
- '''Returns a string with the string prefix |source_dir| removed from
- |directory|.'''
- prefixes_to_cut = ["build/src/", "valgrind/coregrind/", "out/Release/../../"]
-
- if source_dir:
- prefixes_to_cut.append(source_dir)
-
- for p in prefixes_to_cut:
- index = directory.rfind(p)
- if index != -1:
- directory = directory[index + len(p):]
-
- return directory
-
-# Constants that give real names to the abbreviations in valgrind XML output.
-INSTRUCTION_POINTER = "ip"
-OBJECT_FILE = "obj"
-FUNCTION_NAME = "fn"
-SRC_FILE_DIR = "dir"
-SRC_FILE_NAME = "file"
-SRC_LINE = "line"
-
-def gatherFrames(node, source_dir):
- frames = []
- for frame in node.getElementsByTagName("frame"):
- frame_dict = {
- INSTRUCTION_POINTER : getTextOf(frame, INSTRUCTION_POINTER),
- OBJECT_FILE : getTextOf(frame, OBJECT_FILE),
- FUNCTION_NAME : getTextOf(frame, FUNCTION_NAME),
- SRC_FILE_DIR : shortenFilePath(
- source_dir, getTextOf(frame, SRC_FILE_DIR)),
- SRC_FILE_NAME : getTextOf(frame, SRC_FILE_NAME),
- SRC_LINE : getTextOf(frame, SRC_LINE)
- }
-
- # Ignore this frame and all the following if it's a "boring" function.
- enough_frames = False
- for regexp in _BORING_CALLERS:
- if re.match("^%s$" % regexp, frame_dict[FUNCTION_NAME]):
- enough_frames = True
- break
- if enough_frames:
- break
-
- frames += [frame_dict]
-
- global TheAddressTable
- if TheAddressTable != None and frame_dict[SRC_LINE] == "":
- # Try using gdb
- TheAddressTable.Add(frame_dict[OBJECT_FILE],
- frame_dict[INSTRUCTION_POINTER])
- return frames
-
-class ValgrindError:
- ''' Takes a <DOM Element: error> node and reads all the data from it. A
- ValgrindError is immutable and is hashed on its pretty printed output.
- '''
-
- def __init__(self, source_dir, error_node, commandline, testcase):
- ''' Copies all the relevant information out of the DOM and into object
- properties.
-
- Args:
- error_node: The <error></error> DOM node we're extracting from.
- source_dir: Prefix that should be stripped from the <dir> node.
- commandline: The command that was run under valgrind
- testcase: The test case name, if known.
- '''
-
- # Valgrind errors contain one <what><stack> pair, plus an optional
- # <auxwhat><stack> pair, plus an optional <origin><what><stack></origin>,
- # plus (since 3.5.0) a <suppression></suppression> pair.
- # (Origin is nicely enclosed; too bad the other two aren't.)
- # The most common way to see all three in one report is
- # a syscall with a parameter that points to uninitialized memory, e.g.
- # Format:
- # <error>
- # <unique>0x6d</unique>
- # <tid>1</tid>
- # <kind>SyscallParam</kind>
- # <what>Syscall param write(buf) points to uninitialised byte(s)</what>
- # <stack>
- # <frame>
- # ...
- # </frame>
- # </stack>
- # <auxwhat>Address 0x5c9af4f is 7 bytes inside a block of ...</auxwhat>
- # <stack>
- # <frame>
- # ...
- # </frame>
- # </stack>
- # <origin>
- # <what>Uninitialised value was created by a heap allocation</what>
- # <stack>
- # <frame>
- # ...
- # </frame>
- # </stack>
- # </origin>
- # <suppression>
- # <sname>insert_a_suppression_name_here</sname>
- # <skind>Memcheck:Param</skind>
- # <skaux>write(buf)</skaux>
- # <sframe> <fun>__write_nocancel</fun> </sframe>
- # ...
- # <sframe> <fun>main</fun> </sframe>
- # <rawtext>
- # <![CDATA[
- # {
- # <insert_a_suppression_name_here>
- # Memcheck:Param
- # write(buf)
- # fun:__write_nocancel
- # ...
- # fun:main
- # }
- # ]]>
- # </rawtext>
- # </suppression>
- # </error>
- #
- # Each frame looks like this:
- # <frame>
- # <ip>0x83751BC</ip>
- # <obj>/data/dkegel/chrome-build/src/out/Release/base_unittests</obj>
- # <fn>_ZN7testing8internal12TestInfoImpl7RunTestEPNS_8TestInfoE</fn>
- # <dir>/data/dkegel/chrome-build/src/testing/gtest/src</dir>
- # <file>gtest-internal-inl.h</file>
- # <line>655</line>
- # </frame>
- # although the dir, file, and line elements are missing if there is
- # no debug info.
-
- self._kind = getTextOf(error_node, "kind")
- self._backtraces = []
- self._suppression = None
- self._commandline = commandline
- self._testcase = testcase
- self._additional = []
-
- # Iterate through the nodes, parsing <what|auxwhat><stack> pairs.
- description = None
- for node in error_node.childNodes:
- if node.localName == "what" or node.localName == "auxwhat":
- description = "".join([n.data for n in node.childNodes
- if n.nodeType == n.TEXT_NODE])
- elif node.localName == "xwhat":
- description = getTextOf(node, "text")
- elif node.localName == "stack":
- assert description
- self._backtraces.append([description, gatherFrames(node, source_dir)])
- description = None
- elif node.localName == "origin":
- description = getTextOf(node, "what")
- stack = node.getElementsByTagName("stack")[0]
- frames = gatherFrames(stack, source_dir)
- self._backtraces.append([description, frames])
- description = None
- stack = None
- frames = None
- elif description and node.localName != None:
- # The lastest description has no stack, e.g. "Address 0x28 is unknown"
- self._additional.append(description)
- description = None
-
- if node.localName == "suppression":
- self._suppression = getCDATAOf(node, "rawtext");
-
- def __str__(self):
- ''' Pretty print the type and backtrace(s) of this specific error,
- including suppression (which is just a mangled backtrace).'''
- output = ""
- output += "\n" # Make sure the ### is at the beginning of line.
- output += "### BEGIN MEMORY TOOL REPORT (error hash=#%016X#)\n" % \
- self.ErrorHash()
- if (self._commandline):
- output += self._commandline + "\n"
-
- output += self._kind + "\n"
- for backtrace in self._backtraces:
- output += backtrace[0] + "\n"
- filter = subprocess.Popen("c++filt -n", stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- shell=True,
- close_fds=True)
- buf = ""
- for frame in backtrace[1]:
- buf += (frame[FUNCTION_NAME] or frame[INSTRUCTION_POINTER]) + "\n"
- (stdoutbuf, stderrbuf) = filter.communicate(buf.encode('latin-1'))
- demangled_names = stdoutbuf.split("\n")
-
- i = 0
- for frame in backtrace[1]:
- output += (" " + demangled_names[i])
- i = i + 1
-
- global TheAddressTable
- if TheAddressTable != None and frame[SRC_FILE_DIR] == "":
- # Try using gdb
- foo = TheAddressTable.GetFileLine(frame[OBJECT_FILE],
- frame[INSTRUCTION_POINTER])
- if foo[0] != None:
- output += (" (" + foo[0] + ":" + foo[1] + ")")
- elif frame[SRC_FILE_DIR] != "":
- output += (" (" + frame[SRC_FILE_DIR] + "/" + frame[SRC_FILE_NAME] +
- ":" + frame[SRC_LINE] + ")")
- else:
- output += " (" + frame[OBJECT_FILE] + ")"
- output += "\n"
-
- for additional in self._additional:
- output += additional + "\n"
-
- assert self._suppression != None, "Your Valgrind doesn't generate " \
- "suppressions - is it too old?"
-
- if self._testcase:
- output += "The report came from the `%s` test.\n" % self._testcase
- output += "Suppression (error hash=#%016X#):\n" % self.ErrorHash()
- output += (" For more info on using suppressions see "
- "http://dev.chromium.org/developers/tree-sheriffs/sheriff-details-chromium/memory-sheriff#TOC-Suppressing-memory-reports")
-
- # Widen suppression slightly to make portable between mac and linux
- # TODO(timurrrr): Oops, these transformations should happen
- # BEFORE calculating the hash!
- supp = self._suppression;
- supp = supp.replace("fun:_Znwj", "fun:_Znw*")
- supp = supp.replace("fun:_Znwm", "fun:_Znw*")
- supp = supp.replace("fun:_Znaj", "fun:_Zna*")
- supp = supp.replace("fun:_Znam", "fun:_Zna*")
-
- # Make suppressions even less platform-dependent.
- for sz in [1, 2, 4, 8]:
- supp = supp.replace("Memcheck:Addr%d" % sz, "Memcheck:Unaddressable")
- supp = supp.replace("Memcheck:Value%d" % sz, "Memcheck:Uninitialized")
- supp = supp.replace("Memcheck:Cond", "Memcheck:Uninitialized")
-
- # Split into lines so we can enforce length limits
- supplines = supp.split("\n")
- supp = None # to avoid re-use
-
- # Truncate at line 26 (VG_MAX_SUPP_CALLERS plus 2 for name and type)
- # or at the first 'boring' caller.
- # (https://bugs.kde.org/show_bug.cgi?id=199468 proposes raising
- # VG_MAX_SUPP_CALLERS, but we're probably fine with it as is.)
- newlen = min(26, len(supplines));
-
- # Drop boring frames and all the following.
- enough_frames = False
- for frameno in range(newlen):
- for boring_caller in _BORING_CALLERS:
- if re.match("^ +fun:%s$" % boring_caller, supplines[frameno]):
- newlen = frameno
- enough_frames = True
- break
- if enough_frames:
- break
- if (len(supplines) > newlen):
- supplines = supplines[0:newlen]
- supplines.append("}")
-
- for frame in range(len(supplines)):
- # Replace the always-changing anonymous namespace prefix with "*".
- m = re.match("( +fun:)_ZN.*_GLOBAL__N_.*\.cc_" +
- "[0-9a-fA-F]{8}_[0-9a-fA-F]{8}(.*)",
- supplines[frame])
- if m:
- supplines[frame] = "*".join(m.groups())
-
- output += "\n".join(supplines) + "\n"
- output += "### END MEMORY TOOL REPORT (error hash=#%016X#)\n" % \
- self.ErrorHash()
-
- return output
-
- def UniqueString(self):
- ''' String to use for object identity. Don't print this, use str(obj)
- instead.'''
- rep = self._kind + " "
- for backtrace in self._backtraces:
- for frame in backtrace[1]:
- rep += frame[FUNCTION_NAME]
-
- if frame[SRC_FILE_DIR] != "":
- rep += frame[SRC_FILE_DIR] + "/" + frame[SRC_FILE_NAME]
- else:
- rep += frame[OBJECT_FILE]
-
- return rep
-
- # This is a device-independent hash identifying the suppression.
- # By printing out this hash we can find duplicate reports between tests and
- # different shards running on multiple buildbots
- def ErrorHash(self):
- return int(hashlib.md5(self.UniqueString()).hexdigest()[:16], 16)
-
- def __hash__(self):
- return hash(self.UniqueString())
- def __eq__(self, rhs):
- return self.UniqueString() == rhs
-
-def log_is_finished(f, force_finish):
- f.seek(0)
- prev_line = ""
- while True:
- line = f.readline()
- if line == "":
- if not force_finish:
- return False
- # Okay, the log is not finished but we can make it up to be parseable:
- if prev_line.strip() in ["</error>", "</errorcounts>", "</status>"]:
- f.write("</valgrindoutput>\n")
- return True
- return False
- if '</valgrindoutput>' in line:
- # Valgrind often has garbage after </valgrindoutput> upon crash.
- f.truncate()
- return True
- prev_line = line
-
-class MemcheckAnalyzer:
- ''' Given a set of Valgrind XML files, parse all the errors out of them,
- unique them and output the results.'''
-
- SANITY_TEST_SUPPRESSIONS = {
- "Memcheck sanity test 01 (memory leak).": 1,
- "Memcheck sanity test 02 (malloc/read left).": 1,
- "Memcheck sanity test 03 (malloc/read right).": 1,
- "Memcheck sanity test 04 (malloc/write left).": 1,
- "Memcheck sanity test 05 (malloc/write right).": 1,
- "Memcheck sanity test 06 (new/read left).": 1,
- "Memcheck sanity test 07 (new/read right).": 1,
- "Memcheck sanity test 08 (new/write left).": 1,
- "Memcheck sanity test 09 (new/write right).": 1,
- "Memcheck sanity test 10 (write after free).": 1,
- "Memcheck sanity test 11 (write after delete).": 1,
- "Memcheck sanity test 12 (array deleted without []).": 1,
- "Memcheck sanity test 13 (single element deleted with []).": 1,
- "Memcheck sanity test 14 (malloc/read uninit).": 1,
- "Memcheck sanity test 15 (new/read uninit).": 1,
- }
-
- # Max time to wait for memcheck logs to complete.
- LOG_COMPLETION_TIMEOUT = 180.0
-
- def __init__(self, source_dir, show_all_leaks=False, use_gdb=False):
- '''Create a parser for Memcheck logs.
-
- Args:
- source_dir: Path to top of source tree for this build
- show_all_leaks: Whether to show even less important leaks
- use_gdb: Whether to use gdb to resolve source filenames and line numbers
- in the report stacktraces
- '''
- self._source_dir = source_dir
- self._show_all_leaks = show_all_leaks
- self._use_gdb = use_gdb
-
- # Contains the set of unique errors
- self._errors = set()
-
- # Contains the time when the we started analyzing the first log file.
- # This variable is used to skip incomplete logs after some timeout.
- self._analyze_start_time = None
-
-
- def Report(self, files, testcase, check_sanity=False):
- '''Reads in a set of files and prints Memcheck report.
-
- Args:
- files: A list of filenames.
- check_sanity: if true, search for SANITY_TEST_SUPPRESSIONS
- '''
- # Beyond the detailed errors parsed by ValgrindError above,
- # the xml file contain records describing suppressions that were used:
- # <suppcounts>
- # <pair>
- # <count>28</count>
- # <name>pango_font_leak_todo</name>
- # </pair>
- # <pair>
- # <count>378</count>
- # <name>bug_13243</name>
- # </pair>
- # </suppcounts
- # Collect these and print them at the end.
- #
- # With our patch for https://bugs.kde.org/show_bug.cgi?id=205000 in,
- # the file also includes records of the form
- # <load_obj><obj>/usr/lib/libgcc_s.1.dylib</obj><ip>0x27000</ip></load_obj>
- # giving the filename and load address of each binary that was mapped
- # into the process.
-
- global TheAddressTable
- if self._use_gdb:
- TheAddressTable = gdb_helper.AddressTable()
- else:
- TheAddressTable = None
- cur_report_errors = set()
- suppcounts = defaultdict(int)
- badfiles = set()
-
- if self._analyze_start_time == None:
- self._analyze_start_time = time.time()
- start_time = self._analyze_start_time
-
- parse_failed = False
- for file in files:
- # Wait up to three minutes for valgrind to finish writing all files,
- # but after that, just skip incomplete files and warn.
- f = open(file, "r+")
- pid = re.match(".*\.([0-9]+)$", file)
- if pid:
- pid = pid.groups()[0]
- found = False
- running = True
- firstrun = True
- skip = False
- origsize = os.path.getsize(file)
- while (running and not found and not skip and
- (firstrun or
- ((time.time() - start_time) < self.LOG_COMPLETION_TIMEOUT))):
- firstrun = False
- f.seek(0)
- if pid:
- # Make sure the process is still running so we don't wait for
- # 3 minutes if it was killed. See http://crbug.com/17453
- ps_out = subprocess.Popen("ps p %s" % pid, shell=True,
- stdout=subprocess.PIPE).stdout
- if len(ps_out.readlines()) < 2:
- running = False
- else:
- skip = True
- running = False
- found = log_is_finished(f, False)
- if not running and not found:
- logging.warn("Valgrind process PID = %s is not running but its "
- "XML log has not been finished correctly.\n"
- "Make it up by adding some closing tags manually." % pid)
- found = log_is_finished(f, not running)
- if running and not found:
- time.sleep(1)
- f.close()
- if not found:
- badfiles.add(file)
- else:
- newsize = os.path.getsize(file)
- if origsize > newsize+1:
- logging.warn(str(origsize - newsize) +
- " bytes of junk were after </valgrindoutput> in %s!" %
- file)
- try:
- parsed_file = parse(file);
- except ExpatError, e:
- parse_failed = True
- logging.warn("could not parse %s: %s" % (file, e))
- lineno = e.lineno - 1
- context_lines = 5
- context_start = max(0, lineno - context_lines)
- context_end = lineno + context_lines + 1
- context_file = open(file, "r")
- for i in range(0, context_start):
- context_file.readline()
- for i in range(context_start, context_end):
- context_data = context_file.readline().rstrip()
- if i != lineno:
- logging.warn(" %s" % context_data)
- else:
- logging.warn("> %s" % context_data)
- context_file.close()
- continue
- if TheAddressTable != None:
- load_objs = parsed_file.getElementsByTagName("load_obj")
- for load_obj in load_objs:
- obj = getTextOf(load_obj, "obj")
- ip = getTextOf(load_obj, "ip")
- TheAddressTable.AddBinaryAt(obj, ip)
-
- commandline = None
- preamble = parsed_file.getElementsByTagName("preamble")[0];
- for node in preamble.getElementsByTagName("line"):
- if node.localName == "line":
- for x in node.childNodes:
- if x.nodeType == node.TEXT_NODE and "Command" in x.data:
- commandline = x.data
- break
-
- raw_errors = parsed_file.getElementsByTagName("error")
- for raw_error in raw_errors:
- # Ignore "possible" leaks for now by default.
- if (self._show_all_leaks or
- getTextOf(raw_error, "kind") != "Leak_PossiblyLost"):
- error = ValgrindError(self._source_dir,
- raw_error, commandline, testcase)
- if error not in cur_report_errors:
- # We haven't seen such errors doing this report yet...
- if error in self._errors:
- # ... but we saw it in earlier reports, e.g. previous UI test
- cur_report_errors.add("This error was already printed in "
- "some other test, see 'hash=#%016X#'" % \
- error.ErrorHash())
- else:
- # ... and we haven't seen it in other tests as well
- self._errors.add(error)
- cur_report_errors.add(error)
-
- suppcountlist = parsed_file.getElementsByTagName("suppcounts")
- if len(suppcountlist) > 0:
- suppcountlist = suppcountlist[0]
- for node in suppcountlist.getElementsByTagName("pair"):
- count = getTextOf(node, "count");
- name = getTextOf(node, "name");
- suppcounts[name] += int(count)
-
- if len(badfiles) > 0:
- logging.warn("valgrind didn't finish writing %d files?!" % len(badfiles))
- for file in badfiles:
- logging.warn("Last 20 lines of %s :" % file)
- os.system("tail -n 20 '%s' 1>&2" % file)
-
- if parse_failed:
- logging.error("FAIL! Couldn't parse Valgrind output file")
- return -2
-
- common.PrintUsedSuppressionsList(suppcounts)
-
- retcode = 0
- if cur_report_errors:
- logging.error("FAIL! There were %s errors: " % len(cur_report_errors))
-
- if TheAddressTable != None:
- TheAddressTable.ResolveAll()
-
- for error in cur_report_errors:
- logging.error(error)
-
- retcode = -1
-
- # Report tool's insanity even if there were errors.
- if check_sanity:
- remaining_sanity_supp = MemcheckAnalyzer.SANITY_TEST_SUPPRESSIONS
- for (name, count) in suppcounts.iteritems():
- # Workaround for http://crbug.com/334074
- if (name in remaining_sanity_supp and
- remaining_sanity_supp[name] <= count):
- del remaining_sanity_supp[name]
- if remaining_sanity_supp:
- logging.error("FAIL! Sanity check failed!")
- logging.info("The following test errors were not handled: ")
- for (name, count) in remaining_sanity_supp.iteritems():
- logging.info(" * %dx %s" % (count, name))
- retcode = -3
-
- if retcode != 0:
- return retcode
-
- logging.info("PASS! No errors found!")
- return 0
-
-
-def _main():
- '''For testing only. The MemcheckAnalyzer class should be imported instead.'''
- parser = optparse.OptionParser("usage: %prog [options] <files to analyze>")
- parser.add_option("", "--source-dir",
- help="path to top of source tree for this build"
- "(used to normalize source paths in baseline)")
-
- (options, args) = parser.parse_args()
- if len(args) == 0:
- parser.error("no filename specified")
- filenames = args
-
- analyzer = MemcheckAnalyzer(options.source_dir, use_gdb=True)
- return analyzer.Report(filenames, None)
-
-
-if __name__ == "__main__":
- sys.exit(_main())
diff --git a/libs/libyuv/tools_libyuv/valgrind/valgrind.sh b/libs/libyuv/tools_libyuv/valgrind/valgrind.sh
deleted file mode 100755
index 7f3f792666..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/valgrind.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2017 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# This is a small script for manually launching valgrind, along with passing
-# it the suppression file, and some helpful arguments (automatically attaching
-# the debugger on failures, etc). Run it from your repo root, something like:
-# $ sh ./tools/valgrind/valgrind.sh ./out/Debug/chrome
-#
-# This is mostly intended for running the chrome browser interactively.
-# To run unit tests, you probably want to run chrome_tests.sh instead.
-# That's the script used by the valgrind buildbot.
-
-export THISDIR=`dirname $0`
-
-setup_memcheck() {
- RUN_COMMAND="valgrind"
-
- # Prompt to attach gdb when there was an error detected.
- DEFAULT_TOOL_FLAGS=("--db-command=gdb -nw %f %p" "--db-attach=yes" \
- # Keep the registers in gdb in sync with the code.
- "--vex-iropt-register-updates=allregs-at-mem-access" \
- # Overwrite newly allocated or freed objects
- # with 0x41 to catch inproper use.
- "--malloc-fill=41" "--free-fill=41" \
- # Increase the size of stacks being tracked.
- "--num-callers=30")
-}
-
-setup_unknown() {
- echo "Unknown tool \"$TOOL_NAME\" specified, the result is not guaranteed"
- DEFAULT_TOOL_FLAGS=()
-}
-
-set -e
-
-if [ $# -eq 0 ]; then
- echo "usage: <command to run> <arguments ...>"
- exit 1
-fi
-
-TOOL_NAME="memcheck"
-declare -a DEFAULT_TOOL_FLAGS[0]
-
-# Select a tool different from memcheck with --tool=TOOL as a first argument
-TMP_STR=`echo $1 | sed 's/^\-\-tool=//'`
-if [ "$TMP_STR" != "$1" ]; then
- TOOL_NAME="$TMP_STR"
- shift
-fi
-
-if echo "$@" | grep "\-\-tool" ; then
- echo "--tool=TOOL must be the first argument" >&2
- exit 1
-fi
-
-case $TOOL_NAME in
- memcheck*) setup_memcheck "$1";;
- *) setup_unknown;;
-esac
-
-
-SUPPRESSIONS="$THISDIR/$TOOL_NAME/suppressions.txt"
-
-CHROME_VALGRIND=`sh $THISDIR/locate_valgrind.sh`
-if [ "$CHROME_VALGRIND" = "" ]
-then
- # locate_valgrind.sh failed
- exit 1
-fi
-echo "Using valgrind binaries from ${CHROME_VALGRIND}"
-
-set -x
-PATH="${CHROME_VALGRIND}/bin:$PATH"
-# We need to set these variables to override default lib paths hard-coded into
-# Valgrind binary.
-export VALGRIND_LIB="$CHROME_VALGRIND/lib/valgrind"
-export VALGRIND_LIB_INNER="$CHROME_VALGRIND/lib/valgrind"
-
-# G_SLICE=always-malloc: make glib use system malloc
-# NSS_DISABLE_UNLOAD=1: make nss skip dlclosing dynamically loaded modules,
-# which would result in "obj:*" in backtraces.
-# NSS_DISABLE_ARENA_FREE_LIST=1: make nss use system malloc
-# G_DEBUG=fatal_warnings: make GTK abort on any critical or warning assertions.
-# If it crashes on you in the Options menu, you hit bug 19751,
-# comment out the G_DEBUG=fatal_warnings line.
-#
-# GTEST_DEATH_TEST_USE_FORK=1: make gtest death tests valgrind-friendly
-#
-# When everyone has the latest valgrind, we might want to add
-# --show-possibly-lost=no
-# to ignore possible but not definite leaks.
-
-G_SLICE=always-malloc \
-NSS_DISABLE_UNLOAD=1 \
-NSS_DISABLE_ARENA_FREE_LIST=1 \
-G_DEBUG=fatal_warnings \
-GTEST_DEATH_TEST_USE_FORK=1 \
-$RUN_COMMAND \
- --trace-children=yes \
- --leak-check=yes \
- --suppressions="$SUPPRESSIONS" \
- "${DEFAULT_TOOL_FLAGS[@]}" \
- "$@"
diff --git a/libs/libyuv/tools_libyuv/valgrind/valgrind_test.py b/libs/libyuv/tools_libyuv/valgrind/valgrind_test.py
deleted file mode 100755
index 0fd3d97f11..0000000000
--- a/libs/libyuv/tools_libyuv/valgrind/valgrind_test.py
+++ /dev/null
@@ -1,517 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-"""Runs an exe through Valgrind and puts the intermediate files in a
-directory.
-"""
-
-import datetime
-import glob
-import logging
-import optparse
-import os
-import re
-import shutil
-import stat
-import subprocess
-import sys
-import tempfile
-
-import common
-
-import memcheck_analyze
-
-class BaseTool(object):
- """Abstract class for running dynamic error detection tools.
-
- Always subclass this and implement ToolCommand with framework- and
- tool-specific stuff.
- """
-
- def __init__(self):
- temp_parent_dir = None
- self.log_parent_dir = ""
- if common.IsWindows():
- # gpu process on Windows Vista+ runs at Low Integrity and can only
- # write to certain directories (http://crbug.com/119131)
- #
- # TODO(bruening): if scripts die in middle and don't clean up temp
- # dir, we'll accumulate files in profile dir. should remove
- # really old files automatically.
- profile = os.getenv("USERPROFILE")
- if profile:
- self.log_parent_dir = profile + "\\AppData\\LocalLow\\"
- if os.path.exists(self.log_parent_dir):
- self.log_parent_dir = common.NormalizeWindowsPath(self.log_parent_dir)
- temp_parent_dir = self.log_parent_dir
- # Generated every time (even when overridden)
- self.temp_dir = tempfile.mkdtemp(prefix="vg_logs_", dir=temp_parent_dir)
- self.log_dir = self.temp_dir # overridable by --keep_logs
- self.option_parser_hooks = []
- # TODO(glider): we may not need some of the env vars on some of the
- # platforms.
- self._env = {
- "G_SLICE" : "always-malloc",
- "NSS_DISABLE_UNLOAD" : "1",
- "NSS_DISABLE_ARENA_FREE_LIST" : "1",
- "GTEST_DEATH_TEST_USE_FORK": "1",
- }
-
- def ToolName(self):
- raise NotImplementedError, "This method should be implemented " \
- "in the tool-specific subclass"
-
- def Analyze(self, check_sanity=False):
- raise NotImplementedError, "This method should be implemented " \
- "in the tool-specific subclass"
-
- def RegisterOptionParserHook(self, hook):
- # Frameworks and tools can add their own flags to the parser.
- self.option_parser_hooks.append(hook)
-
- def CreateOptionParser(self):
- # Defines Chromium-specific flags.
- self._parser = optparse.OptionParser("usage: %prog [options] <program to "
- "test>")
- self._parser.disable_interspersed_args()
- self._parser.add_option("-t", "--timeout",
- dest="timeout", metavar="TIMEOUT", default=10000,
- help="timeout in seconds for the run (default 10000)")
- self._parser.add_option("", "--build-dir",
- help="the location of the compiler output")
- self._parser.add_option("", "--source-dir",
- help="path to top of source tree for this build"
- "(used to normalize source paths in baseline)")
- self._parser.add_option("", "--gtest_filter", default="",
- help="which test case to run")
- self._parser.add_option("", "--gtest_repeat",
- help="how many times to run each test")
- self._parser.add_option("", "--gtest_print_time", action="store_true",
- default=False,
- help="show how long each test takes")
- self._parser.add_option("", "--ignore_exit_code", action="store_true",
- default=False,
- help="ignore exit code of the test "
- "(e.g. test failures)")
- self._parser.add_option("", "--keep_logs", action="store_true",
- default=False,
- help="store memory tool logs in the <tool>.logs "
- "directory instead of /tmp.\nThis can be "
- "useful for tool developers/maintainers.\n"
- "Please note that the <tool>.logs directory "
- "will be clobbered on tool startup.")
-
- # To add framework- or tool-specific flags, please add a hook using
- # RegisterOptionParserHook in the corresponding subclass.
- # See ValgrindTool for an example.
- for hook in self.option_parser_hooks:
- hook(self, self._parser)
-
- def ParseArgv(self, args):
- self.CreateOptionParser()
-
- # self._tool_flags will store those tool flags which we don't parse
- # manually in this script.
- self._tool_flags = []
- known_args = []
-
- """ We assume that the first argument not starting with "-" is a program
- name and all the following flags should be passed to the program.
- TODO(timurrrr): customize optparse instead
- """
- while len(args) > 0 and args[0][:1] == "-":
- arg = args[0]
- if (arg == "--"):
- break
- if self._parser.has_option(arg.split("=")[0]):
- known_args += [arg]
- else:
- self._tool_flags += [arg]
- args = args[1:]
-
- if len(args) > 0:
- known_args += args
-
- self._options, self._args = self._parser.parse_args(known_args)
-
- self._timeout = int(self._options.timeout)
- self._source_dir = self._options.source_dir
- if self._options.keep_logs:
- # log_parent_dir has trailing slash if non-empty
- self.log_dir = self.log_parent_dir + "%s.logs" % self.ToolName()
- if os.path.exists(self.log_dir):
- shutil.rmtree(self.log_dir)
- os.mkdir(self.log_dir)
- logging.info("Logs are in " + self.log_dir)
-
- self._ignore_exit_code = self._options.ignore_exit_code
- if self._options.gtest_filter != "":
- self._args.append("--gtest_filter=%s" % self._options.gtest_filter)
- if self._options.gtest_repeat:
- self._args.append("--gtest_repeat=%s" % self._options.gtest_repeat)
- if self._options.gtest_print_time:
- self._args.append("--gtest_print_time")
-
- return True
-
- def Setup(self, args):
- return self.ParseArgv(args)
-
- def ToolCommand(self):
- raise NotImplementedError, "This method should be implemented " \
- "in the tool-specific subclass"
-
- def Cleanup(self):
- # You may override it in the tool-specific subclass
- pass
-
- def Execute(self):
- """ Execute the app to be tested after successful instrumentation.
- Full execution command-line provided by subclassers via proc."""
- logging.info("starting execution...")
- proc = self.ToolCommand()
- for var in self._env:
- common.PutEnvAndLog(var, self._env[var])
- return common.RunSubprocess(proc, self._timeout)
-
- def RunTestsAndAnalyze(self, check_sanity):
- exec_retcode = self.Execute()
- analyze_retcode = self.Analyze(check_sanity)
-
- if analyze_retcode:
- logging.error("Analyze failed.")
- logging.info("Search the log for '[ERROR]' to see the error reports.")
- return analyze_retcode
-
- if exec_retcode:
- if self._ignore_exit_code:
- logging.info("Test execution failed, but the exit code is ignored.")
- else:
- logging.error("Test execution failed.")
- return exec_retcode
- else:
- logging.info("Test execution completed successfully.")
-
- if not analyze_retcode:
- logging.info("Analysis completed successfully.")
-
- return 0
-
- def Main(self, args, check_sanity, min_runtime_in_seconds):
- """Call this to run through the whole process: Setup, Execute, Analyze"""
- start_time = datetime.datetime.now()
- retcode = -1
- if self.Setup(args):
- retcode = self.RunTestsAndAnalyze(check_sanity)
- shutil.rmtree(self.temp_dir, ignore_errors=True)
- self.Cleanup()
- else:
- logging.error("Setup failed")
- end_time = datetime.datetime.now()
- runtime_in_seconds = (end_time - start_time).seconds
- hours = runtime_in_seconds / 3600
- seconds = runtime_in_seconds % 3600
- minutes = seconds / 60
- seconds = seconds % 60
- logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds))
- if (min_runtime_in_seconds > 0 and
- runtime_in_seconds < min_runtime_in_seconds):
- logging.error("Layout tests finished too quickly. "
- "It should have taken at least %d seconds. "
- "Something went wrong?" % min_runtime_in_seconds)
- retcode = -1
- return retcode
-
- def Run(self, args, module, min_runtime_in_seconds=0):
- MODULES_TO_SANITY_CHECK = ["base"]
-
- check_sanity = module in MODULES_TO_SANITY_CHECK
- return self.Main(args, check_sanity, min_runtime_in_seconds)
-
-
-class ValgrindTool(BaseTool):
- """Abstract class for running Valgrind tools.
-
- Always subclass this and implement ToolSpecificFlags() and
- ExtendOptionParser() for tool-specific stuff.
- """
- def __init__(self):
- super(ValgrindTool, self).__init__()
- self.RegisterOptionParserHook(ValgrindTool.ExtendOptionParser)
-
- def UseXML(self):
- # Override if tool prefers nonxml output
- return True
-
- def ExtendOptionParser(self, parser):
- parser.add_option("", "--suppressions", default=[],
- action="append",
- help="path to a valgrind suppression file")
- parser.add_option("", "--indirect", action="store_true",
- default=False,
- help="set BROWSER_WRAPPER rather than "
- "running valgrind directly")
- parser.add_option("", "--indirect_webkit_layout", action="store_true",
- default=False,
- help="set --wrapper rather than running Dr. Memory "
- "directly.")
- parser.add_option("", "--trace_children", action="store_true",
- default=False,
- help="also trace child processes")
- parser.add_option("", "--num-callers",
- dest="num_callers", default=30,
- help="number of callers to show in stack traces")
- parser.add_option("", "--generate_dsym", action="store_true",
- default=False,
- help="Generate .dSYM file on Mac if needed. Slow!")
-
- def Setup(self, args):
- if not BaseTool.Setup(self, args):
- return False
- return True
-
- def ToolCommand(self):
- """Get the valgrind command to run."""
- # Note that self._args begins with the exe to be run.
- tool_name = self.ToolName()
-
- # Construct the valgrind command.
- if 'CHROME_VALGRIND' in os.environ:
- path = os.path.join(os.environ['CHROME_VALGRIND'], "bin", "valgrind")
- else:
- path = "valgrind"
- proc = [path, "--tool=%s" % tool_name]
-
- proc += ["--num-callers=%i" % int(self._options.num_callers)]
-
- if self._options.trace_children:
- proc += ["--trace-children=yes"]
- proc += ["--trace-children-skip='*dbus-daemon*'"]
- proc += ["--trace-children-skip='*dbus-launch*'"]
- proc += ["--trace-children-skip='*perl*'"]
- proc += ["--trace-children-skip='*python*'"]
- # This is really Python, but for some reason Valgrind follows it.
- proc += ["--trace-children-skip='*lsb_release*'"]
-
- proc += self.ToolSpecificFlags()
- proc += self._tool_flags
-
- suppression_count = 0
- for suppression_file in self._options.suppressions:
- if os.path.exists(suppression_file):
- suppression_count += 1
- proc += ["--suppressions=%s" % suppression_file]
-
- if not suppression_count:
- logging.warning("WARNING: NOT USING SUPPRESSIONS!")
-
- logfilename = self.log_dir + ("/%s." % tool_name) + "%p"
- if self.UseXML():
- proc += ["--xml=yes", "--xml-file=" + logfilename]
- else:
- proc += ["--log-file=" + logfilename]
-
- # The Valgrind command is constructed.
-
- # Handle --indirect_webkit_layout separately.
- if self._options.indirect_webkit_layout:
- # Need to create the wrapper before modifying |proc|.
- wrapper = self.CreateBrowserWrapper(proc, webkit=True)
- proc = self._args
- proc.append("--wrapper")
- proc.append(wrapper)
- return proc
-
- if self._options.indirect:
- wrapper = self.CreateBrowserWrapper(proc)
- os.environ["BROWSER_WRAPPER"] = wrapper
- logging.info('export BROWSER_WRAPPER=' + wrapper)
- proc = []
- proc += self._args
- return proc
-
- def ToolSpecificFlags(self):
- raise NotImplementedError, "This method should be implemented " \
- "in the tool-specific subclass"
-
- def CreateBrowserWrapper(self, proc, webkit=False):
- """The program being run invokes Python or something else that can't stand
- to be valgrinded, and also invokes the Chrome browser. In this case, use a
- magic wrapper to only valgrind the Chrome browser. Build the wrapper here.
- Returns the path to the wrapper. It's up to the caller to use the wrapper
- appropriately.
- """
- command = " ".join(proc)
- # Add the PID of the browser wrapper to the logfile names so we can
- # separate log files for different UI tests at the analyze stage.
- command = command.replace("%p", "$$.%p")
-
- (fd, indirect_fname) = tempfile.mkstemp(dir=self.log_dir,
- prefix="browser_wrapper.",
- text=True)
- f = os.fdopen(fd, "w")
- f.write('#!/bin/bash\n'
- 'echo "Started Valgrind wrapper for this test, PID=$$" >&2\n')
-
- f.write('DIR=`dirname $0`\n'
- 'TESTNAME_FILE=$DIR/testcase.$$.name\n\n')
-
- if webkit:
- # Webkit layout_tests pass the URL as the first line of stdin.
- f.write('tee $TESTNAME_FILE | %s "$@"\n' % command)
- else:
- # Try to get the test case name by looking at the program arguments.
- # i.e. Chromium ui_tests used --test-name arg.
- # TODO(timurrrr): This doesn't handle "--test-name Test.Name"
- # TODO(timurrrr): ui_tests are dead. Where do we use the non-webkit
- # wrapper now? browser_tests? What do they do?
- f.write('for arg in $@\ndo\n'
- ' if [[ "$arg" =~ --test-name=(.*) ]]\n then\n'
- ' echo ${BASH_REMATCH[1]} >$TESTNAME_FILE\n'
- ' fi\n'
- 'done\n\n'
- '%s "$@"\n' % command)
-
- f.close()
- os.chmod(indirect_fname, stat.S_IRUSR|stat.S_IXUSR)
- return indirect_fname
-
- def CreateAnalyzer(self):
- raise NotImplementedError, "This method should be implemented " \
- "in the tool-specific subclass"
-
- def GetAnalyzeResults(self, check_sanity=False):
- # Glob all the files in the log directory
- filenames = glob.glob(self.log_dir + "/" + self.ToolName() + ".*")
-
- # If we have browser wrapper, the logfiles are named as
- # "toolname.wrapper_PID.valgrind_PID".
- # Let's extract the list of wrapper_PIDs and name it ppids
- ppids = set([int(f.split(".")[-2]) \
- for f in filenames if re.search("\.[0-9]+\.[0-9]+$", f)])
-
- analyzer = self.CreateAnalyzer()
- if len(ppids) == 0:
- # Fast path - no browser wrapper was set.
- return analyzer.Report(filenames, None, check_sanity)
-
- ret = 0
- for ppid in ppids:
- testcase_name = None
- try:
- f = open(self.log_dir + ("/testcase.%d.name" % ppid))
- testcase_name = f.read().strip()
- f.close()
- wk_layout_prefix="third_party/WebKit/LayoutTests/"
- wk_prefix_at = testcase_name.rfind(wk_layout_prefix)
- if wk_prefix_at != -1:
- testcase_name = testcase_name[wk_prefix_at + len(wk_layout_prefix):]
- except IOError:
- pass
- print "====================================================="
- print " Below is the report for valgrind wrapper PID=%d." % ppid
- if testcase_name:
- print " It was used while running the `%s` test." % testcase_name
- else:
- print " You can find the corresponding test"
- print " by searching the above log for 'PID=%d'" % ppid
- sys.stdout.flush()
-
- ppid_filenames = [f for f in filenames \
- if re.search("\.%d\.[0-9]+$" % ppid, f)]
- # check_sanity won't work with browser wrappers
- assert check_sanity == False
- ret |= analyzer.Report(ppid_filenames, testcase_name)
- print "====================================================="
- sys.stdout.flush()
-
- if ret != 0:
- print ""
- print "The Valgrind reports are grouped by test names."
- print "Each test has its PID printed in the log when the test was run"
- print "and at the beginning of its Valgrind report."
- print "Hint: you can search for the reports by Ctrl+F -> `=#`"
- sys.stdout.flush()
-
- return ret
-
-
-# TODO(timurrrr): Split into a separate file.
-class Memcheck(ValgrindTool):
- """Memcheck
- Dynamic memory error detector for Linux & Mac
-
- http://valgrind.org/info/tools.html#memcheck
- """
-
- def __init__(self):
- super(Memcheck, self).__init__()
- self.RegisterOptionParserHook(Memcheck.ExtendOptionParser)
-
- def ToolName(self):
- return "memcheck"
-
- def ExtendOptionParser(self, parser):
- parser.add_option("--leak-check", "--leak_check", type="string",
- default="yes", # --leak-check=yes is equivalent of =full
- help="perform leak checking at the end of the run")
- parser.add_option("", "--show_all_leaks", action="store_true",
- default=False,
- help="also show less blatant leaks")
- parser.add_option("", "--track_origins", action="store_true",
- default=False,
- help="Show whence uninitialized bytes came. 30% slower.")
-
- def ToolSpecificFlags(self):
- ret = ["--gen-suppressions=all", "--demangle=no"]
- ret += ["--leak-check=%s" % self._options.leak_check]
-
- if self._options.show_all_leaks:
- ret += ["--show-reachable=yes"]
- else:
- ret += ["--show-possibly-lost=no"]
-
- if self._options.track_origins:
- ret += ["--track-origins=yes"]
-
- # TODO(glider): this is a temporary workaround for http://crbug.com/51716
- # Let's see whether it helps.
- if common.IsMac():
- ret += ["--smc-check=all"]
-
- return ret
-
- def CreateAnalyzer(self):
- use_gdb = common.IsMac()
- return memcheck_analyze.MemcheckAnalyzer(self._source_dir,
- self._options.show_all_leaks,
- use_gdb=use_gdb)
-
- def Analyze(self, check_sanity=False):
- ret = self.GetAnalyzeResults(check_sanity)
-
- if ret != 0:
- logging.info("Please see http://dev.chromium.org/developers/how-tos/"
- "using-valgrind for the info on Memcheck/Valgrind")
- return ret
-
-
-class ToolFactory:
- def Create(self, tool_name):
- if tool_name == "memcheck":
- return Memcheck()
- try:
- platform_name = common.PlatformNames()[0]
- except common.NotImplementedError:
- platform_name = sys.platform + "(Unknown)"
- raise RuntimeError, "Unknown tool (tool=%s, platform=%s)" % (tool_name,
- platform_name)
-
-def CreateTool(tool):
- return ToolFactory().Create(tool)
diff --git a/libs/libyuv/unit_test/basictypes_test.cc b/libs/libyuv/unit_test/basictypes_test.cc
index 9aaa2dcd98..3c9068c1ff 100644
--- a/libs/libyuv/unit_test/basictypes_test.cc
+++ b/libs/libyuv/unit_test/basictypes_test.cc
@@ -8,8 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <stdint.h>
+
#include "../unit_test/unit_test.h"
-#include "libyuv/basic_types.h"
namespace libyuv {
@@ -22,22 +23,22 @@ TEST_F(LibYUVBaseTest, SizeOfTypes) {
uint32_t u32 = 1u;
int64_t i64 = -1;
uint64_t u64 = 1u;
- EXPECT_EQ(1u, sizeof(i8));
- EXPECT_EQ(1u, sizeof(u8));
- EXPECT_EQ(2u, sizeof(i16));
- EXPECT_EQ(2u, sizeof(u16));
- EXPECT_EQ(4u, sizeof(i32));
- EXPECT_EQ(4u, sizeof(u32));
- EXPECT_EQ(8u, sizeof(i64));
- EXPECT_EQ(8u, sizeof(u64));
- EXPECT_GT(0, i8);
- EXPECT_LT(0u, u8);
- EXPECT_GT(0, i16);
- EXPECT_LT(0u, u16);
- EXPECT_GT(0, i32);
- EXPECT_LT(0u, u32);
- EXPECT_GT(0, i64);
- EXPECT_LT(0u, u64);
+ ASSERT_EQ(1u, sizeof(i8));
+ ASSERT_EQ(1u, sizeof(u8));
+ ASSERT_EQ(2u, sizeof(i16));
+ ASSERT_EQ(2u, sizeof(u16));
+ ASSERT_EQ(4u, sizeof(i32));
+ ASSERT_EQ(4u, sizeof(u32));
+ ASSERT_EQ(8u, sizeof(i64));
+ ASSERT_EQ(8u, sizeof(u64));
+ ASSERT_GT(0, i8);
+ ASSERT_LT(0u, u8);
+ ASSERT_GT(0, i16);
+ ASSERT_LT(0u, u16);
+ ASSERT_GT(0, i32);
+ ASSERT_LT(0u, u32);
+ ASSERT_GT(0, i64);
+ ASSERT_LT(0u, u64);
}
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/color_test.cc b/libs/libyuv/unit_test/color_test.cc
index 4bb448d56f..2f46832e3e 100644
--- a/libs/libyuv/unit_test/color_test.cc
+++ b/libs/libyuv/unit_test/color_test.cc
@@ -11,7 +11,6 @@
#include <stdlib.h>
#include "../unit_test/unit_test.h"
-#include "libyuv/basic_types.h"
#include "libyuv/convert.h"
#include "libyuv/convert_argb.h"
#include "libyuv/convert_from.h"
@@ -20,20 +19,16 @@
namespace libyuv {
-// TODO(fbarchard): Port high accuracy YUV to RGB to Neon.
-#if !defined(LIBYUV_DISABLE_NEON) && \
- (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
-#define ERROR_R 1
-#define ERROR_G 1
-#define ERROR_B 3
+// TODO(fbarchard): clang x86 has a higher accuracy YUV to RGB.
+// Port to Visual C and other CPUs
#define ERROR_FULL 6
-#define ERROR_J420 5
-#else
+#define ERROR_J420 6
#define ERROR_R 1
#define ERROR_G 1
-#define ERROR_B 3
-#define ERROR_FULL 5
-#define ERROR_J420 3
+#ifdef LIBYUV_UNLIMITED_DATA
+#define ERROR_B 1
+#else
+#define ERROR_B 18
#endif
#define TESTCS(TESTNAME, YUVTOARGB, ARGBTOYUV, HS1, HS, HN, DIFF) \
@@ -117,11 +112,11 @@ namespace libyuv {
} \
/* Test C and SIMD match. */ \
for (int i = 0; i < kPixels * 4; ++i) { \
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); \
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); \
} \
/* Test SIMD is close to original. */ \
for (int i = 0; i < kPixels * 4; ++i) { \
- EXPECT_NEAR(static_cast<int>(orig_pixels[i]), \
+ ASSERT_NEAR(static_cast<int>(orig_pixels[i]), \
static_cast<int>(dst_pixels_opt[i]), DIFF); \
} \
\
@@ -154,6 +149,7 @@ static void YUVToRGB(int y, int u, int v, int* r, int* g, int* b) {
memset(orig_y, y, kPixels);
memset(orig_u, u, kHalfPixels);
memset(orig_v, v, kHalfPixels);
+ memset(orig_pixels, 0, sizeof(orig_pixels));
/* YUV converted to ARGB. */
I422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
@@ -177,6 +173,7 @@ static void YUVJToRGB(int y, int u, int v, int* r, int* g, int* b) {
memset(orig_y, y, kPixels);
memset(orig_u, u, kHalfPixels);
memset(orig_v, v, kHalfPixels);
+ memset(orig_pixels, 0, sizeof(orig_pixels));
/* YUV converted to ARGB. */
J422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
@@ -187,6 +184,104 @@ static void YUVJToRGB(int y, int u, int v, int* r, int* g, int* b) {
*r = orig_pixels[2];
}
+static void YUVHToRGB(int y, int u, int v, int* r, int* g, int* b) {
+ const int kWidth = 16;
+ const int kHeight = 1;
+ const int kPixels = kWidth * kHeight;
+ const int kHalfPixels = ((kWidth + 1) / 2) * ((kHeight + 1) / 2);
+
+ SIMD_ALIGNED(uint8_t orig_y[16]);
+ SIMD_ALIGNED(uint8_t orig_u[8]);
+ SIMD_ALIGNED(uint8_t orig_v[8]);
+ SIMD_ALIGNED(uint8_t orig_pixels[16 * 4]);
+ memset(orig_y, y, kPixels);
+ memset(orig_u, u, kHalfPixels);
+ memset(orig_v, v, kHalfPixels);
+
+ /* YUV converted to ARGB. */
+ H422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
+ orig_pixels, kWidth * 4, kWidth, kHeight);
+
+ *b = orig_pixels[0];
+ *g = orig_pixels[1];
+ *r = orig_pixels[2];
+}
+
+#define F422ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvF709Constants, i, j)
+
+static void YUVFToRGB(int y, int u, int v, int* r, int* g, int* b) {
+ const int kWidth = 16;
+ const int kHeight = 1;
+ const int kPixels = kWidth * kHeight;
+ const int kHalfPixels = ((kWidth + 1) / 2) * ((kHeight + 1) / 2);
+
+ SIMD_ALIGNED(uint8_t orig_y[16]);
+ SIMD_ALIGNED(uint8_t orig_u[8]);
+ SIMD_ALIGNED(uint8_t orig_v[8]);
+ SIMD_ALIGNED(uint8_t orig_pixels[16 * 4]);
+ memset(orig_y, y, kPixels);
+ memset(orig_u, u, kHalfPixels);
+ memset(orig_v, v, kHalfPixels);
+
+ /* YUV converted to ARGB. */
+ F422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
+ orig_pixels, kWidth * 4, kWidth, kHeight);
+
+ *b = orig_pixels[0];
+ *g = orig_pixels[1];
+ *r = orig_pixels[2];
+}
+
+static void YUVUToRGB(int y, int u, int v, int* r, int* g, int* b) {
+ const int kWidth = 16;
+ const int kHeight = 1;
+ const int kPixels = kWidth * kHeight;
+ const int kHalfPixels = ((kWidth + 1) / 2) * ((kHeight + 1) / 2);
+
+ SIMD_ALIGNED(uint8_t orig_y[16]);
+ SIMD_ALIGNED(uint8_t orig_u[8]);
+ SIMD_ALIGNED(uint8_t orig_v[8]);
+ SIMD_ALIGNED(uint8_t orig_pixels[16 * 4]);
+ memset(orig_y, y, kPixels);
+ memset(orig_u, u, kHalfPixels);
+ memset(orig_v, v, kHalfPixels);
+
+ /* YUV converted to ARGB. */
+ U422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
+ orig_pixels, kWidth * 4, kWidth, kHeight);
+
+ *b = orig_pixels[0];
+ *g = orig_pixels[1];
+ *r = orig_pixels[2];
+}
+
+#define V422ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvV2020Constants, i, j)
+
+static void YUVVToRGB(int y, int u, int v, int* r, int* g, int* b) {
+ const int kWidth = 16;
+ const int kHeight = 1;
+ const int kPixels = kWidth * kHeight;
+ const int kHalfPixels = ((kWidth + 1) / 2) * ((kHeight + 1) / 2);
+
+ SIMD_ALIGNED(uint8_t orig_y[16]);
+ SIMD_ALIGNED(uint8_t orig_u[8]);
+ SIMD_ALIGNED(uint8_t orig_v[8]);
+ SIMD_ALIGNED(uint8_t orig_pixels[16 * 4]);
+ memset(orig_y, y, kPixels);
+ memset(orig_u, u, kHalfPixels);
+ memset(orig_v, v, kHalfPixels);
+
+ /* YUV converted to ARGB. */
+ V422ToARGB(orig_y, kWidth, orig_u, (kWidth + 1) / 2, orig_v, (kWidth + 1) / 2,
+ orig_pixels, kWidth * 4, kWidth, kHeight);
+
+ *b = orig_pixels[0];
+ *g = orig_pixels[1];
+ *r = orig_pixels[2];
+}
+
static void YToRGB(int y, int* r, int* g, int* b) {
const int kWidth = 16;
const int kHeight = 1;
@@ -331,65 +426,103 @@ TEST_F(LibYUVColorTest, TestRoundToByte) {
allb |= b;
}
}
- EXPECT_GE(allb, 0);
- EXPECT_LE(allb, 255);
+ ASSERT_GE(allb, 0);
+ ASSERT_LE(allb, 255);
}
+// BT.601 limited range YUV to RGB reference
static void YUVToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
- *r = RoundToByte((y - 16) * 1.164 - (v - 128) * -1.596);
- *g = RoundToByte((y - 16) * 1.164 - (u - 128) * 0.391 - (v - 128) * 0.813);
- *b = RoundToByte((y - 16) * 1.164 - (u - 128) * -2.018);
+ double y1 = (y - 16) * 1.164;
+ *r = RoundToByte(y1 - (v - 128) * -1.596);
+ *g = RoundToByte(y1 - (u - 128) * 0.391 - (v - 128) * 0.813);
+ *b = RoundToByte(y1 - (u - 128) * -2.018);
}
+// BT.601 full range YUV to RGB reference (aka JPEG)
static void YUVJToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
*r = RoundToByte(y - (v - 128) * -1.40200);
*g = RoundToByte(y - (u - 128) * 0.34414 - (v - 128) * 0.71414);
*b = RoundToByte(y - (u - 128) * -1.77200);
}
+// BT.709 limited range YUV to RGB reference
+// See also http://www.equasys.de/colorconversion.html
+static void YUVHToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
+ double y1 = (y - 16) * 1.164;
+ *r = RoundToByte(y1 - (v - 128) * -1.793);
+ *g = RoundToByte(y1 - (u - 128) * 0.213 - (v - 128) * 0.533);
+ *b = RoundToByte(y1 - (u - 128) * -2.112);
+}
+
+// BT.709 full range YUV to RGB reference
+static void YUVFToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
+ *r = RoundToByte(y - (v - 128) * -1.5748);
+ *g = RoundToByte(y - (u - 128) * 0.18732 - (v - 128) * 0.46812);
+ *b = RoundToByte(y - (u - 128) * -1.8556);
+}
+
+// BT.2020 limited range YUV to RGB reference
+static void YUVUToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
+ double y1 = (y - 16) * 1.164384;
+ *r = RoundToByte(y1 - (v - 128) * -1.67867);
+ *g = RoundToByte(y1 - (u - 128) * 0.187326 - (v - 128) * 0.65042);
+ *b = RoundToByte(y1 - (u - 128) * -2.14177);
+}
+
+// BT.2020 full range YUV to RGB reference
+static void YUVVToRGBReference(int y, int u, int v, int* r, int* g, int* b) {
+ *r = RoundToByte(y + (v - 128) * 1.474600);
+ *g = RoundToByte(y - (u - 128) * 0.164553 - (v - 128) * 0.571353);
+ *b = RoundToByte(y + (u - 128) * 1.881400);
+}
+
TEST_F(LibYUVColorTest, TestYUV) {
int r0, g0, b0, r1, g1, b1;
// cyan (less red)
YUVToRGBReference(240, 255, 0, &r0, &g0, &b0);
- EXPECT_EQ(56, r0);
- EXPECT_EQ(255, g0);
- EXPECT_EQ(255, b0);
+ ASSERT_EQ(56, r0);
+ ASSERT_EQ(255, g0);
+ ASSERT_EQ(255, b0);
YUVToRGB(240, 255, 0, &r1, &g1, &b1);
- EXPECT_EQ(57, r1);
- EXPECT_EQ(255, g1);
- EXPECT_EQ(255, b1);
+ ASSERT_EQ(57, r1);
+ ASSERT_EQ(255, g1);
+ ASSERT_EQ(255, b1);
// green (less red and blue)
YUVToRGBReference(240, 0, 0, &r0, &g0, &b0);
- EXPECT_EQ(56, r0);
- EXPECT_EQ(255, g0);
- EXPECT_EQ(2, b0);
+ ASSERT_EQ(56, r0);
+ ASSERT_EQ(255, g0);
+ ASSERT_EQ(2, b0);
YUVToRGB(240, 0, 0, &r1, &g1, &b1);
- EXPECT_EQ(57, r1);
- EXPECT_EQ(255, g1);
- EXPECT_EQ(5, b1);
+ ASSERT_EQ(57, r1);
+ ASSERT_EQ(255, g1);
+#ifdef LIBYUV_UNLIMITED_DATA
+ ASSERT_EQ(3, b1);
+#else
+ ASSERT_EQ(5, b1);
+#endif
for (int i = 0; i < 256; ++i) {
YUVToRGBReference(i, 128, 128, &r0, &g0, &b0);
YUVToRGB(i, 128, 128, &r1, &g1, &b1);
- EXPECT_NEAR(r0, r1, ERROR_R);
- EXPECT_NEAR(g0, g1, ERROR_G);
- EXPECT_NEAR(b0, b1, ERROR_B);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
YUVToRGBReference(i, 0, 0, &r0, &g0, &b0);
YUVToRGB(i, 0, 0, &r1, &g1, &b1);
- EXPECT_NEAR(r0, r1, ERROR_R);
- EXPECT_NEAR(g0, g1, ERROR_G);
- EXPECT_NEAR(b0, b1, ERROR_B);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
YUVToRGBReference(i, 0, 255, &r0, &g0, &b0);
YUVToRGB(i, 0, 255, &r1, &g1, &b1);
- EXPECT_NEAR(r0, r1, ERROR_R);
- EXPECT_NEAR(g0, g1, ERROR_G);
- EXPECT_NEAR(b0, b1, ERROR_B);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
}
}
@@ -398,74 +531,74 @@ TEST_F(LibYUVColorTest, TestGreyYUV) {
// black
YUVToRGBReference(16, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(0, r0);
- EXPECT_EQ(0, g0);
- EXPECT_EQ(0, b0);
+ ASSERT_EQ(0, r0);
+ ASSERT_EQ(0, g0);
+ ASSERT_EQ(0, b0);
YUVToRGB(16, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(0, r1);
- EXPECT_EQ(0, g1);
- EXPECT_EQ(0, b1);
+ ASSERT_EQ(0, r1);
+ ASSERT_EQ(0, g1);
+ ASSERT_EQ(0, b1);
// white
YUVToRGBReference(240, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(255, r0);
- EXPECT_EQ(255, g0);
- EXPECT_EQ(255, b0);
+ ASSERT_EQ(255, r0);
+ ASSERT_EQ(255, g0);
+ ASSERT_EQ(255, b0);
YUVToRGB(240, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(255, r1);
- EXPECT_EQ(255, g1);
- EXPECT_EQ(255, b1);
+ ASSERT_EQ(255, r1);
+ ASSERT_EQ(255, g1);
+ ASSERT_EQ(255, b1);
// grey
YUVToRGBReference(128, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(130, r0);
- EXPECT_EQ(130, g0);
- EXPECT_EQ(130, b0);
+ ASSERT_EQ(130, r0);
+ ASSERT_EQ(130, g0);
+ ASSERT_EQ(130, b0);
YUVToRGB(128, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(130, r1);
- EXPECT_EQ(130, g1);
- EXPECT_EQ(130, b1);
+ ASSERT_EQ(130, r1);
+ ASSERT_EQ(130, g1);
+ ASSERT_EQ(130, b1);
for (int y = 0; y < 256; ++y) {
YUVToRGBReference(y, 128, 128, &r0, &g0, &b0);
YUVToRGB(y, 128, 128, &r1, &g1, &b1);
YToRGB(y, &r2, &g2, &b2);
- EXPECT_EQ(r0, r1);
- EXPECT_EQ(g0, g1);
- EXPECT_EQ(b0, b1);
- EXPECT_EQ(r0, r2);
- EXPECT_EQ(g0, g2);
- EXPECT_EQ(b0, b2);
+ ASSERT_EQ(r0, r1);
+ ASSERT_EQ(g0, g1);
+ ASSERT_EQ(b0, b1);
+ ASSERT_EQ(r0, r2);
+ ASSERT_EQ(g0, g2);
+ ASSERT_EQ(b0, b2);
}
}
static void PrintHistogram(int rh[256], int gh[256], int bh[256]) {
int i;
- printf("hist");
+ printf("hist ");
for (i = 0; i < 256; ++i) {
if (rh[i] || gh[i] || bh[i]) {
- printf("\t%8d", i - 128);
+ printf(" %8d", i - 128);
}
}
- printf("\nred");
+ printf("\nred ");
for (i = 0; i < 256; ++i) {
if (rh[i] || gh[i] || bh[i]) {
- printf("\t%8d", rh[i]);
+ printf(" %8d", rh[i]);
}
}
printf("\ngreen");
for (i = 0; i < 256; ++i) {
if (rh[i] || gh[i] || bh[i]) {
- printf("\t%8d", gh[i]);
+ printf(" %8d", gh[i]);
}
}
- printf("\nblue");
+ printf("\nblue ");
for (i = 0; i < 256; ++i) {
if (rh[i] || gh[i] || bh[i]) {
- printf("\t%8d", bh[i]);
+ printf(" %8d", bh[i]);
}
}
printf("\n");
@@ -473,7 +606,14 @@ static void PrintHistogram(int rh[256], int gh[256], int bh[256]) {
// Step by 5 on inner loop goes from 0 to 255 inclusive.
// Set to 1 for better converage. 3, 5 or 17 for faster testing.
+#ifdef DISABLE_SLOW_TESTS
#define FASTSTEP 5
+#else
+#define FASTSTEP 3
+#endif
+
+// BT.601 limited range.
+#ifndef DISABLE_SLOW_TESTS
TEST_F(LibYUVColorTest, TestFullYUV) {
int rh[256] = {
0,
@@ -484,16 +624,16 @@ TEST_F(LibYUVColorTest, TestFullYUV) {
int bh[256] = {
0,
};
- for (int u = 0; u < 256; ++u) {
- for (int v = 0; v < 256; ++v) {
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
int r0, g0, b0, r1, g1, b1;
int y = RANDOM256(y2);
YUVToRGBReference(y, u, v, &r0, &g0, &b0);
YUVToRGB(y, u, v, &r1, &g1, &b1);
- EXPECT_NEAR(r0, r1, ERROR_R);
- EXPECT_NEAR(g0, g1, ERROR_G);
- EXPECT_NEAR(b0, b1, ERROR_B);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
++rh[r1 - r0 + 128];
++gh[g1 - g0 + 128];
++bh[b1 - b0 + 128];
@@ -503,6 +643,7 @@ TEST_F(LibYUVColorTest, TestFullYUV) {
PrintHistogram(rh, gh, bh);
}
+// BT.601 full range.
TEST_F(LibYUVColorTest, TestFullYUVJ) {
int rh[256] = {
0,
@@ -513,16 +654,76 @@ TEST_F(LibYUVColorTest, TestFullYUVJ) {
int bh[256] = {
0,
};
- for (int u = 0; u < 256; ++u) {
- for (int v = 0; v < 256; ++v) {
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
int r0, g0, b0, r1, g1, b1;
int y = RANDOM256(y2);
YUVJToRGBReference(y, u, v, &r0, &g0, &b0);
YUVJToRGB(y, u, v, &r1, &g1, &b1);
- EXPECT_NEAR(r0, r1, 1);
- EXPECT_NEAR(g0, g1, 1);
- EXPECT_NEAR(b0, b1, 1);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
+ ++rh[r1 - r0 + 128];
+ ++gh[g1 - g0 + 128];
+ ++bh[b1 - b0 + 128];
+ }
+ }
+ }
+ PrintHistogram(rh, gh, bh);
+}
+
+// BT.709 limited range.
+TEST_F(LibYUVColorTest, TestFullYUVH) {
+ int rh[256] = {
+ 0,
+ };
+ int gh[256] = {
+ 0,
+ };
+ int bh[256] = {
+ 0,
+ };
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
+ for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
+ int r0, g0, b0, r1, g1, b1;
+ int y = RANDOM256(y2);
+ YUVHToRGBReference(y, u, v, &r0, &g0, &b0);
+ YUVHToRGB(y, u, v, &r1, &g1, &b1);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
+ ++rh[r1 - r0 + 128];
+ ++gh[g1 - g0 + 128];
+ ++bh[b1 - b0 + 128];
+ }
+ }
+ }
+ PrintHistogram(rh, gh, bh);
+}
+
+// BT.709 full range.
+TEST_F(LibYUVColorTest, TestFullYUVF) {
+ int rh[256] = {
+ 0,
+ };
+ int gh[256] = {
+ 0,
+ };
+ int bh[256] = {
+ 0,
+ };
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
+ for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
+ int r0, g0, b0, r1, g1, b1;
+ int y = RANDOM256(y2);
+ YUVFToRGBReference(y, u, v, &r0, &g0, &b0);
+ YUVFToRGB(y, u, v, &r1, &g1, &b1);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
++rh[r1 - r0 + 128];
++gh[g1 - g0 + 128];
++bh[b1 - b0 + 128];
@@ -531,6 +732,68 @@ TEST_F(LibYUVColorTest, TestFullYUVJ) {
}
PrintHistogram(rh, gh, bh);
}
+
+// BT.2020 limited range.
+TEST_F(LibYUVColorTest, TestFullYUVU) {
+ int rh[256] = {
+ 0,
+ };
+ int gh[256] = {
+ 0,
+ };
+ int bh[256] = {
+ 0,
+ };
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
+ for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
+ int r0, g0, b0, r1, g1, b1;
+ int y = RANDOM256(y2);
+ YUVUToRGBReference(y, u, v, &r0, &g0, &b0);
+ YUVUToRGB(y, u, v, &r1, &g1, &b1);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, ERROR_G);
+ ASSERT_NEAR(b0, b1, ERROR_B);
+ ++rh[r1 - r0 + 128];
+ ++gh[g1 - g0 + 128];
+ ++bh[b1 - b0 + 128];
+ }
+ }
+ }
+ PrintHistogram(rh, gh, bh);
+}
+
+// BT.2020 full range.
+TEST_F(LibYUVColorTest, TestFullYUVV) {
+ int rh[256] = {
+ 0,
+ };
+ int gh[256] = {
+ 0,
+ };
+ int bh[256] = {
+ 0,
+ };
+ for (int u = 0; u < 256; u += FASTSTEP) {
+ for (int v = 0; v < 256; v += FASTSTEP) {
+ for (int y2 = 0; y2 < 256; y2 += FASTSTEP) {
+ int r0, g0, b0, r1, g1, b1;
+ int y = RANDOM256(y2);
+ YUVVToRGBReference(y, u, v, &r0, &g0, &b0);
+ YUVVToRGB(y, u, v, &r1, &g1, &b1);
+ ASSERT_NEAR(r0, r1, ERROR_R);
+ ASSERT_NEAR(g0, g1, 2);
+ ASSERT_NEAR(b0, b1, ERROR_B);
+ ++rh[r1 - r0 + 128];
+ ++gh[g1 - g0 + 128];
+ ++bh[b1 - b0 + 128];
+ }
+ }
+ }
+ PrintHistogram(rh, gh, bh);
+}
+#endif // DISABLE_SLOW_TESTS
+
#undef FASTSTEP
TEST_F(LibYUVColorTest, TestGreyYUVJ) {
@@ -538,47 +801,47 @@ TEST_F(LibYUVColorTest, TestGreyYUVJ) {
// black
YUVJToRGBReference(0, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(0, r0);
- EXPECT_EQ(0, g0);
- EXPECT_EQ(0, b0);
+ ASSERT_EQ(0, r0);
+ ASSERT_EQ(0, g0);
+ ASSERT_EQ(0, b0);
YUVJToRGB(0, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(0, r1);
- EXPECT_EQ(0, g1);
- EXPECT_EQ(0, b1);
+ ASSERT_EQ(0, r1);
+ ASSERT_EQ(0, g1);
+ ASSERT_EQ(0, b1);
// white
YUVJToRGBReference(255, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(255, r0);
- EXPECT_EQ(255, g0);
- EXPECT_EQ(255, b0);
+ ASSERT_EQ(255, r0);
+ ASSERT_EQ(255, g0);
+ ASSERT_EQ(255, b0);
YUVJToRGB(255, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(255, r1);
- EXPECT_EQ(255, g1);
- EXPECT_EQ(255, b1);
+ ASSERT_EQ(255, r1);
+ ASSERT_EQ(255, g1);
+ ASSERT_EQ(255, b1);
// grey
YUVJToRGBReference(128, 128, 128, &r0, &g0, &b0);
- EXPECT_EQ(128, r0);
- EXPECT_EQ(128, g0);
- EXPECT_EQ(128, b0);
+ ASSERT_EQ(128, r0);
+ ASSERT_EQ(128, g0);
+ ASSERT_EQ(128, b0);
YUVJToRGB(128, 128, 128, &r1, &g1, &b1);
- EXPECT_EQ(128, r1);
- EXPECT_EQ(128, g1);
- EXPECT_EQ(128, b1);
+ ASSERT_EQ(128, r1);
+ ASSERT_EQ(128, g1);
+ ASSERT_EQ(128, b1);
for (int y = 0; y < 256; ++y) {
YUVJToRGBReference(y, 128, 128, &r0, &g0, &b0);
YUVJToRGB(y, 128, 128, &r1, &g1, &b1);
YJToRGB(y, &r2, &g2, &b2);
- EXPECT_EQ(r0, r1);
- EXPECT_EQ(g0, g1);
- EXPECT_EQ(b0, b1);
- EXPECT_EQ(r0, r2);
- EXPECT_EQ(g0, g2);
- EXPECT_EQ(b0, b2);
+ ASSERT_EQ(r0, r1);
+ ASSERT_EQ(g0, g1);
+ ASSERT_EQ(b0, b1);
+ ASSERT_EQ(r0, r2);
+ ASSERT_EQ(g0, g2);
+ ASSERT_EQ(b0, b2);
}
}
diff --git a/libs/libyuv/unit_test/compare_test.cc b/libs/libyuv/unit_test/compare_test.cc
index bd99cdd3ac..a3a402aa15 100644
--- a/libs/libyuv/unit_test/compare_test.cc
+++ b/libs/libyuv/unit_test/compare_test.cc
@@ -13,7 +13,6 @@
#include <time.h>
#include "../unit_test/unit_test.h"
-#include "libyuv/basic_types.h"
#include "libyuv/compare.h"
#include "libyuv/cpu_id.h"
#include "libyuv/video_common.h"
@@ -48,7 +47,7 @@ TEST_F(LibYUVCompareTest, Djb2_Test) {
" together with Hermann Zapf";
uint32_t foxhash = HashDjb2(reinterpret_cast<const uint8_t*>(fox), 131, 5381);
const uint32_t kExpectedFoxHash = 2611006483u;
- EXPECT_EQ(kExpectedFoxHash, foxhash);
+ ASSERT_EQ(kExpectedFoxHash, foxhash);
for (int i = 0; i < kMaxTest; ++i) {
src_a[i] = (fastrand() & 0xff);
@@ -57,13 +56,13 @@ TEST_F(LibYUVCompareTest, Djb2_Test) {
// Compare different buffers. Expect hash is different.
uint32_t h1 = HashDjb2(src_a, kMaxTest, 5381);
uint32_t h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make last half same. Expect hash is different.
memcpy(src_a + kMaxTest / 2, src_b + kMaxTest / 2, kMaxTest / 2);
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make first half same. Expect hash is different.
memcpy(src_a + kMaxTest / 2, src_a, kMaxTest / 2);
@@ -71,52 +70,52 @@ TEST_F(LibYUVCompareTest, Djb2_Test) {
memcpy(src_a, src_b, kMaxTest / 2);
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make same. Expect hash is same.
memcpy(src_a, src_b, kMaxTest);
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_EQ(h1, h2);
+ ASSERT_EQ(h1, h2);
// Mask seed different. Expect hash is different.
memcpy(src_a, src_b, kMaxTest);
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 1234);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make one byte different in middle. Expect hash is different.
memcpy(src_a, src_b, kMaxTest);
++src_b[kMaxTest / 2];
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make first byte different. Expect hash is different.
memcpy(src_a, src_b, kMaxTest);
++src_b[0];
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make last byte different. Expect hash is different.
memcpy(src_a, src_b, kMaxTest);
++src_b[kMaxTest - 1];
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_b, kMaxTest, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make a zeros. Test different lengths. Expect hash is different.
memset(src_a, 0, kMaxTest);
h1 = HashDjb2(src_a, kMaxTest, 5381);
h2 = HashDjb2(src_a, kMaxTest / 2, 5381);
- EXPECT_NE(h1, h2);
+ ASSERT_NE(h1, h2);
// Make a zeros and seed of zero. Test different lengths. Expect hash is same.
memset(src_a, 0, kMaxTest);
h1 = HashDjb2(src_a, kMaxTest, 0);
h2 = HashDjb2(src_a, kMaxTest / 2, 0);
- EXPECT_EQ(h1, h2);
+ ASSERT_EQ(h1, h2);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -134,7 +133,7 @@ TEST_F(LibYUVCompareTest, BenchmarkDjb2_Opt) {
for (int i = 0; i < benchmark_iterations_; ++i) {
h1 = HashDjb2(src_a, kMaxTest, 5381);
}
- EXPECT_EQ(h1, h2);
+ ASSERT_EQ(h1, h2);
free_aligned_buffer_page_end(src_a);
}
@@ -149,7 +148,7 @@ TEST_F(LibYUVCompareTest, BenchmarkDjb2_Unaligned) {
for (int i = 0; i < benchmark_iterations_; ++i) {
h1 = HashDjb2(src_a + 1, kMaxTest, 5381);
}
- EXPECT_EQ(h1, h2);
+ ASSERT_EQ(h1, h2);
free_aligned_buffer_page_end(src_a);
}
@@ -164,19 +163,19 @@ TEST_F(LibYUVCompareTest, BenchmarkARGBDetect_Opt) {
src_a[0] = 0;
fourcc = ARGBDetect(src_a, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
- EXPECT_EQ(static_cast<uint32_t>(libyuv::FOURCC_BGRA), fourcc);
+ ASSERT_EQ(static_cast<uint32_t>(libyuv::FOURCC_BGRA), fourcc);
src_a[0] = 255;
src_a[3] = 0;
fourcc = ARGBDetect(src_a, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
- EXPECT_EQ(static_cast<uint32_t>(libyuv::FOURCC_ARGB), fourcc);
+ ASSERT_EQ(static_cast<uint32_t>(libyuv::FOURCC_ARGB), fourcc);
src_a[3] = 255;
for (int i = 0; i < benchmark_iterations_; ++i) {
fourcc = ARGBDetect(src_a, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
}
- EXPECT_EQ(0u, fourcc);
+ ASSERT_EQ(0u, fourcc);
free_aligned_buffer_page_end(src_a);
}
@@ -192,19 +191,74 @@ TEST_F(LibYUVCompareTest, BenchmarkARGBDetect_Unaligned) {
src_a[0 + 1] = 0;
fourcc = ARGBDetect(src_a + 1, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
- EXPECT_EQ(static_cast<uint32_t>(libyuv::FOURCC_BGRA), fourcc);
+ ASSERT_EQ(static_cast<uint32_t>(libyuv::FOURCC_BGRA), fourcc);
src_a[0 + 1] = 255;
src_a[3 + 1] = 0;
fourcc = ARGBDetect(src_a + 1, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
- EXPECT_EQ(static_cast<uint32_t>(libyuv::FOURCC_ARGB), fourcc);
+ ASSERT_EQ(static_cast<uint32_t>(libyuv::FOURCC_ARGB), fourcc);
src_a[3 + 1] = 255;
for (int i = 0; i < benchmark_iterations_; ++i) {
fourcc = ARGBDetect(src_a + 1, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
}
- EXPECT_EQ(0u, fourcc);
+ ASSERT_EQ(0u, fourcc);
+
+ free_aligned_buffer_page_end(src_a);
+}
+
+TEST_F(LibYUVCompareTest, TestARGBDetectAnyWidth) {
+ const int kMaxWidth = 65;
+ const int kBufSize = kMaxWidth * 4 + 4;
+ align_buffer_page_end(src_a, kBufSize);
+ memset(src_a, 255, kBufSize);
+
+ for (int width = 1; width <= kMaxWidth; ++width) {
+ for (int offset = 0; offset < 4; ++offset) {
+ uint8_t* row = src_a + kBufSize - width * 4 - offset;
+ // All 255 -> 0 (unknown).
+ MaskCpuFlags(disable_cpu_flags_);
+ uint32_t c_fourcc = ARGBDetect(row, width * 4, width, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ uint32_t opt_fourcc = ARGBDetect(row, width * 4, width, 1);
+ EXPECT_EQ(0u, c_fourcc);
+ EXPECT_EQ(c_fourcc, opt_fourcc) << "width " << width << " off " << offset;
+
+ for (int x = 0; x < width; ++x) {
+ // Byte 0 != 255 -> FOURCC_BGRA.
+ row[x * 4 + 0] = 128;
+ MaskCpuFlags(disable_cpu_flags_);
+ c_fourcc = ARGBDetect(row, width * 4, width, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ opt_fourcc = ARGBDetect(row, width * 4, width, 1);
+ EXPECT_EQ(static_cast<uint32_t>(FOURCC_BGRA), c_fourcc);
+ EXPECT_EQ(c_fourcc, opt_fourcc)
+ << "width " << width << " x " << x << " byte 0";
+
+ // Both byte 0 and byte 3 != 255 -> FOURCC_BGRA (byte 0 checked first).
+ row[x * 4 + 3] = 64;
+ MaskCpuFlags(disable_cpu_flags_);
+ c_fourcc = ARGBDetect(row, width * 4, width, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ opt_fourcc = ARGBDetect(row, width * 4, width, 1);
+ EXPECT_EQ(static_cast<uint32_t>(FOURCC_BGRA), c_fourcc);
+ EXPECT_EQ(c_fourcc, opt_fourcc)
+ << "width " << width << " x " << x << " byte 0+3";
+
+ // Byte 3 != 255 only -> FOURCC_ARGB.
+ row[x * 4 + 0] = 255;
+ MaskCpuFlags(disable_cpu_flags_);
+ c_fourcc = ARGBDetect(row, width * 4, width, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ opt_fourcc = ARGBDetect(row, width * 4, width, 1);
+ EXPECT_EQ(static_cast<uint32_t>(FOURCC_ARGB), c_fourcc);
+ EXPECT_EQ(c_fourcc, opt_fourcc)
+ << "width " << width << " x " << x << " byte 3";
+ row[x * 4 + 3] = 255;
+ }
+ }
+ }
free_aligned_buffer_page_end(src_a);
}
@@ -221,7 +275,7 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) {
memcpy(src_a, "test0123test4567", 16);
memcpy(src_b, "tick0123tock4567", 16);
uint32_t h1 = HammingDistance_C(src_a, src_b, 16);
- EXPECT_EQ(16u, h1);
+ ASSERT_EQ(16u, h1);
// Test C vs OPT on random buffer
MemRandomize(src_a, kMaxWidth);
@@ -233,37 +287,71 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) {
benchmark_iterations_ *
((benchmark_width_ * benchmark_height_ + kMaxWidth - 1) / kMaxWidth);
for (int i = 0; i < count; ++i) {
-#if defined(HAS_HAMMINGDISTANCE_NEON)
+#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ h1 = HammingDistance_NEON_DotProd(src_a, src_b, kMaxWidth);
+ } else {
+ h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
+ }
+#elif defined(HAS_HAMMINGDISTANCE_NEON)
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ h1 = HammingDistance_AVX512VNNI(src_a, src_b, kMaxWidth);
+ } else if (TestCpuFlag(kCpuHasAVX512BW)) {
+ h1 = HammingDistance_AVX512BW(src_a, src_b, kMaxWidth);
+ } else if (TestCpuFlag(kCpuHasAVX2)) {
+ h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+ } else if (TestCpuFlag(kCpuHasSSE42)) {
+ h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
+#endif
+#if defined(HAS_HAMMINGDISTANCE_SSSE3)
+ } else if (TestCpuFlag(kCpuHasSSSE3)) {
+ h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
+#endif
+ } else {
+ h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
+ }
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- if (has_avx2) {
+ if (TestCpuFlag(kCpuHasAVX2)) {
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+ } else if (TestCpuFlag(kCpuHasSSE42)) {
+ h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
+#endif
+#if defined(HAS_HAMMINGDISTANCE_SSSE3)
+ } else if (TestCpuFlag(kCpuHasSSSE3)) {
+ h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
+#endif
} else {
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- if (has_sse42) {
- h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
- } else {
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- if (has_ssse3) {
- h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
- } else {
- h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
- }
- }
+ h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
}
#elif defined(HAS_HAMMINGDISTANCE_SSE42)
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- if (has_sse42) {
+ if (TestCpuFlag(kCpuHasSSE42)) {
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
} else {
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
}
+#elif defined(HAS_HAMMINGDISTANCE_RVV)
+ h1 = HammingDistance_RVV(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth);
+ } else {
+ h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
+ }
+#elif defined(HAS_HAMMINGDISTANCE_LSX)
+ h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_WASMSIMD)
+ h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_HVX)
+ h1 = HammingDistance_HVX(src_a, src_b, kMaxWidth);
#else
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
#endif
}
- EXPECT_EQ(h0, h1);
+ ASSERT_EQ(h0, h1);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -280,7 +368,7 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_C) {
memcpy(src_a, "test0123test4567", 16);
memcpy(src_b, "tick0123tock4567", 16);
uint32_t h1 = HammingDistance_C(src_a, src_b, 16);
- EXPECT_EQ(16u, h1);
+ ASSERT_EQ(16u, h1);
// Test C vs OPT on random buffer
MemRandomize(src_a, kMaxWidth);
@@ -295,7 +383,7 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_C) {
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
}
- EXPECT_EQ(h0, h1);
+ ASSERT_EQ(h0, h1);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -311,7 +399,7 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance) {
memcpy(src_a, "test0123test4567", 16);
memcpy(src_b, "tick0123tock4567", 16);
uint64_t h1 = ComputeHammingDistance(src_a, src_b, 16);
- EXPECT_EQ(16u, h1);
+ ASSERT_EQ(16u, h1);
// Test C vs OPT on random buffer
MemRandomize(src_a, kMaxWidth);
@@ -326,7 +414,7 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance) {
h1 = ComputeHammingDistance(src_a, src_b, kMaxWidth);
}
- EXPECT_EQ(h0, h1);
+ ASSERT_EQ(h0, h1);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -344,42 +432,76 @@ static const int kMaxOptCount = (1 << (32 - 3)) - 64; // 536870848
TEST_F(LibYUVCompareTest, TestHammingDistance_Opt) {
uint32_t h1 = 0;
- const int kMaxWidth = (benchmark_width_ * benchmark_height_ + 31) & ~31;
+ const int kMaxWidth = (benchmark_width_ * benchmark_height_ + 63) & ~63;
align_buffer_page_end(src_a, kMaxWidth);
align_buffer_page_end(src_b, kMaxWidth);
memset(src_a, 255u, kMaxWidth);
memset(src_b, 0u, kMaxWidth);
uint64_t h0 = ComputeHammingDistance(src_a, src_b, kMaxWidth);
- EXPECT_EQ(kMaxWidth * 8ULL, h0);
+ ASSERT_EQ(kMaxWidth * 8ULL, h0);
for (int i = 0; i < benchmark_iterations_; ++i) {
-#if defined(HAS_HAMMINGDISTANCE_NEON)
+#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ h1 = HammingDistance_NEON_DotProd(src_a, src_b, kMaxWidth);
+ } else {
+ h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
+ }
+#elif defined(HAS_HAMMINGDISTANCE_NEON)
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
+ if (TestCpuFlag(kCpuHasAVX512VNNI)) {
+ h1 = HammingDistance_AVX512VNNI(src_a, src_b, kMaxWidth);
+ } else if (TestCpuFlag(kCpuHasAVX512BW)) {
+ h1 = HammingDistance_AVX512BW(src_a, src_b, kMaxWidth);
+ } else if (TestCpuFlag(kCpuHasAVX2)) {
+ h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+ } else if (TestCpuFlag(kCpuHasSSE42)) {
+ h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
+#endif
+#if defined(HAS_HAMMINGDISTANCE_SSSE3)
+ } else if (TestCpuFlag(kCpuHasSSSE3)) {
+ h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
+#endif
+ } else {
+ h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
+ }
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- if (has_avx2) {
+ if (TestCpuFlag(kCpuHasAVX2)) {
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
+#if defined(HAS_HAMMINGDISTANCE_SSE42)
+ } else if (TestCpuFlag(kCpuHasSSE42)) {
+ h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
+#endif
+#if defined(HAS_HAMMINGDISTANCE_SSSE3)
+ } else if (TestCpuFlag(kCpuHasSSSE3)) {
+ h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
+#endif
} else {
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- if (has_sse42) {
- h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
- } else {
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- if (has_ssse3) {
- h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
- } else {
- h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
- }
- }
+ h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
}
#elif defined(HAS_HAMMINGDISTANCE_SSE42)
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- if (has_sse42) {
+ if (TestCpuFlag(kCpuHasSSE42)) {
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
} else {
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
}
+#elif defined(HAS_HAMMINGDISTANCE_RVV)
+ h1 = HammingDistance_RVV(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth);
+ } else {
+ h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
+ }
+#elif defined(HAS_HAMMINGDISTANCE_LSX)
+ h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_WASMSIMD)
+ h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth);
+#elif defined(HAS_HAMMINGDISTANCE_HVX)
+ h1 = HammingDistance_HVX(src_a, src_b, kMaxWidth);
#else
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
#endif
@@ -389,7 +511,7 @@ TEST_F(LibYUVCompareTest, TestHammingDistance_Opt) {
// result can not be expected to be correct.
// TODO(fbarchard): Consider expecting the low 16 bits to match.
if (kMaxWidth <= kMaxOptCount) {
- EXPECT_EQ(kMaxWidth * 8U, h1);
+ ASSERT_EQ(kMaxWidth * 8U, h1);
} else {
if (kMaxWidth * 8ULL != static_cast<uint64_t>(h1)) {
printf(
@@ -420,7 +542,7 @@ TEST_F(LibYUVCompareTest, TestHammingDistance) {
h1 = ComputeHammingDistance(src_a, src_b,
benchmark_width_ * benchmark_height_);
}
- EXPECT_EQ(benchmark_width_ * benchmark_height_ * 8ULL, h1);
+ ASSERT_EQ(benchmark_width_ * benchmark_height_ * 8ULL, h1);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -436,7 +558,7 @@ TEST_F(LibYUVCompareTest, BenchmarkSumSquareError_Opt) {
memcpy(src_a, "test0123test4567", 16);
memcpy(src_b, "tick0123tock4567", 16);
uint64_t h1 = ComputeSumSquareError(src_a, src_b, 16);
- EXPECT_EQ(790u, h1);
+ ASSERT_EQ(790u, h1);
for (int i = 0; i < kMaxWidth; ++i) {
src_a[i] = i;
@@ -452,7 +574,7 @@ TEST_F(LibYUVCompareTest, BenchmarkSumSquareError_Opt) {
h1 = ComputeSumSquareError(src_a, src_b, kMaxWidth);
}
- EXPECT_EQ(0u, h1);
+ ASSERT_EQ(0u, h1);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -468,18 +590,18 @@ TEST_F(LibYUVCompareTest, SumSquareError) {
uint64_t err;
err = ComputeSumSquareError(src_a, src_b, kMaxWidth);
- EXPECT_EQ(0u, err);
+ ASSERT_EQ(0u, err);
memset(src_a, 1, kMaxWidth);
err = ComputeSumSquareError(src_a, src_b, kMaxWidth);
- EXPECT_EQ(static_cast<int>(err), kMaxWidth);
+ ASSERT_EQ(static_cast<int>(err), kMaxWidth);
memset(src_a, 190, kMaxWidth);
memset(src_b, 193, kMaxWidth);
err = ComputeSumSquareError(src_a, src_b, kMaxWidth);
- EXPECT_EQ(static_cast<int>(err), kMaxWidth * 3 * 3);
+ ASSERT_EQ(static_cast<int>(err), kMaxWidth * 3 * 3);
for (int i = 0; i < kMaxWidth; ++i) {
src_a[i] = (fastrand() & 0xff);
@@ -492,7 +614,51 @@ TEST_F(LibYUVCompareTest, SumSquareError) {
MaskCpuFlags(benchmark_cpu_info_);
uint64_t opt_err = ComputeSumSquareError(src_a, src_b, kMaxWidth);
- EXPECT_EQ(c_err, opt_err);
+ ASSERT_EQ(c_err, opt_err);
+
+ free_aligned_buffer_page_end(src_a);
+ free_aligned_buffer_page_end(src_b);
+}
+
+// Test every count, aligned and unaligned, ending at the end of a page.
+TEST_F(LibYUVCompareTest, TestCompareAnyCount) {
+ const int kMaxCount = 64 * 5 + 1;
+ align_buffer_page_end(src_a, kMaxCount + 1);
+ align_buffer_page_end(src_b, kMaxCount + 1);
+ MemRandomize(src_a, kMaxCount + 1);
+ MemRandomize(src_b, kMaxCount + 1);
+
+ const int kCpuMasks[] = {
+ benchmark_cpu_info_,
+ benchmark_cpu_info_ & ~kCpuHasAVX512VNNI,
+ benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VNNI),
+ benchmark_cpu_info_ &
+ ~(kCpuHasAVX2 | kCpuHasAVX512BW | kCpuHasAVX512VNNI),
+ };
+
+ for (int count = 0; count <= kMaxCount; ++count) {
+ for (int offset = 0; offset < 2; ++offset) {
+ const uint8_t* a = src_a + kMaxCount + 1 - count - offset;
+ const uint8_t* b = src_b + kMaxCount + 1 - count - offset;
+ MaskCpuFlags(disable_cpu_flags_);
+ uint32_t c_hash = HashDjb2(a, count, 5381);
+ uint64_t c_diff = ComputeHammingDistance(a, b, count);
+ uint64_t c_err = ComputeSumSquareError(a, b, count);
+ for (size_t m = 0; m < sizeof(kCpuMasks) / sizeof(kCpuMasks[0]); ++m) {
+ MaskCpuFlags(kCpuMasks[m]);
+ uint32_t opt_hash = HashDjb2(a, count, 5381);
+ uint64_t opt_diff = ComputeHammingDistance(a, b, count);
+ uint64_t opt_err = ComputeSumSquareError(a, b, count);
+ EXPECT_EQ(c_hash, opt_hash)
+ << "count " << count << " offset " << offset << " mask " << m;
+ EXPECT_EQ(c_diff, opt_diff)
+ << "count " << count << " offset " << offset << " mask " << m;
+ EXPECT_EQ(c_err, opt_err)
+ << "count " << count << " offset " << offset << " mask " << m;
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+ }
+ }
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -517,7 +683,7 @@ TEST_F(LibYUVCompareTest, BenchmarkPsnr_Opt) {
opt_time = (get_time() - opt_time) / benchmark_iterations_;
printf("BenchmarkPsnr_Opt - %8.2f us opt\n", opt_time * 1e6);
- EXPECT_EQ(0, 0);
+ ASSERT_EQ(0, 0);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -542,7 +708,7 @@ TEST_F(LibYUVCompareTest, BenchmarkPsnr_Unaligned) {
opt_time = (get_time() - opt_time) / benchmark_iterations_;
printf("BenchmarkPsnr_Opt - %8.2f us opt\n", opt_time * 1e6);
- EXPECT_EQ(0, 0);
+ ASSERT_EQ(0, 0);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -564,7 +730,7 @@ TEST_F(LibYUVCompareTest, Psnr) {
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
kSrcHeight);
- EXPECT_EQ(err, kMaxPsnr);
+ ASSERT_EQ(err, kMaxPsnr);
memset(src_a, 255, kSrcPlaneSize);
@@ -572,7 +738,7 @@ TEST_F(LibYUVCompareTest, Psnr) {
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
kSrcHeight);
- EXPECT_EQ(err, 0.0);
+ ASSERT_EQ(err, 0.0);
memset(src_a, 1, kSrcPlaneSize);
@@ -580,8 +746,8 @@ TEST_F(LibYUVCompareTest, Psnr) {
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
kSrcHeight);
- EXPECT_GT(err, 48.0);
- EXPECT_LT(err, 49.0);
+ ASSERT_GT(err, 48.0);
+ ASSERT_LT(err, 49.0);
for (int i = 0; i < kSrcPlaneSize; ++i) {
src_a[i] = i;
@@ -591,9 +757,9 @@ TEST_F(LibYUVCompareTest, Psnr) {
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
kSrcHeight);
- EXPECT_GT(err, 2.0);
+ ASSERT_GT(err, 2.0);
if (kSrcWidth * kSrcHeight >= 256) {
- EXPECT_LT(err, 6.0);
+ ASSERT_LT(err, 6.0);
}
memset(src_a, 0, kSrcPlaneSize);
@@ -619,13 +785,13 @@ TEST_F(LibYUVCompareTest, Psnr) {
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
kSrcHeight);
- EXPECT_EQ(opt_err, c_err);
+ ASSERT_EQ(opt_err, c_err);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
}
-TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) {
+TEST_F(LibYUVCompareTest, BenchmarkSsim_Opt) {
align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_);
align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_);
for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
@@ -635,16 +801,82 @@ TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) {
MaskCpuFlags(benchmark_cpu_info_);
- double opt_time = get_time();
+ double ssim = 0.0;
for (int i = 0; i < benchmark_iterations_; ++i) {
- CalcFrameSsim(src_a, benchmark_width_, src_b, benchmark_width_,
- benchmark_width_, benchmark_height_);
+ ssim = CalcFrameSsim(src_a, benchmark_width_, src_b, benchmark_width_,
+ benchmark_width_, benchmark_height_);
}
- opt_time = (get_time() - opt_time) / benchmark_iterations_;
- printf("BenchmarkSsim_Opt - %8.2f us opt\n", opt_time * 1e6);
+ if (benchmark_width_ > 8 && benchmark_height_ > 8) {
+ EXPECT_EQ(1.0, ssim);
+ } else {
+ EXPECT_EQ(0.0, ssim);
+ }
+
+ free_aligned_buffer_page_end(src_a);
+ free_aligned_buffer_page_end(src_b);
+}
+
+TEST_F(LibYUVCompareTest, BenchmarkSsim_Unaligned) {
+ align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_ + 1);
+ align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_);
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ src_a[i + 1] = i;
+ src_b[i] = i;
+ }
+
+ MaskCpuFlags(benchmark_cpu_info_);
- EXPECT_EQ(0, 0); // Pass if we get this far.
+ double ssim = 0.0;
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ ssim = CalcFrameSsim(src_a + 1, benchmark_width_, src_b, benchmark_width_,
+ benchmark_width_, benchmark_height_);
+ }
+
+ if (benchmark_width_ > 8 && benchmark_height_ > 8) {
+ EXPECT_EQ(1.0, ssim);
+ } else {
+ EXPECT_EQ(0.0, ssim);
+ }
+
+ free_aligned_buffer_page_end(src_a);
+ free_aligned_buffer_page_end(src_b);
+}
+
+TEST_F(LibYUVCompareTest, TestSsimAnyWidth) {
+ const int kMaxWidth = 140;
+ const int kHeight = 16;
+ const int kPlaneSize = kMaxWidth * kHeight + 1;
+ align_buffer_page_end(src_a, kPlaneSize);
+ align_buffer_page_end(src_b, kPlaneSize);
+ for (int i = 0; i < kPlaneSize; ++i) {
+ src_a[i] = (i * 73 + 19) & 0xff;
+ src_b[i] = (i * 151 + 43) & 0xff;
+ }
+
+ const int kCpuMasks[] = {
+ benchmark_cpu_info_,
+ benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VNNI),
+ benchmark_cpu_info_ &
+ ~(kCpuHasAVX2 | kCpuHasAVX512BW | kCpuHasAVX512VNNI),
+ };
+
+ for (int width = 9; width <= kMaxWidth; ++width) {
+ for (int offset = 0; offset < 2; ++offset) {
+ const int size = width * kHeight;
+ const uint8_t* a = src_a + kPlaneSize - size - offset;
+ const uint8_t* b = src_b + kPlaneSize - size - offset;
+ MaskCpuFlags(disable_cpu_flags_);
+ double c_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
+ for (size_t m = 0; m < sizeof(kCpuMasks) / sizeof(kCpuMasks[0]); ++m) {
+ MaskCpuFlags(kCpuMasks[m]);
+ double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
+ EXPECT_NEAR(c_ssim, opt_ssim, 1e-5)
+ << "width " << width << " offset " << offset << " mask " << m;
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+ }
+ }
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
@@ -665,6 +897,7 @@ TEST_F(LibYUVCompareTest, Ssim) {
printf("warning - Ssim size too small. Testing function executes.\n");
}
+ // Identical images: SSIM == 1.0.
double err;
err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
@@ -674,6 +907,7 @@ TEST_F(LibYUVCompareTest, Ssim) {
EXPECT_EQ(err, 1.0);
}
+ // Completely different constant images (255 vs 0): SSIM near 0.
memset(src_a, 255, kSrcPlaneSize);
err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
@@ -684,6 +918,7 @@ TEST_F(LibYUVCompareTest, Ssim) {
EXPECT_LT(err, 0.0001);
}
+ // Slightly different constant images (1 vs 0): SSIM ~ 0.84.
memset(src_a, 1, kSrcPlaneSize);
err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
@@ -691,10 +926,11 @@ TEST_F(LibYUVCompareTest, Ssim) {
kSrcHeight);
if (kSrcWidth > 8 && kSrcHeight > 8) {
- EXPECT_GT(err, 0.0001);
+ EXPECT_GT(err, 0.8);
EXPECT_LT(err, 0.9);
}
+ // Gradient vs constant 0: SSIM near 0.
for (int i = 0; i < kSrcPlaneSize; ++i) {
src_a[i] = i;
}
@@ -708,28 +944,30 @@ TEST_F(LibYUVCompareTest, Ssim) {
EXPECT_LT(err, 0.01);
}
+ // Similar images (deterministic gradient + small perturbation): SSIM ~ 0.99.
for (int i = b; i < (kSrcHeight + b); ++i) {
for (int j = b; j < (kSrcWidth + b); ++j) {
- src_a[(i * kSrcStride) + j] = (fastrand() & 0xff);
- src_b[(i * kSrcStride) + j] = (fastrand() & 0xff);
+ uint8_t val = static_cast<uint8_t>((i * 3 + j * 5) & 0xff);
+ src_a[(i * kSrcStride) + j] = val;
+ src_b[(i * kSrcStride) + j] =
+ static_cast<uint8_t>((val + ((i ^ j) & 3)) & 0xff);
}
}
MaskCpuFlags(disable_cpu_flags_);
- double c_err, opt_err;
-
- c_err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
- src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
- kSrcHeight);
+ double c_err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
+ src_b + kSrcStride * b + b, kSrcStride,
+ kSrcWidth, kSrcHeight);
MaskCpuFlags(benchmark_cpu_info_);
-
- opt_err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
- src_b + kSrcStride * b + b, kSrcStride, kSrcWidth,
- kSrcHeight);
+ double opt_err = CalcFrameSsim(src_a + kSrcStride * b + b, kSrcStride,
+ src_b + kSrcStride * b + b, kSrcStride,
+ kSrcWidth, kSrcHeight);
if (kSrcWidth > 8 && kSrcHeight > 8) {
- EXPECT_EQ(opt_err, c_err);
+ EXPECT_GT(opt_err, 0.9);
+ EXPECT_LT(opt_err, 1.0);
+ EXPECT_NEAR(c_err, opt_err, 1e-5);
}
free_aligned_buffer_page_end(src_a);
diff --git a/libs/libyuv/unit_test/convert_argb_test.cc b/libs/libyuv/unit_test/convert_argb_test.cc
new file mode 100644
index 0000000000..a2e1ad547b
--- /dev/null
+++ b/libs/libyuv/unit_test/convert_argb_test.cc
@@ -0,0 +1,3165 @@
+/*
+ * Copyright 2023 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "../unit_test/unit_test.h"
+#include "libyuv/compare.h"
+#include "libyuv/convert.h"
+#include "libyuv/convert_argb.h"
+#include "libyuv/convert_from.h"
+#include "libyuv/convert_from_argb.h"
+#include "libyuv/cpu_id.h"
+#include "libyuv/planar_functions.h"
+#include "libyuv/rotate.h"
+#include "libyuv/video_common.h"
+
+#ifdef ENABLE_ROW_TESTS
+#include "libyuv/row.h" /* For ARGBToAR30Row_AVX2 */
+#endif
+
+#if (defined(__riscv) && !defined(__clang__)) || defined(__hexagon__)
+#define DISABLE_SLOW_TESTS
+#undef ENABLE_FULL_TESTS
+#undef ENABLE_ROW_TESTS
+#define LEAN_TESTS
+#endif
+
+// Some functions fail on big endian. Enable these tests on all cpus except
+// PowerPC, but they are not optimized so disabled by default.
+#if !defined(DISABLE_SLOW_TESTS) && !defined(__powerpc__)
+#define LITTLE_ENDIAN_ONLY_TEST 1
+#endif
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
+namespace libyuv {
+
+// Alias to copy pixels as is
+#define AR30ToAR30 ARGBCopy
+#define ABGRToABGR ARGBCopy
+
+// subsample amount uses a divide.
+#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))
+
+#define ALIGNINT(V, ALIGN) (((V) + (ALIGN) - 1) / (ALIGN) * (ALIGN))
+
+#define TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, W1280, N, NEG, OFF, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
+ static_assert(SRC_BPC == 1 || SRC_BPC == 2, "SRC BPC unsupported"); \
+ static_assert(DST_BPC == 1 || DST_BPC == 2, "DST BPC unsupported"); \
+ static_assert(SRC_SUBSAMP_X == 1 || SRC_SUBSAMP_X == 2, \
+ "SRC_SUBSAMP_X unsupported"); \
+ static_assert(SRC_SUBSAMP_Y == 1 || SRC_SUBSAMP_Y == 2, \
+ "SRC_SUBSAMP_Y unsupported"); \
+ static_assert(DST_SUBSAMP_X == 1 || DST_SUBSAMP_X == 2, \
+ "DST_SUBSAMP_X unsupported"); \
+ static_assert(DST_SUBSAMP_Y == 1 || DST_SUBSAMP_Y == 2, \
+ "DST_SUBSAMP_Y unsupported"); \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kSrcHalfWidth = SUBSAMPLE(kWidth, SRC_SUBSAMP_X); \
+ const int kDstHalfWidth = SUBSAMPLE(kWidth, DST_SUBSAMP_X); \
+ const int kDstHalfHeight = SUBSAMPLE(kHeight, DST_SUBSAMP_Y); \
+ const int kPaddedWidth = (kWidth + (TILE_WIDTH - 1)) & ~(TILE_WIDTH - 1); \
+ const int kPaddedHeight = \
+ (kHeight + (TILE_HEIGHT - 1)) & ~(TILE_HEIGHT - 1); \
+ const int kSrcHalfPaddedWidth = SUBSAMPLE(kPaddedWidth, SRC_SUBSAMP_X); \
+ const int kSrcHalfPaddedHeight = SUBSAMPLE(kPaddedHeight, SRC_SUBSAMP_Y); \
+ align_buffer_page_end(src_y, \
+ kPaddedWidth * kPaddedHeight * SRC_BPC + OFF); \
+ align_buffer_page_end( \
+ src_uv, \
+ kSrcHalfPaddedWidth * kSrcHalfPaddedHeight * SRC_BPC * 2 + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_u_c, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_v_c, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_u_opt, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_v_opt, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ SRC_T* src_y_p = reinterpret_cast<SRC_T*>(src_y + OFF); \
+ SRC_T* src_uv_p = reinterpret_cast<SRC_T*>(src_uv + OFF); \
+ for (int i = 0; i < kPaddedWidth * kPaddedHeight; ++i) { \
+ src_y_p[i] = \
+ (fastrand() & (((SRC_T)(-1)) << ((8 * SRC_BPC) - SRC_DEPTH))); \
+ } \
+ for (int i = 0; i < kSrcHalfPaddedWidth * kSrcHalfPaddedHeight * 2; ++i) { \
+ src_uv_p[i] = \
+ (fastrand() & (((SRC_T)(-1)) << ((8 * SRC_BPC) - SRC_DEPTH))); \
+ } \
+ memset(dst_y_c, 1, kWidth * kHeight * DST_BPC); \
+ memset(dst_u_c, 2, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_v_c, 3, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_y_opt, 101, kWidth * kHeight * DST_BPC); \
+ memset(dst_u_opt, 102, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_v_opt, 103, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ SRC_FMT_PLANAR##To##FMT_PLANAR( \
+ src_y_p, kWidth, src_uv_p, kSrcHalfWidth * 2, \
+ reinterpret_cast<DST_T*>(dst_y_c), kWidth, \
+ reinterpret_cast<DST_T*>(dst_u_c), kDstHalfWidth, \
+ reinterpret_cast<DST_T*>(dst_v_c), kDstHalfWidth, kWidth, \
+ NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ SRC_FMT_PLANAR##To##FMT_PLANAR( \
+ src_y_p, kWidth, src_uv_p, kSrcHalfWidth * 2, \
+ reinterpret_cast<DST_T*>(dst_y_opt), kWidth, \
+ reinterpret_cast<DST_T*>(dst_u_opt), kDstHalfWidth, \
+ reinterpret_cast<DST_T*>(dst_v_opt), kDstHalfWidth, kWidth, \
+ NEG kHeight); \
+ } \
+ for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
+ ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
+ } \
+ for (int i = 0; i < kDstHalfWidth * kDstHalfHeight * DST_BPC; ++i) { \
+ ASSERT_EQ(dst_u_c[i], dst_u_opt[i]); \
+ ASSERT_EQ(dst_v_c[i], dst_v_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(dst_y_c); \
+ free_aligned_buffer_page_end(dst_u_c); \
+ free_aligned_buffer_page_end(dst_v_c); \
+ free_aligned_buffer_page_end(dst_y_opt); \
+ free_aligned_buffer_page_end(dst_u_opt); \
+ free_aligned_buffer_page_end(dst_v_opt); \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_uv); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTBPTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT) \
+ TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT)
+#else
+#define TESTBPTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT) \
+ TESTBPTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT)
+#endif
+
+TESTBPTOP(NV12, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8, 1, 1)
+TESTBPTOP(NV21, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8, 1, 1)
+TESTBPTOP(MM21, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8, 16, 32)
+TESTBPTOP(P010, uint16_t, 2, 2, 2, I010, uint16_t, 2, 2, 2, 10, 1, 1)
+TESTBPTOP(P012, uint16_t, 2, 2, 2, I012, uint16_t, 2, 2, 2, 12, 1, 1)
+TESTBPTOP(P210, uint16_t, 2, 2, 1, I210, uint16_t, 2, 2, 1, 10, 1, 1)
+TESTBPTOP(P410, uint16_t, 2, 1, 1, I410, uint16_t, 2, 1, 1, 10, 1, 1)
+TESTBPTOP(P010, uint16_t, 2, 2, 2, I420, uint8_t, 1, 2, 2, 10, 1, 1)
+TESTBPTOP(P210, uint16_t, 2, 2, 1, I420, uint8_t, 1, 2, 2, 10, 1, 1)
+TESTBPTOP(P410, uint16_t, 2, 1, 1, I420, uint8_t, 1, 2, 2, 10, 1, 1)
+
+// Provide matrix wrappers for full range bt.709
+#define F420ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I420ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuF709Constants, i, j)
+#define F420ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I420ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvF709Constants, i, j)
+#define F422ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuF709Constants, i, j)
+#define F422ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvF709Constants, i, j)
+#define F444ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I444ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuF709Constants, i, j)
+#define F444ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I444ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvF709Constants, i, j)
+
+// Provide matrix wrappers for full range bt.2020
+#define V420ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I420ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuV2020Constants, i, j)
+#define V420ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I420ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvV2020Constants, i, j)
+#define V422ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuV2020Constants, i, j)
+#define V422ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvV2020Constants, i, j)
+#define V444ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I444ToARGBMatrix(a, b, e, f, c, d, g, h, &kYvuV2020Constants, i, j)
+#define V444ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I444ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvV2020Constants, i, j)
+
+#define I420ToARGBFilter(a, b, c, d, e, f, g, h, i, j) \
+ I420ToARGBMatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I422ToARGBFilter(a, b, c, d, e, f, g, h, i, j) \
+ I422ToARGBMatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I420ToRGB24Filter(a, b, c, d, e, f, g, h, i, j) \
+ I420ToRGB24MatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I422ToRGB24Filter(a, b, c, d, e, f, g, h, i, j) \
+ I420ToRGB24MatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+
+#define TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_u, kSizeUV + OFF); \
+ align_buffer_page_end(src_v, kSizeUV + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ src_u[i + OFF] = (fastrand() & 0xff); \
+ src_v[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ double time0 = get_time(); \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, dst_argb_c + OFF, kStrideB, \
+ kWidth, NEG kHeight); \
+ double time1 = get_time(); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, dst_argb_opt + OFF, \
+ kStrideB, kWidth, NEG kHeight); \
+ } \
+ double time2 = get_time(); \
+ printf(" %8d us C - %8d us OPT\n", \
+ static_cast<int>((time1 - time0) * 1e6), \
+ static_cast<int>((time2 - time1) * 1e6 / benchmark_iterations_)); \
+ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN) \
+ TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_ + 1, _Any, +, 0) \
+ TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Unaligned, +, 4) \
+ TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Invert, -, 0) \
+ TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0)
+#else
+#define TESTPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN) \
+ TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0)
+#endif
+
+#if defined(ENABLE_FULL_TESTS)
+TESTPLANARTOB(I420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(J420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(J420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(F420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(F420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(H420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(H420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(U420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(U420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(V420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(V420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, BGRA, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, RGBA, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, RAW, 3, 3, 1)
+TESTPLANARTOB(I420, 2, 2, RGB24, 3, 3, 1)
+TESTPLANARTOB(J420, 2, 2, RAW, 3, 3, 1)
+TESTPLANARTOB(J420, 2, 2, RGB24, 3, 3, 1)
+TESTPLANARTOB(H420, 2, 2, RAW, 3, 3, 1)
+TESTPLANARTOB(H420, 2, 2, RGB24, 3, 3, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANARTOB(I420, 2, 2, RGB565, 2, 2, 1)
+TESTPLANARTOB(J420, 2, 2, RGB565, 2, 2, 1)
+TESTPLANARTOB(H420, 2, 2, RGB565, 2, 2, 1)
+TESTPLANARTOB(I420, 2, 2, ARGB1555, 2, 2, 1)
+TESTPLANARTOB(I420, 2, 2, ARGB4444, 2, 2, 1)
+TESTPLANARTOB(I422, 2, 1, RGB565, 2, 2, 1)
+#endif
+TESTPLANARTOB(I422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(J422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(J422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(H422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(H422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(U422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(U422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(V422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(V422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, BGRA, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, RGBA, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, RGB24, 3, 3, 1)
+TESTPLANARTOB(I422, 2, 1, RAW, 3, 3, 1)
+TESTPLANARTOB(I444, 1, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(I444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(I444, 1, 1, RGB24, 3, 3, 1)
+TESTPLANARTOB(I444, 1, 1, RAW, 3, 3, 1)
+TESTPLANARTOB(J444, 1, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(J444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(H444, 1, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(H444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(U444, 1, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(U444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(V444, 1, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(V444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, YUY2, 2, 4, 1)
+TESTPLANARTOB(I420, 2, 2, UYVY, 2, 4, 1)
+TESTPLANARTOB(I422, 2, 1, YUY2, 2, 4, 1)
+TESTPLANARTOB(I422, 2, 1, UYVY, 2, 4, 1)
+TESTPLANARTOB(I420, 2, 2, I400, 1, 1, 1)
+TESTPLANARTOB(J420, 2, 2, J400, 1, 1, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANARTOB(I420, 2, 2, AR30, 4, 4, 1)
+TESTPLANARTOB(H420, 2, 2, AR30, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, AB30, 4, 4, 1)
+TESTPLANARTOB(H420, 2, 2, AB30, 4, 4, 1)
+#endif
+TESTPLANARTOB(I420, 2, 2, ARGBFilter, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, ARGBFilter, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, RGB24Filter, 3, 3, 1)
+TESTPLANARTOB(I422, 2, 2, RGB24Filter, 3, 3, 1)
+#else // FULL_TESTS
+TESTPLANARTOB(I420, 2, 2, ABGR, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, ARGB, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, BGRA, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, RAW, 3, 3, 1)
+TESTPLANARTOB(I420, 2, 2, RGB24, 3, 3, 1)
+TESTPLANARTOB(I420, 2, 2, RGBA, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANARTOB(I420, 2, 2, RGB565, 2, 2, 1)
+TESTPLANARTOB(I420, 2, 2, ARGB1555, 2, 2, 1)
+TESTPLANARTOB(I420, 2, 2, ARGB4444, 2, 2, 1)
+TESTPLANARTOB(I422, 2, 1, RGB565, 2, 2, 1)
+#endif
+TESTPLANARTOB(I420, 2, 2, I400, 1, 1, 1)
+TESTPLANARTOB(I420, 2, 2, UYVY, 2, 4, 1)
+TESTPLANARTOB(I420, 2, 2, YUY2, 2, 4, 1)
+TESTPLANARTOB(I422, 2, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, ARGB, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, BGRA, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, RGBA, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, UYVY, 2, 4, 1)
+TESTPLANARTOB(I422, 2, 1, YUY2, 2, 4, 1)
+TESTPLANARTOB(I420, 2, 2, ARGBFilter, 4, 4, 1)
+TESTPLANARTOB(I422, 2, 1, ARGBFilter, 4, 4, 1)
+TESTPLANARTOB(I420, 2, 2, RGB24Filter, 3, 3, 1)
+TESTPLANARTOB(I444, 1, 1, ABGR, 4, 4, 1)
+TESTPLANARTOB(I444, 1, 1, ARGB, 4, 4, 1)
+#endif
+
+#define TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kStrideB = kWidth * BPP_B; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end( \
+ src_uv, kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y) * 2 + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight); \
+ for (int i = 0; i < kHeight; ++i) \
+ for (int j = 0; j < kWidth; ++j) \
+ src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
+ for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
+ for (int j = 0; j < kStrideUV * 2; ++j) { \
+ src_uv[i * kStrideUV * 2 + j + OFF] = (fastrand() & 0xff); \
+ } \
+ } \
+ memset(dst_argb_c, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_uv + OFF, kStrideUV * 2, \
+ dst_argb_c, kWidth * BPP_B, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_uv + OFF, kStrideUV * 2, \
+ dst_argb_opt, kWidth * BPP_B, kWidth, \
+ NEG kHeight); \
+ } \
+ /* Convert to ARGB so 565 is expanded to bytes that can be compared. */ \
+ align_buffer_page_end(dst_argb32_c, kWidth * 4 * kHeight); \
+ align_buffer_page_end(dst_argb32_opt, kWidth * 4 * kHeight); \
+ memset(dst_argb32_c, 2, kWidth * 4 * kHeight); \
+ memset(dst_argb32_opt, 102, kWidth * 4 * kHeight); \
+ FMT_C##ToARGB(dst_argb_c, kStrideB, dst_argb32_c, kWidth * 4, kWidth, \
+ kHeight); \
+ FMT_C##ToARGB(dst_argb_opt, kStrideB, dst_argb32_opt, kWidth * 4, kWidth, \
+ kHeight); \
+ for (int i = 0; i < kHeight; ++i) { \
+ for (int j = 0; j < kWidth * 4; ++j) { \
+ ASSERT_EQ(dst_argb32_c[i * kWidth * 4 + j], \
+ dst_argb32_opt[i * kWidth * 4 + j]); \
+ } \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_uv); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ free_aligned_buffer_page_end(dst_argb32_c); \
+ free_aligned_buffer_page_end(dst_argb32_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTBPTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B) \
+ TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ benchmark_width_ + 1, _Any, +, 0) \
+ TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ benchmark_width_, _Unaligned, +, 2) \
+ TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ benchmark_width_, _Invert, -, 0) \
+ TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ benchmark_width_, _Opt, +, 0)
+#else
+#define TESTBPTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B) \
+ TESTBPTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
+ benchmark_width_, _Opt, +, 0)
+#endif
+
+#define JNV12ToARGB(a, b, c, d, e, f, g, h) \
+ NV12ToARGBMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV21ToARGB(a, b, c, d, e, f, g, h) \
+ NV21ToARGBMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV12ToABGR(a, b, c, d, e, f, g, h) \
+ NV12ToABGRMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV21ToABGR(a, b, c, d, e, f, g, h) \
+ NV21ToABGRMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV12ToRGB24(a, b, c, d, e, f, g, h) \
+ NV12ToRGB24Matrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV21ToRGB24(a, b, c, d, e, f, g, h) \
+ NV21ToRGB24Matrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV12ToRAW(a, b, c, d, e, f, g, h) \
+ NV12ToRAWMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV21ToRAW(a, b, c, d, e, f, g, h) \
+ NV21ToRAWMatrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+#define JNV12ToRGB565(a, b, c, d, e, f, g, h) \
+ NV12ToRGB565Matrix(a, b, c, d, e, f, &kYuvJPEGConstants, g, h)
+
+TESTBPTOB(JNV12, 2, 2, ARGB, ARGB, 4)
+TESTBPTOB(JNV21, 2, 2, ARGB, ARGB, 4)
+TESTBPTOB(JNV12, 2, 2, ABGR, ABGR, 4)
+TESTBPTOB(JNV21, 2, 2, ABGR, ABGR, 4)
+TESTBPTOB(JNV12, 2, 2, RGB24, RGB24, 3)
+TESTBPTOB(JNV21, 2, 2, RGB24, RGB24, 3)
+TESTBPTOB(JNV12, 2, 2, RAW, RAW, 3)
+TESTBPTOB(JNV21, 2, 2, RAW, RAW, 3)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTBPTOB(JNV12, 2, 2, RGB565, RGB565, 2)
+#endif
+
+TESTBPTOB(NV12, 2, 2, ARGB, ARGB, 4)
+TESTBPTOB(NV21, 2, 2, ARGB, ARGB, 4)
+TESTBPTOB(NV12, 2, 2, ABGR, ABGR, 4)
+TESTBPTOB(NV21, 2, 2, ABGR, ABGR, 4)
+TESTBPTOB(NV12, 2, 2, RGB24, RGB24, 3)
+TESTBPTOB(NV21, 2, 2, RGB24, RGB24, 3)
+TESTBPTOB(NV12, 2, 2, RAW, RAW, 3)
+TESTBPTOB(NV21, 2, 2, RAW, RAW, 3)
+TESTBPTOB(NV21, 2, 2, YUV24, RAW, 3)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTBPTOB(NV12, 2, 2, RGB565, RGB565, 2)
+#endif
+
+#define TESTBP16TOBTRANSITIVE(FMT_BP, FMT_PLANAR) \
+ TEST_F(LibYUVConvertTest, FMT_BP##ToARGB_Matches##FMT_PLANAR) { \
+ const int kWidth = benchmark_width_ | 1; \
+ const int kHeight = benchmark_height_; \
+ const int kPixels = kWidth * kHeight; \
+ align_buffer_page_end(src_y, kPixels * 2); \
+ align_buffer_page_end(src_uv, kPixels * 4); \
+ align_buffer_page_end(plane_y, kPixels * 2); \
+ align_buffer_page_end(plane_u, kPixels * 2); \
+ align_buffer_page_end(plane_v, kPixels * 2); \
+ align_buffer_page_end(dst_bp, kPixels * 4); \
+ align_buffer_page_end(dst_planar, kPixels * 4); \
+ uint16_t* src_y16 = reinterpret_cast<uint16_t*>(src_y); \
+ uint16_t* src_uv16 = reinterpret_cast<uint16_t*>(src_uv); \
+ uint16_t* plane_y16 = reinterpret_cast<uint16_t*>(plane_y); \
+ uint16_t* plane_u16 = reinterpret_cast<uint16_t*>(plane_u); \
+ uint16_t* plane_v16 = reinterpret_cast<uint16_t*>(plane_v); \
+ for (int i = 0; i < kPixels; ++i) { \
+ src_y16[i] = static_cast<uint16_t>((fastrand() & 0x3ff) << 6); \
+ } \
+ for (int i = 0; i < kPixels * 2; ++i) { \
+ src_uv16[i] = static_cast<uint16_t>((fastrand() & 0x3ff) << 6); \
+ } \
+ EXPECT_EQ(0, FMT_BP##To##FMT_PLANAR(src_y16, kWidth, src_uv16, kWidth * 2, \
+ plane_y16, kWidth, plane_u16, kWidth, \
+ plane_v16, kWidth, kWidth, kHeight)); \
+ EXPECT_EQ(0, FMT_BP##ToARGBMatrix(src_y16, kWidth, src_uv16, kWidth * 2, \
+ dst_bp, kWidth * 4, &kYuvI601Constants, \
+ kWidth, kHeight)); \
+ EXPECT_EQ( \
+ 0, FMT_PLANAR##ToARGBMatrix(plane_y16, kWidth, plane_u16, kWidth, \
+ plane_v16, kWidth, dst_planar, kWidth * 4, \
+ &kYuvI601Constants, kWidth, kHeight)); \
+ int max_diff = 0; \
+ for (int i = 0; i < kPixels * 4; ++i) { \
+ const int diff = Abs(dst_bp[i] - dst_planar[i]); \
+ if (diff > max_diff) { \
+ max_diff = diff; \
+ } \
+ } \
+ EXPECT_LE(max_diff, 1); \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_uv); \
+ free_aligned_buffer_page_end(plane_y); \
+ free_aligned_buffer_page_end(plane_u); \
+ free_aligned_buffer_page_end(plane_v); \
+ free_aligned_buffer_page_end(dst_bp); \
+ free_aligned_buffer_page_end(dst_planar); \
+ }
+
+TESTBP16TOBTRANSITIVE(P010, I010)
+TESTBP16TOBTRANSITIVE(P210, I210)
+TESTBP16TOBTRANSITIVE(P410, I410)
+
+#define TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
+ const int kStrideA = \
+ (kWidth * EPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideB = \
+ (kWidth * EPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
+ align_buffer_page_end(src_argb, \
+ (size_t)kStrideA * kHeightA * sizeof(TYPE_A) + OFF); \
+ align_buffer_page_end(dst_argb_c, \
+ (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ align_buffer_page_end(dst_argb_opt, \
+ (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA * sizeof(TYPE_A); \
+ ++i) { \
+ src_argb[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c, 1, (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ memset(dst_argb_opt, 101, (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B((TYPE_A*)(src_argb + OFF), kStrideA, (TYPE_B*)dst_argb_c, \
+ kStrideB, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_B((TYPE_A*)(src_argb + OFF), kStrideA, \
+ (TYPE_B*)dst_argb_opt, kStrideB, kWidth, NEG kHeight); \
+ } \
+ for (size_t i = 0; i < (size_t)kStrideB * kHeightB * sizeof(TYPE_B); \
+ ++i) { \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#define TESTATOBRANDOM(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, \
+ TYPE_B, EPP_B, STRIDE_B, HEIGHT_B) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##_Random) { \
+ for (int times = 0; times < benchmark_iterations_; ++times) { \
+ const int kWidth = (fastrand() & 63) + 1; \
+ const int kHeight = (fastrand() & 31) + 1; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
+ const int kStrideA = \
+ (kWidth * EPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideB = \
+ (kWidth * EPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
+ align_buffer_page_end(src_argb, \
+ (size_t)kStrideA * kHeightA * sizeof(TYPE_A)); \
+ align_buffer_page_end(dst_argb_c, \
+ (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ align_buffer_page_end(dst_argb_opt, \
+ (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA * sizeof(TYPE_A); \
+ ++i) { \
+ src_argb[i] = 0xfe; \
+ } \
+ memset(dst_argb_c, 123, (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ memset(dst_argb_opt, 123, (size_t)kStrideB * kHeightB * sizeof(TYPE_B)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B((TYPE_A*)src_argb, kStrideA, (TYPE_B*)dst_argb_c, \
+ kStrideB, kWidth, kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ FMT_A##To##FMT_B((TYPE_A*)src_argb, kStrideA, (TYPE_B*)dst_argb_opt, \
+ kStrideB, kWidth, kHeight); \
+ for (size_t i = 0; i < (size_t)kStrideB * kHeightB * sizeof(TYPE_B); \
+ ++i) { \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ } \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOB(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B) \
+ TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_, _Unaligned, +, 4) \
+ TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_, _Invert, -, 0) \
+ TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_, _Opt, +, 0) \
+ TESTATOBRANDOM(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B)
+#else
+#define TESTATOB(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B) \
+ TESTATOBI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_, _Opt, +, 0)
+#endif
+
+TESTATOB(AB30, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOB(AB30, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(ABGR, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+#endif
+TESTATOB(ABGR, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(AR30, uint8_t, 4, 4, 1, AB30, uint8_t, 4, 4, 1)
+#endif
+TESTATOB(AR30, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(AR30, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+TESTATOB(AR30, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#endif
+TESTATOB(ARGB, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(ARGB, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+#endif
+TESTATOB(ARGB, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, ARGB1555, uint8_t, 2, 2, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, ARGB4444, uint8_t, 2, 2, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, ARGBMirror, uint8_t, 4, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, BGRA, uint8_t, 4, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, I400, uint8_t, 1, 1, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, J400, uint8_t, 1, 1, 1)
+TESTATOB(ABGR, uint8_t, 4, 4, 1, J400, uint8_t, 1, 1, 1)
+TESTATOB(RGBA, uint8_t, 4, 4, 1, J400, uint8_t, 1, 1, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, RAW, uint8_t, 3, 3, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, RGB24, uint8_t, 3, 3, 1)
+TESTATOB(ABGR, uint8_t, 4, 4, 1, RAW, uint8_t, 3, 3, 1)
+TESTATOB(ABGR, uint8_t, 4, 4, 1, RGB24, uint8_t, 3, 3, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(ARGB, uint8_t, 4, 4, 1, RGB565, uint8_t, 2, 2, 1)
+#endif
+TESTATOB(ARGB, uint8_t, 4, 4, 1, RGBA, uint8_t, 4, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, UYVY, uint8_t, 2, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, YUY2, uint8_t, 2, 4, 1)
+TESTATOB(ARGB1555, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(ARGB4444, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(BGRA, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(I400, uint8_t, 1, 1, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(I400, uint8_t, 1, 1, 1, I400, uint8_t, 1, 1, 1)
+TESTATOB(I400, uint8_t, 1, 1, 1, I400Mirror, uint8_t, 1, 1, 1)
+TESTATOB(J400, uint8_t, 1, 1, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(J400, uint8_t, 1, 1, 1, J400, uint8_t, 1, 1, 1)
+TESTATOB(RAW, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(RAW, uint8_t, 3, 3, 1, RGBA, uint8_t, 4, 4, 1)
+TESTATOB(RAW, uint8_t, 3, 3, 1, RGB24, uint8_t, 3, 3, 1)
+TESTATOB(RGB24, uint8_t, 3, 3, 1, RAW, uint8_t, 3, 3, 1)
+TESTATOB(RGB24, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(RGB24, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1)
+TESTATOB(RGB24, uint8_t, 3, 3, 1, RGB24Mirror, uint8_t, 3, 3, 1)
+TESTATOB(RAW, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOB(RGB565, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+#endif
+TESTATOB(RGBA, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(UYVY, uint8_t, 2, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(YUY2, uint8_t, 2, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(YUY2, uint8_t, 2, 4, 1, Y, uint8_t, 1, 1, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+TESTATOB(ARGB, uint8_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+TESTATOB(ABGR, uint8_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+TESTATOB(ABGR, uint8_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+TESTATOB(AR64, uint16_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(AB64, uint16_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOB(AR64, uint16_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOB(AB64, uint16_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOB(AR64, uint16_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+TESTATOB(AB64, uint16_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+
+// in place test
+#define TESTATOAI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
+ const int kStrideA = \
+ (kWidth * EPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideB = \
+ (kWidth * EPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
+ align_buffer_page_end(src_argb, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A) + OFF); \
+ align_buffer_page_end(dst_argb_c, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A) + OFF); \
+ align_buffer_page_end(dst_argb_opt, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A) + OFF); \
+ for (int i = 0; i < kStrideA * kHeightA * (int)sizeof(TYPE_A); ++i) { \
+ src_argb[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memcpy(dst_argb_c + OFF, src_argb, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A)); \
+ memcpy(dst_argb_opt + OFF, src_argb, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B((TYPE_A*)(dst_argb_c /* src */ + OFF), kStrideA, \
+ (TYPE_B*)dst_argb_c, kStrideB, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_B((TYPE_A*)(dst_argb_opt /* src */ + OFF), kStrideA, \
+ (TYPE_B*)dst_argb_opt, kStrideB, kWidth, NEG kHeight); \
+ } \
+ memcpy(dst_argb_opt + OFF, src_argb, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A)); \
+ FMT_A##To##FMT_B((TYPE_A*)(dst_argb_opt /* src */ + OFF), kStrideA, \
+ (TYPE_B*)dst_argb_opt, kStrideB, kWidth, NEG kHeight); \
+ for (int i = 0; i < kStrideB * kHeightB * (int)sizeof(TYPE_B); ++i) { \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#define TESTATOA(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, \
+ EPP_B, STRIDE_B, HEIGHT_B) \
+ TESTATOAI(FMT_A, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, FMT_B, TYPE_B, EPP_B, \
+ STRIDE_B, HEIGHT_B, benchmark_width_, _Inplace, +, 0)
+
+TESTATOA(AB30, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOA(AB30, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOA(ABGR, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+#endif
+TESTATOA(ABGR, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOA(AR30, uint8_t, 4, 4, 1, AB30, uint8_t, 4, 4, 1)
+#endif
+TESTATOA(AR30, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOA(AR30, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+TESTATOA(AR30, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+#endif
+TESTATOA(ARGB, uint8_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOA(ARGB, uint8_t, 4, 4, 1, AR30, uint8_t, 4, 4, 1)
+#endif
+TESTATOA(ARGB, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, ARGB1555, uint8_t, 2, 2, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, ARGB4444, uint8_t, 2, 2, 1)
+// TODO(fbarchard): Support in place for mirror.
+// TESTATOA(ARGB, uint8_t, 4, 4, 1, ARGBMirror, uint8_t, 4, 4, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, BGRA, uint8_t, 4, 4, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, I400, uint8_t, 1, 1, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, J400, uint8_t, 1, 1, 1)
+TESTATOA(RGBA, uint8_t, 4, 4, 1, J400, uint8_t, 1, 1, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, RAW, uint8_t, 3, 3, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, RGB24, uint8_t, 3, 3, 1)
+TESTATOA(ABGR, uint8_t, 4, 4, 1, RAW, uint8_t, 3, 3, 1)
+TESTATOA(ABGR, uint8_t, 4, 4, 1, RGB24, uint8_t, 3, 3, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOA(ARGB, uint8_t, 4, 4, 1, RGB565, uint8_t, 2, 2, 1)
+#endif
+TESTATOA(ARGB, uint8_t, 4, 4, 1, RGBA, uint8_t, 4, 4, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, UYVY, uint8_t, 2, 4, 1)
+TESTATOA(ARGB, uint8_t, 4, 4, 1, YUY2, uint8_t, 2, 4, 1)
+// TODO(fbarchard): Support in place for conversions that increase bpp.
+// TESTATOA(ARGB1555, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+// TESTATOA(ARGB4444, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(BGRA, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+// TESTATOA(I400, uint8_t, 1, 1, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(I400, uint8_t, 1, 1, 1, I400, uint8_t, 1, 1, 1)
+// TESTATOA(I400, uint8_t, 1, 1, 1, I400Mirror, uint8_t, 1, 1, 1)
+// TESTATOA(J400, uint8_t, 1, 1, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(J400, uint8_t, 1, 1, 1, J400, uint8_t, 1, 1, 1)
+// TESTATOA(RAW, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
+// TESTATOA(RAW, uint8_t, 3, 3, 1, RGBA, uint8_t, 4, 4, 1)
+TESTATOA(RAW, uint8_t, 3, 3, 1, RGB24, uint8_t, 3, 3, 1)
+// TESTATOA(RGB24, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(RGB24, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1)
+// TESTATOA(RGB24, uint8_t, 3, 3, 1, RGB24Mirror, uint8_t, 3, 3, 1)
+TESTATOA(RAW, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+// TESTATOA(RGB565, uint8_t, 2, 2, 1, ARGB, uint8_t, 4, 4, 1)
+#endif
+TESTATOA(RGBA, uint8_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+// TESTATOA(UYVY, uint8_t, 2, 4, 1, ARGB, uint8_t, 4, 4, 1)
+// TESTATOA(YUY2, uint8_t, 2, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(YUY2, uint8_t, 2, 4, 1, Y, uint8_t, 1, 1, 1)
+// TESTATOA(ARGB, uint8_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+// TESTATOA(ARGB, uint8_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+// TESTATOA(ABGR, uint8_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+// TESTATOA(ABGR, uint8_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+TESTATOA(AR64, uint16_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(AB64, uint16_t, 4, 4, 1, ARGB, uint8_t, 4, 4, 1)
+TESTATOA(AR64, uint16_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOA(AB64, uint16_t, 4, 4, 1, ABGR, uint8_t, 4, 4, 1)
+TESTATOA(AR64, uint16_t, 4, 4, 1, AB64, uint16_t, 4, 4, 1)
+TESTATOA(AB64, uint16_t, 4, 4, 1, AR64, uint16_t, 4, 4, 1)
+
+#define TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##Dither##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
+ const int kStrideA = \
+ (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideB = \
+ (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
+ align_buffer_page_end(src_argb, (size_t)kStrideA * kHeightA + OFF); \
+ align_buffer_page_end(dst_argb_c, (size_t)kStrideB * kHeightB); \
+ align_buffer_page_end(dst_argb_opt, (size_t)kStrideB * kHeightB); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA; ++i) { \
+ src_argb[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c, 1, (size_t)kStrideB * kHeightB); \
+ memset(dst_argb_opt, 101, (size_t)kStrideB * kHeightB); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B##Dither(src_argb + OFF, kStrideA, dst_argb_c, kStrideB, \
+ NULL, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_B##Dither(src_argb + OFF, kStrideA, dst_argb_opt, \
+ kStrideB, NULL, kWidth, NEG kHeight); \
+ } \
+ for (size_t i = 0; i < (size_t)kStrideB * kHeightB; ++i) { \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#define TESTATOBDRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, \
+ STRIDE_B, HEIGHT_B) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##Dither_Random) { \
+ for (int times = 0; times < benchmark_iterations_; ++times) { \
+ const int kWidth = (fastrand() & 63) + 1; \
+ const int kHeight = (fastrand() & 31) + 1; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
+ const int kStrideA = \
+ (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideB = \
+ (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
+ align_buffer_page_end(src_argb, (size_t)kStrideA * kHeightA); \
+ align_buffer_page_end(dst_argb_c, (size_t)kStrideB * kHeightB); \
+ align_buffer_page_end(dst_argb_opt, (size_t)kStrideB * kHeightB); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA; ++i) { \
+ src_argb[i] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c, 123, (size_t)kStrideB * kHeightB); \
+ memset(dst_argb_opt, 123, (size_t)kStrideB * kHeightB); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B##Dither(src_argb, kStrideA, dst_argb_c, kStrideB, NULL, \
+ kWidth, kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ FMT_A##To##FMT_B##Dither(src_argb, kStrideA, dst_argb_opt, kStrideB, \
+ NULL, kWidth, kHeight); \
+ for (size_t i = 0; i < (size_t)kStrideB * kHeightB; ++i) { \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ } \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOBD(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B) \
+ TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B, benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B, benchmark_width_, _Unaligned, +, 2) \
+ TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B, benchmark_width_, _Invert, -, 0) \
+ TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B, benchmark_width_, _Opt, +, 0) \
+ TESTATOBDRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B)
+#else
+#define TESTATOBD(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B) \
+ TESTATOBDRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
+ HEIGHT_B)
+#endif
+
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOBD(ARGB, 4, 4, 1, RGB565, 2, 2, 1)
+#endif
+
+// These conversions called twice, produce the original result.
+// e.g. endian swap twice.
+#define TESTENDI(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, W1280, N, NEG, \
+ OFF) \
+ TEST_F(LibYUVConvertTest, FMT_ATOB##_Endswap##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kStrideA = \
+ (kWidth * EPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ align_buffer_page_end(src_argb, \
+ (size_t)kStrideA * kHeightA * sizeof(TYPE_A) + OFF); \
+ align_buffer_page_end(dst_argb_c, \
+ (size_t)kStrideA * kHeightA * sizeof(TYPE_A)); \
+ align_buffer_page_end(dst_argb_opt, \
+ (size_t)kStrideA * kHeightA * sizeof(TYPE_A)); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA * sizeof(TYPE_A); \
+ ++i) { \
+ src_argb[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c, 1, (size_t)kStrideA * kHeightA * sizeof(TYPE_A)); \
+ memset(dst_argb_opt, 101, (size_t)kStrideA * kHeightA * sizeof(TYPE_A)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_ATOB((TYPE_A*)(src_argb + OFF), kStrideA, (TYPE_A*)dst_argb_c, \
+ kStrideA, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_ATOB((TYPE_A*)(src_argb + OFF), kStrideA, (TYPE_A*)dst_argb_opt, \
+ kStrideA, kWidth, NEG kHeight); \
+ } \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_ATOB((TYPE_A*)dst_argb_c, kStrideA, (TYPE_A*)dst_argb_c, kStrideA, \
+ kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ FMT_ATOB((TYPE_A*)dst_argb_opt, kStrideA, (TYPE_A*)dst_argb_opt, kStrideA, \
+ kWidth, NEG kHeight); \
+ for (size_t i = 0; i < (size_t)kStrideA * kHeightA * sizeof(TYPE_A); \
+ ++i) { \
+ ASSERT_EQ(src_argb[i + OFF], dst_argb_opt[i]); \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTEND(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A) \
+ TESTENDI(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, benchmark_width_ + 1, \
+ _Any, +, 0) \
+ TESTENDI(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, benchmark_width_, \
+ _Unaligned, +, 2) \
+ TESTENDI(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, benchmark_width_, \
+ _Opt, +, 0)
+#else
+#define TESTEND(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A) \
+ TESTENDI(FMT_ATOB, TYPE_A, EPP_A, STRIDE_A, HEIGHT_A, benchmark_width_, \
+ _Opt, +, 0)
+#endif
+
+TESTEND(ARGBToBGRA, uint8_t, 4, 4, 1)
+TESTEND(ARGBToABGR, uint8_t, 4, 4, 1)
+TESTEND(BGRAToARGB, uint8_t, 4, 4, 1)
+TESTEND(ABGRToARGB, uint8_t, 4, 4, 1)
+TESTEND(AB64ToAR64, uint16_t, 4, 4, 1)
+
+#define TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, W1280, N, NEG, OFF, ATTEN) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_u, kSizeUV + OFF); \
+ align_buffer_page_end(src_v, kSizeUV + OFF); \
+ align_buffer_page_end(src_a, kWidth * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y[i + OFF] = (fastrand() & 0xff); \
+ src_a[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ src_u[i + OFF] = (fastrand() & 0xff); \
+ src_v[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, src_a + OFF, kWidth, \
+ dst_argb_c + OFF, kStrideB, kWidth, NEG kHeight, \
+ ATTEN); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, src_a + OFF, kWidth, \
+ dst_argb_opt + OFF, kStrideB, kWidth, NEG kHeight, \
+ ATTEN); \
+ } \
+ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(src_a); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTQPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_ + 1, _Any, +, 0, 0) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Unaligned, +, 2, 0) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Invert, -, 0, 0) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, 0) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Premult, +, 0, 1)
+#else
+#define TESTQPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN) \
+ TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, 0)
+#endif
+
+#define J420AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J420AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F420AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F420AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H420AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H420AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U420AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U420AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V420AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V420AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define J422AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J422AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F422AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F422AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H422AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H422AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U422AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U422AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V422AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V422AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define J444AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J444AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F444AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F444AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H444AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H444AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U444AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U444AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V444AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V444AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I444AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+
+#define I420AlphaToARGBFilter(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I420AlphaToARGBMatrixFilter(a, b, c, d, e, f, g, h, i, j, \
+ &kYuvI601Constants, k, l, m, kFilterBilinear)
+#define I422AlphaToARGBFilter(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I422AlphaToARGBMatrixFilter(a, b, c, d, e, f, g, h, i, j, \
+ &kYuvI601Constants, k, l, m, kFilterBilinear)
+
+#if defined(ENABLE_FULL_TESTS)
+TESTQPLANARTOB(I420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(J420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(J420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(H420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(H420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(F420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(F420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(U420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(U420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(V420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(V420Alpha, 2, 2, ABGR, 4, 4, 1)
+TESTQPLANARTOB(I422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(J422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(J422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(H422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(H422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(F422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(F422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(U422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(U422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(V422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(V422Alpha, 2, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(I444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(J444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(J444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(H444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(H444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(F444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(F444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(U444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(U444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(V444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(V444Alpha, 1, 1, ABGR, 4, 4, 1)
+TESTQPLANARTOB(I420Alpha, 2, 2, ARGBFilter, 4, 4, 1)
+TESTQPLANARTOB(I422Alpha, 2, 1, ARGBFilter, 4, 4, 1)
+#else
+TESTQPLANARTOB(I420Alpha, 2, 2, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I422Alpha, 2, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I444Alpha, 1, 1, ARGB, 4, 4, 1)
+TESTQPLANARTOB(I420Alpha, 2, 2, ARGBFilter, 4, 4, 1)
+TESTQPLANARTOB(I422Alpha, 2, 1, ARGBFilter, 4, 4, 1)
+#endif
+
+TEST_F(LibYUVConvertTest, TestYToARGB) {
+ uint8_t y[32];
+ uint8_t expectedg[32];
+ for (int i = 0; i < 32; ++i) {
+ y[i] = i * 5 + 17;
+ expectedg[i] = static_cast<int>((y[i] - 16) * 1.164f + 0.5f);
+ }
+ uint8_t argb[32 * 4];
+ YToARGB(y, 0, argb, 0, 32, 1);
+
+ for (int i = 0; i < 32; ++i) {
+ printf("%2d %d: %d <-> %d,%d,%d,%d\n", i, y[i], expectedg[i],
+ argb[i * 4 + 0], argb[i * 4 + 1], argb[i * 4 + 2], argb[i * 4 + 3]);
+ }
+ for (int i = 0; i < 32; ++i) {
+ ASSERT_EQ(expectedg[i], argb[i * 4 + 0]);
+ }
+}
+
+static const uint8_t kNoDither4x4[16] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+TEST_F(LibYUVConvertTest, TestNoDither) {
+ align_buffer_page_end(src_argb, benchmark_width_ * benchmark_height_ * 4);
+ align_buffer_page_end(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_rgb565dither,
+ benchmark_width_ * benchmark_height_ * 2);
+ MemRandomize(src_argb, benchmark_width_ * benchmark_height_ * 4);
+ MemRandomize(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
+ MemRandomize(dst_rgb565dither, benchmark_width_ * benchmark_height_ * 2);
+ ARGBToRGB565(src_argb, benchmark_width_ * 4, dst_rgb565, benchmark_width_ * 2,
+ benchmark_width_, benchmark_height_);
+ ARGBToRGB565Dither(src_argb, benchmark_width_ * 4, dst_rgb565dither,
+ benchmark_width_ * 2, kNoDither4x4, benchmark_width_,
+ benchmark_height_);
+ for (int i = 0; i < benchmark_width_ * benchmark_height_ * 2; ++i) {
+ ASSERT_EQ(dst_rgb565[i], dst_rgb565dither[i]);
+ }
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_rgb565);
+ free_aligned_buffer_page_end(dst_rgb565dither);
+}
+
+// Ordered 4x4 dither for 888 to 565. Values from 0 to 7.
+static const uint8_t kDither565_4x4[16] = {
+ 0, 4, 1, 5, 6, 2, 7, 3, 1, 5, 0, 4, 7, 3, 6, 2,
+};
+
+TEST_F(LibYUVConvertTest, TestDither) {
+ align_buffer_page_end(src_argb, benchmark_width_ * benchmark_height_ * 4);
+ align_buffer_page_end(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_rgb565dither,
+ benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_argb, benchmark_width_ * benchmark_height_ * 4);
+ align_buffer_page_end(dst_argbdither,
+ benchmark_width_ * benchmark_height_ * 4);
+ MemRandomize(src_argb, benchmark_width_ * benchmark_height_ * 4);
+ MemRandomize(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
+ MemRandomize(dst_rgb565dither, benchmark_width_ * benchmark_height_ * 2);
+ MemRandomize(dst_argb, benchmark_width_ * benchmark_height_ * 4);
+ MemRandomize(dst_argbdither, benchmark_width_ * benchmark_height_ * 4);
+ ARGBToRGB565(src_argb, benchmark_width_ * 4, dst_rgb565, benchmark_width_ * 2,
+ benchmark_width_, benchmark_height_);
+ ARGBToRGB565Dither(src_argb, benchmark_width_ * 4, dst_rgb565dither,
+ benchmark_width_ * 2, kDither565_4x4, benchmark_width_,
+ benchmark_height_);
+ RGB565ToARGB(dst_rgb565, benchmark_width_ * 2, dst_argb, benchmark_width_ * 4,
+ benchmark_width_, benchmark_height_);
+ RGB565ToARGB(dst_rgb565dither, benchmark_width_ * 2, dst_argbdither,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ for (int i = 0; i < benchmark_width_ * benchmark_height_ * 4; ++i) {
+ ASSERT_NEAR(dst_argb[i], dst_argbdither[i], 9);
+ }
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_rgb565);
+ free_aligned_buffer_page_end(dst_rgb565dither);
+ free_aligned_buffer_page_end(dst_argb);
+ free_aligned_buffer_page_end(dst_argbdither);
+}
+
+#define TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, W1280, N, NEG, OFF, FMT_C, BPP_C) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##Dither##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_u, kSizeUV + OFF); \
+ align_buffer_page_end(src_v, kSizeUV + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ src_u[i + OFF] = (fastrand() & 0xff); \
+ src_v[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B##Dither(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, dst_argb_c + OFF, \
+ kStrideB, NULL, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B##Dither( \
+ src_y + OFF, kWidth, src_u + OFF, kStrideUV, src_v + OFF, kStrideUV, \
+ dst_argb_opt + OFF, kStrideB, NULL, kWidth, NEG kHeight); \
+ } \
+ /* Convert to ARGB so 565 is expanded to bytes that can be compared. */ \
+ align_buffer_page_end(dst_argb32_c, kWidth * BPP_C * kHeight); \
+ align_buffer_page_end(dst_argb32_opt, kWidth * BPP_C * kHeight); \
+ memset(dst_argb32_c, 2, kWidth * BPP_C * kHeight); \
+ memset(dst_argb32_opt, 102, kWidth * BPP_C * kHeight); \
+ FMT_B##To##FMT_C(dst_argb_c + OFF, kStrideB, dst_argb32_c, kWidth * BPP_C, \
+ kWidth, kHeight); \
+ FMT_B##To##FMT_C(dst_argb_opt + OFF, kStrideB, dst_argb32_opt, \
+ kWidth * BPP_C, kWidth, kHeight); \
+ for (int i = 0; i < kWidth * BPP_C * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb32_c[i], dst_argb32_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ free_aligned_buffer_page_end(dst_argb32_c); \
+ free_aligned_buffer_page_end(dst_argb32_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANARTOBD(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, FMT_C, BPP_C) \
+ TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_ + 1, _Any, +, 0, FMT_C, BPP_C) \
+ TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Unaligned, +, 2, FMT_C, BPP_C) \
+ TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Invert, -, 0, FMT_C, BPP_C) \
+ TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, FMT_C, BPP_C)
+#else
+#define TESTPLANARTOBD(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, FMT_C, BPP_C) \
+ TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, FMT_C, BPP_C)
+#endif
+
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANARTOBD(I420, 2, 2, RGB565, 2, 2, 1, ARGB, 4)
+#endif
+
+// Transitive test. A to B to C is same as A to C.
+// Benchmarks A To B to C for comparison to 1 step, benchmarked elsewhere.
+#define TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ W1280, N, NEG, OFF, FMT_C, BPP_C) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##To##FMT_C##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_u, kSizeUV + OFF); \
+ align_buffer_page_end(src_v, kSizeUV + OFF); \
+ align_buffer_page_end(dst_argb_b, kStrideB * kHeight + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ src_u[i + OFF] = (fastrand() & 0xff); \
+ src_v[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
+ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, dst_argb_b + OFF, kStrideB, \
+ kWidth, NEG kHeight); \
+ /* Convert to a 3rd format in 1 step and 2 steps and compare */ \
+ const int kStrideC = kWidth * BPP_C; \
+ align_buffer_page_end(dst_argb_c, kStrideC * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_bc, kStrideC * kHeight + OFF); \
+ memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
+ memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_C(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
+ src_v + OFF, kStrideUV, dst_argb_c + OFF, \
+ kStrideC, kWidth, NEG kHeight); \
+ /* Convert B to C */ \
+ FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, \
+ kStrideC, kWidth, kHeight); \
+ } \
+ for (int i = 0; i < kStrideC * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_bc[i + OFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst_argb_b); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_bc); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ FMT_C, BPP_C) \
+ TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_ + 1, _Any, +, 0, FMT_C, BPP_C) \
+ TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Unaligned, +, 2, FMT_C, BPP_C) \
+ TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Invert, -, 0, FMT_C, BPP_C) \
+ TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Opt, +, 0, FMT_C, BPP_C)
+#else
+#define TESTPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ FMT_C, BPP_C) \
+ TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Opt, +, 0, FMT_C, BPP_C)
+#endif
+
+#if defined(ENABLE_FULL_TESTS)
+TESTPLANARTOE(I420, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RAW, 3)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RGB24, 3)
+TESTPLANARTOE(I420, 2, 2, BGRA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, RGB24, 3)
+TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, RAW, 3)
+TESTPLANARTOE(I420, 2, 2, RGBA, 1, 4, ARGB, 4)
+TESTPLANARTOE(H420, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(H420, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTPLANARTOE(H420, 2, 2, ARGB, 1, 4, RAW, 3)
+TESTPLANARTOE(H420, 2, 2, ARGB, 1, 4, RGB24, 3)
+TESTPLANARTOE(H420, 2, 2, RAW, 1, 3, ARGB, 4)
+TESTPLANARTOE(H420, 2, 2, RAW, 1, 3, RGB24, 3)
+TESTPLANARTOE(H420, 2, 2, RGB24, 1, 3, ARGB, 4)
+TESTPLANARTOE(H420, 2, 2, RGB24, 1, 3, RAW, 3)
+TESTPLANARTOE(J420, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(J420, 2, 2, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(U420, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(U420, 2, 2, ARGB, 1, 4, ARGB, 4)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RGB565, 2)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB1555, 2)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB4444, 2)
+TESTPLANARTOE(I422, 2, 1, ARGB, 1, 4, RGB565, 2)
+#endif
+TESTPLANARTOE(I422, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTPLANARTOE(I422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(J422, 2, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(J422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(H422, 2, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(H422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(U422, 2, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(U422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(V422, 2, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(V422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, BGRA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, RGBA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I444, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTPLANARTOE(I444, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(J444, 1, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(J444, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(H444, 1, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(H444, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(U444, 1, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(U444, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(V444, 1, 1, ARGB, 1, 4, ARGB, 4)
+TESTPLANARTOE(V444, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, YUY2, 2, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, UYVY, 2, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, YUY2, 2, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, UYVY, 2, 4, ARGB, 4)
+#else
+TESTPLANARTOE(I420, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB1555, 2)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB4444, 2)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RAW, 3)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RGB24, 3)
+TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RGB565, 2)
+TESTPLANARTOE(I420, 2, 2, BGRA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, RGB24, 3)
+TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, RAW, 3)
+TESTPLANARTOE(I420, 2, 2, RGBA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, UYVY, 2, 4, ARGB, 4)
+TESTPLANARTOE(I420, 2, 2, YUY2, 2, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, ARGB, 1, 4, RGB565, 2)
+TESTPLANARTOE(I422, 2, 1, BGRA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, RGBA, 1, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, UYVY, 2, 4, ARGB, 4)
+TESTPLANARTOE(I422, 2, 1, YUY2, 2, 4, ARGB, 4)
+TESTPLANARTOE(I444, 1, 1, ABGR, 1, 4, ARGB, 4)
+#endif
+
+// Transitive test: Compare 1 step vs 2 step conversion for YUVA to ARGB.
+// Benchmark 2 step conversion for comparison to 1 step conversion.
+#define TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ W1280, N, NEG, OFF, FMT_C, BPP_C, ATTEN) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##To##FMT_C##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
+ const int kSizeUV = \
+ SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_u, kSizeUV + OFF); \
+ align_buffer_page_end(src_v, kSizeUV + OFF); \
+ align_buffer_page_end(src_a, kWidth * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_b, kStrideB * kHeight + OFF); \
+ const int kStrideC = kWidth * BPP_C; \
+ align_buffer_page_end(dst_argb_c, kStrideC * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_bc, kStrideC * kHeight + OFF); \
+ memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
+ memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y[i + OFF] = (fastrand() & 0xff); \
+ src_a[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ src_u[i + OFF] = (fastrand() & 0xff); \
+ src_v[i + OFF] = (fastrand() & 0xff); \
+ } \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ /* Convert A to B */ \
+ FMT_PLANAR##To##FMT_B( \
+ src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), \
+ src_v + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), src_a + OFF, kWidth, \
+ dst_argb_b + OFF, kStrideB, kWidth, NEG kHeight, ATTEN); \
+ /* Convert B to C */ \
+ FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, \
+ kStrideC, kWidth, kHeight); \
+ } \
+ /* Convert A to C */ \
+ FMT_PLANAR##To##FMT_C( \
+ src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), \
+ src_v + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), src_a + OFF, kWidth, \
+ dst_argb_c + OFF, kStrideC, kWidth, NEG kHeight, ATTEN); \
+ for (int i = 0; i < kStrideC * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_bc[i + OFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(src_a); \
+ free_aligned_buffer_page_end(dst_argb_b); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_bc); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTQPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ FMT_C, BPP_C) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_ + 1, _Any, +, 0, FMT_C, BPP_C, 0) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Unaligned, +, 2, FMT_C, BPP_C, 0) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Invert, -, 0, FMT_C, BPP_C, 0) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Opt, +, 0, FMT_C, BPP_C, 0) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Premult, +, 0, FMT_C, BPP_C, 1)
+#else
+#define TESTQPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ FMT_C, BPP_C) \
+ TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_, _Opt, +, 0, FMT_C, BPP_C, 0)
+#endif
+
+#if defined(ENABLE_FULL_TESTS)
+TESTQPLANARTOE(I420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(I420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(J420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(J420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(H420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(H420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(F420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(F420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(U420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(U420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(V420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(V420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(I422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(I422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(J422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(J422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(F422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(F422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(H422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(H422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(U422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(U422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(V422Alpha, 2, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(V422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(I444Alpha, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(I444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(J444Alpha, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(J444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(H444Alpha, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(H444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(U444Alpha, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(U444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(V444Alpha, 1, 1, ARGB, 1, 4, ABGR, 4)
+TESTQPLANARTOE(V444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+#else
+TESTQPLANARTOE(I420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(I422Alpha, 2, 1, ABGR, 1, 4, ARGB, 4)
+TESTQPLANARTOE(I444Alpha, 1, 1, ABGR, 1, 4, ARGB, 4)
+#endif
+
+#define TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, W1280, N, NEG, \
+ OFF, FMT_C, BPP_C) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##To##FMT_C##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kStrideA = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \
+ const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
+ align_buffer_page_end(src_argb_a, kStrideA * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_b, kStrideB * kHeight + OFF); \
+ MemRandomize(src_argb_a + OFF, kStrideA * kHeight); \
+ memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
+ FMT_A##To##FMT_B(src_argb_a + OFF, kStrideA, dst_argb_b + OFF, kStrideB, \
+ kWidth, NEG kHeight); \
+ /* Convert to a 3rd format in 1 step and 2 steps and compare */ \
+ const int kStrideC = kWidth * BPP_C; \
+ align_buffer_page_end(dst_argb_c, kStrideC * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_bc, kStrideC * kHeight + OFF); \
+ memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
+ memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_C(src_argb_a + OFF, kStrideA, dst_argb_c + OFF, kStrideC, \
+ kWidth, NEG kHeight); \
+ /* Convert B to C */ \
+ FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, \
+ kStrideC, kWidth, kHeight); \
+ } \
+ for (int i = 0; i < kStrideC * kHeight; i += 4) { \
+ ASSERT_EQ(dst_argb_c[i + OFF + 0], dst_argb_bc[i + OFF + 0]); \
+ ASSERT_EQ(dst_argb_c[i + OFF + 1], dst_argb_bc[i + OFF + 1]); \
+ ASSERT_EQ(dst_argb_c[i + OFF + 2], dst_argb_bc[i + OFF + 2]); \
+ ASSERT_NEAR(dst_argb_c[i + OFF + 3], dst_argb_bc[i + OFF + 3], 64); \
+ } \
+ free_aligned_buffer_page_end(src_argb_a); \
+ free_aligned_buffer_page_end(dst_argb_b); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_bc); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANETOE(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, FMT_C, BPP_C) \
+ TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, \
+ benchmark_width_ + 1, _Any, +, 0, FMT_C, BPP_C) \
+ TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
+ _Unaligned, +, 4, FMT_C, BPP_C) \
+ TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
+ _Invert, -, 0, FMT_C, BPP_C) \
+ TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
+ _Opt, +, 0, FMT_C, BPP_C)
+#else
+#define TESTPLANETOE(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, FMT_C, BPP_C) \
+ TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
+ _Opt, +, 0, FMT_C, BPP_C)
+#endif
+
+// Caveat: Destination needs to be 4 bytes
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANETOE(ARGB, 1, 4, AR30, 1, 4, ARGB, 4)
+TESTPLANETOE(ABGR, 1, 4, AR30, 1, 4, ABGR, 4)
+TESTPLANETOE(AR30, 1, 4, ARGB, 1, 4, ABGR, 4)
+TESTPLANETOE(AR30, 1, 4, ABGR, 1, 4, ARGB, 4)
+TESTPLANETOE(ARGB, 1, 4, AB30, 1, 4, ARGB, 4)
+TESTPLANETOE(ABGR, 1, 4, AB30, 1, 4, ABGR, 4)
+TESTPLANETOE(AB30, 1, 4, ARGB, 1, 4, ABGR, 4)
+TESTPLANETOE(AB30, 1, 4, ABGR, 1, 4, ARGB, 4)
+#endif
+
+TEST_F(LibYUVConvertTest, RotateWithARGBSource) {
+ // 2x2 frames
+ uint32_t src[4];
+ uint32_t dst[4];
+ // some random input
+ src[0] = 0x11000000;
+ src[1] = 0x00450000;
+ src[2] = 0x00009f00;
+ src[3] = 0x000000ff;
+ // zeros on destination
+ dst[0] = 0x00000000;
+ dst[1] = 0x00000000;
+ dst[2] = 0x00000000;
+ dst[3] = 0x00000000;
+
+ int r = ConvertToARGB(reinterpret_cast<uint8_t*>(src),
+ 16, // input size
+ reinterpret_cast<uint8_t*>(dst),
+ 8, // destination stride
+ 0, // crop_x
+ 0, // crop_y
+ 2, // width
+ 2, // height
+ 2, // crop width
+ 2, // crop height
+ kRotate90, FOURCC_ARGB);
+
+ ASSERT_EQ(r, 0);
+ // 90 degrees rotation, no conversion
+ ASSERT_EQ(dst[0], src[2]);
+ ASSERT_EQ(dst[1], src[0]);
+ ASSERT_EQ(dst[2], src[3]);
+ ASSERT_EQ(dst[3], src[1]);
+}
+
+#define TESTCONVERTTOARGBSMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, ConvertToARGB_##FMT) { \
+ const int kWidth = 64; \
+ const int kHeight = 48; \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kSampleSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \
+ align_buffer_page_end(src, kSampleSize); \
+ align_buffer_page_end(dst, kWidth * kHeight * 4); \
+ MemRandomize(src, kSampleSize); \
+ memset(dst, 0, kWidth * kHeight * 4); \
+ EXPECT_EQ( \
+ 0, ConvertToARGB(src, kSampleSize, dst, kWidth * 4, 0, 0, kWidth, \
+ kHeight, kWidth, kHeight, kRotate0, FOURCC_##FMT)); \
+ free_aligned_buffer_page_end(src); \
+ free_aligned_buffer_page_end(dst); \
+ }
+
+TESTCONVERTTOARGBSMOKE(NV16, 1, 2, 1)
+TESTCONVERTTOARGBSMOKE(NV24, 1, 1, 1)
+
+#define TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \
+ NEG) \
+ TEST_F(LibYUVConvertTest, ConvertToARGB_##FMT##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kYPixels = kWidth * kHeight; \
+ const int kUvPixels = kUvStride * kUvHeight; \
+ align_buffer_page_end(src, (kYPixels + kUvPixels) * BPC); \
+ align_buffer_page_end(dst, kWidth * kHeight * 4); \
+ align_buffer_page_end(ref, kWidth * kHeight * 4); \
+ TYPE* src_p = reinterpret_cast<TYPE*>(src); \
+ MemRandomize(src, (kYPixels + kUvPixels) * BPC); \
+ if (BPC == 2) { \
+ for (int i = 0; i < kYPixels + kUvPixels; ++i) { \
+ src_p[i] = static_cast<TYPE>(src_p[i] & 0xffc0); \
+ } \
+ } \
+ memset(dst, 1, kWidth * kHeight * 4); \
+ memset(ref, 2, kWidth * kHeight * 4); \
+ EXPECT_EQ(0, FMT##ToARGBMatrix(src_p, kWidth, src_p + kYPixels, kUvStride, \
+ ref, kWidth * 4, &kYuvI601Constants, \
+ kWidth, NEG kHeight)); \
+ EXPECT_EQ( \
+ 0, ConvertToARGB(src, 0, dst, kWidth * 4, 0, 0, kWidth, NEG kHeight, \
+ kWidth, kHeight, kRotate0, FOURCC_##FMT)); \
+ for (int i = 0; i < kWidth * kHeight * 4; ++i) { \
+ ASSERT_EQ(dst[i], ref[i]); \
+ } \
+ free_aligned_buffer_page_end(src); \
+ free_aligned_buffer_page_end(dst); \
+ free_aligned_buffer_page_end(ref); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +) \
+ TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Invert, -) \
+ TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#else
+#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#endif
+
+TESTCONVERTTOARGB(P010, uint16_t, 2, 2, 2)
+TESTCONVERTTOARGB(P210, uint16_t, 2, 2, 1)
+TESTCONVERTTOARGB(P410, uint16_t, 2, 1, 1)
+
+#ifdef HAS_ARGBTOAR30ROW_AVX2
+TEST_F(LibYUVConvertTest, ARGBToAR30Row_Opt) {
+ // ARGBToAR30Row_AVX2 expects a multiple of 8 pixels.
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7;
+ align_buffer_page_end(src, kPixels * 4);
+ align_buffer_page_end(dst_opt, kPixels * 4);
+ align_buffer_page_end(dst_c, kPixels * 4);
+ MemRandomize(src, kPixels * 4);
+ memset(dst_opt, 0, kPixels * 4);
+ memset(dst_c, 1, kPixels * 4);
+
+ ARGBToAR30Row_C(src, dst_c, kPixels);
+
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ if (has_avx2) {
+ ARGBToAR30Row_AVX2(src, dst_opt, kPixels);
+ } else if (has_ssse3) {
+ ARGBToAR30Row_SSSE3(src, dst_opt, kPixels);
+ } else {
+ ARGBToAR30Row_C(src, dst_opt, kPixels);
+ }
+ }
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_opt[i], dst_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(dst_c);
+}
+#endif // HAS_ARGBTOAR30ROW_AVX2
+
+#ifdef HAS_ABGRTOAR30ROW_AVX2
+TEST_F(LibYUVConvertTest, ABGRToAR30Row_Opt) {
+ // ABGRToAR30Row_AVX2 expects a multiple of 8 pixels.
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7;
+ align_buffer_page_end(src, kPixels * 4);
+ align_buffer_page_end(dst_opt, kPixels * 4);
+ align_buffer_page_end(dst_c, kPixels * 4);
+ MemRandomize(src, kPixels * 4);
+ memset(dst_opt, 0, kPixels * 4);
+ memset(dst_c, 1, kPixels * 4);
+
+ ABGRToAR30Row_C(src, dst_c, kPixels);
+
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ if (has_avx2) {
+ ABGRToAR30Row_AVX2(src, dst_opt, kPixels);
+ } else if (has_ssse3) {
+ ABGRToAR30Row_SSSE3(src, dst_opt, kPixels);
+ } else {
+ ABGRToAR30Row_C(src, dst_opt, kPixels);
+ }
+ }
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_opt[i], dst_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(dst_c);
+}
+#endif // HAS_ABGRTOAR30ROW_AVX2
+
+#if !defined(LEAN_TESTS)
+
+// Provide matrix wrappers for 12 bit YUV
+#define I012ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I012ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define I012ToAR30(a, b, c, d, e, f, g, h, i, j) \
+ I012ToAR30Matrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define I012ToAB30(a, b, c, d, e, f, g, h, i, j) \
+ I012ToAB30Matrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+
+#define I410ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I410ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define I410ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I410ToABGRMatrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define H410ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I410ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuvH709Constants, i, j)
+#define H410ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I410ToABGRMatrix(a, b, c, d, e, f, g, h, &kYuvH709Constants, i, j)
+#define U410ToARGB(a, b, c, d, e, f, g, h, i, j) \
+ I410ToARGBMatrix(a, b, c, d, e, f, g, h, &kYuv2020Constants, i, j)
+#define U410ToABGR(a, b, c, d, e, f, g, h, i, j) \
+ I410ToABGRMatrix(a, b, c, d, e, f, g, h, &kYuv2020Constants, i, j)
+#define I410ToAR30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAR30Matrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define I410ToAB30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAB30Matrix(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j)
+#define H410ToAR30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAR30Matrix(a, b, c, d, e, f, g, h, &kYuvH709Constants, i, j)
+#define H410ToAB30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAB30Matrix(a, b, c, d, e, f, g, h, &kYuvH709Constants, i, j)
+#define U410ToAR30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAR30Matrix(a, b, c, d, e, f, g, h, &kYuv2020Constants, i, j)
+#define U410ToAB30(a, b, c, d, e, f, g, h, i, j) \
+ I410ToAB30Matrix(a, b, c, d, e, f, g, h, &kYuv2020Constants, i, j)
+
+#define I010ToARGBFilter(a, b, c, d, e, f, g, h, i, j) \
+ I010ToARGBMatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I010ToAR30Filter(a, b, c, d, e, f, g, h, i, j) \
+ I010ToAR30MatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I210ToARGBFilter(a, b, c, d, e, f, g, h, i, j) \
+ I210ToARGBMatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+#define I210ToAR30Filter(a, b, c, d, e, f, g, h, i, j) \
+ I210ToAR30MatrixFilter(a, b, c, d, e, f, g, h, &kYuvI601Constants, i, j, \
+ kFilterBilinear)
+
+// TODO(fbarchard): Fix clamping issue affected by U channel.
+#define TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, \
+ BPP_B, ALIGN, YALIGN, W1280, N, NEG, SOFF, DOFF) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kBpc = 2; \
+ align_buffer_page_end(src_y, kWidth * kHeight * kBpc + SOFF); \
+ align_buffer_page_end(src_u, kSizeUV * kBpc + SOFF); \
+ align_buffer_page_end(src_v, kSizeUV * kBpc + SOFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + DOFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + DOFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ reinterpret_cast<uint16_t*>(src_y + SOFF)[i] = (fastrand() & FMT_MASK); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ reinterpret_cast<uint16_t*>(src_u + SOFF)[i] = (fastrand() & FMT_MASK); \
+ reinterpret_cast<uint16_t*>(src_v + SOFF)[i] = (fastrand() & FMT_MASK); \
+ } \
+ memset(dst_argb_c + DOFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + DOFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B( \
+ reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_u + SOFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_v + SOFF), kStrideUV, \
+ dst_argb_c + DOFF, kStrideB, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B( \
+ reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_u + SOFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_v + SOFF), kStrideUV, \
+ dst_argb_opt + DOFF, kStrideB, kWidth, NEG kHeight); \
+ } \
+ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + DOFF], dst_argb_opt[i + DOFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANAR16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, \
+ BPP_B, ALIGN, YALIGN) \
+ TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, BPP_B, \
+ ALIGN, YALIGN, benchmark_width_ + 1, _Any, +, 0, 0) \
+ TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, BPP_B, \
+ ALIGN, YALIGN, benchmark_width_, _Unaligned, +, 4, 4) \
+ TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, BPP_B, \
+ ALIGN, YALIGN, benchmark_width_, _Invert, -, 0, 0) \
+ TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, BPP_B, \
+ ALIGN, YALIGN, benchmark_width_, _Opt, +, 0, 0)
+#else
+#define TESTPLANAR16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, \
+ BPP_B, ALIGN, YALIGN) \
+ TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_MASK, FMT_B, BPP_B, \
+ ALIGN, YALIGN, benchmark_width_, _Opt, +, 0, 0)
+#endif
+
+// These conversions are only optimized for x86
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(H010, 2, 2, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(H010, 2, 2, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(U010, 2, 2, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(U010, 2, 2, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(H210, 2, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(H210, 2, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(U210, 2, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(U210, 2, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(I410, 1, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(I410, 1, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(H410, 1, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(H410, 1, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(U410, 1, 1, 0x3ff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(U410, 1, 1, 0x3ff, ABGR, 4, 4, 1)
+TESTPLANAR16TOB(I012, 2, 2, 0xfff, ARGB, 4, 4, 1)
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, ARGBFilter, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, ARGBFilter, 4, 4, 1)
+
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(H010, 2, 2, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(H010, 2, 2, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(U010, 2, 2, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(U010, 2, 2, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(H210, 2, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(H210, 2, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(U210, 2, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(U210, 2, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(I410, 1, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(I410, 1, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(H410, 1, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(H410, 1, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(U410, 1, 1, 0x3ff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(U410, 1, 1, 0x3ff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(I012, 2, 2, 0xfff, AR30, 4, 4, 1)
+TESTPLANAR16TOB(I012, 2, 2, 0xfff, AB30, 4, 4, 1)
+TESTPLANAR16TOB(I010, 2, 2, 0x3ff, AR30Filter, 4, 4, 1)
+TESTPLANAR16TOB(I210, 2, 1, 0x3ff, AR30Filter, 4, 4, 1)
+#endif // LITTLE_ENDIAN_ONLY_TEST
+#endif // DISABLE_SLOW_TESTS
+
+#define TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, \
+ ALIGN, YALIGN, W1280, N, NEG, OFF, ATTEN, S_DEPTH) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kBpc = 2; \
+ align_buffer_page_end(src_y, kWidth * kHeight * kBpc + OFF); \
+ align_buffer_page_end(src_u, kSizeUV * kBpc + OFF); \
+ align_buffer_page_end(src_v, kSizeUV * kBpc + OFF); \
+ align_buffer_page_end(src_a, kWidth * kHeight * kBpc + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ reinterpret_cast<uint16_t*>(src_y + OFF)[i] = \
+ (fastrand() & ((1 << S_DEPTH) - 1)); \
+ reinterpret_cast<uint16_t*>(src_a + OFF)[i] = \
+ (fastrand() & ((1 << S_DEPTH) - 1)); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ reinterpret_cast<uint16_t*>(src_u + OFF)[i] = \
+ (fastrand() & ((1 << S_DEPTH) - 1)); \
+ reinterpret_cast<uint16_t*>(src_v + OFF)[i] = \
+ (fastrand() & ((1 << S_DEPTH) - 1)); \
+ } \
+ memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B(reinterpret_cast<uint16_t*>(src_y + OFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_u + OFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_v + OFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_a + OFF), kWidth, \
+ dst_argb_c + OFF, kStrideB, kWidth, NEG kHeight, \
+ ATTEN); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B( \
+ reinterpret_cast<uint16_t*>(src_y + OFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_u + OFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_v + OFF), kStrideUV, \
+ reinterpret_cast<uint16_t*>(src_a + OFF), kWidth, \
+ dst_argb_opt + OFF, kStrideB, kWidth, NEG kHeight, ATTEN); \
+ } \
+ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(src_a); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTQPLANAR16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, \
+ ALIGN, YALIGN, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_ + 1, _Any, +, 0, 0, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Unaligned, +, 2, 0, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Invert, -, 0, 0, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, 0, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Premult, +, 0, 1, S_DEPTH)
+#else
+#define TESTQPLANAR16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, \
+ ALIGN, YALIGN, S_DEPTH) \
+ TESTQPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, benchmark_width_, _Opt, +, 0, 0, S_DEPTH)
+#endif
+
+#define I010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define I010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define J010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V010AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V010AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define I210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define I210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define J210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V210AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V210AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I210AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define I410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define I410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvI601Constants, k, \
+ l, m)
+#define J410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define J410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvJPEGConstants, k, \
+ l, m)
+#define F410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define F410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvF709Constants, k, \
+ l, m)
+#define H410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define H410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvH709Constants, k, \
+ l, m)
+#define U410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define U410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuv2020Constants, k, \
+ l, m)
+#define V410AlphaToARGB(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToARGBMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define V410AlphaToABGR(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I410AlphaToABGRMatrix(a, b, c, d, e, f, g, h, i, j, &kYuvV2020Constants, k, \
+ l, m)
+#define I010AlphaToARGBFilter(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrixFilter(a, b, c, d, e, f, g, h, i, j, \
+ &kYuvI601Constants, k, l, m, kFilterBilinear)
+#define I210AlphaToARGBFilter(a, b, c, d, e, f, g, h, i, j, k, l, m) \
+ I010AlphaToARGBMatrixFilter(a, b, c, d, e, f, g, h, i, j, \
+ &kYuvI601Constants, k, l, m, kFilterBilinear)
+
+// These conversions are only optimized for x86
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+TESTQPLANAR16TOB(I010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V010Alpha, 2, 2, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V010Alpha, 2, 2, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V210Alpha, 2, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V210Alpha, 2, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(J410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(H410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(F410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(U410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V410Alpha, 1, 1, ARGB, 4, 4, 1, 10)
+TESTQPLANAR16TOB(V410Alpha, 1, 1, ABGR, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I010Alpha, 2, 2, ARGBFilter, 4, 4, 1, 10)
+TESTQPLANAR16TOB(I210Alpha, 2, 1, ARGBFilter, 4, 4, 1, 10)
+#endif // DISABLE_SLOW_TESTS
+
+#define TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, W1280, N, NEG, SOFF, DOFF, S_DEPTH) \
+ TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
+ const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X) * 2; \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y) * 2; \
+ const int kBpc = 2; \
+ align_buffer_page_end(src_y, kWidth * kHeight * kBpc + SOFF); \
+ align_buffer_page_end(src_uv, kSizeUV * kBpc + SOFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight + DOFF); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight + DOFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ reinterpret_cast<uint16_t*>(src_y + SOFF)[i] = \
+ (fastrand() & (((uint16_t)(-1)) << (16 - S_DEPTH))); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ reinterpret_cast<uint16_t*>(src_uv + SOFF)[i] = \
+ (fastrand() & (((uint16_t)(-1)) << (16 - S_DEPTH))); \
+ } \
+ memset(dst_argb_c + DOFF, 1, kStrideB * kHeight); \
+ memset(dst_argb_opt + DOFF, 101, kStrideB * kHeight); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_PLANAR##To##FMT_B(reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_uv + SOFF), \
+ kStrideUV, dst_argb_c + DOFF, kStrideB, kWidth, \
+ NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_PLANAR##To##FMT_B(reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
+ reinterpret_cast<uint16_t*>(src_uv + SOFF), \
+ kStrideUV, dst_argb_opt + DOFF, kStrideB, kWidth, \
+ NEG kHeight); \
+ } \
+ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
+ ASSERT_EQ(dst_argb_c[i + DOFF], dst_argb_opt[i + DOFF]); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_uv); \
+ free_aligned_buffer_page_end(dst_argb_c); \
+ free_aligned_buffer_page_end(dst_argb_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTBP16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, S_DEPTH) \
+ TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, YALIGN, \
+ benchmark_width_ + 1, _Any, +, 0, 0, S_DEPTH) \
+ TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, YALIGN, \
+ benchmark_width_, _Unaligned, +, 4, 4, S_DEPTH) \
+ TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, YALIGN, \
+ benchmark_width_, _Invert, -, 0, 0, S_DEPTH) \
+ TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, YALIGN, \
+ benchmark_width_, _Opt, +, 0, 0, S_DEPTH)
+#else
+#define TESTBP16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
+ YALIGN, S_DEPTH) \
+ TESTBP16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, YALIGN, \
+ benchmark_width_, _Opt, +, 0, 0, S_DEPTH)
+#endif
+
+#define P010ToARGB(a, b, c, d, e, f, g, h) \
+ P010ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P210ToARGB(a, b, c, d, e, f, g, h) \
+ P210ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P410ToARGB(a, b, c, d, e, f, g, h) \
+ P410ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P010ToAR30(a, b, c, d, e, f, g, h) \
+ P010ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P210ToAR30(a, b, c, d, e, f, g, h) \
+ P210ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P410ToAR30(a, b, c, d, e, f, g, h) \
+ P410ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+
+#define P012ToARGB(a, b, c, d, e, f, g, h) \
+ P012ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P212ToARGB(a, b, c, d, e, f, g, h) \
+ P212ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P412ToARGB(a, b, c, d, e, f, g, h) \
+ P412ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P012ToAR30(a, b, c, d, e, f, g, h) \
+ P012ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P212ToAR30(a, b, c, d, e, f, g, h) \
+ P212ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P412ToAR30(a, b, c, d, e, f, g, h) \
+ P412ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+
+#define P016ToARGB(a, b, c, d, e, f, g, h) \
+ P016ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P216ToARGB(a, b, c, d, e, f, g, h) \
+ P216ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P416ToARGB(a, b, c, d, e, f, g, h) \
+ P416ToARGBMatrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P016ToAR30(a, b, c, d, e, f, g, h) \
+ P016ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P216ToAR30(a, b, c, d, e, f, g, h) \
+ P216ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+#define P416ToAR30(a, b, c, d, e, f, g, h) \
+ P416ToAR30Matrix(a, b, c, d, e, f, &kYuvH709Constants, g, h)
+
+#define P010ToARGBFilter(a, b, c, d, e, f, g, h) \
+ P010ToARGBMatrixFilter(a, b, c, d, e, f, &kYuvH709Constants, g, h, \
+ kFilterBilinear)
+#define P210ToARGBFilter(a, b, c, d, e, f, g, h) \
+ P210ToARGBMatrixFilter(a, b, c, d, e, f, &kYuvH709Constants, g, h, \
+ kFilterBilinear)
+#define P010ToAR30Filter(a, b, c, d, e, f, g, h) \
+ P010ToAR30MatrixFilter(a, b, c, d, e, f, &kYuvH709Constants, g, h, \
+ kFilterBilinear)
+#define P210ToAR30Filter(a, b, c, d, e, f, g, h) \
+ P210ToAR30MatrixFilter(a, b, c, d, e, f, &kYuvH709Constants, g, h, \
+ kFilterBilinear)
+
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+TESTBP16TOB(P010, 2, 2, ARGB, 4, 4, 1, 10)
+TESTBP16TOB(P210, 2, 1, ARGB, 4, 4, 1, 10)
+TESTBP16TOB(P410, 1, 1, ARGB, 4, 4, 1, 10)
+TESTBP16TOB(P012, 2, 2, ARGB, 4, 4, 1, 12)
+TESTBP16TOB(P212, 2, 1, ARGB, 4, 4, 1, 12)
+TESTBP16TOB(P412, 1, 1, ARGB, 4, 4, 1, 12)
+TESTBP16TOB(P016, 2, 2, ARGB, 4, 4, 1, 16)
+TESTBP16TOB(P216, 2, 1, ARGB, 4, 4, 1, 16)
+TESTBP16TOB(P416, 1, 1, ARGB, 4, 4, 1, 16)
+TESTBP16TOB(P010, 2, 2, ARGBFilter, 4, 4, 1, 10)
+TESTBP16TOB(P210, 2, 1, ARGBFilter, 4, 4, 1, 10)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTBP16TOB(P010, 2, 2, AR30, 4, 4, 1, 10)
+TESTBP16TOB(P210, 2, 1, AR30, 4, 4, 1, 10)
+TESTBP16TOB(P410, 1, 1, AR30, 4, 4, 1, 10)
+TESTBP16TOB(P012, 2, 2, AR30, 4, 4, 1, 12)
+TESTBP16TOB(P212, 2, 1, AR30, 4, 4, 1, 12)
+TESTBP16TOB(P412, 1, 1, AR30, 4, 4, 1, 12)
+TESTBP16TOB(P016, 2, 2, AR30, 4, 4, 1, 16)
+TESTBP16TOB(P216, 2, 1, AR30, 4, 4, 1, 16)
+TESTBP16TOB(P416, 1, 1, AR30, 4, 4, 1, 16)
+TESTBP16TOB(P010, 2, 2, AR30Filter, 4, 4, 1, 10)
+TESTBP16TOB(P210, 2, 1, AR30Filter, 4, 4, 1, 10)
+#endif // LITTLE_ENDIAN_ONLY_TEST
+#endif // DISABLE_SLOW_TESTS
+
+static int Clamp(int y) {
+ if (y < 0) {
+ y = 0;
+ }
+ if (y > 255) {
+ y = 255;
+ }
+ return y;
+}
+
+static int Clamp10(int y) {
+ if (y < 0) {
+ y = 0;
+ }
+ if (y > 1023) {
+ y = 1023;
+ }
+ return y;
+}
+
+// Test 8 bit YUV to 8 bit RGB
+TEST_F(LibYUVConvertTest, TestH420ToARGB) {
+ const int kSize = 256;
+ int histogram_b[256];
+ int histogram_g[256];
+ int histogram_r[256];
+ memset(histogram_b, 0, sizeof(histogram_b));
+ memset(histogram_g, 0, sizeof(histogram_g));
+ memset(histogram_r, 0, sizeof(histogram_r));
+ align_buffer_page_end(orig_yuv, kSize + kSize / 2 * 2);
+ align_buffer_page_end(argb_pixels, kSize * 4);
+ uint8_t* orig_y = orig_yuv;
+ uint8_t* orig_u = orig_y + kSize;
+ uint8_t* orig_v = orig_u + kSize / 2;
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_y[i] = i;
+ }
+ for (int i = 0; i < kSize / 2; ++i) {
+ orig_u[i] = 128; // 128 is 0.
+ orig_v[i] = 128;
+ }
+
+ H420ToARGB(orig_y, 0, orig_u, 0, orig_v, 0, argb_pixels, 0, kSize, 1);
+
+ for (int i = 0; i < kSize; ++i) {
+ int b = argb_pixels[i * 4 + 0];
+ int g = argb_pixels[i * 4 + 1];
+ int r = argb_pixels[i * 4 + 2];
+ int a = argb_pixels[i * 4 + 3];
+ ++histogram_b[b];
+ ++histogram_g[g];
+ ++histogram_r[r];
+ // Reference formula for Y channel contribution in YUV to RGB conversions:
+ int expected_y = Clamp(static_cast<int>((i - 16) * 1.164f + 0.5f));
+ ASSERT_EQ(b, expected_y);
+ ASSERT_EQ(g, expected_y);
+ ASSERT_EQ(r, expected_y);
+ ASSERT_EQ(a, 255);
+ }
+
+ int count_b = 0;
+ int count_g = 0;
+ int count_r = 0;
+ for (int i = 0; i < kSize; ++i) {
+ if (histogram_b[i]) {
+ ++count_b;
+ }
+ if (histogram_g[i]) {
+ ++count_g;
+ }
+ if (histogram_r[i]) {
+ ++count_r;
+ }
+ }
+ printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
+
+ free_aligned_buffer_page_end(orig_yuv);
+ free_aligned_buffer_page_end(argb_pixels);
+}
+
+// Test 10 bit YUV to 8 bit RGB
+TEST_F(LibYUVConvertTest, TestH010ToARGB) {
+ const int kSize = 1024;
+ int histogram_b[1024];
+ int histogram_g[1024];
+ int histogram_r[1024];
+ memset(histogram_b, 0, sizeof(histogram_b));
+ memset(histogram_g, 0, sizeof(histogram_g));
+ memset(histogram_r, 0, sizeof(histogram_r));
+ align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
+ align_buffer_page_end(argb_pixels, kSize * 4);
+ uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
+ uint16_t* orig_u = orig_y + kSize;
+ uint16_t* orig_v = orig_u + kSize / 2;
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_y[i] = i;
+ }
+ for (int i = 0; i < kSize / 2; ++i) {
+ orig_u[i] = 512; // 512 is 0.
+ orig_v[i] = 512;
+ }
+
+ H010ToARGB(orig_y, 0, orig_u, 0, orig_v, 0, argb_pixels, 0, kSize, 1);
+
+ for (int i = 0; i < kSize; ++i) {
+ int b = argb_pixels[i * 4 + 0];
+ int g = argb_pixels[i * 4 + 1];
+ int r = argb_pixels[i * 4 + 2];
+ int a = argb_pixels[i * 4 + 3];
+ ++histogram_b[b];
+ ++histogram_g[g];
+ ++histogram_r[r];
+ int expected_y = Clamp(static_cast<int>((i - 64) * 1.164f / 4));
+ ASSERT_NEAR(b, expected_y, 1);
+ ASSERT_NEAR(g, expected_y, 1);
+ ASSERT_NEAR(r, expected_y, 1);
+ ASSERT_EQ(a, 255);
+ }
+
+ int count_b = 0;
+ int count_g = 0;
+ int count_r = 0;
+ for (int i = 0; i < kSize; ++i) {
+ if (histogram_b[i]) {
+ ++count_b;
+ }
+ if (histogram_g[i]) {
+ ++count_g;
+ }
+ if (histogram_r[i]) {
+ ++count_r;
+ }
+ }
+ printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
+
+ free_aligned_buffer_page_end(orig_yuv);
+ free_aligned_buffer_page_end(argb_pixels);
+}
+
+// Test 10 bit YUV to 10 bit RGB
+// Caveat: Result is near due to float rounding in expected
+// result.
+TEST_F(LibYUVConvertTest, TestH010ToAR30) {
+ const int kSize = 1024;
+ int histogram_b[1024];
+ int histogram_g[1024];
+ int histogram_r[1024];
+ memset(histogram_b, 0, sizeof(histogram_b));
+ memset(histogram_g, 0, sizeof(histogram_g));
+ memset(histogram_r, 0, sizeof(histogram_r));
+
+ align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
+ align_buffer_page_end(ar30_pixels, kSize * 4);
+ uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
+ uint16_t* orig_u = orig_y + kSize;
+ uint16_t* orig_v = orig_u + kSize / 2;
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_y[i] = i;
+ }
+ for (int i = 0; i < kSize / 2; ++i) {
+ orig_u[i] = 512; // 512 is 0.
+ orig_v[i] = 512;
+ }
+
+ H010ToAR30(orig_y, 0, orig_u, 0, orig_v, 0, ar30_pixels, 0, kSize, 1);
+
+ for (int i = 0; i < kSize; ++i) {
+ int b10 = reinterpret_cast<uint32_t*>(ar30_pixels)[i] & 1023;
+ int g10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 10) & 1023;
+ int r10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 20) & 1023;
+ int a2 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 30) & 3;
+ ++histogram_b[b10];
+ ++histogram_g[g10];
+ ++histogram_r[r10];
+ int expected_y = Clamp10(static_cast<int>((i - 64) * 1.164f + 0.5));
+ ASSERT_NEAR(b10, expected_y, 4);
+ ASSERT_NEAR(g10, expected_y, 4);
+ ASSERT_NEAR(r10, expected_y, 4);
+ ASSERT_EQ(a2, 3);
+ }
+
+ int count_b = 0;
+ int count_g = 0;
+ int count_r = 0;
+ for (int i = 0; i < kSize; ++i) {
+ if (histogram_b[i]) {
+ ++count_b;
+ }
+ if (histogram_g[i]) {
+ ++count_g;
+ }
+ if (histogram_r[i]) {
+ ++count_r;
+ }
+ }
+ printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
+
+ free_aligned_buffer_page_end(orig_yuv);
+ free_aligned_buffer_page_end(ar30_pixels);
+}
+
+// Test 10 bit YUV to 10 bit RGB
+// Caveat: Result is near due to float rounding in expected
+// result.
+TEST_F(LibYUVConvertTest, TestH010ToAB30) {
+ const int kSize = 1024;
+ int histogram_b[1024];
+ int histogram_g[1024];
+ int histogram_r[1024];
+ memset(histogram_b, 0, sizeof(histogram_b));
+ memset(histogram_g, 0, sizeof(histogram_g));
+ memset(histogram_r, 0, sizeof(histogram_r));
+
+ align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
+ align_buffer_page_end(ab30_pixels, kSize * 4);
+ uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
+ uint16_t* orig_u = orig_y + kSize;
+ uint16_t* orig_v = orig_u + kSize / 2;
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_y[i] = i;
+ }
+ for (int i = 0; i < kSize / 2; ++i) {
+ orig_u[i] = 512; // 512 is 0.
+ orig_v[i] = 512;
+ }
+
+ H010ToAB30(orig_y, 0, orig_u, 0, orig_v, 0, ab30_pixels, 0, kSize, 1);
+
+ for (int i = 0; i < kSize; ++i) {
+ int r10 = reinterpret_cast<uint32_t*>(ab30_pixels)[i] & 1023;
+ int g10 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 10) & 1023;
+ int b10 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 20) & 1023;
+ int a2 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 30) & 3;
+ ++histogram_b[b10];
+ ++histogram_g[g10];
+ ++histogram_r[r10];
+ int expected_y = Clamp10(static_cast<int>((i - 64) * 1.164f + 0.5));
+ ASSERT_NEAR(r10, expected_y, 4);
+ ASSERT_NEAR(g10, expected_y, 4);
+ ASSERT_NEAR(b10, expected_y, 4);
+ ASSERT_EQ(a2, 3);
+ }
+
+ int count_b = 0;
+ int count_g = 0;
+ int count_r = 0;
+ for (int i = 0; i < kSize; ++i) {
+ if (histogram_b[i]) {
+ ++count_b;
+ }
+ if (histogram_g[i]) {
+ ++count_g;
+ }
+ if (histogram_r[i]) {
+ ++count_r;
+ }
+ }
+ printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
+
+ free_aligned_buffer_page_end(orig_yuv);
+ free_aligned_buffer_page_end(ab30_pixels);
+}
+
+// Test 8 bit YUV to 10 bit RGB
+TEST_F(LibYUVConvertTest, TestH420ToAR30) {
+ const int kSize = 256;
+ const int kHistSize = 1024;
+ int histogram_b[kHistSize];
+ int histogram_g[kHistSize];
+ int histogram_r[kHistSize];
+ memset(histogram_b, 0, sizeof(histogram_b));
+ memset(histogram_g, 0, sizeof(histogram_g));
+ memset(histogram_r, 0, sizeof(histogram_r));
+ align_buffer_page_end(orig_yuv, kSize + kSize / 2 * 2);
+ align_buffer_page_end(ar30_pixels, kSize * 4);
+ uint8_t* orig_y = orig_yuv;
+ uint8_t* orig_u = orig_y + kSize;
+ uint8_t* orig_v = orig_u + kSize / 2;
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_y[i] = i;
+ }
+ for (int i = 0; i < kSize / 2; ++i) {
+ orig_u[i] = 128; // 128 is 0.
+ orig_v[i] = 128;
+ }
+
+ H420ToAR30(orig_y, 0, orig_u, 0, orig_v, 0, ar30_pixels, 0, kSize, 1);
+
+ for (int i = 0; i < kSize; ++i) {
+ int b10 = reinterpret_cast<uint32_t*>(ar30_pixels)[i] & 1023;
+ int g10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 10) & 1023;
+ int r10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 20) & 1023;
+ int a2 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 30) & 3;
+ ++histogram_b[b10];
+ ++histogram_g[g10];
+ ++histogram_r[r10];
+ int expected_y = Clamp10(static_cast<int>((i - 16) * 1.164f * 4.f + 0.5));
+ ASSERT_NEAR(b10, expected_y, 4);
+ ASSERT_NEAR(g10, expected_y, 4);
+ ASSERT_NEAR(r10, expected_y, 4);
+ ASSERT_EQ(a2, 3);
+ }
+
+ int count_b = 0;
+ int count_g = 0;
+ int count_r = 0;
+ for (int i = 0; i < kHistSize; ++i) {
+ if (histogram_b[i]) {
+ ++count_b;
+ }
+ if (histogram_g[i]) {
+ ++count_g;
+ }
+ if (histogram_r[i]) {
+ ++count_r;
+ }
+ }
+ printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
+
+ free_aligned_buffer_page_end(orig_yuv);
+ free_aligned_buffer_page_end(ar30_pixels);
+}
+
+// Test I400 with jpeg matrix is same as J400
+TEST_F(LibYUVConvertTest, TestI400) {
+ const int kSize = 256;
+ align_buffer_page_end(orig_i400, kSize);
+ align_buffer_page_end(argb_pixels_i400, kSize * 4);
+ align_buffer_page_end(argb_pixels_j400, kSize * 4);
+ align_buffer_page_end(argb_pixels_jpeg_i400, kSize * 4);
+ align_buffer_page_end(argb_pixels_h709_i400, kSize * 4);
+ align_buffer_page_end(argb_pixels_2020_i400, kSize * 4);
+
+ // Test grey scale
+ for (int i = 0; i < kSize; ++i) {
+ orig_i400[i] = i;
+ }
+
+ J400ToARGB(orig_i400, 0, argb_pixels_j400, 0, kSize, 1);
+ I400ToARGB(orig_i400, 0, argb_pixels_i400, 0, kSize, 1);
+ I400ToARGBMatrix(orig_i400, 0, argb_pixels_jpeg_i400, 0, &kYuvJPEGConstants,
+ kSize, 1);
+ I400ToARGBMatrix(orig_i400, 0, argb_pixels_h709_i400, 0, &kYuvH709Constants,
+ kSize, 1);
+ I400ToARGBMatrix(orig_i400, 0, argb_pixels_2020_i400, 0, &kYuv2020Constants,
+ kSize, 1);
+
+ ASSERT_EQ(0, argb_pixels_i400[0]);
+ ASSERT_EQ(0, argb_pixels_j400[0]);
+ ASSERT_EQ(0, argb_pixels_jpeg_i400[0]);
+ ASSERT_EQ(0, argb_pixels_h709_i400[0]);
+ ASSERT_EQ(0, argb_pixels_2020_i400[0]);
+ ASSERT_EQ(0, argb_pixels_i400[16 * 4]);
+ ASSERT_EQ(16, argb_pixels_j400[16 * 4]);
+ ASSERT_EQ(16, argb_pixels_jpeg_i400[16 * 4]);
+ ASSERT_EQ(0, argb_pixels_h709_i400[16 * 4]);
+ ASSERT_EQ(0, argb_pixels_2020_i400[16 * 4]);
+ ASSERT_EQ(130, argb_pixels_i400[128 * 4]);
+ ASSERT_EQ(128, argb_pixels_j400[128 * 4]);
+ ASSERT_EQ(128, argb_pixels_jpeg_i400[128 * 4]);
+ ASSERT_EQ(130, argb_pixels_h709_i400[128 * 4]);
+ ASSERT_EQ(130, argb_pixels_2020_i400[128 * 4]);
+ ASSERT_EQ(255, argb_pixels_i400[255 * 4]);
+ ASSERT_EQ(255, argb_pixels_j400[255 * 4]);
+ ASSERT_EQ(255, argb_pixels_jpeg_i400[255 * 4]);
+ ASSERT_EQ(255, argb_pixels_h709_i400[255 * 4]);
+ ASSERT_EQ(255, argb_pixels_2020_i400[255 * 4]);
+
+ for (int i = 0; i < kSize * 4; ++i) {
+ if ((i & 3) == 3) {
+ ASSERT_EQ(255, argb_pixels_j400[i]);
+ } else {
+ ASSERT_EQ(i / 4, argb_pixels_j400[i]);
+ }
+ ASSERT_EQ(argb_pixels_jpeg_i400[i], argb_pixels_j400[i]);
+ }
+
+ free_aligned_buffer_page_end(orig_i400);
+ free_aligned_buffer_page_end(argb_pixels_i400);
+ free_aligned_buffer_page_end(argb_pixels_j400);
+ free_aligned_buffer_page_end(argb_pixels_jpeg_i400);
+ free_aligned_buffer_page_end(argb_pixels_h709_i400);
+ free_aligned_buffer_page_end(argb_pixels_2020_i400);
+}
+
+// Test RGB24 to ARGB and back to RGB24
+TEST_F(LibYUVConvertTest, TestARGBToRGB24) {
+ const int kSize = 256;
+ align_buffer_page_end(orig_rgb24, kSize * 3);
+ align_buffer_page_end(argb_pixels, kSize * 4);
+ align_buffer_page_end(dest_rgb24, kSize * 3);
+
+ // Test grey scale
+ for (int i = 0; i < kSize * 3; ++i) {
+ orig_rgb24[i] = i;
+ }
+
+ RGB24ToARGB(orig_rgb24, 0, argb_pixels, 0, kSize, 1);
+ ARGBToRGB24(argb_pixels, 0, dest_rgb24, 0, kSize, 1);
+
+ for (int i = 0; i < kSize * 3; ++i) {
+ ASSERT_EQ(orig_rgb24[i], dest_rgb24[i]);
+ }
+
+ free_aligned_buffer_page_end(orig_rgb24);
+ free_aligned_buffer_page_end(argb_pixels);
+ free_aligned_buffer_page_end(dest_rgb24);
+}
+
+TEST_F(LibYUVConvertTest, TestARGBToRGB565) {
+ SIMD_ALIGNED(uint8_t orig_pixels[256][4]);
+ SIMD_ALIGNED(uint8_t dest_rgb565[256][2]);
+
+ for (int i = 0; i < 256; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ orig_pixels[i][j] = i;
+ }
+ }
+ ARGBToRGB565(&orig_pixels[0][0], 0, &dest_rgb565[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_rgb565[0][0], sizeof(dest_rgb565), 5381);
+ ASSERT_EQ(610919429u, checksum);
+}
+
+TEST_F(LibYUVConvertTest, TestYUY2ToARGB) {
+ SIMD_ALIGNED(uint8_t orig_pixels[256][2]);
+ SIMD_ALIGNED(uint8_t dest_argb[256][4]);
+
+ for (int i = 0; i < 256; ++i) {
+ for (int j = 0; j < 2; ++j) {
+ orig_pixels[i][j] = i;
+ }
+ }
+ YUY2ToARGB(&orig_pixels[0][0], 0, &dest_argb[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_argb[0][0], sizeof(dest_argb), 5381);
+#if defined(LIBYUV_UNLIMITED_DATA)
+ ASSERT_EQ(10343289u, checksum);
+#else
+ ASSERT_EQ(3486643515u, checksum);
+#endif
+}
+
+TEST_F(LibYUVConvertTest, TestUYVYToARGB) {
+ SIMD_ALIGNED(uint8_t orig_pixels[256][2]);
+ SIMD_ALIGNED(uint8_t dest_argb[256][4]);
+
+ for (int i = 0; i < 256; ++i) {
+ for (int j = 0; j < 2; ++j) {
+ orig_pixels[i][j] = i;
+ }
+ }
+ UYVYToARGB(&orig_pixels[0][0], 0, &dest_argb[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_argb[0][0], sizeof(dest_argb), 5381);
+#if defined(LIBYUV_UNLIMITED_DATA)
+ ASSERT_EQ(10343289u, checksum);
+#else
+ ASSERT_EQ(3486643515u, checksum);
+#endif
+}
+
+// Call P010ToARGBMatrixFilter() with a width that is not a multiple of 16.
+// Verify there is no stack buffer overflow in P410ToARGBRow_Any_AVX2().
+TEST_F(LibYUVConvertTest, P010ToARGBMatrixFilterOverflow) {
+ int width = 1024 + 15; // 1039
+ int height = 4;
+ int dst_stride_argb = width * 4;
+ int src_stride_uv = (width + 1) & ~1;
+ int src_uv_height = (height + 1) / 2;
+ align_buffer_page_end_16(src_y, width * height);
+ align_buffer_page_end_16(src_uv, src_stride_uv * src_uv_height);
+ align_buffer_page_end(dst_argb, dst_stride_argb * height);
+ memset(src_y, 0x02, width * height * sizeof(uint16_t));
+ memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t));
+ EXPECT_EQ(
+ 0, P010ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, kFilterBilinear));
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(src_uv);
+ free_aligned_buffer_page_end(dst_argb);
+}
+
+// Call P210ToARGBMatrixFilter() with a width that is not a multiple of 16.
+// Verify there is no stack buffer overflow in P410ToARGBRow_Any_AVX2().
+TEST_F(LibYUVConvertTest, P210ToARGBMatrixFilterOverflow) {
+ int width = 1024 + 15; // 1039
+ int height = 4;
+ int dst_stride_argb = width * 4;
+ int src_stride_uv = (width + 1) & ~1;
+ align_buffer_page_end_16(src_y, width * height);
+ align_buffer_page_end_16(src_uv, src_stride_uv * height);
+ align_buffer_page_end(dst_argb, dst_stride_argb * height);
+ memset(src_y, 0x02, width * height * sizeof(uint16_t));
+ memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t));
+ EXPECT_EQ(
+ 0, P210ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, kFilterBilinear));
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(src_uv);
+ free_aligned_buffer_page_end(dst_argb);
+}
+
+// Call P010ToAR30MatrixFilter() with a width that is not a multiple of 16.
+// Verify there is no stack buffer overflow in P410ToAR30Row_Any_AVX2().
+TEST_F(LibYUVConvertTest, P010ToAR30MatrixFilterOverflow) {
+ int width = 1024 + 15; // 1039
+ int height = 4;
+ int dst_stride_argb = width * 4;
+ int src_stride_uv = (width + 1) & ~1;
+ int src_uv_height = (height + 1) / 2;
+ align_buffer_page_end_16(src_y, width * height);
+ align_buffer_page_end_16(src_uv, src_stride_uv * src_uv_height);
+ align_buffer_page_end(dst_argb, dst_stride_argb * height);
+ memset(src_y, 0x02, width * height * sizeof(uint16_t));
+ memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t));
+ EXPECT_EQ(
+ 0, P010ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
+ dst_stride_argb, &kYuvI601Constants, width,
+ height, kFilterBilinear));
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(src_uv);
+ free_aligned_buffer_page_end(dst_argb);
+}
+
+// Call P210ToAR30MatrixFilter(() with a width that is not a multiple of 16.
+// Verify there is no stack buffer overflow in P410ToAR30Row_Any_AVX2().
+TEST_F(LibYUVConvertTest, P210ToAR30MatrixFilterOverflow) {
+ int width = 1024 + 15; // 1039
+ int height = 4;
+ int dst_stride_ar30 = width * 4;
+ int src_stride_uv = (width + 1) & ~1;
+ align_buffer_page_end_16(src_y, width * height);
+ align_buffer_page_end_16(src_uv, src_stride_uv * height);
+ align_buffer_page_end(dst_ar30, dst_stride_ar30 * height);
+ memset(src_y, 0x02, width * height * sizeof(uint16_t));
+ memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t));
+ EXPECT_EQ(
+ 0, P210ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_ar30,
+ dst_stride_ar30, &kYuvI601Constants, width,
+ height, kFilterBilinear));
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(src_uv);
+ free_aligned_buffer_page_end(dst_ar30);
+}
+
+#ifdef ENABLE_ROW_TESTS
+TEST_F(LibYUVConvertTest, TestARGBToUVMatrixRow_Opt) {
+ const int kMaxWidth = 128;
+ SIMD_ALIGNED(uint8_t orig_argb_pixels[kMaxWidth * 4 * 2]);
+ SIMD_ALIGNED(uint8_t dest_u_c[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dest_v_c[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dest_u_opt[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dest_v_opt[kMaxWidth]);
+
+ for (int i = 0; i < kMaxWidth * 4 * 2; ++i) {
+ orig_argb_pixels[i] = i * 43;
+ }
+
+#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ if (has_neon) {
+ for (int width = 1; width <= kMaxWidth; ++width) {
+ for (int height = 1; height <= 2; ++height) {
+ memset(dest_u_c, 0, sizeof(dest_u_c));
+ memset(dest_v_c, 0, sizeof(dest_v_c));
+ memset(dest_u_opt, 0, sizeof(dest_u_opt));
+ memset(dest_v_opt, 0, sizeof(dest_v_opt));
+
+ int src_stride = (height == 1) ? 0 : kMaxWidth * 4;
+
+ ARGBToUVMatrixRow_C(&orig_argb_pixels[0], src_stride, &dest_u_c[0],
+ &dest_v_c[0], width, &kArgbI601Constants);
+ ARGBToUVMatrixRow_Any_NEON(&orig_argb_pixels[0], src_stride,
+ &dest_u_opt[0], &dest_v_opt[0], width,
+ &kArgbI601Constants);
+
+ int half_width = (width + 1) / 2;
+ for (int i = 0; i < half_width; ++i) {
+ ASSERT_EQ(dest_u_c[i], dest_u_opt[i])
+ << "u mismatch at " << i << " width " << width << " height "
+ << height;
+ ASSERT_EQ(dest_v_c[i], dest_v_opt[i])
+ << "v mismatch at " << i << " width " << width << " height "
+ << height;
+ }
+ }
+ }
+ }
+#endif
+}
+#endif
+
+#if !defined(DISABLE_SLOW_TESTS) && \
+ (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__))
+// TODO(fbarchard): Consider _set_new_mode(0) to make malloc return NULL
+
+#ifndef DISABLE_SLOW_TESTS
+TEST_F(LibYUVConvertTest, TestI400LargeSize) {
+ // The width and height are chosen as follows:
+ // - kWidth * kHeight is not a multiple of 8: This lets us to use the Any
+ // variant of the conversion function.
+ const int kWidth = 1073741823;
+ const int kHeight = 2;
+
+#if defined(__aarch64__)
+ // Infer malloc can accept a large size for cpu with dot product (a76/a55)
+ int has_large_malloc = TestCpuFlag(kCpuHasNeonDotProd);
+#elif defined(__SANITIZE_MEMORY__)
+ int has_large_malloc = 0;
+#elif defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+ int has_large_malloc = 0;
+#else
+ int has_large_malloc = 1;
+#endif
+#else
+ int has_large_malloc = 1;
+#endif
+ if (!has_large_malloc) {
+ GTEST_SKIP() << "WARNING: Large allocation may assert for "
+ << (size_t)kWidth * kHeight << " bytes";
+ }
+
+ // Allocate one extra column so that the coalesce optimizations do not trigger
+ // in convert_argb.cc (they are triggered only when stride is equal to width).
+ const size_t kStride = kWidth + 1;
+
+ printf("WARNING: attempting to allocate I400 image of %zd bytes\n",
+ (size_t)kWidth * kHeight);
+ fflush(stdout);
+ align_buffer_page_end(orig_i400, (size_t)kWidth * kHeight);
+ if (!orig_i400) {
+ GTEST_SKIP() << "WARNING: unable to allocate I400 image of "
+ << (size_t)kWidth * kHeight << " bytes";
+ }
+ printf("INFO: allocate I400 image returned %p\n", orig_i400);
+ fflush(stdout);
+ align_buffer_page_end(dest_argb, (size_t)kWidth * kHeight * 4);
+ if (!dest_argb) {
+ free_aligned_buffer_page_end(orig_i400);
+ GTEST_SKIP() << "WARNING: unable to allocate ARGB image of "
+ << (size_t)kWidth * kHeight * 4 << " bytes";
+ }
+ printf("INFO: allocate ARGB image returned %p\n", dest_argb);
+ fflush(stdout);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ orig_i400[i] = i % 256;
+ }
+ ASSERT_EQ(I400ToARGBMatrix(orig_i400, kStride, dest_argb, kWidth,
+ &kYuvJPEGConstants, kWidth, kHeight),
+ 0);
+ free_aligned_buffer_page_end(dest_argb);
+ free_aligned_buffer_page_end(orig_i400);
+}
+#endif // DISABLE_SLOW_TESTS
+#endif // !defined(DISABLE_SLOW_TESTS) && \
+ // (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__))
+
+#endif // !defined(LEAN_TESTS)
+
+#define TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
+ const int kStrideA = \
+ (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
+ const int kStrideY = kWidth; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X) * 2; \
+ const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_argb, \
+ kStrideA * kHeightA * (int)sizeof(TYPE_A) + OFF); \
+ align_buffer_page_end(dst_y_c, kStrideY * kHeight); \
+ align_buffer_page_end(dst_uv_c, kSizeUV); \
+ align_buffer_page_end(dst_y_opt, kStrideY * kHeight); \
+ align_buffer_page_end(dst_uv_opt, kSizeUV); \
+ for (int i = 0; i < kStrideA * kHeightA * (int)sizeof(TYPE_A); ++i) { \
+ src_argb[i + OFF] = (fastrand() & 0xff); \
+ } \
+ memset(dst_y_c, 1, kStrideY * kHeight); \
+ memset(dst_uv_c, 2, kSizeUV); \
+ memset(dst_y_opt, 101, kStrideY * kHeight); \
+ memset(dst_uv_opt, 102, kSizeUV); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_B((TYPE_A*)(src_argb + OFF), kStrideA, dst_y_c, kStrideY, \
+ dst_uv_c, kStrideUV, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_B((TYPE_A*)(src_argb + OFF), kStrideA, dst_y_opt, \
+ kStrideY, dst_uv_opt, kStrideUV, kWidth, NEG kHeight); \
+ } \
+ for (int i = 0; i < kStrideY * kHeight; ++i) { \
+ ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
+ } \
+ for (int i = 0; i < kSizeUV; ++i) { \
+ ASSERT_EQ(dst_uv_c[i], dst_uv_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_y_c); \
+ free_aligned_buffer_page_end(dst_uv_c); \
+ free_aligned_buffer_page_end(dst_y_opt); \
+ free_aligned_buffer_page_end(dst_uv_opt); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOBP(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y) \
+ TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, benchmark_width_, _Unaligned, +, 4) \
+ TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, benchmark_width_, _Invert, -, 0) \
+ TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, benchmark_width_, _Opt, +, 0)
+#else
+#define TESTATOBP(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y) \
+ TESTATOBPI(FMT_A, TYPE_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, SUBSAMP_X, \
+ SUBSAMP_Y, benchmark_width_, _Opt, +, 0)
+#endif
+
+TESTATOBP(RAW, uint8_t, 3, 3, 1, NV21, 2, 2)
+TESTATOBP(RGB24, uint8_t, 3, 3, 1, NV12, 2, 2)
+TESTATOBP(RAW, uint8_t, 3, 3, 1, JNV21, 2, 2)
+
+} // namespace libyuv
diff --git a/libs/libyuv/unit_test/convert_test.cc b/libs/libyuv/unit_test/convert_test.cc
index b676954b21..aaba43f29d 100644
--- a/libs/libyuv/unit_test/convert_test.cc
+++ b/libs/libyuv/unit_test/convert_test.cc
@@ -10,9 +10,9 @@
#include <assert.h>
#include <stdlib.h>
+#include <string.h>
#include <time.h>
-#include "libyuv/basic_types.h"
#include "libyuv/compare.h"
#include "libyuv/convert.h"
#include "libyuv/convert_argb.h"
@@ -27,21 +27,21 @@
#include "libyuv/rotate.h"
#include "libyuv/video_common.h"
-#ifdef ENABLE_ROW_TESTS
-#include "libyuv/row.h" /* For ARGBToAR30Row_AVX2 */
+#if (defined(__riscv) && !defined(__clang__)) || defined(__hexagon__)
+#define DISABLE_SLOW_TESTS
+#undef ENABLE_FULL_TESTS
+#define LEAN_TESTS
#endif
-#if defined(__arm__) || defined(__aarch64__)
-// arm version subsamples by summing 4 pixels then multiplying by matrix with
-// 4x smaller coefficients which are rounded to nearest integer.
-#define ARM_YUV_ERROR 4
-#else
-#define ARM_YUV_ERROR 0
+// Some functions fail on big endian. Enable these tests on all cpus except
+// PowerPC, but they are not optimized so disabled by default.
+#if !defined(DISABLE_SLOW_TESTS) && !defined(__powerpc__)
+#define LITTLE_ENDIAN_ONLY_TEST 1
#endif
-
-// Some functions fail on big endian. Enable these tests on all cpus except PowerPC
-#if !defined(__powerpc__)
-#define LITTLE_ENDIAN_TEST 1
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
#endif
namespace libyuv {
@@ -50,76 +50,90 @@ namespace libyuv {
#define AR30ToAR30 ARGBCopy
#define ABGRToABGR ARGBCopy
-#define SUBSAMPLE(v, a) ((((v) + (a)-1)) / (a))
+// subsample amount uses a divide.
+#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))
+
+#define ALIGNINT(V, ALIGN) (((V) + (ALIGN) - 1) / (ALIGN) * (ALIGN))
// Planar test
#define TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
- DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF) \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, DOY, \
+ SRC_DEPTH) \
TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
static_assert(SRC_BPC == 1 || SRC_BPC == 2, "SRC BPC unsupported"); \
static_assert(DST_BPC == 1 || DST_BPC == 2, "DST BPC unsupported"); \
static_assert(SRC_SUBSAMP_X == 1 || SRC_SUBSAMP_X == 2, \
- "DST SRC_SUBSAMP_X unsupported"); \
+ "SRC_SUBSAMP_X unsupported"); \
static_assert(SRC_SUBSAMP_Y == 1 || SRC_SUBSAMP_Y == 2, \
- "DST SRC_SUBSAMP_Y unsupported"); \
+ "SRC_SUBSAMP_Y unsupported"); \
static_assert(DST_SUBSAMP_X == 1 || DST_SUBSAMP_X == 2, \
- "DST DST_SUBSAMP_X unsupported"); \
+ "DST_SUBSAMP_X unsupported"); \
static_assert(DST_SUBSAMP_Y == 1 || DST_SUBSAMP_Y == 2, \
- "DST DST_SUBSAMP_Y unsupported"); \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ "DST_SUBSAMP_Y unsupported"); \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
const int kSrcHalfWidth = SUBSAMPLE(kWidth, SRC_SUBSAMP_X); \
const int kSrcHalfHeight = SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); \
const int kDstHalfWidth = SUBSAMPLE(kWidth, DST_SUBSAMP_X); \
const int kDstHalfHeight = SUBSAMPLE(kHeight, DST_SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight* SRC_BPC + OFF); \
+ align_buffer_page_end(src_y, kWidth * kHeight * SRC_BPC + OFF); \
align_buffer_page_end(src_u, \
- kSrcHalfWidth* kSrcHalfHeight* SRC_BPC + OFF); \
+ kSrcHalfWidth * kSrcHalfHeight * SRC_BPC + OFF); \
align_buffer_page_end(src_v, \
- kSrcHalfWidth* kSrcHalfHeight* SRC_BPC + OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight* DST_BPC); \
- align_buffer_page_end(dst_u_c, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- align_buffer_page_end(dst_v_c, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight* DST_BPC); \
- align_buffer_page_end(dst_u_opt, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- align_buffer_page_end(dst_v_opt, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
+ kSrcHalfWidth * kSrcHalfHeight * SRC_BPC + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_u_c, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_v_c, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_u_opt, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_v_opt, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC); \
MemRandomize(src_y + OFF, kWidth * kHeight * SRC_BPC); \
MemRandomize(src_u + OFF, kSrcHalfWidth * kSrcHalfHeight * SRC_BPC); \
MemRandomize(src_v + OFF, kSrcHalfWidth * kSrcHalfHeight * SRC_BPC); \
- memset(dst_y_c, 1, kWidth* kHeight* DST_BPC); \
- memset(dst_u_c, 2, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- memset(dst_v_c, 3, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- memset(dst_y_opt, 101, kWidth* kHeight* DST_BPC); \
- memset(dst_u_opt, 102, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
- memset(dst_v_opt, 103, kDstHalfWidth* kDstHalfHeight* DST_BPC); \
+ SRC_T* src_y_p = reinterpret_cast<SRC_T*>(src_y + OFF); \
+ SRC_T* src_u_p = reinterpret_cast<SRC_T*>(src_u + OFF); \
+ SRC_T* src_v_p = reinterpret_cast<SRC_T*>(src_v + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y_p[i] = src_y_p[i] & ((1 << SRC_DEPTH) - 1); \
+ } \
+ for (int i = 0; i < kSrcHalfWidth * kSrcHalfHeight; ++i) { \
+ src_u_p[i] = src_u_p[i] & ((1 << SRC_DEPTH) - 1); \
+ src_v_p[i] = src_v_p[i] & ((1 << SRC_DEPTH) - 1); \
+ } \
+ memset(dst_y_c, 1, kWidth * kHeight * DST_BPC); \
+ memset(dst_u_c, 2, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_v_c, 3, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_y_opt, 101, kWidth * kHeight * DST_BPC); \
+ memset(dst_u_opt, 102, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_v_opt, 103, kDstHalfWidth * kDstHalfHeight * DST_BPC); \
MaskCpuFlags(disable_cpu_flags_); \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
- reinterpret_cast<SRC_T*>(src_y + OFF), kWidth, \
- reinterpret_cast<SRC_T*>(src_u + OFF), kSrcHalfWidth, \
- reinterpret_cast<SRC_T*>(src_v + OFF), kSrcHalfWidth, \
- reinterpret_cast<DST_T*>(dst_y_c), kWidth, \
+ src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \
+ DOY ? reinterpret_cast<DST_T*>(dst_y_c) : NULL, kWidth, \
reinterpret_cast<DST_T*>(dst_u_c), kDstHalfWidth, \
reinterpret_cast<DST_T*>(dst_v_c), kDstHalfWidth, kWidth, \
NEG kHeight); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
- reinterpret_cast<SRC_T*>(src_y + OFF), kWidth, \
- reinterpret_cast<SRC_T*>(src_u + OFF), kSrcHalfWidth, \
- reinterpret_cast<SRC_T*>(src_v + OFF), kSrcHalfWidth, \
- reinterpret_cast<DST_T*>(dst_y_opt), kWidth, \
+ src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \
+ DOY ? reinterpret_cast<DST_T*>(dst_y_opt) : NULL, kWidth, \
reinterpret_cast<DST_T*>(dst_u_opt), kDstHalfWidth, \
reinterpret_cast<DST_T*>(dst_v_opt), kDstHalfWidth, kWidth, \
NEG kHeight); \
} \
- for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
- EXPECT_EQ(dst_y_c[i], dst_y_opt[i]); \
+ if (DOY) { \
+ for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
+ ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
+ } \
} \
for (int i = 0; i < kDstHalfWidth * kDstHalfHeight * DST_BPC; ++i) { \
- EXPECT_EQ(dst_u_c[i], dst_u_opt[i]); \
- EXPECT_EQ(dst_v_c[i], dst_v_opt[i]); \
+ ASSERT_EQ(dst_u_c[i], dst_u_opt[i]); \
+ ASSERT_EQ(dst_v_c[i], dst_v_opt[i]); \
} \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_u_c); \
@@ -132,55 +146,91 @@ namespace libyuv {
free_aligned_buffer_page_end(src_v); \
}
+#if defined(ENABLE_FULL_TESTS)
#define TESTPLANARTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
- DST_SUBSAMP_X, DST_SUBSAMP_Y) \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
- benchmark_width_ - 4, _Any, +, 0) \
+ benchmark_width_ + 1, _Any, +, 0, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
- benchmark_width_, _Unaligned, +, 1) \
+ benchmark_width_, _Unaligned, +, 2, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
- benchmark_width_, _Invert, -, 0) \
+ benchmark_width_, _Invert, -, 0, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
- benchmark_width_, _Opt, +, 0)
+ benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH) \
+ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH)
+#else
+#define TESTPLANARTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
+ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH) \
+ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH)
+#endif
-TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2)
-TESTPLANARTOP(I422, uint8_t, 1, 2, 1, I420, uint8_t, 1, 2, 2)
-TESTPLANARTOP(I444, uint8_t, 1, 1, 1, I420, uint8_t, 1, 2, 2)
-TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I422, uint8_t, 1, 2, 1)
-TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I444, uint8_t, 1, 1, 1)
-TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420Mirror, uint8_t, 1, 2, 2)
-TESTPLANARTOP(I422, uint8_t, 1, 2, 1, I422, uint8_t, 1, 2, 1)
-TESTPLANARTOP(I444, uint8_t, 1, 1, 1, I444, uint8_t, 1, 1, 1)
-TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I010, uint16_t, 2, 2, 2)
-TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I420, uint8_t, 1, 2, 2)
-TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I010, uint16_t, 2, 2, 2)
-TESTPLANARTOP(H010, uint16_t, 2, 2, 2, H010, uint16_t, 2, 2, 2)
-TESTPLANARTOP(H010, uint16_t, 2, 2, 2, H420, uint8_t, 1, 2, 2)
-TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOP(I422, uint8_t, 1, 2, 1, I420, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOP(I444, uint8_t, 1, 1, 1, I420, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I422, uint8_t, 1, 2, 1, 8)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I444, uint8_t, 1, 1, 1, 8)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420Mirror, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOP(I422, uint8_t, 1, 2, 1, I422, uint8_t, 1, 2, 1, 8)
+TESTPLANARTOP(I422, uint8_t, 1, 2, 1, I444, uint8_t, 1, 1, 1, 8)
+TESTPLANARTOP(I444, uint8_t, 1, 1, 1, I444, uint8_t, 1, 1, 1, 8)
+TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I010, uint16_t, 2, 2, 2, 10)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I010, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I012, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOP(H010, uint16_t, 2, 2, 2, H010, uint16_t, 2, 2, 2, 10)
+TESTPLANARTOP(H010, uint16_t, 2, 2, 2, H420, uint8_t, 1, 2, 2, 10)
+TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H012, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOP(J420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I410, uint16_t, 2, 1, 1, 10)
+TESTPLANARTOP(I210, uint16_t, 2, 2, 1, I410, uint16_t, 2, 1, 1, 10)
+TESTPLANARTOP(I012, uint16_t, 2, 2, 2, I412, uint16_t, 2, 1, 1, 12)
+TESTPLANARTOP(I212, uint16_t, 2, 2, 1, I412, uint16_t, 2, 1, 1, 12)
+TESTPLANARTOP(I410, uint16_t, 2, 1, 1, I010, uint16_t, 2, 2, 2, 10)
+TESTPLANARTOP(I210, uint16_t, 2, 2, 1, I010, uint16_t, 2, 2, 2, 10)
+TESTPLANARTOP(I412, uint16_t, 2, 1, 1, I012, uint16_t, 2, 2, 2, 12)
+TESTPLANARTOP(I212, uint16_t, 2, 2, 1, I012, uint16_t, 2, 2, 2, 12)
+TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I420, uint8_t, 1, 2, 2, 10)
+TESTPLANARTOP(I210, uint16_t, 2, 2, 1, I420, uint8_t, 1, 2, 2, 10)
+TESTPLANARTOP(I210, uint16_t, 2, 2, 1, I422, uint8_t, 1, 2, 1, 10)
+TESTPLANARTOP(I410, uint16_t, 2, 1, 1, I420, uint8_t, 1, 2, 2, 10)
+TESTPLANARTOP(I410, uint16_t, 2, 1, 1, I444, uint8_t, 1, 1, 1, 10)
+TESTPLANARTOP(I012, uint16_t, 2, 2, 2, I420, uint8_t, 1, 2, 2, 12)
+TESTPLANARTOP(I212, uint16_t, 2, 2, 1, I420, uint8_t, 1, 2, 2, 12)
+TESTPLANARTOP(I212, uint16_t, 2, 2, 1, I422, uint8_t, 1, 2, 1, 12)
+TESTPLANARTOP(I412, uint16_t, 2, 1, 1, I420, uint8_t, 1, 2, 2, 12)
+TESTPLANARTOP(I412, uint16_t, 2, 1, 1, I444, uint8_t, 1, 1, 1, 12)
// Test Android 420 to I420
#define TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, \
SRC_SUBSAMP_Y, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
W1280, N, NEG, OFF, PN, OFF_U, OFF_V) \
- TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##_##PN##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##To##PN##N) { \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
const int kSizeUV = \
SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
align_buffer_page_end(src_uv, \
- kSizeUV*((PIXEL_STRIDE == 3) ? 3 : 2) + OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
+ kSizeUV * ((PIXEL_STRIDE == 3) ? 3 : 2) + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight); \
align_buffer_page_end(dst_u_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
SUBSAMPLE(kHeight, SUBSAMP_Y)); \
align_buffer_page_end(dst_v_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight); \
align_buffer_page_end(dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
SUBSAMPLE(kHeight, SUBSAMP_Y)); \
align_buffer_page_end(dst_v_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
@@ -199,12 +249,12 @@ TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2)
(fastrand() & 0xff); \
} \
} \
- memset(dst_y_c, 1, kWidth* kHeight); \
+ memset(dst_y_c, 1, kWidth * kHeight); \
memset(dst_u_c, 2, \
SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
memset(dst_v_c, 3, \
SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
+ memset(dst_y_opt, 101, kWidth * kHeight); \
memset(dst_u_opt, 102, \
SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
memset(dst_v_opt, 103, \
@@ -223,41 +273,23 @@ TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2)
dst_y_opt, kWidth, dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X), \
dst_v_opt, SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight); \
} \
- int max_diff = 0; \
for (int i = 0; i < kHeight; ++i) { \
for (int j = 0; j < kWidth; ++j) { \
- int abs_diff = abs(static_cast<int>(dst_y_c[i * kWidth + j]) - \
- static_cast<int>(dst_y_opt[i * kWidth + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
} \
} \
- EXPECT_EQ(0, max_diff); \
for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
- int abs_diff = abs( \
- static_cast<int>(dst_u_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]) - \
- static_cast<int>( \
- dst_u_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_u_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j], \
+ dst_u_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]); \
} \
} \
- EXPECT_LE(max_diff, 3); \
for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
- int abs_diff = abs( \
- static_cast<int>(dst_v_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]) - \
- static_cast<int>( \
- dst_v_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_v_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j], \
+ dst_v_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]); \
} \
} \
- EXPECT_LE(max_diff, 3); \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_u_c); \
free_aligned_buffer_page_end(dst_v_c); \
@@ -268,114 +300,123 @@ TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2)
free_aligned_buffer_page_end(src_uv); \
}
+#if defined(ENABLE_FULL_TESTS)
#define TESTAPLANARTOP(SRC_FMT_PLANAR, PN, PIXEL_STRIDE, OFF_U, OFF_V, \
SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, SUBSAMP_X, \
SUBSAMP_Y) \
TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_ - 4, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_ + 1, \
_Any, +, 0, PN, OFF_U, OFF_V) \
TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
- _Unaligned, +, 1, PN, OFF_U, OFF_V) \
+ _Unaligned, +, 2, PN, OFF_U, OFF_V) \
TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Invert, \
-, 0, PN, OFF_U, OFF_V) \
TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, \
0, PN, OFF_U, OFF_V)
+#else
+#define TESTAPLANARTOP(SRC_FMT_PLANAR, PN, PIXEL_STRIDE, OFF_U, OFF_V, \
+ SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, SUBSAMP_X, \
+ SUBSAMP_Y) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, \
+ 0, PN, OFF_U, OFF_V)
+#endif
TESTAPLANARTOP(Android420, I420, 1, 0, 0, 2, 2, I420, 2, 2)
TESTAPLANARTOP(Android420, NV12, 2, 0, 1, 2, 2, I420, 2, 2)
TESTAPLANARTOP(Android420, NV21, 2, 1, 0, 2, 2, I420, 2, 2)
+#undef TESTAPLANARTOP
+#undef TESTAPLANARTOPI
// wrapper to keep API the same
-int I400ToNV21(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* /* src_u */,
- int /* src_stride_u */,
- const uint8_t* /* src_v */,
- int /* src_stride_v */,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
+static int I400ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* /* src_u */,
+ int /* src_stride_u */,
+ const uint8_t* /* src_v */,
+ int /* src_stride_v */,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
return I400ToNV21(src_y, src_stride_y, dst_y, dst_stride_y, dst_vu,
dst_stride_vu, width, height);
}
-#define TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, W1280, N, NEG, OFF) \
+#define TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, \
+ SRC_DEPTH) \
TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ static_assert(SRC_BPC == 1 || SRC_BPC == 2, "SRC BPC unsupported"); \
+ static_assert(DST_BPC == 1 || DST_BPC == 2, "DST BPC unsupported"); \
+ static_assert(SRC_SUBSAMP_X == 1 || SRC_SUBSAMP_X == 2, \
+ "SRC_SUBSAMP_X unsupported"); \
+ static_assert(SRC_SUBSAMP_Y == 1 || SRC_SUBSAMP_Y == 2, \
+ "SRC_SUBSAMP_Y unsupported"); \
+ static_assert(DST_SUBSAMP_X == 1 || DST_SUBSAMP_X == 2, \
+ "DST_SUBSAMP_X unsupported"); \
+ static_assert(DST_SUBSAMP_Y == 1 || DST_SUBSAMP_Y == 2, \
+ "DST_SUBSAMP_Y unsupported"); \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SRC_SUBSAMP_Y) + \
- OFF); \
- align_buffer_page_end(src_v, SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SRC_SUBSAMP_Y) + \
- OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
- align_buffer_page_end(dst_uv_c, SUBSAMPLE(kWidth, SUBSAMP_X) * 2 * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
- align_buffer_page_end(dst_uv_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * 2 * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- for (int i = 0; i < kHeight; ++i) \
- for (int j = 0; j < kWidth; ++j) \
- src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); ++i) { \
- for (int j = 0; j < SUBSAMPLE(kWidth, SRC_SUBSAMP_X); ++j) { \
- src_u[(i * SUBSAMPLE(kWidth, SRC_SUBSAMP_X)) + j + OFF] = \
- (fastrand() & 0xff); \
- src_v[(i * SUBSAMPLE(kWidth, SRC_SUBSAMP_X)) + j + OFF] = \
- (fastrand() & 0xff); \
- } \
+ const int kSrcHalfWidth = SUBSAMPLE(kWidth, SRC_SUBSAMP_X); \
+ const int kSrcHalfHeight = SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); \
+ const int kDstHalfWidth = SUBSAMPLE(kWidth, DST_SUBSAMP_X); \
+ const int kDstHalfHeight = SUBSAMPLE(kHeight, DST_SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight * SRC_BPC + OFF); \
+ align_buffer_page_end(src_u, \
+ kSrcHalfWidth * kSrcHalfHeight * SRC_BPC + OFF); \
+ align_buffer_page_end(src_v, \
+ kSrcHalfWidth * kSrcHalfHeight * SRC_BPC + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_uv_c, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC * 2); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_uv_opt, \
+ kDstHalfWidth * kDstHalfHeight * DST_BPC * 2); \
+ MemRandomize(src_y + OFF, kWidth * kHeight * SRC_BPC); \
+ MemRandomize(src_u + OFF, kSrcHalfWidth * kSrcHalfHeight * SRC_BPC); \
+ MemRandomize(src_v + OFF, kSrcHalfWidth * kSrcHalfHeight * SRC_BPC); \
+ SRC_T* src_y_p = reinterpret_cast<SRC_T*>(src_y + OFF); \
+ SRC_T* src_u_p = reinterpret_cast<SRC_T*>(src_u + OFF); \
+ SRC_T* src_v_p = reinterpret_cast<SRC_T*>(src_v + OFF); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ src_y_p[i] = src_y_p[i] & ((1 << SRC_DEPTH) - 1); \
+ } \
+ for (int i = 0; i < kSrcHalfWidth * kSrcHalfHeight; ++i) { \
+ src_u_p[i] = src_u_p[i] & ((1 << SRC_DEPTH) - 1); \
+ src_v_p[i] = src_v_p[i] & ((1 << SRC_DEPTH) - 1); \
} \
- memset(dst_y_c, 1, kWidth* kHeight); \
- memset(dst_uv_c, 2, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
- memset(dst_uv_opt, 102, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_y_c, 1, kWidth * kHeight * DST_BPC); \
+ memset(dst_uv_c, 2, kDstHalfWidth * kDstHalfHeight * DST_BPC * 2); \
+ memset(dst_y_opt, 101, kWidth * kHeight * DST_BPC); \
+ memset(dst_uv_opt, 102, kDstHalfWidth * kDstHalfHeight * DST_BPC * 2); \
MaskCpuFlags(disable_cpu_flags_); \
- SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), \
- src_v + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), dst_y_c, kWidth, \
- dst_uv_c, SUBSAMPLE(kWidth, SUBSAMP_X) * 2, kWidth, NEG kHeight); \
+ SRC_FMT_PLANAR##To##FMT_PLANAR(src_y_p, kWidth, src_u_p, kSrcHalfWidth, \
+ src_v_p, kSrcHalfWidth, \
+ reinterpret_cast<DST_T*>(dst_y_c), kWidth, \
+ reinterpret_cast<DST_T*>(dst_uv_c), \
+ kDstHalfWidth * 2, kWidth, NEG kHeight); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), \
- src_v + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), dst_y_opt, kWidth, \
- dst_uv_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * 2, kWidth, NEG kHeight); \
+ src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \
+ reinterpret_cast<DST_T*>(dst_y_opt), kWidth, \
+ reinterpret_cast<DST_T*>(dst_uv_opt), kDstHalfWidth * 2, kWidth, \
+ NEG kHeight); \
} \
- int max_diff = 0; \
- for (int i = 0; i < kHeight; ++i) { \
- for (int j = 0; j < kWidth; ++j) { \
- int abs_diff = abs(static_cast<int>(dst_y_c[i * kWidth + j]) - \
- static_cast<int>(dst_y_opt[i * kWidth + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
+ for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
+ ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
} \
- EXPECT_LE(max_diff, 1); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X) * 2; ++j) { \
- int abs_diff = \
- abs(static_cast<int>( \
- dst_uv_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) * 2 + j]) - \
- static_cast<int>( \
- dst_uv_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) * 2 + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
+ for (int i = 0; i < kDstHalfWidth * kDstHalfHeight * DST_BPC * 2; ++i) { \
+ ASSERT_EQ(dst_uv_c[i], dst_uv_opt[i]); \
} \
- EXPECT_LE(max_diff, 1); \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_uv_c); \
free_aligned_buffer_page_end(dst_y_opt); \
@@ -385,93 +426,315 @@ int I400ToNV21(const uint8_t* src_y,
free_aligned_buffer_page_end(src_v); \
}
-#define TESTPLANARTOBP(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
- TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_ - 4, _Any, +, 0) \
- TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Unaligned, +, 1) \
- TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Invert, -, 0) \
- TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, 0)
-
-TESTPLANARTOBP(I420, 2, 2, NV12, 2, 2)
-TESTPLANARTOBP(I420, 2, 2, NV21, 2, 2)
-TESTPLANARTOBP(I422, 2, 1, NV21, 2, 2)
-TESTPLANARTOBP(I444, 1, 1, NV21, 2, 2)
-TESTPLANARTOBP(I400, 2, 2, NV21, 2, 2)
-
-#define TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, W1280, N, NEG, \
- OFF, DOY) \
+#if defined(ENABLE_FULL_TESTS)
+#define TESTPLANARTOBP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
+ TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, benchmark_width_ + 1, _Any, +, 0, SRC_DEPTH) \
+ TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, benchmark_width_, _Unaligned, +, 2, \
+ SRC_DEPTH) \
+ TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, benchmark_width_, _Invert, -, 0, SRC_DEPTH) \
+ TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, benchmark_width_, _Opt, +, 0, SRC_DEPTH)
+#else
+#define TESTPLANARTOBP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
+ DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
+ TESTPLANARTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, benchmark_width_, _Opt, +, 0, SRC_DEPTH)
+#endif
+
+TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, NV12, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, NV21, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I422, uint8_t, 1, 2, 1, NV21, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I444, uint8_t, 1, 1, 1, NV12, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I444, uint8_t, 1, 1, 1, NV21, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I400, uint8_t, 1, 2, 2, NV21, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I010, uint16_t, 2, 2, 2, NV12, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I010, uint16_t, 2, 2, 2, P010, uint16_t, 2, 2, 2, 10)
+TESTPLANARTOBP(I210, uint16_t, 2, 2, 1, P210, uint16_t, 2, 2, 1, 10)
+TESTPLANARTOBP(I012, uint16_t, 2, 2, 2, P012, uint16_t, 2, 2, 2, 12)
+TESTPLANARTOBP(I212, uint16_t, 2, 2, 1, P212, uint16_t, 2, 2, 1, 12)
+TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, P010, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOBP(I422, uint8_t, 1, 2, 1, P210, uint16_t, 2, 2, 1, 8)
+TESTPLANARTOBP(I444, uint8_t, 1, 1, 1, P410, uint16_t, 2, 1, 1, 8)
+TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, P210, uint16_t, 2, 2, 1, 8)
+TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, P410, uint16_t, 2, 1, 1, 8)
+TESTPLANARTOBP(I410, uint16_t, 2, 1, 1, P410, uint16_t, 2, 1, 1, 10)
+TESTPLANARTOBP(I412, uint16_t, 2, 1, 1, P412, uint16_t, 2, 1, 1, 12)
+
+#define TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \
+ NEG) \
+ TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHalfWidth = SUBSAMPLE(kWidth, 2); \
+ const int kHalfHeight = SUBSAMPLE(kHeight, 2); \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kSamplePixels = kWidth * kHeight + kUvStride * kUvHeight; \
+ align_buffer_page_end(src_y, kWidth * kHeight); \
+ align_buffer_page_end(src_u, kHalfWidth * kHalfHeight); \
+ align_buffer_page_end(src_v, kHalfWidth * kHalfHeight); \
+ align_buffer_page_end(dst, kSamplePixels * BPC); \
+ align_buffer_page_end(ref, kSamplePixels * BPC); \
+ MemRandomize(src_y, kWidth * kHeight); \
+ MemRandomize(src_u, kHalfWidth * kHalfHeight); \
+ MemRandomize(src_v, kHalfWidth * kHalfHeight); \
+ memset(dst, 0, kSamplePixels * BPC); \
+ memset(ref, 1, kSamplePixels * BPC); \
+ TYPE* dst_p = reinterpret_cast<TYPE*>(dst); \
+ TYPE* ref_p = reinterpret_cast<TYPE*>(ref); \
+ EXPECT_EQ(0, \
+ I420To##FMT(src_y, kWidth, src_u, kHalfWidth, src_v, kHalfWidth, \
+ ref_p, kWidth, ref_p + kWidth * kHeight, kUvStride, \
+ kWidth, NEG kHeight)); \
+ EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \
+ kHalfWidth, dst, 0, kWidth, NEG kHeight, \
+ FOURCC_##FMT)); \
+ for (int i = 0; i < kSamplePixels; ++i) { \
+ ASSERT_EQ(dst_p[i], ref_p[i]); \
+ } \
+ if (BPC == 2) { \
+ EXPECT_EQ(-1, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \
+ kHalfWidth, dst, 1, kWidth, kHeight, \
+ FOURCC_##FMT)); \
+ } \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst); \
+ free_aligned_buffer_page_end(ref); \
+ }
+
+#define TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \
+ NEG) \
+ TEST_F(LibYUVConvertTest, ConvertToI420_##FMT##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kHalfWidth = SUBSAMPLE(kWidth, 2); \
+ const int kHalfHeight = SUBSAMPLE(kHeight, 2); \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kYPixels = kWidth * kHeight; \
+ const int kUvPixels = kUvStride * kUvHeight; \
+ align_buffer_page_end(src, (kYPixels + kUvPixels) * BPC); \
+ align_buffer_page_end(dst_y, kWidth * kHeight); \
+ align_buffer_page_end(dst_u, kHalfWidth * kHalfHeight); \
+ align_buffer_page_end(dst_v, kHalfWidth * kHalfHeight); \
+ align_buffer_page_end(ref_y, kWidth * kHeight); \
+ align_buffer_page_end(ref_u, kHalfWidth * kHalfHeight); \
+ align_buffer_page_end(ref_v, kHalfWidth * kHalfHeight); \
+ TYPE* src_p = reinterpret_cast<TYPE*>(src); \
+ MemRandomize(src, (kYPixels + kUvPixels) * BPC); \
+ if (BPC == 2) { \
+ for (int i = 0; i < kYPixels + kUvPixels; ++i) { \
+ src_p[i] = static_cast<TYPE>(src_p[i] & 0xffc0); \
+ } \
+ } \
+ memset(dst_y, 1, kWidth * kHeight); \
+ memset(dst_u, 2, kHalfWidth * kHalfHeight); \
+ memset(dst_v, 3, kHalfWidth * kHalfHeight); \
+ memset(ref_y, 4, kWidth * kHeight); \
+ memset(ref_u, 5, kHalfWidth * kHalfHeight); \
+ memset(ref_v, 6, kHalfWidth * kHalfHeight); \
+ EXPECT_EQ(0, FMT##ToI420(src_p, kWidth, src_p + kYPixels, kUvStride, \
+ ref_y, kWidth, ref_u, kHalfWidth, ref_v, \
+ kHalfWidth, kWidth, NEG kHeight)); \
+ EXPECT_EQ(0, ConvertToI420(src, 0, dst_y, kWidth, dst_u, kHalfWidth, \
+ dst_v, kHalfWidth, 0, 0, kWidth, NEG kHeight, \
+ kWidth, kHeight, kRotate0, FOURCC_##FMT)); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ ASSERT_EQ(dst_y[i], ref_y[i]); \
+ } \
+ for (int i = 0; i < kHalfWidth * kHalfHeight; ++i) { \
+ ASSERT_EQ(dst_u[i], ref_u[i]); \
+ ASSERT_EQ(dst_v[i], ref_v[i]); \
+ } \
+ free_aligned_buffer_page_end(src); \
+ free_aligned_buffer_page_end(dst_y); \
+ free_aligned_buffer_page_end(dst_u); \
+ free_aligned_buffer_page_end(dst_v); \
+ free_aligned_buffer_page_end(ref_y); \
+ free_aligned_buffer_page_end(ref_u); \
+ free_aligned_buffer_page_end(ref_v); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +) \
+ TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Invert, -) \
+ TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +) \
+ TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Invert, -) \
+ TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#else
+#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Opt, +)
+#endif
+
+TESTCONVERTFROMI420(P010, uint16_t, 2, 2, 2)
+TESTCONVERTFROMI420(P210, uint16_t, 2, 2, 1)
+TESTCONVERTFROMI420(P410, uint16_t, 2, 1, 1)
+TESTCONVERTTOI420(P010, uint16_t, 2, 2, 2)
+TESTCONVERTTOI420(P210, uint16_t, 2, 2, 1)
+TESTCONVERTTOI420(P410, uint16_t, 2, 1, 1)
+
+#define TESTCONVERTTOI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, ConvertToI420_##FMT) { \
+ const int kWidth = 64; \
+ const int kHeight = 48; \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kSampleSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \
+ align_buffer_page_end(src, kSampleSize); \
+ align_buffer_page_end(dst_y, kWidth * kHeight); \
+ align_buffer_page_end(dst_u, (kWidth / 2) * (kHeight / 2)); \
+ align_buffer_page_end(dst_v, (kWidth / 2) * (kHeight / 2)); \
+ MemRandomize(src, kSampleSize); \
+ memset(dst_y, 0, kWidth * kHeight); \
+ memset(dst_u, 0, (kWidth / 2) * (kHeight / 2)); \
+ memset(dst_v, 0, (kWidth / 2) * (kHeight / 2)); \
+ EXPECT_EQ( \
+ 0, ConvertToI420(src, kSampleSize, dst_y, kWidth, dst_u, kWidth / 2, \
+ dst_v, kWidth / 2, 0, 0, kWidth, kHeight, kWidth, \
+ kHeight, kRotate0, FOURCC_##FMT)); \
+ free_aligned_buffer_page_end(src); \
+ free_aligned_buffer_page_end(dst_y); \
+ free_aligned_buffer_page_end(dst_u); \
+ free_aligned_buffer_page_end(dst_v); \
+ }
+
+#define TESTCONVERTFROMI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT) { \
+ const int kWidth = 64; \
+ const int kHeight = 48; \
+ const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \
+ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kDstSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \
+ align_buffer_page_end(src_y, kWidth * kHeight); \
+ align_buffer_page_end(src_u, (kWidth / 2) * (kHeight / 2)); \
+ align_buffer_page_end(src_v, (kWidth / 2) * (kHeight / 2)); \
+ align_buffer_page_end(dst, kDstSize); \
+ MemRandomize(src_y, kWidth * kHeight); \
+ MemRandomize(src_u, (kWidth / 2) * (kHeight / 2)); \
+ MemRandomize(src_v, (kWidth / 2) * (kHeight / 2)); \
+ memset(dst, 0, kDstSize); \
+ EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kWidth / 2, src_v, \
+ kWidth / 2, dst, kWidth * BPC, kWidth, \
+ kHeight, FOURCC_##FMT)); \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_u); \
+ free_aligned_buffer_page_end(src_v); \
+ free_aligned_buffer_page_end(dst); \
+ }
+
+TESTCONVERTTOI420SMOKE(NV16, 1, 2, 1)
+TESTCONVERTTOI420SMOKE(NV24, 1, 1, 1)
+TESTCONVERTFROMI420SMOKE(NV16, 1, 2, 1)
+TESTCONVERTFROMI420SMOKE(NV24, 1, 1, 1)
+
+#define TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, W1280, N, NEG, OFF, DOY, SRC_DEPTH, \
+ TILE_WIDTH, TILE_HEIGHT) \
TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ static_assert(DST_BPC == 1 || DST_BPC == 2, "DST BPC unsupported"); \
+ static_assert(SRC_SUBSAMP_X == 1 || SRC_SUBSAMP_X == 2, \
+ "SRC_SUBSAMP_X unsupported"); \
+ static_assert(SRC_SUBSAMP_Y == 1 || SRC_SUBSAMP_Y == 2, \
+ "SRC_SUBSAMP_Y unsupported"); \
+ static_assert(DST_SUBSAMP_X == 1 || DST_SUBSAMP_X == 2, \
+ "DST_SUBSAMP_X unsupported"); \
+ static_assert(DST_SUBSAMP_Y == 1 || DST_SUBSAMP_Y == 2, \
+ "DST_SUBSAMP_Y unsupported"); \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_uv, 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SRC_SUBSAMP_Y) + \
- OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
- align_buffer_page_end(dst_uv_c, 2 * SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
- align_buffer_page_end(dst_uv_opt, 2 * SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- for (int i = 0; i < kHeight; ++i) \
- for (int j = 0; j < kWidth; ++j) \
- src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); ++i) { \
- for (int j = 0; j < 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X); ++j) { \
- src_uv[(i * 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X)) + j + OFF] = \
- (fastrand() & 0xff); \
- } \
+ const int kSrcHalfWidth = SUBSAMPLE(kWidth, SRC_SUBSAMP_X); \
+ const int kDstHalfWidth = SUBSAMPLE(kWidth, DST_SUBSAMP_X); \
+ const int kDstHalfHeight = SUBSAMPLE(kHeight, DST_SUBSAMP_Y); \
+ const int kPaddedWidth = (kWidth + (TILE_WIDTH - 1)) & ~(TILE_WIDTH - 1); \
+ const int kPaddedHeight = \
+ (kHeight + (TILE_HEIGHT - 1)) & ~(TILE_HEIGHT - 1); \
+ const int kSrcHalfPaddedWidth = SUBSAMPLE(kPaddedWidth, SRC_SUBSAMP_X); \
+ const int kSrcHalfPaddedHeight = SUBSAMPLE(kPaddedHeight, SRC_SUBSAMP_Y); \
+ align_buffer_page_end(src_y, \
+ kPaddedWidth * kPaddedHeight * SRC_BPC + OFF); \
+ align_buffer_page_end( \
+ src_uv, \
+ 2 * kSrcHalfPaddedWidth * kSrcHalfPaddedHeight * SRC_BPC + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_uv_c, \
+ 2 * kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight * DST_BPC); \
+ align_buffer_page_end(dst_uv_opt, \
+ 2 * kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ SRC_T* src_y_p = reinterpret_cast<SRC_T*>(src_y + OFF); \
+ SRC_T* src_uv_p = reinterpret_cast<SRC_T*>(src_uv + OFF); \
+ for (int i = 0; \
+ i < kPaddedWidth * kPaddedHeight * SRC_BPC / (int)sizeof(SRC_T); \
+ ++i) { \
+ src_y_p[i] = \
+ (fastrand() & (((SRC_T)(-1)) << ((8 * SRC_BPC) - SRC_DEPTH))); \
+ } \
+ for (int i = 0; i < kSrcHalfPaddedWidth * kSrcHalfPaddedHeight * 2 * \
+ SRC_BPC / (int)sizeof(SRC_T); \
+ ++i) { \
+ src_uv_p[i] = \
+ (fastrand() & (((SRC_T)(-1)) << ((8 * SRC_BPC) - SRC_DEPTH))); \
} \
- memset(dst_y_c, 1, kWidth* kHeight); \
- memset(dst_uv_c, 2, \
- 2 * SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
- memset(dst_uv_opt, 102, \
- 2 * SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_y_c, 1, kWidth * kHeight * DST_BPC); \
+ memset(dst_uv_c, 2, 2 * kDstHalfWidth * kDstHalfHeight * DST_BPC); \
+ memset(dst_y_opt, 101, kWidth * kHeight * DST_BPC); \
+ memset(dst_uv_opt, 102, 2 * kDstHalfWidth * kDstHalfHeight * DST_BPC); \
MaskCpuFlags(disable_cpu_flags_); \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_uv + OFF, \
- 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X), DOY ? dst_y_c : NULL, kWidth, \
- dst_uv_c, 2 * SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight); \
+ src_y_p, kWidth * SRC_BPC / (int)sizeof(SRC_T), src_uv_p, \
+ 2 * kSrcHalfWidth * SRC_BPC / (int)sizeof(SRC_T), \
+ DOY ? reinterpret_cast<DST_T*>(dst_y_c) : NULL, kWidth, \
+ reinterpret_cast<DST_T*>(dst_uv_c), 2 * kDstHalfWidth, kWidth, \
+ NEG kHeight); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_uv + OFF, \
- 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X), DOY ? dst_y_opt : NULL, \
- kWidth, dst_uv_opt, 2 * SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, \
+ src_y_p, kWidth * SRC_BPC / (int)sizeof(SRC_T), src_uv_p, \
+ 2 * kSrcHalfWidth * SRC_BPC / (int)sizeof(SRC_T), \
+ DOY ? reinterpret_cast<DST_T*>(dst_y_opt) : NULL, kWidth, \
+ reinterpret_cast<DST_T*>(dst_uv_opt), 2 * kDstHalfWidth, kWidth, \
NEG kHeight); \
} \
- int max_diff = 0; \
if (DOY) { \
for (int i = 0; i < kHeight; ++i) { \
for (int j = 0; j < kWidth; ++j) { \
- int abs_diff = abs(static_cast<int>(dst_y_c[i * kWidth + j]) - \
- static_cast<int>(dst_y_opt[i * kWidth + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
} \
} \
- EXPECT_LE(max_diff, 1); \
} \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < 2 * SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
- int abs_diff = \
- abs(static_cast<int>( \
- dst_uv_c[i * 2 * SUBSAMPLE(kWidth, SUBSAMP_X) + j]) - \
- static_cast<int>( \
- dst_uv_opt[i * 2 * SUBSAMPLE(kWidth, SUBSAMP_X) + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ for (int i = 0; i < kDstHalfHeight; ++i) { \
+ for (int j = 0; j < 2 * kDstHalfWidth; ++j) { \
+ ASSERT_EQ(dst_uv_c[i * 2 * kDstHalfWidth + j], \
+ dst_uv_opt[i * 2 * kDstHalfWidth + j]); \
} \
} \
- EXPECT_LE(max_diff, 1); \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_uv_c); \
free_aligned_buffer_page_end(dst_y_opt); \
@@ -480,565 +743,244 @@ TESTPLANARTOBP(I400, 2, 2, NV21, 2, 2)
free_aligned_buffer_page_end(src_uv); \
}
-#define TESTBIPLANARTOBP(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
- TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_ - 4, _Any, +, 0, 1) \
- TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Unaligned, +, 1, \
- 1) \
- TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Invert, -, 0, 1) \
- TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, 0, 1) \
- TESTBIPLANARTOBPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _NullY, +, 0, 0)
-
-TESTBIPLANARTOBP(NV21, 2, 2, NV12, 2, 2)
-
-#define TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, W1280, N, NEG, OFF, \
- DOY) \
- TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_uv, 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SRC_SUBSAMP_Y) + \
- OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
- align_buffer_page_end(dst_u_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_v_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
- align_buffer_page_end(dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_v_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
- SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- for (int i = 0; i < kHeight; ++i) \
- for (int j = 0; j < kWidth; ++j) \
- src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); ++i) { \
- for (int j = 0; j < 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X); ++j) { \
- src_uv[(i * 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X)) + j + OFF] = \
- (fastrand() & 0xff); \
- } \
- } \
- memset(dst_y_c, 1, kWidth* kHeight); \
- memset(dst_u_c, 2, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_v_c, 3, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
- memset(dst_u_opt, 102, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_v_opt, 103, \
- SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- MaskCpuFlags(disable_cpu_flags_); \
- SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_uv + OFF, \
- 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X), DOY ? dst_y_c : NULL, kWidth, \
- dst_u_c, SUBSAMPLE(kWidth, SUBSAMP_X), dst_v_c, \
- SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- SRC_FMT_PLANAR##To##FMT_PLANAR( \
- src_y + OFF, kWidth, src_uv + OFF, \
- 2 * SUBSAMPLE(kWidth, SRC_SUBSAMP_X), DOY ? dst_y_opt : NULL, \
- kWidth, dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X), dst_v_opt, \
- SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight); \
- } \
- int max_diff = 0; \
- if (DOY) { \
- for (int i = 0; i < kHeight; ++i) { \
- for (int j = 0; j < kWidth; ++j) { \
- int abs_diff = abs(static_cast<int>(dst_y_c[i * kWidth + j]) - \
- static_cast<int>(dst_y_opt[i * kWidth + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- } \
- EXPECT_LE(max_diff, 1); \
- } \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
- int abs_diff = abs( \
- static_cast<int>(dst_u_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]) - \
- static_cast<int>( \
- dst_u_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- } \
- EXPECT_LE(max_diff, 1); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
- int abs_diff = abs( \
- static_cast<int>(dst_v_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]) - \
- static_cast<int>( \
- dst_v_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- } \
- EXPECT_LE(max_diff, 1); \
- free_aligned_buffer_page_end(dst_y_c); \
- free_aligned_buffer_page_end(dst_u_c); \
- free_aligned_buffer_page_end(dst_v_c); \
- free_aligned_buffer_page_end(dst_y_opt); \
- free_aligned_buffer_page_end(dst_u_opt); \
- free_aligned_buffer_page_end(dst_v_opt); \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_uv); \
- }
-
-#define TESTBIPLANARTOP(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
- FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
- TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_ - 4, _Any, +, 0, 1) \
- TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Unaligned, +, 1, \
- 1) \
- TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Invert, -, 0, 1) \
- TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, 0, 1) \
- TESTBIPLANARTOPI(SRC_FMT_PLANAR, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, \
- SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _NullY, +, 0, 0)
-
-TESTBIPLANARTOP(NV12, 2, 2, I420, 2, 2)
-TESTBIPLANARTOP(NV21, 2, 2, I420, 2, 2)
-
-#define ALIGNINT(V, ALIGN) (((V) + (ALIGN)-1) / (ALIGN) * (ALIGN))
-
-#define TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, W1280, N, NEG, OFF) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
- const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, kSizeUV + OFF); \
- align_buffer_page_end(src_v, kSizeUV + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight + OFF); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight + OFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- src_y[i + OFF] = (fastrand() & 0xff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- src_u[i + OFF] = (fastrand() & 0xff); \
- src_v[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
- memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
- MaskCpuFlags(disable_cpu_flags_); \
- double time0 = get_time(); \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, dst_argb_c + OFF, kStrideB, \
- kWidth, NEG kHeight); \
- double time1 = get_time(); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, dst_argb_opt + OFF, \
- kStrideB, kWidth, NEG kHeight); \
- } \
- double time2 = get_time(); \
- printf(" %8d us C - %8d us OPT\n", \
- static_cast<int>((time1 - time0) * 1e6), \
- static_cast<int>((time2 - time1) * 1e6 / benchmark_iterations_)); \
- for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
- EXPECT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \
- } \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN) \
- TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_ - 4, _Any, +, 0) \
- TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, _Unaligned, +, 1) \
- TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, _Invert, -, 0) \
- TESTPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, _Opt, +, 0)
-
-TESTPLANARTOB(I420, 2, 2, ARGB, 4, 4, 1)
-TESTPLANARTOB(J420, 2, 2, ARGB, 4, 4, 1)
-TESTPLANARTOB(J420, 2, 2, ABGR, 4, 4, 1)
-TESTPLANARTOB(H420, 2, 2, ARGB, 4, 4, 1)
-TESTPLANARTOB(H420, 2, 2, ABGR, 4, 4, 1)
-TESTPLANARTOB(I420, 2, 2, BGRA, 4, 4, 1)
-TESTPLANARTOB(I420, 2, 2, ABGR, 4, 4, 1)
-TESTPLANARTOB(I420, 2, 2, RGBA, 4, 4, 1)
-TESTPLANARTOB(I420, 2, 2, RAW, 3, 3, 1)
-TESTPLANARTOB(I420, 2, 2, RGB24, 3, 3, 1)
-TESTPLANARTOB(H420, 2, 2, RAW, 3, 3, 1)
-TESTPLANARTOB(H420, 2, 2, RGB24, 3, 3, 1)
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANARTOB(I420, 2, 2, RGB565, 2, 2, 1)
-TESTPLANARTOB(J420, 2, 2, RGB565, 2, 2, 1)
-TESTPLANARTOB(H420, 2, 2, RGB565, 2, 2, 1)
-TESTPLANARTOB(I420, 2, 2, ARGB1555, 2, 2, 1)
-TESTPLANARTOB(I420, 2, 2, ARGB4444, 2, 2, 1)
-TESTPLANARTOB(I422, 2, 1, RGB565, 2, 2, 1)
-#endif
-TESTPLANARTOB(I422, 2, 1, ARGB, 4, 4, 1)
-TESTPLANARTOB(J422, 2, 1, ARGB, 4, 4, 1)
-TESTPLANARTOB(J422, 2, 1, ABGR, 4, 4, 1)
-TESTPLANARTOB(H422, 2, 1, ARGB, 4, 4, 1)
-TESTPLANARTOB(H422, 2, 1, ABGR, 4, 4, 1)
-TESTPLANARTOB(I422, 2, 1, BGRA, 4, 4, 1)
-TESTPLANARTOB(I422, 2, 1, ABGR, 4, 4, 1)
-TESTPLANARTOB(I422, 2, 1, RGBA, 4, 4, 1)
-TESTPLANARTOB(I444, 1, 1, ARGB, 4, 4, 1)
-TESTPLANARTOB(J444, 1, 1, ARGB, 4, 4, 1)
-TESTPLANARTOB(I444, 1, 1, ABGR, 4, 4, 1)
-TESTPLANARTOB(I420, 2, 2, YUY2, 2, 4, 1)
-TESTPLANARTOB(I420, 2, 2, UYVY, 2, 4, 1)
-TESTPLANARTOB(I422, 2, 1, YUY2, 2, 4, 1)
-TESTPLANARTOB(I422, 2, 1, UYVY, 2, 4, 1)
-TESTPLANARTOB(I420, 2, 2, I400, 1, 1, 1)
-TESTPLANARTOB(J420, 2, 2, J400, 1, 1, 1)
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANARTOB(I420, 2, 2, AR30, 4, 4, 1)
-TESTPLANARTOB(H420, 2, 2, AR30, 4, 4, 1)
+#if defined(ENABLE_FULL_TESTS)
+#define TESTBPTOBP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0, 1, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2, 1, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0, 1, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT)
+#else
+#define TESTBPTOBP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, \
+ DST_SUBSAMP_Y, SRC_DEPTH, TILE_WIDTH, TILE_HEIGHT) \
+ TESTBPTOBPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
+ benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH, TILE_WIDTH, \
+ TILE_HEIGHT)
#endif
-#define TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, W1280, DIFF, N, NEG, OFF, ATTEN) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
- const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, kSizeUV + OFF); \
- align_buffer_page_end(src_v, kSizeUV + OFF); \
- align_buffer_page_end(src_a, kWidth* kHeight + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight + OFF); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight + OFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- src_y[i + OFF] = (fastrand() & 0xff); \
- src_a[i + OFF] = (fastrand() & 0xff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- src_u[i + OFF] = (fastrand() & 0xff); \
- src_v[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
- memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, src_a + OFF, kWidth, \
- dst_argb_c + OFF, kStrideB, kWidth, NEG kHeight, \
- ATTEN); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, src_a + OFF, kWidth, \
- dst_argb_opt + OFF, kStrideB, kWidth, NEG kHeight, \
- ATTEN); \
- } \
- int max_diff = 0; \
- for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb_c[i + OFF]) - \
- static_cast<int>(dst_argb_opt[i + OFF])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(src_a); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTQPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, DIFF) \
- TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_ - 4, DIFF, _Any, +, 0, 0) \
- TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Unaligned, +, 1, 0) \
- TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Invert, -, 0, 0) \
- TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Opt, +, 0, 0) \
- TESTQPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Premult, +, 0, 1)
-
-TESTQPLANARTOB(I420Alpha, 2, 2, ARGB, 4, 4, 1, 2)
-TESTQPLANARTOB(I420Alpha, 2, 2, ABGR, 4, 4, 1, 2)
-
-#define TESTBIPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, \
- BPP_B, W1280, DIFF, N, NEG, OFF) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kStrideB = kWidth * BPP_B; \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_uv, \
- kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y) * 2 + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight); \
- for (int i = 0; i < kHeight; ++i) \
- for (int j = 0; j < kWidth; ++j) \
- src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < kStrideUV * 2; ++j) { \
- src_uv[i * kStrideUV * 2 + j + OFF] = (fastrand() & 0xff); \
- } \
- } \
- memset(dst_argb_c, 1, kStrideB* kHeight); \
- memset(dst_argb_opt, 101, kStrideB* kHeight); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_uv + OFF, kStrideUV * 2, \
- dst_argb_c, kWidth * BPP_B, kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_uv + OFF, kStrideUV * 2, \
- dst_argb_opt, kWidth * BPP_B, kWidth, \
- NEG kHeight); \
- } \
- /* Convert to ARGB so 565 is expanded to bytes that can be compared. */ \
- align_buffer_page_end(dst_argb32_c, kWidth * 4 * kHeight); \
- align_buffer_page_end(dst_argb32_opt, kWidth * 4 * kHeight); \
- memset(dst_argb32_c, 2, kWidth * 4 * kHeight); \
- memset(dst_argb32_opt, 102, kWidth * 4 * kHeight); \
- FMT_C##ToARGB(dst_argb_c, kStrideB, dst_argb32_c, kWidth * 4, kWidth, \
- kHeight); \
- FMT_C##ToARGB(dst_argb_opt, kStrideB, dst_argb32_opt, kWidth * 4, kWidth, \
- kHeight); \
- int max_diff = 0; \
- for (int i = 0; i < kHeight; ++i) { \
- for (int j = 0; j < kWidth * 4; ++j) { \
- int abs_diff = \
- abs(static_cast<int>(dst_argb32_c[i * kWidth * 4 + j]) - \
- static_cast<int>(dst_argb32_opt[i * kWidth * 4 + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_uv); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- free_aligned_buffer_page_end(dst_argb32_c); \
- free_aligned_buffer_page_end(dst_argb32_opt); \
- }
-
-#define TESTBIPLANARTOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
- DIFF) \
- TESTBIPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
- benchmark_width_ - 4, DIFF, _Any, +, 0) \
- TESTBIPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
- benchmark_width_, DIFF, _Unaligned, +, 1) \
- TESTBIPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
- benchmark_width_, DIFF, _Invert, -, 0) \
- TESTBIPLANARTOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, FMT_C, BPP_B, \
- benchmark_width_, DIFF, _Opt, +, 0)
-
-TESTBIPLANARTOB(NV12, 2, 2, ARGB, ARGB, 4, 2)
-TESTBIPLANARTOB(NV21, 2, 2, ARGB, ARGB, 4, 2)
-TESTBIPLANARTOB(NV12, 2, 2, ABGR, ABGR, 4, 2)
-TESTBIPLANARTOB(NV21, 2, 2, ABGR, ABGR, 4, 2)
-TESTBIPLANARTOB(NV12, 2, 2, RGB24, RGB24, 3, 2)
-TESTBIPLANARTOB(NV21, 2, 2, RGB24, RGB24, 3, 2)
-TESTBIPLANARTOB(NV12, 2, 2, RAW, RAW, 3, 2)
-TESTBIPLANARTOB(NV21, 2, 2, RAW, RAW, 3, 2)
-#ifdef LITTLE_ENDIAN_TEST
-TESTBIPLANARTOB(NV12, 2, 2, RGB565, RGB565, 2, 9)
-#endif
-TESTBIPLANARTOB(NV21, 2, 2, YUV24, RAW, 3, 2)
+TESTBPTOBP(NV12, uint8_t, 1, 2, 2, NV21, uint8_t, 1, 2, 2, 8, 1, 1)
+TESTBPTOBP(NV21, uint8_t, 1, 2, 2, NV12, uint8_t, 1, 2, 2, 8, 1, 1)
+TESTBPTOBP(NV12, uint8_t, 1, 2, 2, NV12Mirror, uint8_t, 1, 2, 2, 8, 1, 1)
+TESTBPTOBP(NV12, uint8_t, 1, 2, 2, NV24, uint8_t, 1, 1, 1, 8, 1, 1)
+TESTBPTOBP(NV16, uint8_t, 1, 2, 1, NV24, uint8_t, 1, 1, 1, 8, 1, 1)
+TESTBPTOBP(P010, uint16_t, 2, 2, 2, P410, uint16_t, 2, 1, 1, 10, 1, 1)
+TESTBPTOBP(P210, uint16_t, 2, 2, 1, P410, uint16_t, 2, 1, 1, 10, 1, 1)
+TESTBPTOBP(P012, uint16_t, 2, 2, 2, P412, uint16_t, 2, 1, 1, 10, 1, 1)
+TESTBPTOBP(P212, uint16_t, 2, 2, 1, P412, uint16_t, 2, 1, 1, 12, 1, 1)
+TESTBPTOBP(P016, uint16_t, 2, 2, 2, P416, uint16_t, 2, 1, 1, 12, 1, 1)
+TESTBPTOBP(P216, uint16_t, 2, 2, 1, P416, uint16_t, 2, 1, 1, 12, 1, 1)
+TESTBPTOBP(MM21, uint8_t, 1, 2, 2, NV12, uint8_t, 1, 2, 2, 8, 16, 32)
+TESTBPTOBP(MT2T, uint8_t, 10 / 8, 2, 2, P010, uint16_t, 2, 2, 2, 10, 16, 32)
+TESTBPTOBP(P010, uint16_t, 2, 2, 2, NV12, uint8_t, 1, 2, 2, 8, 1, 1)
-#ifdef DO_THREE_PLANES
-// Do 3 allocations for yuv. conventional but slower.
#define TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- W1280, DIFF, N, NEG, OFF) \
+ W1280, N, NEG, OFF) \
TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ const int kWidth = W1280; \
const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
const int kStride = (kStrideUV * SUBSAMP_X * 8 * BPP_A + 7) / 8; \
- align_buffer_page_end(src_argb, kStride* kHeight + OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
- align_buffer_page_end(dst_u_c, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_v_c, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
- align_buffer_page_end(dst_u_opt, \
- kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_v_opt, \
- kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_c, 1, kWidth* kHeight); \
- memset(dst_u_c, 2, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_v_c, 3, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
- memset(dst_u_opt, 102, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_v_opt, 103, kStrideUV* SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ align_buffer_page_end(src_argb, kStride * kHeight + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight); \
+ align_buffer_page_end(dst_uv_c, \
+ kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight); \
+ align_buffer_page_end(dst_uv_opt, \
+ kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_y_c, 1, kWidth * kHeight); \
+ memset(dst_uv_c, 2, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_y_opt, 101, kWidth * kHeight); \
+ memset(dst_uv_opt, 102, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
for (int i = 0; i < kHeight; ++i) \
for (int j = 0; j < kStride; ++j) \
src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \
MaskCpuFlags(disable_cpu_flags_); \
- FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_c, kWidth, dst_u_c, \
- kStrideUV, dst_v_c, kStrideUV, kWidth, NEG kHeight); \
+ FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_c, kWidth, dst_uv_c, \
+ kStrideUV * 2, dst_uv_c + kStrideUV, kStrideUV * 2, \
+ kWidth, NEG kHeight); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_opt, kWidth, \
- dst_u_opt, kStrideUV, dst_v_opt, kStrideUV, \
- kWidth, NEG kHeight); \
+ dst_uv_opt, kStrideUV * 2, dst_uv_opt + kStrideUV, \
+ kStrideUV * 2, kWidth, NEG kHeight); \
} \
for (int i = 0; i < kHeight; ++i) { \
for (int j = 0; j < kWidth; ++j) { \
- EXPECT_NEAR(static_cast<int>(dst_y_c[i * kWidth + j]), \
- static_cast<int>(dst_y_opt[i * kWidth + j]), DIFF); \
- } \
- } \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
- for (int j = 0; j < kStrideUV; ++j) { \
- EXPECT_NEAR(static_cast<int>(dst_u_c[i * kStrideUV + j]), \
- static_cast<int>(dst_u_opt[i * kStrideUV + j]), DIFF); \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
} \
} \
- for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
+ for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y) * 2; ++i) { \
for (int j = 0; j < kStrideUV; ++j) { \
- EXPECT_NEAR(static_cast<int>(dst_v_c[i * kStrideUV + j]), \
- static_cast<int>(dst_v_opt[i * kStrideUV + j]), DIFF); \
+ ASSERT_EQ(dst_uv_c[i * kStrideUV + j], dst_uv_opt[i * kStrideUV + j]); \
} \
} \
free_aligned_buffer_page_end(dst_y_c); \
- free_aligned_buffer_page_end(dst_u_c); \
- free_aligned_buffer_page_end(dst_v_c); \
+ free_aligned_buffer_page_end(dst_uv_c); \
free_aligned_buffer_page_end(dst_y_opt); \
- free_aligned_buffer_page_end(dst_u_opt); \
- free_aligned_buffer_page_end(dst_v_opt); \
+ free_aligned_buffer_page_end(dst_uv_opt); \
free_aligned_buffer_page_end(src_argb); \
}
+
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOPLANAR(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2) \
+ TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0) \
+ TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
#else
-#define TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- W1280, DIFF, N, NEG, OFF) \
+#define TESTATOPLANAR(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
+#endif
+
+TESTATOPLANAR(ABGR, 4, 1, I420, 2, 2)
+TESTATOPLANAR(ARGB, 4, 1, I420, 2, 2)
+TESTATOPLANAR(ARGB, 4, 1, I422, 2, 1)
+TESTATOPLANAR(ARGB, 4, 1, I444, 1, 1)
+TESTATOPLANAR(ARGB, 4, 1, J420, 2, 2)
+TESTATOPLANAR(ARGB, 4, 1, J422, 2, 1)
+TESTATOPLANAR(ARGB, 4, 1, J444, 1, 1)
+TESTATOPLANAR(ABGR, 4, 1, J420, 2, 2)
+TESTATOPLANAR(ABGR, 4, 1, J422, 2, 1)
+#ifdef LITTLE_ENDIAN_ONLY_TEST
+TESTATOPLANAR(ARGB4444, 2, 1, I420, 2, 2)
+TESTATOPLANAR(RGB565, 2, 1, I420, 2, 2)
+TESTATOPLANAR(ARGB1555, 2, 1, I420, 2, 2)
+#endif
+TESTATOPLANAR(BGRA, 4, 1, I420, 2, 2)
+TESTATOPLANAR(I400, 1, 1, I420, 2, 2)
+TESTATOPLANAR(J400, 1, 1, J420, 2, 2)
+TESTATOPLANAR(RAW, 3, 1, I420, 2, 2)
+TESTATOPLANAR(RAW, 3, 1, I444, 1, 1)
+TESTATOPLANAR(RAW, 3, 1, J420, 2, 2)
+TESTATOPLANAR(RAW, 3, 1, J444, 1, 1)
+TESTATOPLANAR(RGB24, 3, 1, I420, 2, 2)
+TESTATOPLANAR(RGB24, 3, 1, J420, 2, 2)
+TESTATOPLANAR(RGBA, 4, 1, I420, 2, 2)
+TESTATOPLANAR(AR30, 4, 1, I420, 2, 2)
+TESTATOPLANAR(AB30, 4, 1, I420, 2, 2)
+TESTATOPLANAR(UYVY, 2, 1, I420, 2, 2)
+TESTATOPLANAR(UYVY, 2, 1, I422, 2, 1)
+TESTATOPLANAR(YUY2, 2, 1, I420, 2, 2)
+TESTATOPLANAR(YUY2, 2, 1, I422, 2, 1)
+
+#define TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, \
+ SUBSAMP_Y, W1280, N, NEG, OFF) \
TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ const int kWidth = W1280; \
const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
const int kStride = (kStrideUV * SUBSAMP_X * 8 * BPP_A + 7) / 8; \
- align_buffer_page_end(src_argb, kStride* kHeight + OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
+ align_buffer_page_end(src_argb, kStride * kHeight + OFF); \
+ align_buffer_page_end(dst_a_c, kWidth * kHeight); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight); \
align_buffer_page_end(dst_uv_c, \
kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
+ align_buffer_page_end(dst_a_opt, kWidth * kHeight); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight); \
align_buffer_page_end(dst_uv_opt, \
kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_c, 1, kWidth* kHeight); \
- memset(dst_uv_c, 2, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
- memset(dst_uv_opt, 102, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_a_c, 1, kWidth * kHeight); \
+ memset(dst_y_c, 2, kWidth * kHeight); \
+ memset(dst_uv_c, 3, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_a_opt, 101, kWidth * kHeight); \
+ memset(dst_y_opt, 102, kWidth * kHeight); \
+ memset(dst_uv_opt, 103, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
for (int i = 0; i < kHeight; ++i) \
for (int j = 0; j < kStride; ++j) \
src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \
MaskCpuFlags(disable_cpu_flags_); \
FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_c, kWidth, dst_uv_c, \
kStrideUV * 2, dst_uv_c + kStrideUV, kStrideUV * 2, \
- kWidth, NEG kHeight); \
+ dst_a_c, kWidth, kWidth, NEG kHeight); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_opt, kWidth, \
dst_uv_opt, kStrideUV * 2, dst_uv_opt + kStrideUV, \
- kStrideUV * 2, kWidth, NEG kHeight); \
+ kStrideUV * 2, dst_a_opt, kWidth, kWidth, \
+ NEG kHeight); \
} \
for (int i = 0; i < kHeight; ++i) { \
for (int j = 0; j < kWidth; ++j) { \
- EXPECT_NEAR(static_cast<int>(dst_y_c[i * kWidth + j]), \
- static_cast<int>(dst_y_opt[i * kWidth + j]), DIFF); \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
+ ASSERT_EQ(dst_a_c[i * kWidth + j], dst_a_opt[i * kWidth + j]); \
} \
} \
for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y) * 2; ++i) { \
for (int j = 0; j < kStrideUV; ++j) { \
- EXPECT_NEAR(static_cast<int>(dst_uv_c[i * kStrideUV + j]), \
- static_cast<int>(dst_uv_opt[i * kStrideUV + j]), DIFF); \
+ ASSERT_EQ(dst_uv_c[i * kStrideUV + j], dst_uv_opt[i * kStrideUV + j]); \
} \
} \
+ free_aligned_buffer_page_end(dst_a_c); \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_uv_c); \
+ free_aligned_buffer_page_end(dst_a_opt); \
free_aligned_buffer_page_end(dst_y_opt); \
free_aligned_buffer_page_end(dst_uv_opt); \
free_aligned_buffer_page_end(src_argb); \
}
-#endif
-#define TESTATOPLANAR(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- DIFF) \
- TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_ - 4, DIFF, _Any, +, 0) \
- TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, DIFF, _Unaligned, +, 1) \
- TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, DIFF, _Invert, -, 0) \
- TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, DIFF, _Opt, +, 0)
-
-TESTATOPLANAR(ABGR, 4, 1, I420, 2, 2, 4)
-TESTATOPLANAR(ARGB, 4, 1, I420, 2, 2, 4)
-TESTATOPLANAR(ARGB, 4, 1, I422, 2, 1, 2)
-TESTATOPLANAR(ARGB, 4, 1, I444, 1, 1, 2)
-TESTATOPLANAR(ARGB, 4, 1, J420, 2, 2, ARM_YUV_ERROR)
-TESTATOPLANAR(ARGB, 4, 1, J422, 2, 1, ARM_YUV_ERROR)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOPLANAR(ARGB1555, 2, 1, I420, 2, 2, 15)
-TESTATOPLANAR(ARGB4444, 2, 1, I420, 2, 2, 17)
-#endif
-TESTATOPLANAR(BGRA, 4, 1, I420, 2, 2, 4)
-TESTATOPLANAR(I400, 1, 1, I420, 2, 2, 2)
-TESTATOPLANAR(J400, 1, 1, J420, 2, 2, 2)
-TESTATOPLANAR(RAW, 3, 1, I420, 2, 2, 4)
-TESTATOPLANAR(RGB24, 3, 1, I420, 2, 2, 4)
-TESTATOPLANAR(RGB24, 3, 1, J420, 2, 2, ARM_YUV_ERROR)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOPLANAR(RGB565, 2, 1, I420, 2, 2, 5)
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOPLANARA(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2) \
+ TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0) \
+ TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
+#else
+#define TESTATOPLANARA(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOPLANARAI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
#endif
-TESTATOPLANAR(RGBA, 4, 1, I420, 2, 2, 4)
-TESTATOPLANAR(UYVY, 2, 1, I420, 2, 2, 2)
-TESTATOPLANAR(UYVY, 2, 1, I422, 2, 1, 2)
-TESTATOPLANAR(YUY2, 2, 1, I420, 2, 2, 2)
-TESTATOPLANAR(YUY2, 2, 1, I422, 2, 1, 2)
-
-#define TESTATOBIPLANARI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, \
- SUBSAMP_Y, W1280, N, NEG, OFF) \
+
+TESTATOPLANARA(ARGB, 4, 1, I420Alpha, 2, 2)
+
+#define TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ W1280, N, NEG, OFF) \
TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
const int kStride = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \
const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- align_buffer_page_end(src_argb, kStride* kHeight + OFF); \
- align_buffer_page_end(dst_y_c, kWidth* kHeight); \
+ align_buffer_page_end(src_argb, kStride * kHeight + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight); \
align_buffer_page_end(dst_uv_c, \
kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- align_buffer_page_end(dst_y_opt, kWidth* kHeight); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight); \
align_buffer_page_end(dst_uv_opt, \
kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
for (int i = 0; i < kHeight; ++i) \
for (int j = 0; j < kStride; ++j) \
src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \
- memset(dst_y_c, 1, kWidth* kHeight); \
+ memset(dst_y_c, 1, kWidth * kHeight); \
memset(dst_uv_c, 2, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
- memset(dst_y_opt, 101, kWidth* kHeight); \
+ memset(dst_y_opt, 101, kWidth * kHeight); \
memset(dst_uv_opt, 102, kStrideUV * 2 * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
MaskCpuFlags(disable_cpu_flags_); \
FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_c, kWidth, dst_uv_c, \
@@ -1048,28 +990,17 @@ TESTATOPLANAR(YUY2, 2, 1, I422, 2, 1, 2)
FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, dst_y_opt, kWidth, \
dst_uv_opt, kStrideUV * 2, kWidth, NEG kHeight); \
} \
- int max_diff = 0; \
for (int i = 0; i < kHeight; ++i) { \
for (int j = 0; j < kWidth; ++j) { \
- int abs_diff = abs(static_cast<int>(dst_y_c[i * kWidth + j]) - \
- static_cast<int>(dst_y_opt[i * kWidth + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
} \
} \
- EXPECT_LE(max_diff, 4); \
for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
for (int j = 0; j < kStrideUV * 2; ++j) { \
- int abs_diff = \
- abs(static_cast<int>(dst_uv_c[i * kStrideUV * 2 + j]) - \
- static_cast<int>(dst_uv_opt[i * kStrideUV * 2 + j])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ ASSERT_EQ(dst_uv_c[i * kStrideUV * 2 + j], \
+ dst_uv_opt[i * kStrideUV * 2 + j]); \
} \
} \
- EXPECT_LE(max_diff, 4); \
free_aligned_buffer_page_end(dst_y_c); \
free_aligned_buffer_page_end(dst_uv_c); \
free_aligned_buffer_page_end(dst_y_opt); \
@@ -1077,330 +1008,283 @@ TESTATOPLANAR(YUY2, 2, 1, I422, 2, 1, 2)
free_aligned_buffer_page_end(src_argb); \
}
-#define TESTATOBIPLANAR(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
- TESTATOBIPLANARI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_ - 4, _Any, +, 0) \
- TESTATOBIPLANARI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, _Unaligned, +, 1) \
- TESTATOBIPLANARI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, _Invert, -, 0) \
- TESTATOBIPLANARI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
- benchmark_width_, _Opt, +, 0)
-
-TESTATOBIPLANAR(ARGB, 1, 4, NV12, 2, 2)
-TESTATOBIPLANAR(ARGB, 1, 4, NV21, 2, 2)
-TESTATOBIPLANAR(ABGR, 1, 4, NV12, 2, 2)
-TESTATOBIPLANAR(ABGR, 1, 4, NV21, 2, 2)
-TESTATOBIPLANAR(YUY2, 2, 4, NV12, 2, 2)
-TESTATOBIPLANAR(UYVY, 2, 4, NV12, 2, 2)
-TESTATOBIPLANAR(AYUV, 1, 4, NV12, 2, 2)
-TESTATOBIPLANAR(AYUV, 1, 4, NV21, 2, 2)
-
-#define TESTATOBI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, W1280, DIFF, N, NEG, OFF) \
- TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
- const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
- const int kStrideA = \
- (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
- const int kStrideB = \
- (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
- align_buffer_page_end(src_argb, kStrideA* kHeightA + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeightB); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeightB); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- src_argb[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c, 1, kStrideB* kHeightB); \
- memset(dst_argb_opt, 101, kStrideB* kHeightB); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_A##To##FMT_B(src_argb + OFF, kStrideA, dst_argb_c, kStrideB, kWidth, \
- NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_A##To##FMT_B(src_argb + OFF, kStrideA, dst_argb_opt, kStrideB, \
- kWidth, NEG kHeight); \
- } \
- int max_diff = 0; \
- for (int i = 0; i < kStrideB * kHeightB; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb_c[i]) - \
- static_cast<int>(dst_argb_opt[i])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_argb); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTATOBRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, \
- STRIDE_B, HEIGHT_B, DIFF) \
- TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##_Random) { \
- for (int times = 0; times < benchmark_iterations_; ++times) { \
- const int kWidth = (fastrand() & 63) + 1; \
- const int kHeight = (fastrand() & 31) + 1; \
- const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
- const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
- const int kStrideA = \
- (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
- const int kStrideB = \
- (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
- align_buffer_page_end(src_argb, kStrideA* kHeightA); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeightB); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeightB); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- src_argb[i] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c, 123, kStrideB* kHeightB); \
- memset(dst_argb_opt, 123, kStrideB* kHeightB); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_A##To##FMT_B(src_argb, kStrideA, dst_argb_c, kStrideB, kWidth, \
- kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- FMT_A##To##FMT_B(src_argb, kStrideA, dst_argb_opt, kStrideB, kWidth, \
- kHeight); \
- for (int i = 0; i < kStrideB * kHeightB; ++i) { \
- EXPECT_NEAR(dst_argb_c[i], dst_argb_opt[i], DIFF); \
- } \
- free_aligned_buffer_page_end(src_argb); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- } \
- }
-
-#define TESTATOB(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, DIFF) \
- TESTATOBI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_ - 4, DIFF, _Any, +, 0) \
- TESTATOBI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Unaligned, +, 1) \
- TESTATOBI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Invert, -, 0) \
- TESTATOBI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Opt, +, 0) \
- TESTATOBRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, DIFF)
-
-// TODO(fbarchard): make ARM version of C code that matches NEON.
-TESTATOB(AB30, 4, 4, 1, ABGR, 4, 4, 1, 0)
-TESTATOB(AB30, 4, 4, 1, ARGB, 4, 4, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(ABGR, 4, 4, 1, AR30, 4, 4, 1, 0)
-#endif
-TESTATOB(ABGR, 4, 4, 1, ARGB, 4, 4, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(AR30, 4, 4, 1, AB30, 4, 4, 1, 0)
-#endif
-TESTATOB(AR30, 4, 4, 1, ABGR, 4, 4, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(AR30, 4, 4, 1, AR30, 4, 4, 1, 0)
-TESTATOB(AR30, 4, 4, 1, ARGB, 4, 4, 1, 0)
-#endif
-TESTATOB(ARGB, 4, 4, 1, ABGR, 4, 4, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(ARGB, 4, 4, 1, AR30, 4, 4, 1, 0)
-#endif
-TESTATOB(ARGB, 4, 4, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, ARGB1555, 2, 2, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, ARGB4444, 2, 2, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, ARGBMirror, 4, 4, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, BGRA, 4, 4, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, I400, 1, 1, 1, 2)
-TESTATOB(ARGB, 4, 4, 1, J400, 1, 1, 1, 2)
-TESTATOB(ARGB, 4, 4, 1, RAW, 3, 3, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, RGB24, 3, 3, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(ARGB, 4, 4, 1, RGB565, 2, 2, 1, 0)
-#endif
-TESTATOB(ARGB, 4, 4, 1, RGBA, 4, 4, 1, 0)
-TESTATOB(ARGB, 4, 4, 1, UYVY, 2, 4, 1, 4)
-TESTATOB(ARGB, 4, 4, 1, YUY2, 2, 4, 1, 4)
-TESTATOB(ARGB1555, 2, 2, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(ARGB4444, 2, 2, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(BGRA, 4, 4, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(I400, 1, 1, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(I400, 1, 1, 1, I400, 1, 1, 1, 0)
-TESTATOB(I400, 1, 1, 1, I400Mirror, 1, 1, 1, 0)
-TESTATOB(J400, 1, 1, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(J400, 1, 1, 1, J400, 1, 1, 1, 0)
-TESTATOB(RAW, 3, 3, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(RAW, 3, 3, 1, RGB24, 3, 3, 1, 0)
-TESTATOB(RGB24, 3, 3, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(RGB24, 3, 3, 1, J400, 1, 1, 1, 0)
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOB(RGB565, 2, 2, 1, ARGB, 4, 4, 1, 0)
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOBP(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2) \
+ TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0) \
+ TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
+#else
+#define TESTATOBP(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOBPI(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
#endif
-TESTATOB(RGBA, 4, 4, 1, ARGB, 4, 4, 1, 0)
-TESTATOB(UYVY, 2, 4, 1, ARGB, 4, 4, 1, ARM_YUV_ERROR)
-TESTATOB(YUY2, 2, 4, 1, ARGB, 4, 4, 1, ARM_YUV_ERROR)
-TESTATOB(YUY2, 2, 4, 1, Y, 1, 1, 1, 0)
-
-#define TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, W1280, DIFF, N, NEG, OFF) \
- TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##Dither##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
- const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
- const int kStrideA = \
- (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
- const int kStrideB = \
- (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
- align_buffer_page_end(src_argb, kStrideA* kHeightA + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeightB); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeightB); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- src_argb[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c, 1, kStrideB* kHeightB); \
- memset(dst_argb_opt, 101, kStrideB* kHeightB); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_A##To##FMT_B##Dither(src_argb + OFF, kStrideA, dst_argb_c, kStrideB, \
- NULL, kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_A##To##FMT_B##Dither(src_argb + OFF, kStrideA, dst_argb_opt, \
- kStrideB, NULL, kWidth, NEG kHeight); \
- } \
- int max_diff = 0; \
- for (int i = 0; i < kStrideB * kHeightB; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb_c[i]) - \
- static_cast<int>(dst_argb_opt[i])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_argb); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTATOBDRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, \
- STRIDE_B, HEIGHT_B, DIFF) \
- TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##Dither_Random) { \
- for (int times = 0; times < benchmark_iterations_; ++times) { \
- const int kWidth = (fastrand() & 63) + 1; \
- const int kHeight = (fastrand() & 31) + 1; \
- const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
- const int kHeightB = (kHeight + HEIGHT_B - 1) / HEIGHT_B * HEIGHT_B; \
- const int kStrideA = \
- (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
- const int kStrideB = \
- (kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
- align_buffer_page_end(src_argb, kStrideA* kHeightA); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeightB); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeightB); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- src_argb[i] = (fastrand() & 0xff); \
+
+TESTATOBP(ARGB, 1, 4, NV12, 2, 2)
+TESTATOBP(ARGB, 1, 4, NV21, 2, 2)
+TESTATOBP(ARGB, 1, 4, NV16, 2, 1)
+TESTATOBP(ARGB, 1, 4, NV24, 1, 1)
+TESTATOBP(ABGR, 1, 4, NV12, 2, 2)
+TESTATOBP(ABGR, 1, 4, NV21, 2, 2)
+TESTATOBP(YUY2, 2, 4, NV12, 2, 2)
+TESTATOBP(UYVY, 2, 4, NV12, 2, 2)
+TESTATOBP(AYUV, 1, 4, NV12, 2, 2)
+TESTATOBP(AYUV, 1, 4, NV21, 2, 2)
+
+// Matrix API with I601 constants matches the non-matrix wrapper.
+#define TESTATOBPMATRIX(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \
+ const int kWidth = 16; \
+ const int kHeight = 16; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \
+ align_buffer_page_end(dst_y, kWidth * kHeight); \
+ align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV); \
+ align_buffer_page_end(ref_y, kWidth * kHeight); \
+ align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV); \
+ \
+ MemRandomize(src_argb, kWidth * kHeight * BPP_A); \
+ \
+ FMT_A##To##FMT_B##Matrix(src_argb, kWidth * BPP_A, dst_y, kWidth, dst_uv, \
+ kStrideUV * 2, &kArgbI601Constants, kWidth, \
+ kHeight); \
+ FMT_A##To##FMT_B(src_argb, kWidth * BPP_A, ref_y, kWidth, ref_uv, \
+ kStrideUV * 2, kWidth, kHeight); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ ASSERT_EQ(dst_y[i], ref_y[i]); \
+ } \
+ for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \
+ ASSERT_EQ(dst_uv[i], ref_uv[i]); \
+ } \
+ \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_y); \
+ free_aligned_buffer_page_end(dst_uv); \
+ free_aligned_buffer_page_end(ref_y); \
+ free_aligned_buffer_page_end(ref_uv); \
+ }
+
+TESTATOBPMATRIX(ARGB, 4, NV12, 2, 2)
+TESTATOBPMATRIX(ARGB, 4, NV16, 2, 1)
+TESTATOBPMATRIX(ARGB, 4, NV24, 1, 1)
+
+#define TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ W1280, N, NEG, OFF) \
+ TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kStride = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_argb, kStride * kHeight + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight * 2); \
+ align_buffer_page_end(dst_uv_c, kStrideUV * 2 * kHeightUV * 2); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight * 2); \
+ align_buffer_page_end(dst_uv_opt, kStrideUV * 2 * kHeightUV * 2); \
+ for (int i = 0; i < kHeight; ++i) \
+ for (int j = 0; j < kStride; ++j) \
+ src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \
+ memset(dst_y_c, 1, kWidth * kHeight * 2); \
+ memset(dst_uv_c, 2, kStrideUV * 2 * kHeightUV * 2); \
+ memset(dst_y_opt, 101, kWidth * kHeight * 2); \
+ memset(dst_uv_opt, 102, kStrideUV * 2 * kHeightUV * 2); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \
+ reinterpret_cast<uint16_t*>(dst_y_c), kWidth, \
+ reinterpret_cast<uint16_t*>(dst_uv_c), \
+ kStrideUV * 2, kWidth, NEG kHeight); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \
+ reinterpret_cast<uint16_t*>(dst_y_opt), kWidth, \
+ reinterpret_cast<uint16_t*>(dst_uv_opt), \
+ kStrideUV * 2, kWidth, NEG kHeight); \
+ } \
+ const uint16_t* y_c = reinterpret_cast<const uint16_t*>(dst_y_c); \
+ const uint16_t* y_opt = reinterpret_cast<const uint16_t*>(dst_y_opt); \
+ const uint16_t* uv_c = reinterpret_cast<const uint16_t*>(dst_uv_c); \
+ const uint16_t* uv_opt = reinterpret_cast<const uint16_t*>(dst_uv_opt); \
+ for (int i = 0; i < kHeight; ++i) { \
+ for (int j = 0; j < kWidth; ++j) { \
+ ASSERT_EQ(y_c[i * kWidth + j], y_opt[i * kWidth + j]); \
} \
- memset(dst_argb_c, 123, kStrideB* kHeightB); \
- memset(dst_argb_opt, 123, kStrideB* kHeightB); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_A##To##FMT_B##Dither(src_argb, kStrideA, dst_argb_c, kStrideB, NULL, \
- kWidth, kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- FMT_A##To##FMT_B##Dither(src_argb, kStrideA, dst_argb_opt, kStrideB, \
- NULL, kWidth, kHeight); \
- int max_diff = 0; \
- for (int i = 0; i < kStrideB * kHeightB; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb_c[i]) - \
- static_cast<int>(dst_argb_opt[i])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
+ } \
+ for (int i = 0; i < kHeightUV; ++i) { \
+ for (int j = 0; j < kStrideUV * 2; ++j) { \
+ ASSERT_EQ(uv_c[i * kStrideUV * 2 + j], uv_opt[i * kStrideUV * 2 + j]); \
} \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_argb); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
} \
+ free_aligned_buffer_page_end(dst_y_c); \
+ free_aligned_buffer_page_end(dst_uv_c); \
+ free_aligned_buffer_page_end(dst_y_opt); \
+ free_aligned_buffer_page_end(dst_uv_opt); \
+ free_aligned_buffer_page_end(src_argb); \
}
-#define TESTATOBD(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, DIFF) \
- TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_ - 4, DIFF, _Any, +, 0) \
- TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Unaligned, +, 1) \
- TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Invert, -, 0) \
- TESTATOBDI(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, benchmark_width_, DIFF, _Opt, +, 0) \
- TESTATOBDRANDOM(FMT_A, BPP_A, STRIDE_A, HEIGHT_A, FMT_B, BPP_B, STRIDE_B, \
- HEIGHT_B, DIFF)
-
-#ifdef LITTLE_ENDIAN_TEST
-TESTATOBD(ARGB, 4, 4, 1, RGB565, 2, 2, 1, 0)
+#if defined(ENABLE_FULL_TESTS)
+#define TESTATOBP16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_ + 1, _Any, +, 0) \
+ TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Unaligned, +, 2) \
+ TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Invert, -, 0) \
+ TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
+#else
+#define TESTATOBP16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y) \
+ TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ benchmark_width_, _Opt, +, 0)
#endif
-#define TESTSYMI(FMT_ATOB, BPP_A, STRIDE_A, HEIGHT_A, W1280, N, NEG, OFF) \
- TEST_F(LibYUVConvertTest, FMT_ATOB##_Symetric##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+TESTATOBP16(ARGB, 1, 4, P010, 2, 2)
+TESTATOBP16(ARGB, 1, 4, P210, 2, 1)
+TESTATOBP16(ARGB, 1, 4, P410, 1, 1)
+
+#define TESTATOBPMATRIX16(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \
+ const int kWidth = 16; \
+ const int kHeight = 16; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \
+ align_buffer_page_end(dst_y, kWidth * kHeight * 2); \
+ align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV * 2); \
+ align_buffer_page_end(ref_y, kWidth * kHeight * 2); \
+ align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV * 2); \
+ \
+ MemRandomize(src_argb, kWidth * kHeight * BPP_A); \
+ \
+ FMT_A##To##FMT_B##Matrix( \
+ src_argb, kWidth * BPP_A, reinterpret_cast<uint16_t*>(dst_y), kWidth, \
+ reinterpret_cast<uint16_t*>(dst_uv), kStrideUV * 2, \
+ &kArgbI601Constants, kWidth, kHeight); \
+ FMT_A##To##FMT_B( \
+ src_argb, kWidth * BPP_A, reinterpret_cast<uint16_t*>(ref_y), kWidth, \
+ reinterpret_cast<uint16_t*>(ref_uv), kStrideUV * 2, kWidth, kHeight); \
+ const uint16_t* y = reinterpret_cast<const uint16_t*>(dst_y); \
+ const uint16_t* y_ref = reinterpret_cast<const uint16_t*>(ref_y); \
+ const uint16_t* uv = reinterpret_cast<const uint16_t*>(dst_uv); \
+ const uint16_t* uv_ref = reinterpret_cast<const uint16_t*>(ref_uv); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ ASSERT_EQ(y[i], y_ref[i]); \
+ } \
+ for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \
+ ASSERT_EQ(uv[i], uv_ref[i]); \
+ } \
+ \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst_y); \
+ free_aligned_buffer_page_end(dst_uv); \
+ free_aligned_buffer_page_end(ref_y); \
+ free_aligned_buffer_page_end(ref_uv); \
+ }
+
+TESTATOBPMATRIX16(ARGB, 4, P010, 2, 2)
+TESTATOBPMATRIX16(ARGB, 4, P210, 2, 1)
+TESTATOBPMATRIX16(ARGB, 4, P410, 1, 1)
+
+#define TESTATOBP16FROM8(FMT_A, FMT_16, FMT_8, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_16##From##FMT_8) { \
+ const int kWidth = 16; \
+ const int kHeight = 16; \
+ const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4); \
+ align_buffer_page_end(dst8_y, kWidth * kHeight); \
+ align_buffer_page_end(dst8_uv, kStrideUV * 2 * kHeightUV); \
+ align_buffer_page_end(dst16_y, kWidth * kHeight * 2); \
+ align_buffer_page_end(dst16_uv, kStrideUV * 2 * kHeightUV * 2); \
+ MemRandomize(src_argb, kWidth * kHeight * 4); \
+ FMT_A##To##FMT_8(src_argb, kWidth * 4, dst8_y, kWidth, dst8_uv, \
+ kStrideUV * 2, kWidth, kHeight); \
+ FMT_A##To##FMT_16(src_argb, kWidth * 4, \
+ reinterpret_cast<uint16_t*>(dst16_y), kWidth, \
+ reinterpret_cast<uint16_t*>(dst16_uv), kStrideUV * 2, \
+ kWidth, kHeight); \
+ const uint16_t* y16 = reinterpret_cast<const uint16_t*>(dst16_y); \
+ const uint16_t* uv16 = reinterpret_cast<const uint16_t*>(dst16_uv); \
+ for (int i = 0; i < kWidth * kHeight; ++i) { \
+ ASSERT_EQ(y16[i], static_cast<uint16_t>( \
+ ((dst8_y[i] * 1024u * 0x0101u) >> 16) << 6)); \
+ } \
+ for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \
+ ASSERT_EQ(uv16[i], static_cast<uint16_t>( \
+ ((dst8_uv[i] * 1024u * 0x0101u) >> 16) << 6)); \
+ } \
+ free_aligned_buffer_page_end(src_argb); \
+ free_aligned_buffer_page_end(dst8_y); \
+ free_aligned_buffer_page_end(dst8_uv); \
+ free_aligned_buffer_page_end(dst16_y); \
+ free_aligned_buffer_page_end(dst16_uv); \
+ }
+
+TESTATOBP16FROM8(ARGB, P010, NV12, 2, 2)
+TESTATOBP16FROM8(ARGB, P210, NV16, 2, 1)
+TESTATOBP16FROM8(ARGB, P410, NV24, 1, 1)
+
+#define TESTATOBP16TRANSITIVE(FMT_A, FMT_I, FMT_P, SUBSAMP_X, SUBSAMP_Y) \
+ TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_P##Transitive) { \
+ const int kWidth = benchmark_width_ | 1; \
const int kHeight = benchmark_height_; \
- const int kHeightA = (kHeight + HEIGHT_A - 1) / HEIGHT_A * HEIGHT_A; \
- const int kStrideA = \
- (kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
- align_buffer_page_end(src_argb, kStrideA* kHeightA + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideA* kHeightA); \
- align_buffer_page_end(dst_argb_opt, kStrideA* kHeightA); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- src_argb[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c, 1, kStrideA* kHeightA); \
- memset(dst_argb_opt, 101, kStrideA* kHeightA); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_ATOB(src_argb + OFF, kStrideA, dst_argb_c, kStrideA, kWidth, \
- NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_ATOB(src_argb + OFF, kStrideA, dst_argb_opt, kStrideA, kWidth, \
- NEG kHeight); \
+ const int kWidthUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
+ const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \
+ const int kYSize = kWidth * kHeight; \
+ const int kUVSize = kWidthUV * kHeightUV; \
+ align_buffer_page_end(src_argb, kYSize * 4); \
+ align_buffer_page_end(dst_p_y, kYSize * 2); \
+ align_buffer_page_end(dst_p_uv, kUVSize * 2 * 2); \
+ align_buffer_page_end(ref_y, kYSize); \
+ align_buffer_page_end(ref_u, kUVSize); \
+ align_buffer_page_end(ref_v, kUVSize); \
+ align_buffer_page_end(ref_p_y, kYSize * 2); \
+ align_buffer_page_end(ref_p_uv, kUVSize * 2 * 2); \
+ uint16_t* dst_p_y16 = reinterpret_cast<uint16_t*>(dst_p_y); \
+ uint16_t* dst_p_uv16 = reinterpret_cast<uint16_t*>(dst_p_uv); \
+ uint16_t* ref_p_y16 = reinterpret_cast<uint16_t*>(ref_p_y); \
+ uint16_t* ref_p_uv16 = reinterpret_cast<uint16_t*>(ref_p_uv); \
+ MemRandomize(src_argb, kYSize * 4); \
+ memset(dst_p_y, 1, kYSize * 2); \
+ memset(dst_p_uv, 2, kUVSize * 2 * 2); \
+ memset(ref_y, 3, kYSize); \
+ memset(ref_u, 4, kUVSize); \
+ memset(ref_v, 5, kUVSize); \
+ memset(ref_p_y, 6, kYSize * 2); \
+ memset(ref_p_uv, 7, kUVSize * 2 * 2); \
+ EXPECT_EQ(0, FMT_A##To##FMT_P(src_argb, kWidth * 4, dst_p_y16, kWidth, \
+ dst_p_uv16, kWidthUV * 2, kWidth, kHeight)); \
+ EXPECT_EQ(0, \
+ FMT_A##To##FMT_I(src_argb, kWidth * 4, ref_y, kWidth, ref_u, \
+ kWidthUV, ref_v, kWidthUV, kWidth, kHeight)); \
+ EXPECT_EQ(0, FMT_I##To##FMT_P(ref_y, kWidth, ref_u, kWidthUV, ref_v, \
+ kWidthUV, ref_p_y16, kWidth, ref_p_uv16, \
+ kWidthUV * 2, kWidth, kHeight)); \
+ for (int i = 0; i < kYSize; ++i) { \
+ ASSERT_EQ(dst_p_y16[i], ref_p_y16[i]); \
} \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_ATOB(dst_argb_c, kStrideA, dst_argb_c, kStrideA, kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- FMT_ATOB(dst_argb_opt, kStrideA, dst_argb_opt, kStrideA, kWidth, \
- NEG kHeight); \
- for (int i = 0; i < kStrideA * kHeightA; ++i) { \
- EXPECT_EQ(src_argb[i + OFF], dst_argb_opt[i]); \
- EXPECT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ for (int i = 0; i < kUVSize * 2; ++i) { \
+ ASSERT_EQ(dst_p_uv16[i], ref_p_uv16[i]); \
} \
free_aligned_buffer_page_end(src_argb); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTSYM(FMT_ATOB, BPP_A, STRIDE_A, HEIGHT_A) \
- TESTSYMI(FMT_ATOB, BPP_A, STRIDE_A, HEIGHT_A, benchmark_width_ - 4, _Any, +, \
- 0) \
- TESTSYMI(FMT_ATOB, BPP_A, STRIDE_A, HEIGHT_A, benchmark_width_, _Unaligned, \
- +, 1) \
- TESTSYMI(FMT_ATOB, BPP_A, STRIDE_A, HEIGHT_A, benchmark_width_, _Opt, +, 0)
-
-TESTSYM(ARGBToARGB, 4, 4, 1)
-TESTSYM(ARGBToBGRA, 4, 4, 1)
-TESTSYM(ARGBToABGR, 4, 4, 1)
-TESTSYM(BGRAToARGB, 4, 4, 1)
-TESTSYM(ABGRToARGB, 4, 4, 1)
-
-TEST_F(LibYUVConvertTest, Test565) {
- SIMD_ALIGNED(uint8_t orig_pixels[256][4]);
- SIMD_ALIGNED(uint8_t pixels565[256][2]);
-
- for (int i = 0; i < 256; ++i) {
- for (int j = 0; j < 4; ++j) {
- orig_pixels[i][j] = i;
- }
+ free_aligned_buffer_page_end(dst_p_y); \
+ free_aligned_buffer_page_end(dst_p_uv); \
+ free_aligned_buffer_page_end(ref_y); \
+ free_aligned_buffer_page_end(ref_u); \
+ free_aligned_buffer_page_end(ref_v); \
+ free_aligned_buffer_page_end(ref_p_y); \
+ free_aligned_buffer_page_end(ref_p_uv); \
}
- ARGBToRGB565(&orig_pixels[0][0], 0, &pixels565[0][0], 0, 256, 1);
- uint32_t checksum = HashDjb2(&pixels565[0][0], sizeof(pixels565), 5381);
- EXPECT_EQ(610919429u, checksum);
-}
+
+TESTATOBP16TRANSITIVE(ARGB, I420, P010, 2, 2)
+TESTATOBP16TRANSITIVE(ARGB, I422, P210, 2, 1)
+TESTATOBP16TRANSITIVE(ARGB, I444, P410, 1, 1)
+
+#if !defined(LEAN_TESTS)
#ifdef HAVE_JPEG
TEST_F(LibYUVConvertTest, ValidateJpeg) {
@@ -1414,11 +1298,11 @@ TEST_F(LibYUVConvertTest, ValidateJpeg) {
// No SOI or EOI. Expect fail.
memset(orig_pixels, 0, kSize);
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kSize));
// Test special value that matches marker start.
memset(orig_pixels, 0xff, kSize);
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kSize));
// EOI, SOI. Expect pass.
orig_pixels[0] = 0xff;
@@ -1427,7 +1311,7 @@ TEST_F(LibYUVConvertTest, ValidateJpeg) {
orig_pixels[kSize - kOff + 0] = 0xff;
orig_pixels[kSize - kOff + 1] = 0xd9; // EOI.
for (int times = 0; times < benchmark_iterations_; ++times) {
- EXPECT_TRUE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_TRUE(ValidateJpeg(orig_pixels, kSize));
}
free_aligned_buffer_page_end(orig_pixels);
}
@@ -1445,7 +1329,7 @@ TEST_F(LibYUVConvertTest, ValidateJpegLarge) {
// No SOI or EOI. Expect fail.
memset(orig_pixels, 0, kBufSize);
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kBufSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kBufSize));
// EOI, SOI. Expect pass.
orig_pixels[0] = 0xff;
@@ -1454,7 +1338,7 @@ TEST_F(LibYUVConvertTest, ValidateJpegLarge) {
orig_pixels[kSize - kOff + 0] = 0xff;
orig_pixels[kSize - kOff + 1] = 0xd9; // EOI.
for (int times = 0; times < benchmark_iterations_; ++times) {
- EXPECT_TRUE(ValidateJpeg(orig_pixels, kBufSize));
+ ASSERT_TRUE(ValidateJpeg(orig_pixels, kBufSize));
}
free_aligned_buffer_page_end(orig_pixels);
}
@@ -1469,24 +1353,24 @@ TEST_F(LibYUVConvertTest, InvalidateJpeg) {
align_buffer_page_end(orig_pixels, kSize);
// NULL pointer. Expect fail.
- EXPECT_FALSE(ValidateJpeg(NULL, kSize));
+ ASSERT_FALSE(ValidateJpeg(NULL, kSize));
// Negative size. Expect fail.
- EXPECT_FALSE(ValidateJpeg(orig_pixels, -1));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, -1));
// Too large size. Expect fail.
- EXPECT_FALSE(ValidateJpeg(orig_pixels, 0xfb000000ull));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, 0xfb000000ull));
// No SOI or EOI. Expect fail.
memset(orig_pixels, 0, kSize);
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kSize));
// SOI but no EOI. Expect fail.
orig_pixels[0] = 0xff;
orig_pixels[1] = 0xd8; // SOI.
orig_pixels[2] = 0xff;
for (int times = 0; times < benchmark_iterations_; ++times) {
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kSize));
}
// EOI but no SOI. Expect fail.
@@ -1494,7 +1378,7 @@ TEST_F(LibYUVConvertTest, InvalidateJpeg) {
orig_pixels[1] = 0;
orig_pixels[kSize - kOff + 0] = 0xff;
orig_pixels[kSize - kOff + 1] = 0xd9; // EOI.
- EXPECT_FALSE(ValidateJpeg(orig_pixels, kSize));
+ ASSERT_FALSE(ValidateJpeg(orig_pixels, kSize));
free_aligned_buffer_page_end(orig_pixels);
}
@@ -1818,7 +1702,7 @@ TEST_F(LibYUVConvertTest, TestMJPGSize) {
int width = 0;
int height = 0;
int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
printf("test jpeg size %d x %d\n", width, height);
}
@@ -1827,12 +1711,15 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420) {
int width = 0;
int height = 0;
int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_u, half_width * half_height);
@@ -1842,15 +1729,15 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420) {
dst_v, half_width, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Test result matches known hash value.
uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
uint32_t dst_u_hash = HashDjb2(dst_u, half_width * half_height, 5381);
uint32_t dst_v_hash = HashDjb2(dst_v, half_width * half_height, 5381);
- EXPECT_EQ(dst_y_hash, 2682851208u);
- EXPECT_EQ(dst_u_hash, 2501859930u);
- EXPECT_EQ(dst_v_hash, 2126459123u);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_u_hash, 2501859930u);
+ ASSERT_EQ(dst_v_hash, 2126459123u);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_u);
@@ -1861,12 +1748,15 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420_NV21) {
int width = 0;
int height = 0;
int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
// Convert to NV21
align_buffer_page_end(dst_y, width * height);
@@ -1877,7 +1767,7 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420_NV21) {
half_width * 2, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Convert to I420
align_buffer_page_end(dst2_y, width * height);
@@ -1888,7 +1778,7 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420_NV21) {
dst2_v, half_width, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Convert I420 to NV21
align_buffer_page_end(dst3_y, width * height);
@@ -1898,11 +1788,11 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420_NV21) {
width, dst3_vu, half_width * 2, width, height);
for (int i = 0; i < width * height; ++i) {
- EXPECT_EQ(dst_y[i], dst3_y[i]);
+ ASSERT_EQ(dst_y[i], dst3_y[i]);
}
for (int i = 0; i < half_width * half_height * 2; ++i) {
- EXPECT_EQ(dst_vu[i], dst3_vu[i]);
- EXPECT_EQ(dst_vu[i], dst3_vu[i]);
+ ASSERT_EQ(dst_vu[i], dst3_vu[i]);
+ ASSERT_EQ(dst_vu[i], dst3_vu[i]);
}
free_aligned_buffer_page_end(dst3_y);
@@ -1916,134 +1806,360 @@ TEST_F(LibYUVConvertTest, TestMJPGToI420_NV21) {
free_aligned_buffer_page_end(dst_vu);
}
-TEST_F(LibYUVConvertTest, TestMJPGToNV21_420) {
+TEST_F(LibYUVConvertTest, TestMJPGToI420_NV12) {
+ int width = 0;
+ int height = 0;
+ int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
+
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
+ benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
+
+ // Convert to NV12
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, half_width * half_height * 2);
+
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToNV12(kTest2Jpg, kTest2JpgLen, dst_y, width, dst_uv,
+ half_width * 2, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Convert to I420
+ align_buffer_page_end(dst2_y, width * height);
+ align_buffer_page_end(dst2_u, half_width * half_height);
+ align_buffer_page_end(dst2_v, half_width * half_height);
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToI420(kTest2Jpg, kTest2JpgLen, dst2_y, width, dst2_u, half_width,
+ dst2_v, half_width, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Convert I420 to NV12
+ align_buffer_page_end(dst3_y, width * height);
+ align_buffer_page_end(dst3_uv, half_width * half_height * 2);
+
+ I420ToNV12(dst2_y, width, dst2_u, half_width, dst2_v, half_width, dst3_y,
+ width, dst3_uv, half_width * 2, width, height);
+
+ for (int i = 0; i < width * height; ++i) {
+ ASSERT_EQ(dst_y[i], dst3_y[i]);
+ }
+ for (int i = 0; i < half_width * half_height * 2; ++i) {
+ ASSERT_EQ(dst_uv[i], dst3_uv[i]);
+ ASSERT_EQ(dst_uv[i], dst3_uv[i]);
+ }
+
+ free_aligned_buffer_page_end(dst3_y);
+ free_aligned_buffer_page_end(dst3_uv);
+
+ free_aligned_buffer_page_end(dst2_y);
+ free_aligned_buffer_page_end(dst2_u);
+ free_aligned_buffer_page_end(dst2_v);
+
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+}
+
+TEST_F(LibYUVConvertTest, TestMJPGToNV21_420) {
+ int width = 0;
+ int height = 0;
+ int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
+
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
+ benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
+
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, half_width * half_height * 2);
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToNV21(kTest2Jpg, kTest2JpgLen, dst_y, width, dst_uv,
+ half_width * 2, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Test result matches known hash value.
+ uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
+ uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_uv_hash, 1069662856u);
+
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+}
+
+TEST_F(LibYUVConvertTest, TestMJPGToNV12_420) {
+ int width = 0;
+ int height = 0;
+ int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
+
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
+ benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
+
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, half_width * half_height * 2);
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToNV12(kTest2Jpg, kTest2JpgLen, dst_y, width, dst_uv,
+ half_width * 2, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Test result matches known hash value. Hashes are for VU so flip the plane.
+ uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
+ align_buffer_page_end(dst_vu, half_width * half_height * 2);
+ SwapUVPlane(dst_uv, half_width * 2, dst_vu, half_width * 2, half_width,
+ half_height);
+ uint32_t dst_vu_hash = HashDjb2(dst_vu, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_vu_hash, 1069662856u);
+
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(dst_vu);
+}
+
+// TODO(fbarchard): Improve test to compare against I422, not checksum
+TEST_F(LibYUVConvertTest, DISABLED_TestMJPGToNV21_422) {
+ int width = 0;
+ int height = 0;
+ int ret = MJPGSize(kTest3Jpg, kTest3JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
+
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
+ benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
+
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, half_width * half_height * 2);
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToNV21(kTest3Jpg, kTest3JpgLen, dst_y, width, dst_uv,
+ half_width * 2, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Test result matches known hash value.
+ uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
+ uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_uv_hash, 493520167u);
+
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+}
+
+TEST_F(LibYUVConvertTest, DISABLED_TestMJPGToNV12_422) {
+ int width = 0;
+ int height = 0;
+ int ret = MJPGSize(kTest3Jpg, kTest3JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
+
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
+ benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
+
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, half_width * half_height * 2);
+ for (int times = 0; times < benchmark_iterations; ++times) {
+ ret = MJPGToNV12(kTest3Jpg, kTest3JpgLen, dst_y, width, dst_uv,
+ half_width * 2, width, height, width, height);
+ }
+ // Expect sucesss
+ ASSERT_EQ(0, ret);
+
+ // Test result matches known hash value. Hashes are for VU so flip the plane.
+ uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
+ align_buffer_page_end(dst_vu, half_width * half_height * 2);
+ SwapUVPlane(dst_uv, half_width * 2, dst_vu, half_width * 2, half_width,
+ half_height);
+ uint32_t dst_vu_hash = HashDjb2(dst_vu, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_vu_hash, 493520167u);
+
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(dst_vu);
+}
+
+TEST_F(LibYUVConvertTest, TestMJPGToNV21_400) {
int width = 0;
int height = 0;
- int ret = MJPGSize(kTest2Jpg, kTest2JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ int ret = MJPGSize(kTest0Jpg, kTest0JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_uv, half_width * half_height * 2);
for (int times = 0; times < benchmark_iterations; ++times) {
- ret = MJPGToNV21(kTest2Jpg, kTest2JpgLen, dst_y, width, dst_uv,
+ ret = MJPGToNV21(kTest0Jpg, kTest0JpgLen, dst_y, width, dst_uv,
half_width * 2, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Test result matches known hash value.
uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
- EXPECT_EQ(dst_y_hash, 2682851208u);
- EXPECT_EQ(dst_uv_hash, 1069662856u);
+ ASSERT_EQ(dst_y_hash, 330644005u);
+ ASSERT_EQ(dst_uv_hash, 135214341u);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_uv);
}
-TEST_F(LibYUVConvertTest, TestMJPGToNV21_422) {
+TEST_F(LibYUVConvertTest, TestMJPGToNV12_400) {
int width = 0;
int height = 0;
- int ret = MJPGSize(kTest3Jpg, kTest3JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ int ret = MJPGSize(kTest0Jpg, kTest0JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_uv, half_width * half_height * 2);
for (int times = 0; times < benchmark_iterations; ++times) {
- ret = MJPGToNV21(kTest3Jpg, kTest3JpgLen, dst_y, width, dst_uv,
+ ret = MJPGToNV12(kTest0Jpg, kTest0JpgLen, dst_y, width, dst_uv,
half_width * 2, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
- // Test result matches known hash value.
+ // Test result matches known hash value. Hashes are for VU so flip the plane.
uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
- uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
- EXPECT_EQ(dst_y_hash, 2682851208u);
- EXPECT_EQ(dst_uv_hash, 3543430771u);
+ align_buffer_page_end(dst_vu, half_width * half_height * 2);
+ SwapUVPlane(dst_uv, half_width * 2, dst_vu, half_width * 2, half_width,
+ half_height);
+ uint32_t dst_vu_hash = HashDjb2(dst_vu, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 330644005u);
+ ASSERT_EQ(dst_vu_hash, 135214341u);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(dst_vu);
}
-TEST_F(LibYUVConvertTest, TestMJPGToNV21_400) {
+TEST_F(LibYUVConvertTest, TestMJPGToNV21_444) {
int width = 0;
int height = 0;
- int ret = MJPGSize(kTest0Jpg, kTest0JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ int ret = MJPGSize(kTest1Jpg, kTest1JpgLen, &width, &height);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_uv, half_width * half_height * 2);
for (int times = 0; times < benchmark_iterations; ++times) {
- ret = MJPGToNV21(kTest0Jpg, kTest0JpgLen, dst_y, width, dst_uv,
+ ret = MJPGToNV21(kTest1Jpg, kTest1JpgLen, dst_y, width, dst_uv,
half_width * 2, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Test result matches known hash value.
uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
- EXPECT_EQ(dst_y_hash, 330644005u);
- EXPECT_EQ(dst_uv_hash, 135214341u);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_uv_hash, 506143297u);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_uv);
}
-TEST_F(LibYUVConvertTest, TestMJPGToNV21_444) {
+TEST_F(LibYUVConvertTest, TestMJPGToNV12_444) {
int width = 0;
int height = 0;
int ret = MJPGSize(kTest1Jpg, kTest1JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_uv, half_width * half_height * 2);
for (int times = 0; times < benchmark_iterations; ++times) {
- ret = MJPGToNV21(kTest1Jpg, kTest1JpgLen, dst_y, width, dst_uv,
+ ret = MJPGToNV12(kTest1Jpg, kTest1JpgLen, dst_y, width, dst_uv,
half_width * 2, width, height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
- // Test result matches known hash value.
+ // Test result matches known hash value. Hashes are for VU so flip the plane.
uint32_t dst_y_hash = HashDjb2(dst_y, width * height, 5381);
- uint32_t dst_uv_hash = HashDjb2(dst_uv, half_width * half_height * 2, 5381);
- EXPECT_EQ(dst_y_hash, 2682851208u);
- EXPECT_EQ(dst_uv_hash, 506143297u);
+ align_buffer_page_end(dst_vu, half_width * half_height * 2);
+ SwapUVPlane(dst_uv, half_width * 2, dst_vu, half_width * 2, half_width,
+ half_height);
+ uint32_t dst_vu_hash = HashDjb2(dst_vu, half_width * half_height * 2, 5381);
+ ASSERT_EQ(dst_y_hash, 2682851208u);
+ ASSERT_EQ(dst_vu_hash, 506143297u);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(dst_vu);
}
TEST_F(LibYUVConvertTest, TestMJPGToARGB) {
int width = 0;
int height = 0;
int ret = MJPGSize(kTest3Jpg, kTest3JpgLen, &width, &height);
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
int benchmark_iterations = benchmark_iterations_ * benchmark_width_ *
benchmark_height_ / (width * height);
+ if (benchmark_iterations < 1) {
+ benchmark_iterations = 1;
+ }
align_buffer_page_end(dst_argb, width * height * 4);
for (int times = 0; times < benchmark_iterations; ++times) {
@@ -2051,18 +2167,22 @@ TEST_F(LibYUVConvertTest, TestMJPGToARGB) {
height, width, height);
}
// Expect sucesss
- EXPECT_EQ(0, ret);
+ ASSERT_EQ(0, ret);
// Test result matches known hash value.
uint32_t dst_argb_hash = HashDjb2(dst_argb, width * height, 5381);
- EXPECT_EQ(dst_argb_hash, 2355976473u);
+#ifdef LIBYUV_UNLIMITED_DATA
+ ASSERT_EQ(dst_argb_hash, 3900633302u);
+#else
+ ASSERT_EQ(dst_argb_hash, 2355976473u);
+#endif
free_aligned_buffer_page_end(dst_argb);
}
-static int ShowJPegInfo(const uint8_t* sample, size_t sample_size) {
+static bool ShowJPegInfo(const uint8_t* sample, size_t sample_size) {
MJpegDecoder mjpeg_decoder;
- LIBYUV_BOOL ret = mjpeg_decoder.LoadFrame(sample, sample_size);
+ bool ret = mjpeg_decoder.LoadFrame(sample, sample_size);
int width = mjpeg_decoder.GetWidth();
int height = mjpeg_decoder.GetHeight();
@@ -2117,12 +2237,12 @@ static int ShowJPegInfo(const uint8_t* sample, size_t sample_size) {
}
TEST_F(LibYUVConvertTest, TestMJPGInfo) {
- EXPECT_EQ(1, ShowJPegInfo(kTest0Jpg, kTest0JpgLen));
- EXPECT_EQ(1, ShowJPegInfo(kTest1Jpg, kTest1JpgLen));
- EXPECT_EQ(1, ShowJPegInfo(kTest2Jpg, kTest2JpgLen));
- EXPECT_EQ(1, ShowJPegInfo(kTest3Jpg, kTest3JpgLen));
- EXPECT_EQ(1, ShowJPegInfo(kTest4Jpg,
- kTest4JpgLen)); // Valid but unsupported.
+ ASSERT_TRUE(ShowJPegInfo(kTest0Jpg, kTest0JpgLen));
+ ASSERT_TRUE(ShowJPegInfo(kTest1Jpg, kTest1JpgLen));
+ ASSERT_TRUE(ShowJPegInfo(kTest2Jpg, kTest2JpgLen));
+ ASSERT_TRUE(ShowJPegInfo(kTest3Jpg, kTest3JpgLen));
+ ASSERT_TRUE(ShowJPegInfo(kTest4Jpg,
+ kTest4JpgLen)); // Valid but unsupported.
}
#endif // HAVE_JPEG
@@ -2182,18 +2302,18 @@ TEST_F(LibYUVConvertTest, NV12Crop) {
for (int i = 0; i < kDestHeight; ++i) {
for (int j = 0; j < kDestWidth; ++j) {
- EXPECT_EQ(dst_y[i * kWidth + j], dst_y_2[i * kWidth + j]);
+ ASSERT_EQ(dst_y[i * kWidth + j], dst_y_2[i * kWidth + j]);
}
}
for (int i = 0; i < SUBSAMPLE(kDestHeight, SUBSAMP_Y); ++i) {
for (int j = 0; j < SUBSAMPLE(kDestWidth, SUBSAMP_X); ++j) {
- EXPECT_EQ(dst_u[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j],
+ ASSERT_EQ(dst_u[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j],
dst_u_2[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j]);
}
}
for (int i = 0; i < SUBSAMPLE(kDestHeight, SUBSAMP_Y); ++i) {
for (int j = 0; j < SUBSAMPLE(kDestWidth, SUBSAMP_X); ++j) {
- EXPECT_EQ(dst_v[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j],
+ ASSERT_EQ(dst_v[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j],
dst_v_2[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j]);
}
}
@@ -2211,7 +2331,7 @@ TEST_F(LibYUVConvertTest, I420CropOddY) {
const int SUBSAMP_Y = 2;
const int kWidth = benchmark_width_;
const int kHeight = benchmark_height_;
- const int crop_y = 1;
+ const int crop_y = benchmark_height_ > 1 ? 1 : 0;
const int kDestWidth = benchmark_width_;
const int kDestHeight = benchmark_height_ - crop_y * 2;
const int kStrideU = SUBSAMPLE(kWidth, SUBSAMP_X);
@@ -2255,19 +2375,19 @@ TEST_F(LibYUVConvertTest, I420CropOddY) {
for (int i = 0; i < kDestHeight; ++i) {
for (int j = 0; j < kDestWidth; ++j) {
- EXPECT_EQ(src_y[crop_y * kWidth + i * kWidth + j],
+ ASSERT_EQ(src_y[crop_y * kWidth + i * kWidth + j],
dst_y[i * kDestWidth + j]);
}
}
for (int i = 0; i < SUBSAMPLE(kDestHeight, SUBSAMP_Y); ++i) {
for (int j = 0; j < SUBSAMPLE(kDestWidth, SUBSAMP_X); ++j) {
- EXPECT_EQ(src_u[(crop_y / 2 + i) * kStrideU + j],
+ ASSERT_EQ(src_u[(crop_y / 2 + i) * kStrideU + j],
dst_u[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j]);
}
}
for (int i = 0; i < SUBSAMPLE(kDestHeight, SUBSAMP_Y); ++i) {
for (int j = 0; j < SUBSAMPLE(kDestWidth, SUBSAMP_X); ++j) {
- EXPECT_EQ(src_v[(crop_y / 2 + i) * kStrideV + j],
+ ASSERT_EQ(src_v[(crop_y / 2 + i) * kStrideV + j],
dst_v[i * SUBSAMPLE(kDestWidth, SUBSAMP_X) + j]);
}
}
@@ -2278,181 +2398,23 @@ TEST_F(LibYUVConvertTest, I420CropOddY) {
free_aligned_buffer_page_end(src_y);
}
-TEST_F(LibYUVConvertTest, TestYToARGB) {
- uint8_t y[32];
- uint8_t expectedg[32];
- for (int i = 0; i < 32; ++i) {
- y[i] = i * 5 + 17;
- expectedg[i] = static_cast<int>((y[i] - 16) * 1.164f + 0.5f);
- }
- uint8_t argb[32 * 4];
- YToARGB(y, 0, argb, 0, 32, 1);
-
- for (int i = 0; i < 32; ++i) {
- printf("%2d %d: %d <-> %d,%d,%d,%d\n", i, y[i], expectedg[i],
- argb[i * 4 + 0], argb[i * 4 + 1], argb[i * 4 + 2], argb[i * 4 + 3]);
- }
- for (int i = 0; i < 32; ++i) {
- EXPECT_EQ(expectedg[i], argb[i * 4 + 0]);
- }
-}
-
-static const uint8_t kNoDither4x4[16] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-};
-
-TEST_F(LibYUVConvertTest, TestNoDither) {
- align_buffer_page_end(src_argb, benchmark_width_ * benchmark_height_ * 4);
- align_buffer_page_end(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
- align_buffer_page_end(dst_rgb565dither,
- benchmark_width_ * benchmark_height_ * 2);
- MemRandomize(src_argb, benchmark_width_ * benchmark_height_ * 4);
- MemRandomize(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
- MemRandomize(dst_rgb565dither, benchmark_width_ * benchmark_height_ * 2);
- ARGBToRGB565(src_argb, benchmark_width_ * 4, dst_rgb565, benchmark_width_ * 2,
- benchmark_width_, benchmark_height_);
- ARGBToRGB565Dither(src_argb, benchmark_width_ * 4, dst_rgb565dither,
- benchmark_width_ * 2, kNoDither4x4, benchmark_width_,
- benchmark_height_);
- for (int i = 0; i < benchmark_width_ * benchmark_height_ * 2; ++i) {
- EXPECT_EQ(dst_rgb565[i], dst_rgb565dither[i]);
- }
-
- free_aligned_buffer_page_end(src_argb);
- free_aligned_buffer_page_end(dst_rgb565);
- free_aligned_buffer_page_end(dst_rgb565dither);
-}
-
-// Ordered 4x4 dither for 888 to 565. Values from 0 to 7.
-static const uint8_t kDither565_4x4[16] = {
- 0, 4, 1, 5, 6, 2, 7, 3, 1, 5, 0, 4, 7, 3, 6, 2,
-};
-
-TEST_F(LibYUVConvertTest, TestDither) {
- align_buffer_page_end(src_argb, benchmark_width_ * benchmark_height_ * 4);
- align_buffer_page_end(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
- align_buffer_page_end(dst_rgb565dither,
- benchmark_width_ * benchmark_height_ * 2);
- align_buffer_page_end(dst_argb, benchmark_width_ * benchmark_height_ * 4);
- align_buffer_page_end(dst_argbdither,
- benchmark_width_ * benchmark_height_ * 4);
- MemRandomize(src_argb, benchmark_width_ * benchmark_height_ * 4);
- MemRandomize(dst_rgb565, benchmark_width_ * benchmark_height_ * 2);
- MemRandomize(dst_rgb565dither, benchmark_width_ * benchmark_height_ * 2);
- MemRandomize(dst_argb, benchmark_width_ * benchmark_height_ * 4);
- MemRandomize(dst_argbdither, benchmark_width_ * benchmark_height_ * 4);
- ARGBToRGB565(src_argb, benchmark_width_ * 4, dst_rgb565, benchmark_width_ * 2,
- benchmark_width_, benchmark_height_);
- ARGBToRGB565Dither(src_argb, benchmark_width_ * 4, dst_rgb565dither,
- benchmark_width_ * 2, kDither565_4x4, benchmark_width_,
- benchmark_height_);
- RGB565ToARGB(dst_rgb565, benchmark_width_ * 2, dst_argb, benchmark_width_ * 4,
- benchmark_width_, benchmark_height_);
- RGB565ToARGB(dst_rgb565dither, benchmark_width_ * 2, dst_argbdither,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
-
- for (int i = 0; i < benchmark_width_ * benchmark_height_ * 4; ++i) {
- EXPECT_NEAR(dst_argb[i], dst_argbdither[i], 9);
- }
- free_aligned_buffer_page_end(src_argb);
- free_aligned_buffer_page_end(dst_rgb565);
- free_aligned_buffer_page_end(dst_rgb565dither);
- free_aligned_buffer_page_end(dst_argb);
- free_aligned_buffer_page_end(dst_argbdither);
-}
-
-#define TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, W1280, DIFF, N, NEG, OFF, FMT_C, BPP_C) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##Dither##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
- const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, kSizeUV + OFF); \
- align_buffer_page_end(src_v, kSizeUV + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight + OFF); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight + OFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- src_y[i + OFF] = (fastrand() & 0xff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- src_u[i + OFF] = (fastrand() & 0xff); \
- src_v[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
- memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_PLANAR##To##FMT_B##Dither(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, dst_argb_c + OFF, \
- kStrideB, NULL, kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B##Dither( \
- src_y + OFF, kWidth, src_u + OFF, kStrideUV, src_v + OFF, kStrideUV, \
- dst_argb_opt + OFF, kStrideB, NULL, kWidth, NEG kHeight); \
- } \
- int max_diff = 0; \
- /* Convert to ARGB so 565 is expanded to bytes that can be compared. */ \
- align_buffer_page_end(dst_argb32_c, kWidth* BPP_C* kHeight); \
- align_buffer_page_end(dst_argb32_opt, kWidth* BPP_C* kHeight); \
- memset(dst_argb32_c, 2, kWidth* BPP_C* kHeight); \
- memset(dst_argb32_opt, 102, kWidth* BPP_C* kHeight); \
- FMT_B##To##FMT_C(dst_argb_c + OFF, kStrideB, dst_argb32_c, kWidth * BPP_C, \
- kWidth, kHeight); \
- FMT_B##To##FMT_C(dst_argb_opt + OFF, kStrideB, dst_argb32_opt, \
- kWidth * BPP_C, kWidth, kHeight); \
- for (int i = 0; i < kWidth * BPP_C * kHeight; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb32_c[i]) - \
- static_cast<int>(dst_argb32_opt[i])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- free_aligned_buffer_page_end(dst_argb32_c); \
- free_aligned_buffer_page_end(dst_argb32_opt); \
- }
-
-#define TESTPLANARTOBD(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, DIFF, FMT_C, BPP_C) \
- TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_ - 4, DIFF, _Any, +, 0, FMT_C, \
- BPP_C) \
- TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Unaligned, +, 1, FMT_C, \
- BPP_C) \
- TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Invert, -, 0, FMT_C, BPP_C) \
- TESTPLANARTOBID(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Opt, +, 0, FMT_C, BPP_C)
-
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANARTOBD(I420, 2, 2, RGB565, 2, 2, 1, 9, ARGB, 4)
-#endif
#define TESTPTOB(NAME, UYVYTOI420, UYVYTONV12) \
TEST_F(LibYUVConvertTest, NAME) { \
const int kWidth = benchmark_width_; \
const int kHeight = benchmark_height_; \
\
align_buffer_page_end(orig_uyvy, 4 * SUBSAMPLE(kWidth, 2) * kHeight); \
- align_buffer_page_end(orig_y, kWidth* kHeight); \
+ align_buffer_page_end(orig_y, kWidth * kHeight); \
align_buffer_page_end(orig_u, \
SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2)); \
align_buffer_page_end(orig_v, \
SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2)); \
\
- align_buffer_page_end(dst_y_orig, kWidth* kHeight); \
+ align_buffer_page_end(dst_y_orig, kWidth * kHeight); \
align_buffer_page_end(dst_uv_orig, \
2 * SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2)); \
\
- align_buffer_page_end(dst_y, kWidth* kHeight); \
+ align_buffer_page_end(dst_y, kWidth * kHeight); \
align_buffer_page_end(dst_uv, \
2 * SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2)); \
\
@@ -2473,14 +2435,14 @@ TESTPLANARTOBD(I420, 2, 2, RGB565, 2, 2, 1, 9, ARGB, 4)
} \
\
for (int i = 0; i < kWidth * kHeight; ++i) { \
- EXPECT_EQ(orig_y[i], dst_y[i]); \
+ ASSERT_EQ(orig_y[i], dst_y[i]); \
} \
for (int i = 0; i < kWidth * kHeight; ++i) { \
- EXPECT_EQ(dst_y_orig[i], dst_y[i]); \
+ ASSERT_EQ(dst_y_orig[i], dst_y[i]); \
} \
for (int i = 0; i < 2 * SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2); \
++i) { \
- EXPECT_EQ(dst_uv_orig[i], dst_uv[i]); \
+ ASSERT_EQ(dst_uv_orig[i], dst_uv[i]); \
} \
\
free_aligned_buffer_page_end(orig_uyvy); \
@@ -2496,770 +2458,827 @@ TESTPLANARTOBD(I420, 2, 2, RGB565, 2, 2, 1, 9, ARGB, 4)
TESTPTOB(TestYUY2ToNV12, YUY2ToI420, YUY2ToNV12)
TESTPTOB(TestUYVYToNV12, UYVYToI420, UYVYToNV12)
-// Transitive tests. A to B to C is same as A to C.
+TEST_F(LibYUVConvertTest, MM21ToYUY2) {
+ const int kWidth = (benchmark_width_ + 15) & (~15);
+ const int kHeight = (benchmark_height_ + 31) & (~31);
-#define TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- W1280, N, NEG, OFF, FMT_C, BPP_C) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##_##FMT_C##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, kSizeUV + OFF); \
- align_buffer_page_end(src_v, kSizeUV + OFF); \
- align_buffer_page_end(dst_argb_b, kStrideB* kHeight + OFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- src_y[i + OFF] = (fastrand() & 0xff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- src_u[i + OFF] = (fastrand() & 0xff); \
- src_v[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, dst_argb_b + OFF, \
- kStrideB, kWidth, NEG kHeight); \
- } \
- /* Convert to a 3rd format in 1 step and 2 steps and compare */ \
- const int kStrideC = kWidth * BPP_C; \
- align_buffer_page_end(dst_argb_c, kStrideC* kHeight + OFF); \
- align_buffer_page_end(dst_argb_bc, kStrideC* kHeight + OFF); \
- memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
- memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
- FMT_PLANAR##To##FMT_C(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
- src_v + OFF, kStrideUV, dst_argb_c + OFF, kStrideC, \
- kWidth, NEG kHeight); \
- /* Convert B to C */ \
- FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, kStrideC, \
- kWidth, kHeight); \
- for (int i = 0; i < kStrideC * kHeight; ++i) { \
- EXPECT_EQ(dst_argb_c[i + OFF], dst_argb_bc[i + OFF]); \
- } \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(dst_argb_b); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_bc); \
- }
-
-#define TESTPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- FMT_C, BPP_C) \
- TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_ - 4, _Any, +, 0, FMT_C, BPP_C) \
- TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Unaligned, +, 1, FMT_C, BPP_C) \
- TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Invert, -, 0, FMT_C, BPP_C) \
- TESTPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Opt, +, 0, FMT_C, BPP_C)
-
-TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ABGR, 4)
-TESTPLANARTOE(J420, 2, 2, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(J420, 2, 2, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(H420, 2, 2, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(H420, 2, 2, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, BGRA, 1, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, RGBA, 1, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, RGB24, 3)
-TESTPLANARTOE(I420, 2, 2, RGB24, 1, 3, RAW, 3)
-TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RAW, 3)
-TESTPLANARTOE(I420, 2, 2, RAW, 1, 3, ARGB, 4)
-TESTPLANARTOE(H420, 2, 2, RGB24, 1, 3, ARGB, 4)
-TESTPLANARTOE(H420, 2, 2, RAW, 1, 3, RGB24, 3)
-TESTPLANARTOE(H420, 2, 2, RGB24, 1, 3, RAW, 3)
-TESTPLANARTOE(H420, 2, 2, ARGB, 1, 4, RAW, 3)
-TESTPLANARTOE(H420, 2, 2, RAW, 1, 3, ARGB, 4)
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, RGB565, 2)
-TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB1555, 2)
-TESTPLANARTOE(I420, 2, 2, ARGB, 1, 4, ARGB4444, 2)
-TESTPLANARTOE(I422, 2, 1, ARGB, 1, 4, RGB565, 2)
-#endif
-TESTPLANARTOE(J422, 2, 1, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(J422, 2, 1, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(H422, 2, 1, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(H422, 2, 1, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(I422, 2, 1, BGRA, 1, 4, ARGB, 4)
-TESTPLANARTOE(I422, 2, 1, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(I422, 2, 1, RGBA, 1, 4, ARGB, 4)
-TESTPLANARTOE(I444, 1, 1, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(J444, 1, 1, ARGB, 1, 4, ARGB, 4)
-TESTPLANARTOE(I444, 1, 1, ABGR, 1, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, YUY2, 2, 4, ARGB, 4)
-TESTPLANARTOE(I420, 2, 2, UYVY, 2, 4, ARGB, 4)
-TESTPLANARTOE(I422, 2, 1, YUY2, 2, 4, ARGB, 4)
-TESTPLANARTOE(I422, 2, 1, UYVY, 2, 4, ARGB, 4)
-
-#define TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- W1280, N, NEG, OFF, FMT_C, BPP_C, ATTEN) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##_##FMT_C##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
- const int kSizeUV = \
- SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- align_buffer_page_end(src_y, kWidth* kHeight + OFF); \
- align_buffer_page_end(src_u, kSizeUV + OFF); \
- align_buffer_page_end(src_v, kSizeUV + OFF); \
- align_buffer_page_end(src_a, kWidth* kHeight + OFF); \
- align_buffer_page_end(dst_argb_b, kStrideB* kHeight + OFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- src_y[i + OFF] = (fastrand() & 0xff); \
- src_a[i + OFF] = (fastrand() & 0xff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- src_u[i + OFF] = (fastrand() & 0xff); \
- src_v[i + OFF] = (fastrand() & 0xff); \
- } \
- memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B( \
- src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), \
- src_v + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), src_a + OFF, kWidth, \
- dst_argb_b + OFF, kStrideB, kWidth, NEG kHeight, ATTEN); \
- } \
- /* Convert to a 3rd format in 1 step and 2 steps and compare */ \
- const int kStrideC = kWidth * BPP_C; \
- align_buffer_page_end(dst_argb_c, kStrideC* kHeight + OFF); \
- align_buffer_page_end(dst_argb_bc, kStrideC* kHeight + OFF); \
- memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
- memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
- FMT_PLANAR##To##FMT_C( \
- src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), \
- src_v + OFF, SUBSAMPLE(kWidth, SUBSAMP_X), src_a + OFF, kWidth, \
- dst_argb_c + OFF, kStrideC, kWidth, NEG kHeight, ATTEN); \
- /* Convert B to C */ \
- FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, kStrideC, \
- kWidth, kHeight); \
- for (int i = 0; i < kStrideC * kHeight; ++i) { \
- EXPECT_EQ(dst_argb_c[i + OFF], dst_argb_bc[i + OFF]); \
- } \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(src_a); \
- free_aligned_buffer_page_end(dst_argb_b); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_bc); \
- }
-
-#define TESTQPLANARTOE(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- FMT_C, BPP_C) \
- TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_ - 4, _Any, +, 0, FMT_C, BPP_C, 0) \
- TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Unaligned, +, 1, FMT_C, BPP_C, 0) \
- TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Invert, -, 0, FMT_C, BPP_C, 0) \
- TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Opt, +, 0, FMT_C, BPP_C, 0) \
- TESTQPLANARTOEI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, SUB_B, BPP_B, \
- benchmark_width_, _Premult, +, 0, FMT_C, BPP_C, 1)
-
-TESTQPLANARTOE(I420Alpha, 2, 2, ARGB, 1, 4, ABGR, 4)
-TESTQPLANARTOE(I420Alpha, 2, 2, ABGR, 1, 4, ARGB, 4)
-
-#define TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, W1280, N, NEG, \
- OFF, FMT_C, BPP_C) \
- TEST_F(LibYUVConvertTest, FMT_A##To##FMT_B##_##FMT_C##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = benchmark_height_; \
- const int kStrideA = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \
- const int kStrideB = SUBSAMPLE(kWidth, SUB_B) * BPP_B; \
- align_buffer_page_end(src_argb_a, kStrideA* kHeight + OFF); \
- align_buffer_page_end(dst_argb_b, kStrideB* kHeight + OFF); \
- MemRandomize(src_argb_a + OFF, kStrideA * kHeight); \
- memset(dst_argb_b + OFF, 1, kStrideB * kHeight); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_A##To##FMT_B(src_argb_a + OFF, kStrideA, dst_argb_b + OFF, kStrideB, \
- kWidth, NEG kHeight); \
- } \
- /* Convert to a 3rd format in 1 step and 2 steps and compare */ \
- const int kStrideC = kWidth * BPP_C; \
- align_buffer_page_end(dst_argb_c, kStrideC* kHeight + OFF); \
- align_buffer_page_end(dst_argb_bc, kStrideC* kHeight + OFF); \
- memset(dst_argb_c + OFF, 2, kStrideC * kHeight); \
- memset(dst_argb_bc + OFF, 3, kStrideC * kHeight); \
- FMT_A##To##FMT_C(src_argb_a + OFF, kStrideA, dst_argb_c + OFF, kStrideC, \
- kWidth, NEG kHeight); \
- /* Convert B to C */ \
- FMT_B##To##FMT_C(dst_argb_b + OFF, kStrideB, dst_argb_bc + OFF, kStrideC, \
- kWidth, kHeight); \
- for (int i = 0; i < kStrideC * kHeight; i += 4) { \
- EXPECT_EQ(dst_argb_c[i + OFF + 0], dst_argb_bc[i + OFF + 0]); \
- EXPECT_EQ(dst_argb_c[i + OFF + 1], dst_argb_bc[i + OFF + 1]); \
- EXPECT_EQ(dst_argb_c[i + OFF + 2], dst_argb_bc[i + OFF + 2]); \
- EXPECT_NEAR(dst_argb_c[i + OFF + 3], dst_argb_bc[i + OFF + 3], 64); \
- } \
- free_aligned_buffer_page_end(src_argb_a); \
- free_aligned_buffer_page_end(dst_argb_b); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_bc); \
- }
-
-#define TESTPLANETOE(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, FMT_C, BPP_C) \
- TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, \
- benchmark_width_ - 4, _Any, +, 0, FMT_C, BPP_C) \
- TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
- _Unaligned, +, 1, FMT_C, BPP_C) \
- TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
- _Invert, -, 0, FMT_C, BPP_C) \
- TESTPLANETOEI(FMT_A, SUB_A, BPP_A, FMT_B, SUB_B, BPP_B, benchmark_width_, \
- _Opt, +, 0, FMT_C, BPP_C)
-
-// Caveat: Destination needs to be 4 bytes
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANETOE(ARGB, 1, 4, AR30, 1, 4, ARGB, 4)
-TESTPLANETOE(ABGR, 1, 4, AR30, 1, 4, ABGR, 4)
-TESTPLANETOE(AR30, 1, 4, ARGB, 1, 4, ABGR, 4)
-TESTPLANETOE(AR30, 1, 4, ABGR, 1, 4, ARGB, 4)
-TESTPLANETOE(ARGB, 1, 4, AB30, 1, 4, ARGB, 4)
-TESTPLANETOE(ABGR, 1, 4, AB30, 1, 4, ABGR, 4)
-TESTPLANETOE(AB30, 1, 4, ARGB, 1, 4, ABGR, 4)
-TESTPLANETOE(AB30, 1, 4, ABGR, 1, 4, ARGB, 4)
-#endif
+ align_buffer_page_end(orig_y, kWidth * kHeight);
+ align_buffer_page_end(orig_uv,
+ 2 * SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2));
-TEST_F(LibYUVConvertTest, RotateWithARGBSource) {
- // 2x2 frames
- uint32_t src[4];
- uint32_t dst[4];
- // some random input
- src[0] = 0x11000000;
- src[1] = 0x00450000;
- src[2] = 0x00009f00;
- src[3] = 0x000000ff;
- // zeros on destination
- dst[0] = 0x00000000;
- dst[1] = 0x00000000;
- dst[2] = 0x00000000;
- dst[3] = 0x00000000;
-
- int r = ConvertToARGB(reinterpret_cast<uint8_t*>(src),
- 16, // input size
- reinterpret_cast<uint8_t*>(dst),
- 8, // destination stride
- 0, // crop_x
- 0, // crop_y
- 2, // width
- 2, // height
- 2, // crop width
- 2, // crop height
- kRotate90, FOURCC_ARGB);
-
- EXPECT_EQ(r, 0);
- // 90 degrees rotation, no conversion
- EXPECT_EQ(dst[0], src[2]);
- EXPECT_EQ(dst[1], src[0]);
- EXPECT_EQ(dst[2], src[3]);
- EXPECT_EQ(dst[3], src[1]);
-}
+ align_buffer_page_end(tmp_y, kWidth * kHeight);
+ align_buffer_page_end(tmp_u, SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2));
+ align_buffer_page_end(tmp_v, SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2));
+
+ align_buffer_page_end(dst_yuyv, 4 * SUBSAMPLE(kWidth, 2) * kHeight);
+ align_buffer_page_end(golden_yuyv, 4 * SUBSAMPLE(kWidth, 2) * kHeight);
+
+ MemRandomize(orig_y, kWidth * kHeight);
+ MemRandomize(orig_uv, 2 * SUBSAMPLE(kWidth, 2) * SUBSAMPLE(kHeight, 2));
+
+ /* Convert MM21 to YUY2 in 2 steps for reference */
+ libyuv::MM21ToI420(orig_y, kWidth, orig_uv, 2 * SUBSAMPLE(kWidth, 2), tmp_y,
+ kWidth, tmp_u, SUBSAMPLE(kWidth, 2), tmp_v,
+ SUBSAMPLE(kWidth, 2), kWidth, kHeight);
+ libyuv::I420ToYUY2(tmp_y, kWidth, tmp_u, SUBSAMPLE(kWidth, 2), tmp_v,
+ SUBSAMPLE(kWidth, 2), golden_yuyv,
+ 4 * SUBSAMPLE(kWidth, 2), kWidth, kHeight);
-#ifdef HAS_ARGBTOAR30ROW_AVX2
-TEST_F(LibYUVConvertTest, ARGBToAR30Row_Opt) {
- // ARGBToAR30Row_AVX2 expects a multiple of 8 pixels.
- const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7;
- align_buffer_page_end(src, kPixels * 4);
- align_buffer_page_end(dst_opt, kPixels * 4);
- align_buffer_page_end(dst_c, kPixels * 4);
- MemRandomize(src, kPixels * 4);
- memset(dst_opt, 0, kPixels * 4);
- memset(dst_c, 1, kPixels * 4);
-
- ARGBToAR30Row_C(src, dst_c, kPixels);
-
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ /* Convert to NV12 */
for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
- ARGBToAR30Row_AVX2(src, dst_opt, kPixels);
- } else if (has_ssse3) {
- ARGBToAR30Row_SSSE3(src, dst_opt, kPixels);
- } else {
- ARGBToAR30Row_C(src, dst_opt, kPixels);
- }
+ libyuv::MM21ToYUY2(orig_y, kWidth, orig_uv, 2 * SUBSAMPLE(kWidth, 2),
+ dst_yuyv, 4 * SUBSAMPLE(kWidth, 2), kWidth, kHeight);
}
- for (int i = 0; i < kPixels * 4; ++i) {
- EXPECT_EQ(dst_opt[i], dst_c[i]);
+
+ for (int i = 0; i < 4 * SUBSAMPLE(kWidth, 2) * kHeight; ++i) {
+ ASSERT_EQ(dst_yuyv[i], golden_yuyv[i]);
}
- free_aligned_buffer_page_end(src);
- free_aligned_buffer_page_end(dst_opt);
- free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(orig_uv);
+ free_aligned_buffer_page_end(tmp_y);
+ free_aligned_buffer_page_end(tmp_u);
+ free_aligned_buffer_page_end(tmp_v);
+ free_aligned_buffer_page_end(dst_yuyv);
+ free_aligned_buffer_page_end(golden_yuyv);
}
-#endif // HAS_ARGBTOAR30ROW_AVX2
-
-#ifdef HAS_ABGRTOAR30ROW_AVX2
-TEST_F(LibYUVConvertTest, ABGRToAR30Row_Opt) {
- // ABGRToAR30Row_AVX2 expects a multiple of 8 pixels.
- const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7;
- align_buffer_page_end(src, kPixels * 4);
- align_buffer_page_end(dst_opt, kPixels * 4);
- align_buffer_page_end(dst_c, kPixels * 4);
- MemRandomize(src, kPixels * 4);
- memset(dst_opt, 0, kPixels * 4);
- memset(dst_c, 1, kPixels * 4);
-
- ABGRToAR30Row_C(src, dst_c, kPixels);
-
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
- ABGRToAR30Row_AVX2(src, dst_opt, kPixels);
- } else if (has_ssse3) {
- ABGRToAR30Row_SSSE3(src, dst_opt, kPixels);
- } else {
- ABGRToAR30Row_C(src, dst_opt, kPixels);
- }
+
+// Test RGB24 to J420 is exact
+TEST_F(LibYUVConvertTest, TestRGB24ToJ420) {
+ const int kSize = 256;
+ align_buffer_page_end(orig_rgb24, kSize * 3 * 2); // 2 rows of RGB24
+ align_buffer_page_end(dest_j420, kSize * 3 / 2 * 2);
+ int iterations256 = (benchmark_width_ * benchmark_height_ + (kSize * 2 - 1)) /
+ (kSize * 2) * benchmark_iterations_;
+
+ for (int i = 0; i < kSize * 3 * 2; ++i) {
+ orig_rgb24[i] = i;
}
- for (int i = 0; i < kPixels * 4; ++i) {
- EXPECT_EQ(dst_opt[i], dst_c[i]);
+
+ for (int i = 0; i < iterations256; ++i) {
+ RGB24ToJ420(orig_rgb24, kSize * 3, dest_j420, kSize, // Y plane
+ dest_j420 + kSize * 2, kSize / 2, // U plane
+ dest_j420 + kSize * 5 / 2, kSize / 2, // V plane
+ kSize, 2);
}
- free_aligned_buffer_page_end(src);
- free_aligned_buffer_page_end(dst_opt);
- free_aligned_buffer_page_end(dst_c);
+ uint32_t checksum = HashDjb2(dest_j420, kSize * 3 / 2 * 2, 5381);
+ ASSERT_EQ(223551344u, checksum);
+
+ free_aligned_buffer_page_end(orig_rgb24);
+ free_aligned_buffer_page_end(dest_j420);
}
-#endif // HAS_ABGRTOAR30ROW_AVX2
-
-// TODO(fbarchard): Fix clamping issue affected by U channel.
-#define TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, \
- ALIGN, YALIGN, W1280, DIFF, N, NEG, SOFF, DOFF) \
- TEST_F(LibYUVConvertTest, FMT_PLANAR##To##FMT_B##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
- const int kHeight = ALIGNINT(benchmark_height_, YALIGN); \
- const int kStrideB = ALIGNINT(kWidth * BPP_B, ALIGN); \
- const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \
- const int kSizeUV = kStrideUV * SUBSAMPLE(kHeight, SUBSAMP_Y); \
- const int kBpc = 2; \
- align_buffer_page_end(src_y, kWidth* kHeight* kBpc + SOFF); \
- align_buffer_page_end(src_u, kSizeUV* kBpc + SOFF); \
- align_buffer_page_end(src_v, kSizeUV* kBpc + SOFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight + DOFF); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight + DOFF); \
- for (int i = 0; i < kWidth * kHeight; ++i) { \
- reinterpret_cast<uint16_t*>(src_y + SOFF)[i] = (fastrand() & 0x3ff); \
- } \
- for (int i = 0; i < kSizeUV; ++i) { \
- reinterpret_cast<uint16_t*>(src_u + SOFF)[i] = (fastrand() & 0x3ff); \
- reinterpret_cast<uint16_t*>(src_v + SOFF)[i] = (fastrand() & 0x3ff); \
- } \
- memset(dst_argb_c + DOFF, 1, kStrideB * kHeight); \
- memset(dst_argb_opt + DOFF, 101, kStrideB * kHeight); \
- MaskCpuFlags(disable_cpu_flags_); \
- FMT_PLANAR##To##FMT_B( \
- reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
- reinterpret_cast<uint16_t*>(src_u + SOFF), kStrideUV, \
- reinterpret_cast<uint16_t*>(src_v + SOFF), kStrideUV, \
- dst_argb_c + DOFF, kStrideB, kWidth, NEG kHeight); \
- MaskCpuFlags(benchmark_cpu_info_); \
- for (int i = 0; i < benchmark_iterations_; ++i) { \
- FMT_PLANAR##To##FMT_B( \
- reinterpret_cast<uint16_t*>(src_y + SOFF), kWidth, \
- reinterpret_cast<uint16_t*>(src_u + SOFF), kStrideUV, \
- reinterpret_cast<uint16_t*>(src_v + SOFF), kStrideUV, \
- dst_argb_opt + DOFF, kStrideB, kWidth, NEG kHeight); \
- } \
- int max_diff = 0; \
- for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
- int abs_diff = abs(static_cast<int>(dst_argb_c[i + DOFF]) - \
- static_cast<int>(dst_argb_opt[i + DOFF])); \
- if (abs_diff > max_diff) { \
- max_diff = abs_diff; \
- } \
- } \
- EXPECT_LE(max_diff, DIFF); \
- free_aligned_buffer_page_end(src_y); \
- free_aligned_buffer_page_end(src_u); \
- free_aligned_buffer_page_end(src_v); \
- free_aligned_buffer_page_end(dst_argb_c); \
- free_aligned_buffer_page_end(dst_argb_opt); \
- }
-
-#define TESTPLANAR16TOB(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, DIFF) \
- TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_ - 4, DIFF, _Any, +, 0, 0) \
- TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Unaligned, +, 1, 1) \
- TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Invert, -, 0, 0) \
- TESTPLANAR16TOBI(FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, FMT_B, BPP_B, ALIGN, \
- YALIGN, benchmark_width_, DIFF, _Opt, +, 0, 0)
-
-TESTPLANAR16TOB(I010, 2, 2, ARGB, 4, 4, 1, 2)
-TESTPLANAR16TOB(I010, 2, 2, ABGR, 4, 4, 1, 2)
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANAR16TOB(I010, 2, 2, AR30, 4, 4, 1, 2)
-TESTPLANAR16TOB(I010, 2, 2, AB30, 4, 4, 1, 2)
-#endif
-TESTPLANAR16TOB(H010, 2, 2, ARGB, 4, 4, 1, 2)
-TESTPLANAR16TOB(H010, 2, 2, ABGR, 4, 4, 1, 2)
-#ifdef LITTLE_ENDIAN_TEST
-TESTPLANAR16TOB(H010, 2, 2, AR30, 4, 4, 1, 2)
-TESTPLANAR16TOB(H010, 2, 2, AB30, 4, 4, 1, 2)
-#endif
-static int Clamp(int y) {
- if (y < 0) {
- y = 0;
+// Test RGB24 to I420 is exact
+TEST_F(LibYUVConvertTest, TestRGB24ToI420) {
+ const int kSize = 256;
+ align_buffer_page_end(orig_rgb24, kSize * 3 * 2); // 2 rows of RGB24
+ align_buffer_page_end(dest_i420, kSize * 3 / 2 * 2);
+ int iterations256 = (benchmark_width_ * benchmark_height_ + (kSize * 2 - 1)) /
+ (kSize * 2) * benchmark_iterations_;
+
+ for (int i = 0; i < kSize * 3 * 2; ++i) {
+ orig_rgb24[i] = i;
+ }
+
+ for (int i = 0; i < iterations256; ++i) {
+ RGB24ToI420(orig_rgb24, kSize * 3, dest_i420, kSize, // Y plane
+ dest_i420 + kSize * 2, kSize / 2, // U plane
+ dest_i420 + kSize * 5 / 2, kSize / 2, // V plane
+ kSize, 2);
+ }
+
+ uint32_t checksum = HashDjb2(dest_i420, kSize * 3 / 2 * 2, 5381);
+ ASSERT_EQ(4197774805u, checksum);
+
+ free_aligned_buffer_page_end(orig_rgb24);
+ free_aligned_buffer_page_end(dest_i420);
+}
+
+TEST_F(LibYUVConvertTest, TestJ420ToI420) {
+ const uint8_t src_y[12] = {0, 0, 128, 128, 255, 255,
+ 0, 0, 128, 128, 255, 255};
+ const uint8_t src_u[3] = {0, 128, 255};
+ const uint8_t src_v[3] = {0, 128, 255};
+ uint8_t dst_y[12];
+ uint8_t dst_u[3];
+ uint8_t dst_v[3];
+ ASSERT_EQ(J420ToI420(src_y, 6, src_u, 3, src_v, 3, dst_y, 6, dst_u, 3, dst_v,
+ 3, 6, 2),
+ 0);
+ ASSERT_EQ(dst_y[0], 16);
+ ASSERT_EQ(dst_y[2], 126);
+ ASSERT_EQ(dst_y[4], 235);
+ ASSERT_EQ(dst_u[0], 16);
+ ASSERT_EQ(dst_u[1], 128);
+ ASSERT_EQ(dst_u[2], 240);
+ ASSERT_EQ(dst_v[0], 16);
+ ASSERT_EQ(dst_v[1], 128);
+ ASSERT_EQ(dst_v[2], 240);
+}
+
+TEST_F(LibYUVConvertTest, TestABGRToI420Matrix) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_abgr, kWidth * kHeight * 4);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(dst_v, kWidth / 2 * kHeight / 2);
+
+ MemRandomize(src_abgr, kWidth * kHeight * 4);
+
+ // BT.601
+ ARGBToI420Matrix(src_abgr, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kAbgrI601Constants, kWidth, kHeight);
+ // Verify against non-matrix version
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+ ABGRToI420(src_abgr, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ // JPEG
+ ARGBToI420Matrix(src_abgr, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kAbgrJPEGConstants, kWidth, kHeight);
+ // Verify against non-matrix version
+ ABGRToJ420(src_abgr, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ free_aligned_buffer_page_end(src_abgr);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+TEST_F(LibYUVConvertTest, TestRAWToI420) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_raw, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(dst_v, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+
+ MemRandomize(src_raw, kWidth * kHeight * 3);
+
+ RAWToI420(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth / 2, dst_v,
+ kWidth / 2, kWidth, kHeight);
+
+ // Verify against RAWToARGB + ARGBToI420
+ RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight);
+ ARGBToI420(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
}
- if (y > 255) {
- y = 255;
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
}
- return y;
+
+ free_aligned_buffer_page_end(src_raw);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
}
-static int Clamp10(int y) {
- if (y < 0) {
- y = 0;
+TEST_F(LibYUVConvertTest, TestRAWToI444) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_raw, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth * kHeight);
+ align_buffer_page_end(dst_v, kWidth * kHeight);
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth * kHeight);
+ align_buffer_page_end(ref_v, kWidth * kHeight);
+
+ MemRandomize(src_raw, kWidth * kHeight * 3);
+
+ RAWToI444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth,
+ kWidth, kHeight);
+
+ // Verify against RAWToARGB + ARGBToI444
+ RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight);
+ ARGBToI444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth,
+ kWidth, kHeight);
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
}
- if (y > 1023) {
- y = 1023;
+
+ free_aligned_buffer_page_end(src_raw);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+TEST_F(LibYUVConvertTest, TestRAWToJ444) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_raw, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth * kHeight);
+ align_buffer_page_end(dst_v, kWidth * kHeight);
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth * kHeight);
+ align_buffer_page_end(ref_v, kWidth * kHeight);
+
+ MemRandomize(src_raw, kWidth * kHeight * 3);
+
+ RAWToJ444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth,
+ kWidth, kHeight);
+
+ // Verify against RAWToARGB + ARGBToJ444
+ RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight);
+ ARGBToJ444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth,
+ kWidth, kHeight);
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
}
- return y;
+
+ free_aligned_buffer_page_end(src_raw);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
}
-// Test 8 bit YUV to 8 bit RGB
-TEST_F(LibYUVConvertTest, TestH420ToARGB) {
- const int kSize = 256;
- int histogram_b[256];
- int histogram_g[256];
- int histogram_r[256];
- memset(histogram_b, 0, sizeof(histogram_b));
- memset(histogram_g, 0, sizeof(histogram_g));
- memset(histogram_r, 0, sizeof(histogram_r));
- align_buffer_page_end(orig_yuv, kSize + kSize / 2 * 2);
- align_buffer_page_end(argb_pixels, kSize * 4);
- uint8_t* orig_y = orig_yuv;
- uint8_t* orig_u = orig_y + kSize;
- uint8_t* orig_v = orig_u + kSize / 2;
-
- // Test grey scale
- for (int i = 0; i < kSize; ++i) {
- orig_y[i] = i;
- }
- for (int i = 0; i < kSize / 2; ++i) {
- orig_u[i] = 128; // 128 is 0.
- orig_v[i] = 128;
- }
-
- H420ToARGB(orig_y, 0, orig_u, 0, orig_v, 0, argb_pixels, 0, kSize, 1);
-
- for (int i = 0; i < kSize; ++i) {
- int b = argb_pixels[i * 4 + 0];
- int g = argb_pixels[i * 4 + 1];
- int r = argb_pixels[i * 4 + 2];
- int a = argb_pixels[i * 4 + 3];
- ++histogram_b[b];
- ++histogram_g[g];
- ++histogram_r[r];
- int expected_y = Clamp(static_cast<int>((i - 16) * 1.164f));
- EXPECT_NEAR(b, expected_y, 1);
- EXPECT_NEAR(g, expected_y, 1);
- EXPECT_NEAR(r, expected_y, 1);
- EXPECT_EQ(a, 255);
- }
-
- int count_b = 0;
- int count_g = 0;
- int count_r = 0;
- for (int i = 0; i < kSize; ++i) {
- if (histogram_b[i]) {
- ++count_b;
- }
- if (histogram_g[i]) {
- ++count_g;
- }
- if (histogram_r[i]) {
- ++count_r;
- }
+TEST_F(LibYUVConvertTest, TestRAWToNV21) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_raw, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_vu, kWidth * kHeight / 2);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_vu, kWidth * kHeight / 2);
+
+ MemRandomize(src_raw, kWidth * kHeight * 3);
+
+ RAWToNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth,
+ kHeight);
+
+ // Verify against RAWToI420 + interleaving V and U
+ RAWToI420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ref_vu[i * 2] = ref_v[i];
+ ref_vu[i * 2 + 1] = ref_u[i];
+ }
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_vu[i], ref_vu[i]);
}
- printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
- free_aligned_buffer_page_end(orig_yuv);
- free_aligned_buffer_page_end(argb_pixels);
+ free_aligned_buffer_page_end(src_raw);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_vu);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+ free_aligned_buffer_page_end(ref_vu);
}
-// Test 10 bit YUV to 8 bit RGB
-TEST_F(LibYUVConvertTest, TestH010ToARGB) {
- const int kSize = 1024;
- int histogram_b[1024];
- int histogram_g[1024];
- int histogram_r[1024];
- memset(histogram_b, 0, sizeof(histogram_b));
- memset(histogram_g, 0, sizeof(histogram_g));
- memset(histogram_r, 0, sizeof(histogram_r));
- align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
- align_buffer_page_end(argb_pixels, kSize * 4);
- uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
- uint16_t* orig_u = orig_y + kSize;
- uint16_t* orig_v = orig_u + kSize / 2;
-
- // Test grey scale
- for (int i = 0; i < kSize; ++i) {
- orig_y[i] = i;
- }
- for (int i = 0; i < kSize / 2; ++i) {
- orig_u[i] = 512; // 512 is 0.
- orig_v[i] = 512;
- }
-
- H010ToARGB(orig_y, 0, orig_u, 0, orig_v, 0, argb_pixels, 0, kSize, 1);
-
- for (int i = 0; i < kSize; ++i) {
- int b = argb_pixels[i * 4 + 0];
- int g = argb_pixels[i * 4 + 1];
- int r = argb_pixels[i * 4 + 2];
- int a = argb_pixels[i * 4 + 3];
- ++histogram_b[b];
- ++histogram_g[g];
- ++histogram_r[r];
- int expected_y = Clamp(static_cast<int>((i - 64) * 1.164f / 4));
- EXPECT_NEAR(b, expected_y, 1);
- EXPECT_NEAR(g, expected_y, 1);
- EXPECT_NEAR(r, expected_y, 1);
- EXPECT_EQ(a, 255);
- }
-
- int count_b = 0;
- int count_g = 0;
- int count_r = 0;
- for (int i = 0; i < kSize; ++i) {
- if (histogram_b[i]) {
- ++count_b;
- }
- if (histogram_g[i]) {
- ++count_g;
- }
- if (histogram_r[i]) {
- ++count_r;
- }
+TEST_F(LibYUVConvertTest, TestRAWToJNV21) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_raw, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_vu, kWidth * kHeight / 2);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_vu, kWidth * kHeight / 2);
+
+ MemRandomize(src_raw, kWidth * kHeight * 3);
+
+ RAWToJNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth,
+ kHeight);
+
+ // Verify against RAWToJ420 + interleaving V and U
+ RAWToJ420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ref_vu[i * 2] = ref_v[i];
+ ref_vu[i * 2 + 1] = ref_u[i];
+ }
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_vu[i], ref_vu[i]);
}
- printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
- free_aligned_buffer_page_end(orig_yuv);
- free_aligned_buffer_page_end(argb_pixels);
+ free_aligned_buffer_page_end(src_raw);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_vu);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+ free_aligned_buffer_page_end(ref_vu);
}
-// Test 10 bit YUV to 10 bit RGB
-// Caveat: Result is near due to float rounding in expected
-// result.
-TEST_F(LibYUVConvertTest, TestH010ToAR30) {
- const int kSize = 1024;
- int histogram_b[1024];
- int histogram_g[1024];
- int histogram_r[1024];
- memset(histogram_b, 0, sizeof(histogram_b));
- memset(histogram_g, 0, sizeof(histogram_g));
- memset(histogram_r, 0, sizeof(histogram_r));
-
- align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
- align_buffer_page_end(ar30_pixels, kSize * 4);
- uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
- uint16_t* orig_u = orig_y + kSize;
- uint16_t* orig_v = orig_u + kSize / 2;
-
- // Test grey scale
- for (int i = 0; i < kSize; ++i) {
- orig_y[i] = i;
- }
- for (int i = 0; i < kSize / 2; ++i) {
- orig_u[i] = 512; // 512 is 0.
- orig_v[i] = 512;
- }
-
- H010ToAR30(orig_y, 0, orig_u, 0, orig_v, 0, ar30_pixels, 0, kSize, 1);
-
- for (int i = 0; i < kSize; ++i) {
- int b10 = reinterpret_cast<uint32_t*>(ar30_pixels)[i] & 1023;
- int g10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 10) & 1023;
- int r10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 20) & 1023;
- int a2 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 30) & 3;
- ++histogram_b[b10];
- ++histogram_g[g10];
- ++histogram_r[r10];
- int expected_y = Clamp10(static_cast<int>((i - 64) * 1.164f));
- EXPECT_NEAR(b10, expected_y, 4);
- EXPECT_NEAR(g10, expected_y, 4);
- EXPECT_NEAR(r10, expected_y, 4);
- EXPECT_EQ(a2, 3);
- }
-
- int count_b = 0;
- int count_g = 0;
- int count_r = 0;
- for (int i = 0; i < kSize; ++i) {
- if (histogram_b[i]) {
- ++count_b;
- }
- if (histogram_g[i]) {
- ++count_g;
- }
- if (histogram_r[i]) {
- ++count_r;
- }
+TEST_F(LibYUVConvertTest, TestRGB24ToNV12) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_rgb24, kWidth * kHeight * 3);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_uv, kWidth * kHeight / 2);
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_uv, kWidth * kHeight / 2);
+
+ MemRandomize(src_rgb24, kWidth * kHeight * 3);
+
+ RGB24ToNV12(src_rgb24, kWidth * 3, dst_y, kWidth, dst_uv, kWidth, kWidth,
+ kHeight);
+
+ // Verify against RGB24ToI420 + interleaving U and V
+ RGB24ToI420(src_rgb24, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ref_uv[i * 2] = ref_u[i];
+ ref_uv[i * 2 + 1] = ref_v[i];
+ }
+
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_uv[i], ref_uv[i]);
}
- printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
- free_aligned_buffer_page_end(orig_yuv);
- free_aligned_buffer_page_end(ar30_pixels);
+ free_aligned_buffer_page_end(src_rgb24);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+ free_aligned_buffer_page_end(ref_uv);
}
-// Test 10 bit YUV to 10 bit RGB
-// Caveat: Result is near due to float rounding in expected
-// result.
-TEST_F(LibYUVConvertTest, TestH010ToAB30) {
- const int kSize = 1024;
- int histogram_b[1024];
- int histogram_g[1024];
- int histogram_r[1024];
- memset(histogram_b, 0, sizeof(histogram_b));
- memset(histogram_g, 0, sizeof(histogram_g));
- memset(histogram_r, 0, sizeof(histogram_r));
-
- align_buffer_page_end(orig_yuv, kSize * 2 + kSize / 2 * 2 * 2);
- align_buffer_page_end(ab30_pixels, kSize * 4);
- uint16_t* orig_y = reinterpret_cast<uint16_t*>(orig_yuv);
- uint16_t* orig_u = orig_y + kSize;
- uint16_t* orig_v = orig_u + kSize / 2;
-
- // Test grey scale
- for (int i = 0; i < kSize; ++i) {
- orig_y[i] = i;
- }
- for (int i = 0; i < kSize / 2; ++i) {
- orig_u[i] = 512; // 512 is 0.
- orig_v[i] = 512;
- }
-
- H010ToAB30(orig_y, 0, orig_u, 0, orig_v, 0, ab30_pixels, 0, kSize, 1);
-
- for (int i = 0; i < kSize; ++i) {
- int r10 = reinterpret_cast<uint32_t*>(ab30_pixels)[i] & 1023;
- int g10 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 10) & 1023;
- int b10 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 20) & 1023;
- int a2 = (reinterpret_cast<uint32_t*>(ab30_pixels)[i] >> 30) & 3;
- ++histogram_b[b10];
- ++histogram_g[g10];
- ++histogram_r[r10];
- int expected_y = Clamp10(static_cast<int>((i - 64) * 1.164f));
- EXPECT_NEAR(b10, expected_y, 4);
- EXPECT_NEAR(g10, expected_y, 4);
- EXPECT_NEAR(r10, expected_y, 4);
- EXPECT_EQ(a2, 3);
- }
-
- int count_b = 0;
- int count_g = 0;
- int count_r = 0;
- for (int i = 0; i < kSize; ++i) {
- if (histogram_b[i]) {
- ++count_b;
- }
- if (histogram_g[i]) {
- ++count_g;
- }
- if (histogram_r[i]) {
- ++count_r;
- }
+TEST_F(LibYUVConvertTest, TestABGRToNV12Matrix) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_abgr, kWidth * kHeight * 4);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_uv, kWidth * kHeight / 2);
+
+ MemRandomize(src_abgr, kWidth * kHeight * 4);
+
+ // BT.601
+ ARGBToNV12Matrix(src_abgr, kWidth * 4, dst_y, kWidth, dst_uv, kWidth,
+ &kAbgrI601Constants, kWidth, kHeight);
+ // Verify against non-matrix version
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_uv, kWidth * kHeight / 2);
+ ABGRToNV12(src_abgr, kWidth * 4, ref_y, kWidth, ref_uv, kWidth, kWidth,
+ kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_uv[i], ref_uv[i]);
}
- printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
- free_aligned_buffer_page_end(orig_yuv);
- free_aligned_buffer_page_end(ab30_pixels);
+ free_aligned_buffer_page_end(src_abgr);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_uv);
}
-// Test 8 bit YUV to 10 bit RGB
-TEST_F(LibYUVConvertTest, TestH420ToAR30) {
- const int kSize = 256;
- const int kHistSize = 1024;
- int histogram_b[kHistSize];
- int histogram_g[kHistSize];
- int histogram_r[kHistSize];
- memset(histogram_b, 0, sizeof(histogram_b));
- memset(histogram_g, 0, sizeof(histogram_g));
- memset(histogram_r, 0, sizeof(histogram_r));
- align_buffer_page_end(orig_yuv, kSize + kSize / 2 * 2);
- align_buffer_page_end(ar30_pixels, kSize * 4);
- uint8_t* orig_y = orig_yuv;
- uint8_t* orig_u = orig_y + kSize;
- uint8_t* orig_v = orig_u + kSize / 2;
-
- // Test grey scale
- for (int i = 0; i < kSize; ++i) {
- orig_y[i] = i;
- }
- for (int i = 0; i < kSize / 2; ++i) {
- orig_u[i] = 128; // 128 is 0.
- orig_v[i] = 128;
- }
-
- H420ToAR30(orig_y, 0, orig_u, 0, orig_v, 0, ar30_pixels, 0, kSize, 1);
-
- for (int i = 0; i < kSize; ++i) {
- int b10 = reinterpret_cast<uint32_t*>(ar30_pixels)[i] & 1023;
- int g10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 10) & 1023;
- int r10 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 20) & 1023;
- int a2 = (reinterpret_cast<uint32_t*>(ar30_pixels)[i] >> 30) & 3;
- ++histogram_b[b10];
- ++histogram_g[g10];
- ++histogram_r[r10];
- int expected_y = Clamp10(static_cast<int>((i - 16) * 1.164f * 4.f));
- EXPECT_NEAR(b10, expected_y, 4);
- EXPECT_NEAR(g10, expected_y, 4);
- EXPECT_NEAR(r10, expected_y, 4);
- EXPECT_EQ(a2, 3);
- }
-
- int count_b = 0;
- int count_g = 0;
- int count_r = 0;
- for (int i = 0; i < kHistSize; ++i) {
- if (histogram_b[i]) {
- ++count_b;
+TEST_F(LibYUVConvertTest, TestARGBToI420Matrix) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(dst_v, kWidth / 2 * kHeight / 2);
+
+ MemRandomize(src_argb, kWidth * kHeight * 4);
+
+ // BT.601
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbI601Constants, kWidth, kHeight);
+ // Verify against non-matrix version
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
+ ARGBToI420(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ // JPEG
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbJPEGConstants, kWidth, kHeight);
+ // Verify against non-matrix version
+ ARGBToJ420(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ // BT.709
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbH709Constants, kWidth, kHeight);
+ // Just check if it returns 0 for now.
+ // In a real test we'd have reference values.
+
+ // BT.2020
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbU2020Constants, kWidth, kHeight);
+
+ // Reference BT.709 (limited range)
+ // Y = round(0.2126 * 219 / 255 * R + 0.7152 * 219 / 255 * G + 0.0722 * 219 /
+ // 255 * B + 16) Y = round(0.1826 * R + 0.6142 * G + 0.0620 * B + 16) 47 * 255
+ // + 157 * 255 + 16 * 255 + 4224 = 11985 + 40035 + 4080 + 4224 = 60324 60324 /
+ // 256 = 235.64 -> 235. Correct.
+
+ for (int i = 0; i < kWidth * kHeight * 4; ++i)
+ src_argb[i] = 255;
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbH709Constants, kWidth, kHeight);
+ ASSERT_EQ(dst_y[0], 235);
+ ASSERT_EQ(dst_u[0], 128);
+ ASSERT_EQ(dst_v[0], 128);
+
+ for (int i = 0; i < kWidth * kHeight * 4; i += 4) {
+ src_argb[i + 0] = 0; // B
+ src_argb[i + 1] = 0; // G
+ src_argb[i + 2] = 255; // R
+ src_argb[i + 3] = 255; // A
+ }
+ ARGBToI420Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbH709Constants, kWidth, kHeight);
+ // Y = 47 * 255 + 4224 = 11985 + 4224 = 16209. 16209 / 256 = 63.3 -> 63.
+ ASSERT_EQ(dst_y[0], 63);
+ // U = -26 * 255 + 32768 = -6630 + 32768 = 26138. 26138 / 256 = 102.1 -> 102.
+ ASSERT_EQ(dst_u[0], 102);
+ // V = 112 * 255 + 32768 = 28560 + 32768 = 61328. 61328 / 256 = 239.5 -> 239.
+ ASSERT_EQ(dst_v[0], 239);
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+TEST_F(LibYUVConvertTest, TestARGBToI422Matrix) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth / 2 * kHeight);
+ align_buffer_page_end(dst_v, kWidth / 2 * kHeight);
+
+ MemRandomize(src_argb, kWidth * kHeight * 4);
+
+ // BT.601
+ ARGBToI422Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbI601Constants, kWidth, kHeight);
+ // Verify against non-matrix version
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth / 2 * kHeight);
+ align_buffer_page_end(ref_v, kWidth / 2 * kHeight);
+ ARGBToI422(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ // JPEG
+ ARGBToI422Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbJPEGConstants, kWidth, kHeight);
+ // Verify against non-matrix version
+ ARGBToJ422(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+TEST_F(LibYUVConvertTest, TestARGBToI444Matrix) {
+ const int kWidth = 16;
+ const int kHeight = 16;
+ align_buffer_page_end(src_argb, kWidth * kHeight * 4);
+ align_buffer_page_end(dst_y, kWidth * kHeight);
+ align_buffer_page_end(dst_u, kWidth * kHeight);
+ align_buffer_page_end(dst_v, kWidth * kHeight);
+
+ MemRandomize(src_argb, kWidth * kHeight * 4);
+
+ // BT.601
+ ARGBToI444Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth, dst_v,
+ kWidth, &kArgbI601Constants, kWidth, kHeight);
+ // Verify against non-matrix version
+ align_buffer_page_end(ref_y, kWidth * kHeight);
+ align_buffer_page_end(ref_u, kWidth * kHeight);
+ align_buffer_page_end(ref_v, kWidth * kHeight);
+ ARGBToI444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth,
+ kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ // JPEG
+ ARGBToI444Matrix(src_argb, kWidth * 4, dst_y, kWidth, dst_u, kWidth / 2,
+ dst_v, kWidth / 2, &kArgbJPEGConstants, kWidth, kHeight);
+ // Verify against non-matrix version
+ ARGBToJ444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
+ kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
+ }
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+template <typename ConvertToYUV, typename ConvertToARGB>
+static void TestRGBToI420(ConvertToYUV convert_to_yuv,
+ ConvertToARGB convert_to_argb,
+ int width,
+ int height,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ align_buffer_page_end(src_rgb, width * height * 4);
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_u, (width + 1) / 2 * (height + 1) / 2);
+ align_buffer_page_end(dst_v, (width + 1) / 2 * (height + 1) / 2);
+
+ align_buffer_page_end(tmp_argb, width * height * 4);
+ align_buffer_page_end(ref_y, width * height);
+ align_buffer_page_end(ref_u, (width + 1) / 2 * (height + 1) / 2);
+ align_buffer_page_end(ref_v, (width + 1) / 2 * (height + 1) / 2);
+
+ MemRandomize(src_rgb, width * height * 4);
+
+ {
+ SCOPED_TRACE("C_Version");
+ MaskCpuFlags(disable_cpu_flags);
+
+ // Clear buffers
+ memset(dst_y, 0, width * height);
+ memset(dst_u, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(dst_v, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(ref_y, 0, width * height);
+ memset(ref_u, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(ref_v, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(tmp_argb, 0, width * height * 4);
+
+ int r1 =
+ convert_to_yuv(src_rgb, width * 4, dst_y, width, dst_u, (width + 1) / 2,
+ dst_v, (width + 1) / 2, width, height);
+ ASSERT_EQ(r1, 0);
+
+ int r2 =
+ convert_to_argb(src_rgb, width * 4, tmp_argb, width * 4, width, height);
+ ASSERT_EQ(r2, 0);
+
+ int r3 = ARGBToI420(tmp_argb, width * 4, ref_y, width, ref_u,
+ (width + 1) / 2, ref_v, (width + 1) / 2, width, height);
+ ASSERT_EQ(r3, 0);
+
+ for (int i = 0; i < width * height; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
+ }
+ for (int i = 0; i < (width + 1) / 2 * (height + 1) / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
}
- if (histogram_g[i]) {
- ++count_g;
+ }
+
+ {
+ SCOPED_TRACE("SIMD_Version");
+ MaskCpuFlags(benchmark_cpu_info);
+
+ // Clear buffers
+ memset(dst_y, 0, width * height);
+ memset(dst_u, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(dst_v, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(ref_y, 0, width * height);
+ memset(ref_u, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(ref_v, 0, (width + 1) / 2 * (height + 1) / 2);
+ memset(tmp_argb, 0, width * height * 4);
+
+ int r1 =
+ convert_to_yuv(src_rgb, width * 4, dst_y, width, dst_u, (width + 1) / 2,
+ dst_v, (width + 1) / 2, width, height);
+ ASSERT_EQ(r1, 0);
+
+ int r2 =
+ convert_to_argb(src_rgb, width * 4, tmp_argb, width * 4, width, height);
+ ASSERT_EQ(r2, 0);
+
+ int r3 = ARGBToI420(tmp_argb, width * 4, ref_y, width, ref_u,
+ (width + 1) / 2, ref_v, (width + 1) / 2, width, height);
+ ASSERT_EQ(r3, 0);
+
+ for (int i = 0; i < width * height; ++i) {
+ ASSERT_EQ(dst_y[i], ref_y[i]);
}
- if (histogram_r[i]) {
- ++count_r;
+ for (int i = 0; i < (width + 1) / 2 * (height + 1) / 2; ++i) {
+ ASSERT_EQ(dst_u[i], ref_u[i]);
+ ASSERT_EQ(dst_v[i], ref_v[i]);
}
}
- printf("uniques: B %d, G, %d, R %d\n", count_b, count_g, count_r);
- free_aligned_buffer_page_end(orig_yuv);
- free_aligned_buffer_page_end(ar30_pixels);
+ free_aligned_buffer_page_end(src_rgb);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(tmp_argb);
+ free_aligned_buffer_page_end(ref_y);
+ free_aligned_buffer_page_end(ref_u);
+ free_aligned_buffer_page_end(ref_v);
+}
+
+TEST_F(LibYUVConvertTest, BGRAToI420_Check) {
+ TestRGBToI420(BGRAToI420, BGRAToARGB, 16, 16, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(BGRAToI420, BGRAToARGB, 17, 17, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(BGRAToI420, BGRAToARGB, 1280, 720, disable_cpu_flags_,
+ benchmark_cpu_info_);
}
-// Test RGB24 to ARGB and back to RGB24
-TEST_F(LibYUVConvertTest, TestARGBToRGB24) {
- const int kSize = 256;
- align_buffer_page_end(orig_rgb24, kSize * 3);
- align_buffer_page_end(argb_pixels, kSize * 4);
- align_buffer_page_end(dest_rgb24, kSize * 3);
+TEST_F(LibYUVConvertTest, RGBAToI420_Check) {
+ TestRGBToI420(RGBAToI420, RGBAToARGB, 16, 16, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(RGBAToI420, RGBAToARGB, 17, 17, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(RGBAToI420, RGBAToARGB, 1280, 720, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
- // Test grey scale
- for (int i = 0; i < kSize * 3; ++i) {
- orig_rgb24[i] = i;
- }
+TEST_F(LibYUVConvertTest, ABGRToI420_Check) {
+ TestRGBToI420(ABGRToI420, ABGRToARGB, 16, 16, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(ABGRToI420, ABGRToARGB, 17, 17, disable_cpu_flags_,
+ benchmark_cpu_info_);
+ TestRGBToI420(ABGRToI420, ABGRToARGB, 1280, 720, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
- RGB24ToARGB(orig_rgb24, 0, argb_pixels, 0, kSize, 1);
- ARGBToRGB24(argb_pixels, 0, dest_rgb24, 0, kSize, 1);
+// I210ToI420/I212ToI420 funnel through I21xToI420, which scales chroma
+// vertically. Verify oversize heights do not overflow a signed int.
+TEST_F(LibYUVConvertTest, I21xToI420NoHeightOverflow) {
+ int width = 4;
+ int height = 40000;
+ int half_width = (width + 1) / 2;
+ int half_height = (height + 1) / 2;
+ // I21x: 16-bit 4:2:2
+ align_buffer_page_end_16(src_y, width * height);
+ align_buffer_page_end_16(src_u, half_width * height);
+ align_buffer_page_end_16(src_v, half_width * height);
+ // I420: 8-bit 4:2:0
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_u, half_width * half_height);
+ align_buffer_page_end(dst_v, half_width * half_height);
- for (int i = 0; i < kSize * 3; ++i) {
- EXPECT_EQ(orig_rgb24[i], dest_rgb24[i]);
- }
+ memset(src_y, 0, width * height * sizeof(uint16_t));
+ memset(src_u, 0, half_width * height * sizeof(uint16_t));
+ memset(src_v, 0, half_width * height * sizeof(uint16_t));
+
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32767));
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32768));
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32769));
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32767));
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32768));
+ EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32769));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32767));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32768));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, 32769));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32767));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32768));
+ EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+ dst_v, 2, 4, -32769));
+
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(src_u);
+ free_aligned_buffer_page_end_16(src_v);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_v);
+}
- free_aligned_buffer_page_end(orig_rgb24);
- free_aligned_buffer_page_end(argb_pixels);
- free_aligned_buffer_page_end(dest_rgb24);
+// Call YUY2ToNV12() with a width that is not a multiple of 32. Verify there is
+// no stack buffer overflow in YUY2ToNVUVRow_Any_AVX2().
+TEST_F(LibYUVConvertTest, YUY2ToNV12Overflow) {
+ int width = 40 * 32 + 30; // 1310
+ int height = 4;
+ int src_stride_yuy2 = width * 2;
+ align_buffer_page_end(src_yuy2, src_stride_yuy2 * height);
+ align_buffer_page_end(dst_y, width * height);
+ align_buffer_page_end(dst_uv, width * ((height + 1) / 2));
+ memset(src_yuy2, 0xA5, src_stride_yuy2 * height);
+ EXPECT_EQ(0, YUY2ToNV12(src_yuy2, src_stride_yuy2, dst_y, width, dst_uv,
+ width, width, height));
+ free_aligned_buffer_page_end(src_yuy2);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
}
+#endif // !defined(LEAN_TESTS)
+
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/cpu_test.cc b/libs/libyuv/unit_test/cpu_test.cc
index a7991d2bab..3829f8dd23 100644
--- a/libs/libyuv/unit_test/cpu_test.cc
+++ b/libs/libyuv/unit_test/cpu_test.cc
@@ -8,93 +8,21 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#ifdef __linux__
+#include <ctype.h>
+#include <sys/utsname.h>
+#endif
+
#include "../unit_test/unit_test.h"
-#include "libyuv/basic_types.h"
#include "libyuv/cpu_id.h"
#include "libyuv/version.h"
namespace libyuv {
-TEST_F(LibYUVBaseTest, TestCpuHas) {
- int cpu_flags = TestCpuFlag(-1);
- printf("Cpu Flags %d\n", cpu_flags);
-#if defined(__arm__) || defined(__aarch64__)
- int has_arm = TestCpuFlag(kCpuHasARM);
- printf("Has ARM %d\n", has_arm);
- int has_neon = TestCpuFlag(kCpuHasNEON);
- printf("Has NEON %d\n", has_neon);
-#endif
- int has_x86 = TestCpuFlag(kCpuHasX86);
- int has_sse2 = TestCpuFlag(kCpuHasSSE2);
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- int has_sse41 = TestCpuFlag(kCpuHasSSE41);
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- int has_avx = TestCpuFlag(kCpuHasAVX);
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_erms = TestCpuFlag(kCpuHasERMS);
- int has_fma3 = TestCpuFlag(kCpuHasFMA3);
- int has_f16c = TestCpuFlag(kCpuHasF16C);
- int has_gfni = TestCpuFlag(kCpuHasGFNI);
- int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
- int has_avx512vl = TestCpuFlag(kCpuHasAVX512VL);
- int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI);
- int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
- int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
- int has_avx512vpopcntdq = TestCpuFlag(kCpuHasAVX512VPOPCNTDQ);
- printf("Has X86 %d\n", has_x86);
- printf("Has SSE2 %d\n", has_sse2);
- printf("Has SSSE3 %d\n", has_ssse3);
- printf("Has SSE41 %d\n", has_sse41);
- printf("Has SSE42 %d\n", has_sse42);
- printf("Has AVX %d\n", has_avx);
- printf("Has AVX2 %d\n", has_avx2);
- printf("Has ERMS %d\n", has_erms);
- printf("Has FMA3 %d\n", has_fma3);
- printf("Has F16C %d\n", has_f16c);
- printf("Has GFNI %d\n", has_gfni);
- printf("Has AVX512BW %d\n", has_avx512bw);
- printf("Has AVX512VL %d\n", has_avx512vl);
- printf("Has AVX512VBMI %d\n", has_avx512vbmi);
- printf("Has AVX512VBMI2 %d\n", has_avx512vbmi2);
- printf("Has AVX512VBITALG %d\n", has_avx512vbitalg);
- printf("Has AVX512VPOPCNTDQ %d\n", has_avx512vpopcntdq);
-
-#if defined(__mips__)
- int has_mips = TestCpuFlag(kCpuHasMIPS);
- printf("Has MIPS %d\n", has_mips);
- int has_msa = TestCpuFlag(kCpuHasMSA);
- printf("Has MSA %d\n", has_msa);
- int has_mmi = TestCpuFlag(kCpuHasMMI);
- printf("Has MMI %d\n", has_mmi);
-#endif
-}
-
-TEST_F(LibYUVBaseTest, TestCpuCompilerEnabled) {
-#if defined(__aarch64__)
- printf("Arm64 build\n");
-#endif
-#if defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)
- printf("Neon build enabled\n");
-#endif
-#if defined(__x86_64__) || defined(_M_X64)
- printf("x64 build\n");
-#endif
-#ifdef _MSC_VER
- printf("_MSC_VER %d\n", _MSC_VER);
-#endif
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(GCC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
- defined(VISUALC_HAS_AVX2))
- printf("Has AVX2 1\n");
-#else
- printf("Has AVX2 0\n");
-// If compiler does not support AVX2, the following function not expected:
-#endif
-}
-
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64)
TEST_F(LibYUVBaseTest, TestCpuId) {
@@ -116,9 +44,10 @@ TEST_F(LibYUVBaseTest, TestCpuId) {
cpu_info[0] = cpu_info[1]; // Reorder output
cpu_info[1] = cpu_info[3];
cpu_info[3] = 0;
- printf("Cpu Vendor: %s %x %x %x\n", reinterpret_cast<char*>(&cpu_info[0]),
- cpu_info[0], cpu_info[1], cpu_info[2]);
- EXPECT_EQ(12u, strlen(reinterpret_cast<char*>(&cpu_info[0])));
+ printf("Cpu Vendor: %s 0x%x 0x%x 0x%x\n",
+ reinterpret_cast<char*>(&cpu_info[0]), cpu_info[0], cpu_info[1],
+ cpu_info[2]);
+ ASSERT_EQ(12u, strlen(reinterpret_cast<char*>(&cpu_info[0])));
// CPU Family and Model
// 3:0 - Stepping
@@ -136,6 +65,294 @@ TEST_F(LibYUVBaseTest, TestCpuId) {
}
#endif
+#ifdef __linux__
+static void KernelVersion(int* version) {
+ struct utsname buffer;
+ int i = 0;
+
+ version[0] = version[1] = 0;
+ if (uname(&buffer) == 0) {
+ char* v = buffer.release;
+ for (i = 0; *v && i < 2; ++v) {
+ if (isdigit(*v)) {
+ version[i++] = (int)strtol(v, &v, 10);
+ }
+ }
+ }
+}
+#endif
+
+TEST_F(LibYUVBaseTest, TestCpuHas) {
+#if defined(__linux__)
+ {
+ int kernelversion[2];
+ KernelVersion(kernelversion);
+ printf("Kernel Version %d.%d\n", kernelversion[0], kernelversion[1]);
+ }
+#endif // defined(__linux__)
+
+#if defined(__arm__) || defined(__aarch64__)
+ int has_arm = TestCpuFlag(kCpuHasARM);
+ if (has_arm) {
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ int has_neon_dotprod = TestCpuFlag(kCpuHasNeonDotProd);
+ int has_neon_i8mm = TestCpuFlag(kCpuHasNeonI8MM);
+ int has_sve = TestCpuFlag(kCpuHasSVE);
+ int has_sve2 = TestCpuFlag(kCpuHasSVE2);
+ int has_sve_f32mm = TestCpuFlag(kCpuHasSVEF32MM);
+ int has_sme = TestCpuFlag(kCpuHasSME);
+ int has_sme2 = TestCpuFlag(kCpuHasSME2);
+ printf("Has Arm 0x%x\n", has_arm);
+ printf("Has Neon 0x%x\n", has_neon);
+ printf("Has Neon DotProd 0x%x\n", has_neon_dotprod);
+ printf("Has Neon I8MM 0x%x\n", has_neon_i8mm);
+ printf("Has SVE 0x%x\n", has_sve);
+ printf("Has SVE2 0x%x\n", has_sve2);
+ printf("Has SVE F32MM 0x%x\n", has_sve_f32mm);
+ printf("Has SME 0x%x\n", has_sme);
+ printf("Has SME2 0x%x\n", has_sme2);
+
+#if defined(__aarch64__)
+ // Read and print the SVE and SME vector lengths.
+ if (has_sve) {
+ int sve_vl;
+ asm(".inst 0x04bf5020 \n" // rdvl x0, #1
+ "mov %w[sve_vl], w0 \n"
+ : [sve_vl] "=r"(sve_vl) // %[sve_vl]
+ :
+ : "x0");
+ printf("SVE vector length: %d bytes\n", sve_vl);
+ }
+ if (has_sme) {
+ int sme_vl;
+ asm(".inst 0x04bf5820 \n" // rdsvl x0, #1
+ "mov %w[sme_vl], w0 \n"
+ : [sme_vl] "=r"(sme_vl) // %[sme_vl]
+ :
+ : "x0");
+ printf("SME vector length: %d bytes\n", sme_vl);
+ }
+#endif // defined(__aarch64__)
+ }
+#endif // if defined(__arm__) || defined(__aarch64__)
+
+#if defined(__riscv)
+ int has_riscv = TestCpuFlag(kCpuHasRISCV);
+ if (has_riscv) {
+ int has_rvv = TestCpuFlag(kCpuHasRVV);
+ printf("Has RISCV 0x%x\n", has_riscv);
+ printf("Has RVV 0x%x\n", has_rvv);
+
+ // Read and print the RVV vector length.
+ if (has_rvv) {
+ register uint32_t vlenb __asm__("t0");
+ __asm__(".word 0xC22022F3" /* CSRR t0, vlenb */ : "=r"(vlenb));
+ printf("RVV vector length: %d bytes\n", vlenb);
+ }
+ }
+#endif // defined(__riscv)
+
+#if defined(__loongarch__)
+ int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
+ if (has_loongarch) {
+ int has_lsx = TestCpuFlag(kCpuHasLSX);
+ int has_lasx = TestCpuFlag(kCpuHasLASX);
+ printf("Has LOONGARCH 0x%x\n", has_loongarch);
+ printf("Has LSX 0x%x\n", has_lsx);
+ printf("Has LASX 0x%x\n", has_lasx);
+ }
+#endif // defined(__loongarch__)
+
+#if defined(__wasm__)
+ int has_wasm = TestCpuFlag(kCpuHasWASM);
+ if (has_wasm) {
+ int has_wasm_simd = TestCpuFlag(kCpuHasWASMSIMD);
+ printf("Has WASM 0x%x\n", has_wasm);
+ printf("Has WASM SIMD 0x%x\n", has_wasm_simd);
+ }
+#endif // defined(__wasm__)
+
+#if defined(__hexagon__)
+ int has_hexagon = TestCpuFlag(kCpuHasHexagon);
+ if (has_hexagon) {
+ int has_hvx = TestCpuFlag(kCpuHasHVX);
+ printf("Has Hexagon 0x%x\n", has_hexagon);
+ printf("Has HVX 0x%x\n", has_hvx);
+ }
+#endif // defined(__hexagon__)
+
+#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
+ defined(_M_X64)
+ int has_x86 = TestCpuFlag(kCpuHasX86);
+ if (has_x86) {
+ int has_sse2 = TestCpuFlag(kCpuHasSSE2);
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ int has_sse41 = TestCpuFlag(kCpuHasSSE41);
+ int has_sse42 = TestCpuFlag(kCpuHasSSE42);
+ int has_avx = TestCpuFlag(kCpuHasAVX);
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ int has_erms = TestCpuFlag(kCpuHasERMS);
+ int has_fsmr = TestCpuFlag(kCpuHasFSMR);
+ int has_fma3 = TestCpuFlag(kCpuHasFMA3);
+ int has_f16c = TestCpuFlag(kCpuHasF16C);
+ int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
+ int has_avx512vl = TestCpuFlag(kCpuHasAVX512VL);
+ int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI);
+ int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI);
+ int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
+ int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
+ int has_avx10 = TestCpuFlag(kCpuHasAVX10);
+ int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2);
+ int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI);
+ int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
+ int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
+ int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
+ int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
+ printf("Has X86 0x%x\n", has_x86);
+ printf("Has SSE2 0x%x\n", has_sse2);
+ printf("Has SSSE3 0x%x\n", has_ssse3);
+ printf("Has SSE4.1 0x%x\n", has_sse41);
+ printf("Has SSE4.2 0x%x\n", has_sse42);
+ printf("Has AVX 0x%x\n", has_avx);
+ printf("Has AVX2 0x%x\n", has_avx2);
+ printf("Has ERMS 0x%x\n", has_erms);
+ printf("Has FSMR 0x%x\n", has_fsmr);
+ printf("Has FMA3 0x%x\n", has_fma3);
+ printf("Has F16C 0x%x\n", has_f16c);
+ printf("Has AVX512BW 0x%x\n", has_avx512bw);
+ printf("Has AVX512VL 0x%x\n", has_avx512vl);
+ printf("Has AVX512VNNI 0x%x\n", has_avx512vnni);
+ printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi);
+ printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2);
+ printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg);
+ printf("Has AVX10 0x%x\n", has_avx10);
+ printf("Has AVX10_2 0x%x\n", has_avx10_2);
+ printf("HAS AVXVNNI 0x%x\n", has_avxvnni);
+ printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
+ printf("Has AMXINT8 0x%x\n", has_amxint8);
+ printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
+ printf("Has AMXTF32 0x%x\n", has_amxtf32);
+ }
+#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) ||
+ // defined(_M_X64)
+}
+
+TEST_F(LibYUVBaseTest, TestCompilerMacros) {
+ // Tests all macros used in public headers.
+#ifdef __ATOMIC_RELAXED
+ printf("__ATOMIC_RELAXED %d\n", __ATOMIC_RELAXED);
+#endif
+#ifdef __cplusplus
+ printf("__cplusplus %ld\n", __cplusplus);
+#endif
+#ifdef __clang_major__
+ printf("__clang_major__ %d\n", __clang_major__);
+#endif
+#ifdef __clang_minor__
+ printf("__clang_minor__ %d\n", __clang_minor__);
+#endif
+#ifdef __GNUC__
+ printf("__GNUC__ %d\n", __GNUC__);
+#endif
+#ifdef __GNUC_MINOR__
+ printf("__GNUC_MINOR__ %d\n", __GNUC_MINOR__);
+#endif
+#ifdef __i386__
+ printf("__i386__ %d\n", __i386__);
+#endif
+#ifdef __x86_64__
+ printf("__x86_64__ %d\n", __x86_64__);
+#endif
+#ifdef _M_IX86
+ printf("_M_IX86 %d\n", _M_IX86);
+#endif
+#ifdef _M_X64
+ printf("_M_X64 %d\n", _M_X64);
+#endif
+#ifdef _MSC_VER
+ printf("_MSC_VER %d\n", _MSC_VER);
+#endif
+#ifdef __aarch64__
+ printf("__aarch64__ %d\n", __aarch64__);
+#endif
+#ifdef __arm__
+ printf("__arm__ %d\n", __arm__);
+#endif
+#ifdef __riscv
+ printf("__riscv %d\n", __riscv);
+#endif
+#ifdef __riscv_vector
+ printf("__riscv_vector %d\n", __riscv_vector);
+#endif
+#ifdef __riscv_v_intrinsic
+ printf("__riscv_v_intrinsic %d\n", __riscv_v_intrinsic);
+#endif
+#ifdef __riscv_zve64x
+ printf("__riscv_zve64x %d\n", __riscv_zve64x);
+#endif
+#ifdef __APPLE__
+ printf("__APPLE__ %d\n", __APPLE__);
+#endif
+#ifdef __clang__
+ printf("__clang__ %d\n", __clang__);
+#endif
+#ifdef __CLR_VER
+ printf("__CLR_VER %d\n", __CLR_VER);
+#endif
+#ifdef __CYGWIN__
+ printf("__CYGWIN__ %d\n", __CYGWIN__);
+#endif
+#ifdef __llvm__
+ printf("__llvm__ %d\n", __llvm__);
+#endif
+#ifdef __loongarch__
+ printf("__loongarch__ %d\n", __loongarch__);
+#endif
+#ifdef __wasm__
+ printf("__wasm__ %d\n", __wasm__);
+#endif
+#ifdef __wasm_simd128__
+ printf("__wasm_simd128__ %d\n", __wasm_simd128__);
+#endif
+#ifdef __hexagon__
+ printf("__hexagon__ %d\n", __hexagon__);
+#endif
+#ifdef __HVX__
+ printf("__HVX__ %d\n", __HVX__);
+#endif
+#ifdef __HVX_ARCH__
+ printf("__HVX_ARCH__ %d\n", __HVX_ARCH__);
+#endif
+#ifdef __HVX_LENGTH__
+ printf("__HVX_LENGTH__ %d\n", __HVX_LENGTH__);
+#endif
+#ifdef _WIN32
+ printf("_WIN32 %d\n", _WIN32);
+#endif
+#ifdef __native_client__
+ printf("__native_client__ %d\n", __native_client__);
+#endif
+#ifdef __pic__
+ printf("__pic__ %d\n", __pic__);
+#endif
+#ifdef __pnacl__
+ printf("__pnacl__ %d\n", __pnacl__);
+#endif
+#ifdef GG_LONGLONG
+ printf("GG_LONGLONG %lld\n", GG_LONGLONG(1));
+#endif
+#ifdef INT_TYPES_DEFINED
+ printf("INT_TYPES_DEFINED\n");
+#endif
+#ifdef __has_feature
+ printf("__has_feature\n");
+#if __has_feature(memory_sanitizer)
+ printf("__has_feature(memory_sanitizer) %d\n",
+ __has_feature(memory_sanitizer));
+#endif
+#endif
+}
+
static int FileExists(const char* file_name) {
FILE* f = fopen(file_name, "r");
if (!f) {
@@ -145,22 +362,87 @@ static int FileExists(const char* file_name) {
return 1;
}
-TEST_F(LibYUVBaseTest, TestLinuxNeon) {
+TEST_F(LibYUVBaseTest, DISABLED_TestLinuxArm) {
if (FileExists("../../unit_test/testdata/arm_v7.txt")) {
printf("Note: testing to load \"../../unit_test/testdata/arm_v7.txt\"\n");
- EXPECT_EQ(0, ArmCpuCaps("../../unit_test/testdata/arm_v7.txt"));
- EXPECT_EQ(kCpuHasNEON, ArmCpuCaps("../../unit_test/testdata/tegra3.txt"));
- EXPECT_EQ(kCpuHasNEON, ArmCpuCaps("../../unit_test/testdata/juno.txt"));
+ ASSERT_EQ(0, ArmCpuCaps("../../unit_test/testdata/arm_v7.txt"));
+ ASSERT_EQ(kCpuHasNEON, ArmCpuCaps("../../unit_test/testdata/tegra3.txt"));
} else {
printf("WARNING: unable to load \"../../unit_test/testdata/arm_v7.txt\"\n");
}
-#if defined(__linux__) && defined(__ARM_NEON__)
- EXPECT_EQ(kCpuHasNEON, ArmCpuCaps("/proc/cpuinfo"));
+#if defined(__linux__) && defined(__ARM_NEON__) && !defined(__aarch64__)
+ if (FileExists("/proc/cpuinfo")) {
+ if (kCpuHasNEON != ArmCpuCaps("/proc/cpuinfo")) {
+ // This can happen on Arm emulator but /proc/cpuinfo is from host.
+ printf("WARNING: Neon build enabled but CPU does not have Neon\n");
+ }
+ } else {
+ printf("WARNING: unable to load \"/proc/cpuinfo\"\n");
+ }
+#endif
+}
+
+#if defined(__linux__) && defined(__aarch64__)
+TEST_F(LibYUVBaseTest, TestLinuxAArch64) {
+ // Values taken from a Cortex-A57 machine, only Neon available.
+ ASSERT_EQ(kCpuHasNEON, AArch64CpuCaps(0xffU, 0x0U));
+
+ // Values taken from a Google Pixel 7.
+ int expected = kCpuHasNEON | kCpuHasNeonDotProd;
+ ASSERT_EQ(expected, AArch64CpuCaps(0x119fffU, 0x0U));
+
+ // Values taken from a Google Pixel 8.
+ expected = kCpuHasNEON | kCpuHasNeonDotProd | kCpuHasNeonI8MM | kCpuHasSVE |
+ kCpuHasSVE2;
+ ASSERT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x2f33fU));
+
+ // Values taken from a Neoverse N2 machine.
+ ASSERT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x2f3ffU));
+
+ // Check for SME feature detection.
+ expected |= kCpuHasSME;
+ ASSERT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x82f3ffU));
+
+ // Check for SME2 feature detection.
+ expected |= kCpuHasSME2;
+ ASSERT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x200082f3ffULL));
+}
+#endif
+
+TEST_F(LibYUVBaseTest, DISABLED_TestLinuxRVV) {
+ if (FileExists("../../unit_test/testdata/riscv64.txt")) {
+ printf("Note: testing to load \"../../unit_test/testdata/riscv64.txt\"\n");
+
+ ASSERT_EQ(0, RiscvCpuCaps("../../unit_test/testdata/riscv64.txt"));
+ ASSERT_EQ(kCpuHasRVV,
+ RiscvCpuCaps("../../unit_test/testdata/riscv64_rvv.txt"));
+ ASSERT_EQ(kCpuHasRVV | kCpuHasRVVZVFH,
+ RiscvCpuCaps("../../unit_test/testdata/riscv64_rvv_zvfh.txt"));
+ } else {
+ printf(
+ "WARNING: unable to load "
+ "\"../../unit_test/testdata/riscv64.txt\"\n");
+ }
+#if defined(__linux__) && defined(__riscv)
+ if (FileExists("/proc/cpuinfo")) {
+ if (!(kCpuHasRVV & RiscvCpuCaps("/proc/cpuinfo"))) {
+ // This can happen on RVV emulator but /proc/cpuinfo is from host.
+ printf("WARNING: RVV build enabled but CPU does not have RVV\n");
+ }
+ } else {
+ printf("WARNING: unable to load \"/proc/cpuinfo\"\n");
+ }
#endif
}
-TEST_F(LibYUVBaseTest, TestSetCpuFlags) {
+#ifdef _WIN32
+// This doesn't pass on Windows CQ.
+#define MAYBE_TestSetCpuFlags DISABLED_TestSetCpuFlags
+#else
+#define MAYBE_TestSetCpuFlags TestSetCpuFlags
+#endif
+TEST_F(LibYUVBaseTest, MAYBE_TestSetCpuFlags) {
// Reset any masked flags that may have been set so auto init is enabled.
MaskCpuFlags(0);
@@ -169,15 +451,15 @@ TEST_F(LibYUVBaseTest, TestSetCpuFlags) {
// Test setting different CPU configurations.
int cpu_flags = kCpuHasARM | kCpuHasNEON | kCpuInitialized;
SetCpuFlags(cpu_flags);
- EXPECT_EQ(cpu_flags, TestCpuFlag(-1));
+ ASSERT_EQ(cpu_flags, TestCpuFlag(-1));
cpu_flags = kCpuHasX86 | kCpuInitialized;
SetCpuFlags(cpu_flags);
- EXPECT_EQ(cpu_flags, TestCpuFlag(-1));
+ ASSERT_EQ(cpu_flags, TestCpuFlag(-1));
// Test that setting 0 turns auto-init back on.
SetCpuFlags(0);
- EXPECT_EQ(original_cpu_flags, TestCpuFlag(-1));
+ ASSERT_EQ(original_cpu_flags, TestCpuFlag(-1));
// Restore the CPU flag mask.
MaskCpuFlags(benchmark_cpu_info_);
diff --git a/libs/libyuv/unit_test/cpu_thread_test.cc b/libs/libyuv/unit_test/cpu_thread_test.cc
index 59061b98e0..572074d73f 100644
--- a/libs/libyuv/unit_test/cpu_thread_test.cc
+++ b/libs/libyuv/unit_test/cpu_thread_test.cc
@@ -12,7 +12,7 @@
#include "libyuv/cpu_id.h"
-#if defined(__clang__)
+#if defined(__clang__) && !defined(__wasm__)
#if __has_include(<pthread.h>)
#define LIBYUV_HAVE_PTHREAD 1
#endif
@@ -27,10 +27,10 @@
namespace libyuv {
#ifdef LIBYUV_HAVE_PTHREAD
-void* ThreadMain(void* arg) {
+static void* ThreadMain(void* arg) {
int* flags = static_cast<int*>(arg);
- *flags = TestCpuFlag(kCpuHasSSSE3);
+ *flags = TestCpuFlag(kCpuInitialized);
return nullptr;
}
#endif // LIBYUV_HAVE_PTHREAD
@@ -51,10 +51,10 @@ TEST(LibYUVCpuThreadTest, TestCpuFlagMultipleThreads) {
ret = pthread_create(&thread2, nullptr, ThreadMain, &cpu_flags2);
ASSERT_EQ(ret, 0);
ret = pthread_join(thread1, nullptr);
- EXPECT_EQ(ret, 0);
+ ASSERT_EQ(ret, 0);
ret = pthread_join(thread2, nullptr);
- EXPECT_EQ(ret, 0);
- EXPECT_EQ(cpu_flags1, cpu_flags2);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(cpu_flags1, cpu_flags2);
#else
printf("pthread unavailable; Test skipped.");
#endif // LIBYUV_HAVE_PTHREAD
diff --git a/libs/libyuv/unit_test/math_test.cc b/libs/libyuv/unit_test/math_test.cc
index a1544c122b..1f1a6d8443 100644
--- a/libs/libyuv/unit_test/math_test.cc
+++ b/libs/libyuv/unit_test/math_test.cc
@@ -13,7 +13,6 @@
#include <time.h>
#include "../unit_test/unit_test.h"
-#include "libyuv/basic_types.h"
#include "libyuv/cpu_id.h"
#include "libyuv/scale.h"
@@ -30,44 +29,44 @@ TEST_F(LibYUVBaseTest, TestFixedDiv) {
int result_opt[1280];
int result_c[1280];
- EXPECT_EQ(0x10000, libyuv::FixedDiv(1, 1));
- EXPECT_EQ(0x7fff0000, libyuv::FixedDiv(0x7fff, 1));
+ ASSERT_EQ(0x10000, libyuv::FixedDiv(1, 1));
+ ASSERT_EQ(0x7fff0000, libyuv::FixedDiv(0x7fff, 1));
// TODO(fbarchard): Avoid the following that throw exceptions.
- // EXPECT_EQ(0x100000000, libyuv::FixedDiv(0x10000, 1));
- // EXPECT_EQ(0x80000000, libyuv::FixedDiv(0x8000, 1));
-
- EXPECT_EQ(0x20000, libyuv::FixedDiv(640 * 2, 640));
- EXPECT_EQ(0x30000, libyuv::FixedDiv(640 * 3, 640));
- EXPECT_EQ(0x40000, libyuv::FixedDiv(640 * 4, 640));
- EXPECT_EQ(0x50000, libyuv::FixedDiv(640 * 5, 640));
- EXPECT_EQ(0x60000, libyuv::FixedDiv(640 * 6, 640));
- EXPECT_EQ(0x70000, libyuv::FixedDiv(640 * 7, 640));
- EXPECT_EQ(0x80000, libyuv::FixedDiv(640 * 8, 640));
- EXPECT_EQ(0xa0000, libyuv::FixedDiv(640 * 10, 640));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(960 * 2, 960));
- EXPECT_EQ(0x08000, libyuv::FixedDiv(640 / 2, 640));
- EXPECT_EQ(0x04000, libyuv::FixedDiv(640 / 4, 640));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(1080 * 2, 1080));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(200000, 100000));
- EXPECT_EQ(0x18000, libyuv::FixedDiv(150000, 100000));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(40000, 20000));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(-40000, -20000));
- EXPECT_EQ(-0x20000, libyuv::FixedDiv(40000, -20000));
- EXPECT_EQ(-0x20000, libyuv::FixedDiv(-40000, 20000));
- EXPECT_EQ(0x10000, libyuv::FixedDiv(4095, 4095));
- EXPECT_EQ(0x10000, libyuv::FixedDiv(4096, 4096));
- EXPECT_EQ(0x10000, libyuv::FixedDiv(4097, 4097));
- EXPECT_EQ(123 * 65536, libyuv::FixedDiv(123, 1));
+ // ASSERT_EQ(0x100000000, libyuv::FixedDiv(0x10000, 1));
+ // ASSERT_EQ(0x80000000, libyuv::FixedDiv(0x8000, 1));
+
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(640 * 2, 640));
+ ASSERT_EQ(0x30000, libyuv::FixedDiv(640 * 3, 640));
+ ASSERT_EQ(0x40000, libyuv::FixedDiv(640 * 4, 640));
+ ASSERT_EQ(0x50000, libyuv::FixedDiv(640 * 5, 640));
+ ASSERT_EQ(0x60000, libyuv::FixedDiv(640 * 6, 640));
+ ASSERT_EQ(0x70000, libyuv::FixedDiv(640 * 7, 640));
+ ASSERT_EQ(0x80000, libyuv::FixedDiv(640 * 8, 640));
+ ASSERT_EQ(0xa0000, libyuv::FixedDiv(640 * 10, 640));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(960 * 2, 960));
+ ASSERT_EQ(0x08000, libyuv::FixedDiv(640 / 2, 640));
+ ASSERT_EQ(0x04000, libyuv::FixedDiv(640 / 4, 640));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(1080 * 2, 1080));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(200000, 100000));
+ ASSERT_EQ(0x18000, libyuv::FixedDiv(150000, 100000));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(40000, 20000));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(-40000, -20000));
+ ASSERT_EQ(-0x20000, libyuv::FixedDiv(40000, -20000));
+ ASSERT_EQ(-0x20000, libyuv::FixedDiv(-40000, 20000));
+ ASSERT_EQ(0x10000, libyuv::FixedDiv(4095, 4095));
+ ASSERT_EQ(0x10000, libyuv::FixedDiv(4096, 4096));
+ ASSERT_EQ(0x10000, libyuv::FixedDiv(4097, 4097));
+ ASSERT_EQ(123 * 65536, libyuv::FixedDiv(123, 1));
for (int i = 1; i < 4100; ++i) {
- EXPECT_EQ(0x10000, libyuv::FixedDiv(i, i));
- EXPECT_EQ(0x20000, libyuv::FixedDiv(i * 2, i));
- EXPECT_EQ(0x30000, libyuv::FixedDiv(i * 3, i));
- EXPECT_EQ(0x40000, libyuv::FixedDiv(i * 4, i));
- EXPECT_EQ(0x08000, libyuv::FixedDiv(i, i * 2));
- EXPECT_NEAR(16384 * 65536 / i, libyuv::FixedDiv(16384, i), 1);
+ ASSERT_EQ(0x10000, libyuv::FixedDiv(i, i));
+ ASSERT_EQ(0x20000, libyuv::FixedDiv(i * 2, i));
+ ASSERT_EQ(0x30000, libyuv::FixedDiv(i * 3, i));
+ ASSERT_EQ(0x40000, libyuv::FixedDiv(i * 4, i));
+ ASSERT_EQ(0x08000, libyuv::FixedDiv(i, i * 2));
+ ASSERT_NEAR(16384 * 65536 / i, libyuv::FixedDiv(16384, i), 1);
}
- EXPECT_EQ(123 * 65536, libyuv::FixedDiv(123, 1));
+ ASSERT_EQ(123 * 65536, libyuv::FixedDiv(123, 1));
MemRandomize(reinterpret_cast<uint8_t*>(&num[0]), sizeof(num));
MemRandomize(reinterpret_cast<uint8_t*>(&div[0]), sizeof(div));
@@ -84,7 +83,7 @@ TEST_F(LibYUVBaseTest, TestFixedDiv) {
}
for (int j = 0; j < 1280; ++j) {
result_c[j] = libyuv::FixedDiv_C(num[j], div[j]);
- EXPECT_NEAR(result_c[j], result_opt[j], 1);
+ ASSERT_NEAR(result_c[j], result_opt[j], 1);
}
}
@@ -118,7 +117,7 @@ TEST_F(LibYUVBaseTest, TestFixedDiv_Opt) {
}
for (int j = 0; j < 1280; ++j) {
result_c[j] = libyuv::FixedDiv_C(num[j], div[j]);
- EXPECT_NEAR(result_c[j], result_opt[j], 1);
+ ASSERT_NEAR(result_c[j], result_opt[j], 1);
}
}
@@ -152,7 +151,7 @@ TEST_F(LibYUVBaseTest, TestFixedDiv1_Opt) {
}
for (int j = 0; j < 1280; ++j) {
result_c[j] = libyuv::FixedDiv1_C(num[j], div[j]);
- EXPECT_NEAR(result_c[j], result_opt[j], 1);
+ ASSERT_NEAR(result_c[j], result_opt[j], 1);
}
}
#endif // ENABLE_ROW_TESTS
diff --git a/libs/libyuv/unit_test/planar_test.cc b/libs/libyuv/unit_test/planar_test.cc
index 19232b6fed..5854b8903d 100644
--- a/libs/libyuv/unit_test/planar_test.cc
+++ b/libs/libyuv/unit_test/planar_test.cc
@@ -12,7 +12,6 @@
#include <stdlib.h>
#include <time.h>
-
#include "../unit_test/unit_test.h"
#include "libyuv/compare.h"
#include "libyuv/convert.h"
@@ -22,6 +21,13 @@
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
+#include "libyuv/scale.h"
+#include "libyuv/scale_uv.h"
+
+#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \
+ !defined(ENABLE_ROW_TESTS)
+#define ENABLE_ROW_TESTS
+#endif
#ifdef ENABLE_ROW_TESTS
// row.h defines SIMD_ALIGNED, overriding unit_test.h
@@ -29,6 +35,23 @@
#include "libyuv/row.h" /* For ScaleSumSamples_Neon */
#endif
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#include <sanitizer/msan_interface.h>
+#define MSAN_POISON(ptr, size) __msan_allocated_memory(ptr, size)
+#define MSAN_CHECK_INITIALIZED(ptr, size) \
+ __msan_check_mem_is_initialized(ptr, size)
+#endif
+#endif
+#ifndef MSAN_POISON
+#define MSAN_POISON(ptr, size)
+#endif
+#ifndef MSAN_CHECK_INITIALIZED
+#define MSAN_CHECK_INITIALIZED(ptr, size)
+#endif
+
+#define EXPECTED_UNATTENUATE_DIFF 0
+
namespace libyuv {
TEST_F(LibYUVPlanarTest, TestAttenuate) {
@@ -51,28 +74,74 @@ TEST_F(LibYUVPlanarTest, TestAttenuate) {
orig_pixels[2 * 4 + 0] = 16u;
orig_pixels[2 * 4 + 1] = 64u;
orig_pixels[2 * 4 + 2] = 192u;
- orig_pixels[2 * 4 + 3] = 255u;
+ orig_pixels[2 * 4 + 3] = 128u;
orig_pixels[3 * 4 + 0] = 16u;
orig_pixels[3 * 4 + 1] = 64u;
orig_pixels[3 * 4 + 2] = 192u;
- orig_pixels[3 * 4 + 3] = 128u;
- ARGBUnattenuate(orig_pixels, 0, unatten_pixels, 0, 4, 1);
- EXPECT_EQ(255u, unatten_pixels[0 * 4 + 0]);
- EXPECT_EQ(255u, unatten_pixels[0 * 4 + 1]);
- EXPECT_EQ(254u, unatten_pixels[0 * 4 + 2]);
- EXPECT_EQ(128u, unatten_pixels[0 * 4 + 3]);
- EXPECT_EQ(0u, unatten_pixels[1 * 4 + 0]);
- EXPECT_EQ(0u, unatten_pixels[1 * 4 + 1]);
- EXPECT_EQ(0u, unatten_pixels[1 * 4 + 2]);
- EXPECT_EQ(0u, unatten_pixels[1 * 4 + 3]);
- EXPECT_EQ(16u, unatten_pixels[2 * 4 + 0]);
- EXPECT_EQ(64u, unatten_pixels[2 * 4 + 1]);
- EXPECT_EQ(192u, unatten_pixels[2 * 4 + 2]);
- EXPECT_EQ(255u, unatten_pixels[2 * 4 + 3]);
- EXPECT_EQ(32u, unatten_pixels[3 * 4 + 0]);
- EXPECT_EQ(128u, unatten_pixels[3 * 4 + 1]);
- EXPECT_EQ(255u, unatten_pixels[3 * 4 + 2]);
- EXPECT_EQ(128u, unatten_pixels[3 * 4 + 3]);
+ orig_pixels[3 * 4 + 3] = 255u;
+ orig_pixels[4 * 4 + 0] = 255u;
+ orig_pixels[4 * 4 + 1] = 255u;
+ orig_pixels[4 * 4 + 2] = 255u;
+ orig_pixels[4 * 4 + 3] = 255u;
+
+ ARGBUnattenuate(orig_pixels, 0, unatten_pixels, 0, 5, 1);
+ ASSERT_EQ(255u, unatten_pixels[0 * 4 + 0]);
+ ASSERT_EQ(255u, unatten_pixels[0 * 4 + 1]);
+ ASSERT_EQ(254u, unatten_pixels[0 * 4 + 2]);
+ ASSERT_EQ(128u, unatten_pixels[0 * 4 + 3]);
+ ASSERT_EQ(0u, unatten_pixels[1 * 4 + 0]);
+ ASSERT_EQ(0u, unatten_pixels[1 * 4 + 1]);
+ ASSERT_EQ(0u, unatten_pixels[1 * 4 + 2]);
+ ASSERT_EQ(0u, unatten_pixels[1 * 4 + 3]);
+ ASSERT_EQ(32u, unatten_pixels[2 * 4 + 0]);
+ ASSERT_EQ(128u, unatten_pixels[2 * 4 + 1]);
+ ASSERT_EQ(255u, unatten_pixels[2 * 4 + 2]);
+ ASSERT_EQ(128u, unatten_pixels[2 * 4 + 3]);
+ ASSERT_EQ(16u, unatten_pixels[3 * 4 + 0]);
+ ASSERT_EQ(64u, unatten_pixels[3 * 4 + 1]);
+ ASSERT_EQ(192u, unatten_pixels[3 * 4 + 2]);
+ ASSERT_EQ(255u, unatten_pixels[3 * 4 + 3]);
+ ASSERT_EQ(255u, unatten_pixels[4 * 4 + 0]);
+ ASSERT_EQ(255u, unatten_pixels[4 * 4 + 1]);
+ ASSERT_EQ(255u, unatten_pixels[4 * 4 + 2]);
+ ASSERT_EQ(255u, unatten_pixels[4 * 4 + 3]);
+
+ ARGBAttenuate(orig_pixels, 0, atten_pixels, 0, 5, 1);
+ ASSERT_EQ(100u, atten_pixels[0 * 4 + 0]);
+ ASSERT_EQ(65u, atten_pixels[0 * 4 + 1]);
+ ASSERT_EQ(64u, atten_pixels[0 * 4 + 2]);
+ ASSERT_EQ(128u, atten_pixels[0 * 4 + 3]);
+ ASSERT_EQ(0u, atten_pixels[1 * 4 + 0]);
+ ASSERT_EQ(0u, atten_pixels[1 * 4 + 1]);
+ ASSERT_EQ(0u, atten_pixels[1 * 4 + 2]);
+ ASSERT_EQ(0u, atten_pixels[1 * 4 + 3]);
+ ASSERT_EQ(8u, atten_pixels[2 * 4 + 0]);
+ ASSERT_EQ(32u, atten_pixels[2 * 4 + 1]);
+ ASSERT_EQ(96u, atten_pixels[2 * 4 + 2]);
+ ASSERT_EQ(128u, atten_pixels[2 * 4 + 3]);
+ ASSERT_EQ(16u, atten_pixels[3 * 4 + 0]);
+ ASSERT_EQ(64u, atten_pixels[3 * 4 + 1]);
+ ASSERT_EQ(192u, atten_pixels[3 * 4 + 2]);
+ ASSERT_EQ(255u, atten_pixels[3 * 4 + 3]);
+ ASSERT_EQ(255u, atten_pixels[4 * 4 + 0]);
+ ASSERT_EQ(255u, atten_pixels[4 * 4 + 1]);
+ ASSERT_EQ(255u, atten_pixels[4 * 4 + 2]);
+ ASSERT_EQ(255u, atten_pixels[4 * 4 + 3]);
+
+ // test 255
+ for (int i = 0; i < 256; ++i) {
+ orig_pixels[i * 4 + 0] = i;
+ orig_pixels[i * 4 + 1] = 0;
+ orig_pixels[i * 4 + 2] = 0;
+ orig_pixels[i * 4 + 3] = 255;
+ }
+ ARGBAttenuate(orig_pixels, 0, atten_pixels, 0, 256, 1);
+ for (int i = 0; i < 256; ++i) {
+ ASSERT_EQ(orig_pixels[i * 4 + 0], atten_pixels[i * 4 + 0]);
+ ASSERT_EQ(0, atten_pixels[i * 4 + 1]);
+ ASSERT_EQ(0, atten_pixels[i * 4 + 2]);
+ ASSERT_EQ(255, atten_pixels[i * 4 + 3]);
+ }
for (int i = 0; i < 1280; ++i) {
orig_pixels[i * 4 + 0] = i;
@@ -86,24 +155,24 @@ TEST_F(LibYUVPlanarTest, TestAttenuate) {
ARGBAttenuate(unatten_pixels, 0, atten2_pixels, 0, 1280, 1);
}
for (int i = 0; i < 1280; ++i) {
- EXPECT_NEAR(atten_pixels[i * 4 + 0], atten2_pixels[i * 4 + 0], 2);
- EXPECT_NEAR(atten_pixels[i * 4 + 1], atten2_pixels[i * 4 + 1], 2);
- EXPECT_NEAR(atten_pixels[i * 4 + 2], atten2_pixels[i * 4 + 2], 2);
- EXPECT_NEAR(atten_pixels[i * 4 + 3], atten2_pixels[i * 4 + 3], 2);
+ ASSERT_NEAR(atten_pixels[i * 4 + 0], atten2_pixels[i * 4 + 0], 1);
+ ASSERT_NEAR(atten_pixels[i * 4 + 1], atten2_pixels[i * 4 + 1], 1);
+ ASSERT_NEAR(atten_pixels[i * 4 + 2], atten2_pixels[i * 4 + 2], 1);
+ ASSERT_NEAR(atten_pixels[i * 4 + 3], atten2_pixels[i * 4 + 3], 1);
}
// Make sure transparent, 50% and opaque are fully accurate.
- EXPECT_EQ(0, atten_pixels[0 * 4 + 0]);
- EXPECT_EQ(0, atten_pixels[0 * 4 + 1]);
- EXPECT_EQ(0, atten_pixels[0 * 4 + 2]);
- EXPECT_EQ(0, atten_pixels[0 * 4 + 3]);
- EXPECT_EQ(64, atten_pixels[128 * 4 + 0]);
- EXPECT_EQ(32, atten_pixels[128 * 4 + 1]);
- EXPECT_EQ(21, atten_pixels[128 * 4 + 2]);
- EXPECT_EQ(128, atten_pixels[128 * 4 + 3]);
- EXPECT_NEAR(255, atten_pixels[255 * 4 + 0], 1);
- EXPECT_NEAR(127, atten_pixels[255 * 4 + 1], 1);
- EXPECT_NEAR(85, atten_pixels[255 * 4 + 2], 1);
- EXPECT_EQ(255, atten_pixels[255 * 4 + 3]);
+ ASSERT_EQ(0, atten_pixels[0 * 4 + 0]);
+ ASSERT_EQ(0, atten_pixels[0 * 4 + 1]);
+ ASSERT_EQ(0, atten_pixels[0 * 4 + 2]);
+ ASSERT_EQ(0, atten_pixels[0 * 4 + 3]);
+ ASSERT_EQ(64, atten_pixels[128 * 4 + 0]);
+ ASSERT_EQ(32, atten_pixels[128 * 4 + 1]);
+ ASSERT_EQ(21, atten_pixels[128 * 4 + 2]);
+ ASSERT_EQ(128, atten_pixels[128 * 4 + 3]);
+ ASSERT_EQ(255, atten_pixels[255 * 4 + 0]);
+ ASSERT_EQ(127, atten_pixels[255 * 4 + 1]);
+ ASSERT_EQ(85, atten_pixels[255 * 4 + 2]);
+ ASSERT_EQ(255, atten_pixels[255 * 4 + 3]);
free_aligned_buffer_page_end(atten2_pixels);
free_aligned_buffer_page_end(unatten_pixels);
@@ -123,14 +192,14 @@ static int TestAttenuateI(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb[i + off] = (fastrand() & 0xff);
}
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBAttenuate(src_argb + off, kStride, dst_argb_c, kStride, width,
@@ -141,7 +210,7 @@ static int TestAttenuateI(int width,
invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -155,31 +224,32 @@ static int TestAttenuateI(int width,
}
TEST_F(LibYUVPlanarTest, ARGBAttenuate_Any) {
- int max_diff = TestAttenuateI(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestAttenuateI(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 2);
+
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBAttenuate_Unaligned) {
int max_diff =
TestAttenuateI(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBAttenuate_Invert) {
int max_diff =
TestAttenuateI(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBAttenuate_Opt) {
int max_diff =
TestAttenuateI(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
static int TestUnattenuateI(int width,
@@ -194,16 +264,16 @@ static int TestUnattenuateI(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb[i + off] = (fastrand() & 0xff);
}
ARGBAttenuate(src_argb + off, kStride, src_argb + off, kStride, width,
height);
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBUnattenuate(src_argb + off, kStride, dst_argb_c, kStride, width,
@@ -214,7 +284,7 @@ static int TestUnattenuateI(int width,
invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -228,31 +298,31 @@ static int TestUnattenuateI(int width,
}
TEST_F(LibYUVPlanarTest, ARGBUnattenuate_Any) {
- int max_diff = TestUnattenuateI(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestUnattenuateI(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBUnattenuate_Unaligned) {
int max_diff = TestUnattenuateI(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBUnattenuate_Invert) {
int max_diff = TestUnattenuateI(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBUnattenuate_Opt) {
int max_diff = TestUnattenuateI(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 2);
+ ASSERT_EQ(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, TestARGBComputeCumulativeSum) {
@@ -273,14 +343,15 @@ TEST_F(LibYUVPlanarTest, TestARGBComputeCumulativeSum) {
for (int y = 0; y < 16; ++y) {
for (int x = 0; x < 16; ++x) {
- EXPECT_EQ((x + 1) * (y + 1), added_pixels[y][x][0]);
- EXPECT_EQ((x + 1) * (y + 1) * 2, added_pixels[y][x][1]);
- EXPECT_EQ((x + 1) * (y + 1) * 3, added_pixels[y][x][2]);
- EXPECT_EQ((x + 1) * (y + 1) * 255, added_pixels[y][x][3]);
+ ASSERT_EQ((x + 1) * (y + 1), added_pixels[y][x][0]);
+ ASSERT_EQ((x + 1) * (y + 1) * 2, added_pixels[y][x][1]);
+ ASSERT_EQ((x + 1) * (y + 1) * 3, added_pixels[y][x][2]);
+ ASSERT_EQ((x + 1) * (y + 1) * 255, added_pixels[y][x][3]);
}
}
}
+// near is for legacy platforms.
TEST_F(LibYUVPlanarTest, TestARGBGray) {
SIMD_ALIGNED(uint8_t orig_pixels[1280][4]);
memset(orig_pixels, 0, sizeof(orig_pixels));
@@ -317,30 +388,30 @@ TEST_F(LibYUVPlanarTest, TestARGBGray) {
orig_pixels[5][3] = 224u;
// Do 16 to test asm version.
ARGBGray(&orig_pixels[0][0], 0, 0, 0, 16, 1);
- EXPECT_EQ(30u, orig_pixels[0][0]);
- EXPECT_EQ(30u, orig_pixels[0][1]);
- EXPECT_EQ(30u, orig_pixels[0][2]);
- EXPECT_EQ(128u, orig_pixels[0][3]);
- EXPECT_EQ(149u, orig_pixels[1][0]);
- EXPECT_EQ(149u, orig_pixels[1][1]);
- EXPECT_EQ(149u, orig_pixels[1][2]);
- EXPECT_EQ(0u, orig_pixels[1][3]);
- EXPECT_EQ(76u, orig_pixels[2][0]);
- EXPECT_EQ(76u, orig_pixels[2][1]);
- EXPECT_EQ(76u, orig_pixels[2][2]);
- EXPECT_EQ(255u, orig_pixels[2][3]);
- EXPECT_EQ(0u, orig_pixels[3][0]);
- EXPECT_EQ(0u, orig_pixels[3][1]);
- EXPECT_EQ(0u, orig_pixels[3][2]);
- EXPECT_EQ(255u, orig_pixels[3][3]);
- EXPECT_EQ(255u, orig_pixels[4][0]);
- EXPECT_EQ(255u, orig_pixels[4][1]);
- EXPECT_EQ(255u, orig_pixels[4][2]);
- EXPECT_EQ(255u, orig_pixels[4][3]);
- EXPECT_EQ(96u, orig_pixels[5][0]);
- EXPECT_EQ(96u, orig_pixels[5][1]);
- EXPECT_EQ(96u, orig_pixels[5][2]);
- EXPECT_EQ(224u, orig_pixels[5][3]);
+ ASSERT_NEAR(29u, orig_pixels[0][0], 1);
+ ASSERT_NEAR(29u, orig_pixels[0][1], 1);
+ ASSERT_NEAR(29u, orig_pixels[0][2], 1);
+ ASSERT_EQ(128u, orig_pixels[0][3]);
+ ASSERT_EQ(149u, orig_pixels[1][0]);
+ ASSERT_EQ(149u, orig_pixels[1][1]);
+ ASSERT_EQ(149u, orig_pixels[1][2]);
+ ASSERT_EQ(0u, orig_pixels[1][3]);
+ ASSERT_NEAR(77u, orig_pixels[2][0], 1);
+ ASSERT_NEAR(77u, orig_pixels[2][1], 1);
+ ASSERT_NEAR(77u, orig_pixels[2][2], 1);
+ ASSERT_EQ(255u, orig_pixels[2][3]);
+ ASSERT_EQ(0u, orig_pixels[3][0]);
+ ASSERT_EQ(0u, orig_pixels[3][1]);
+ ASSERT_EQ(0u, orig_pixels[3][2]);
+ ASSERT_EQ(255u, orig_pixels[3][3]);
+ ASSERT_EQ(255u, orig_pixels[4][0]);
+ ASSERT_EQ(255u, orig_pixels[4][1]);
+ ASSERT_EQ(255u, orig_pixels[4][2]);
+ ASSERT_EQ(255u, orig_pixels[4][3]);
+ ASSERT_NEAR(97u, orig_pixels[5][0], 1);
+ ASSERT_NEAR(97u, orig_pixels[5][1], 1);
+ ASSERT_NEAR(97u, orig_pixels[5][2], 1);
+ ASSERT_EQ(224u, orig_pixels[5][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
orig_pixels[i][1] = i / 2;
@@ -389,30 +460,30 @@ TEST_F(LibYUVPlanarTest, TestARGBGrayTo) {
orig_pixels[5][3] = 224u;
// Do 16 to test asm version.
ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 16, 1);
- EXPECT_EQ(30u, gray_pixels[0][0]);
- EXPECT_EQ(30u, gray_pixels[0][1]);
- EXPECT_EQ(30u, gray_pixels[0][2]);
- EXPECT_EQ(128u, gray_pixels[0][3]);
- EXPECT_EQ(149u, gray_pixels[1][0]);
- EXPECT_EQ(149u, gray_pixels[1][1]);
- EXPECT_EQ(149u, gray_pixels[1][2]);
- EXPECT_EQ(0u, gray_pixels[1][3]);
- EXPECT_EQ(76u, gray_pixels[2][0]);
- EXPECT_EQ(76u, gray_pixels[2][1]);
- EXPECT_EQ(76u, gray_pixels[2][2]);
- EXPECT_EQ(255u, gray_pixels[2][3]);
- EXPECT_EQ(0u, gray_pixels[3][0]);
- EXPECT_EQ(0u, gray_pixels[3][1]);
- EXPECT_EQ(0u, gray_pixels[3][2]);
- EXPECT_EQ(255u, gray_pixels[3][3]);
- EXPECT_EQ(255u, gray_pixels[4][0]);
- EXPECT_EQ(255u, gray_pixels[4][1]);
- EXPECT_EQ(255u, gray_pixels[4][2]);
- EXPECT_EQ(255u, gray_pixels[4][3]);
- EXPECT_EQ(96u, gray_pixels[5][0]);
- EXPECT_EQ(96u, gray_pixels[5][1]);
- EXPECT_EQ(96u, gray_pixels[5][2]);
- EXPECT_EQ(224u, gray_pixels[5][3]);
+ ASSERT_NEAR(30u, gray_pixels[0][0], 1);
+ ASSERT_NEAR(30u, gray_pixels[0][1], 1);
+ ASSERT_NEAR(30u, gray_pixels[0][2], 1);
+ ASSERT_NEAR(128u, gray_pixels[0][3], 1);
+ ASSERT_NEAR(149u, gray_pixels[1][0], 1);
+ ASSERT_NEAR(149u, gray_pixels[1][1], 1);
+ ASSERT_NEAR(149u, gray_pixels[1][2], 1);
+ ASSERT_NEAR(0u, gray_pixels[1][3], 1);
+ ASSERT_NEAR(76u, gray_pixels[2][0], 1);
+ ASSERT_NEAR(76u, gray_pixels[2][1], 1);
+ ASSERT_NEAR(76u, gray_pixels[2][2], 1);
+ ASSERT_NEAR(255u, gray_pixels[2][3], 1);
+ ASSERT_NEAR(0u, gray_pixels[3][0], 1);
+ ASSERT_NEAR(0u, gray_pixels[3][1], 1);
+ ASSERT_NEAR(0u, gray_pixels[3][2], 1);
+ ASSERT_NEAR(255u, gray_pixels[3][3], 1);
+ ASSERT_NEAR(255u, gray_pixels[4][0], 1);
+ ASSERT_NEAR(255u, gray_pixels[4][1], 1);
+ ASSERT_NEAR(255u, gray_pixels[4][2], 1);
+ ASSERT_NEAR(255u, gray_pixels[4][3], 1);
+ ASSERT_NEAR(96u, gray_pixels[5][0], 1);
+ ASSERT_NEAR(96u, gray_pixels[5][1], 1);
+ ASSERT_NEAR(96u, gray_pixels[5][2], 1);
+ ASSERT_NEAR(224u, gray_pixels[5][3], 1);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
orig_pixels[i][1] = i / 2;
@@ -422,6 +493,50 @@ TEST_F(LibYUVPlanarTest, TestARGBGrayTo) {
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 1280, 1);
}
+
+ for (int i = 0; i < 256; ++i) {
+ orig_pixels[i][0] = i;
+ orig_pixels[i][1] = i;
+ orig_pixels[i][2] = i;
+ orig_pixels[i][3] = i;
+ }
+ ARGBGray(&orig_pixels[0][0], 0, 0, 0, 256, 1);
+ for (int i = 0; i < 256; ++i) {
+ ASSERT_EQ(i, orig_pixels[i][0]);
+ ASSERT_EQ(i, orig_pixels[i][1]);
+ ASSERT_EQ(i, orig_pixels[i][2]);
+ ASSERT_EQ(i, orig_pixels[i][3]);
+ }
+
+ // Test RGBX input with uninitialized alpha channel under MSan.
+ SIMD_ALIGNED(uint8_t gray_pixels_c[37][4]);
+ for (int i = 0; i < 37; ++i) {
+ orig_pixels[i][0] = static_cast<uint8_t>(i * 7);
+ orig_pixels[i][1] = static_cast<uint8_t>(i * 13);
+ orig_pixels[i][2] = static_cast<uint8_t>(i * 23);
+ orig_pixels[i][3] = 0u;
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels_c[0][0], 0, 37, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < 37; ++i) {
+ MSAN_POISON(&orig_pixels[i][3], 1);
+ }
+ MSAN_POISON(gray_pixels, sizeof(gray_pixels));
+ ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 37, 1);
+ for (int i = 0; i < 37; ++i) {
+ MSAN_CHECK_INITIALIZED(&gray_pixels[i][0], 3);
+ ASSERT_EQ(gray_pixels_c[i][0], gray_pixels[i][0]);
+ ASSERT_EQ(gray_pixels_c[i][1], gray_pixels[i][1]);
+ ASSERT_EQ(gray_pixels_c[i][2], gray_pixels[i][2]);
+ }
+ ARGBGray(&orig_pixels[0][0], 0, 0, 0, 37, 1);
+ for (int i = 0; i < 37; ++i) {
+ MSAN_CHECK_INITIALIZED(&orig_pixels[i][0], 3);
+ ASSERT_EQ(gray_pixels_c[i][0], orig_pixels[i][0]);
+ ASSERT_EQ(gray_pixels_c[i][1], orig_pixels[i][1]);
+ ASSERT_EQ(gray_pixels_c[i][2], orig_pixels[i][2]);
+ }
}
TEST_F(LibYUVPlanarTest, TestARGBSepia) {
@@ -460,30 +575,30 @@ TEST_F(LibYUVPlanarTest, TestARGBSepia) {
orig_pixels[5][3] = 224u;
// Do 16 to test asm version.
ARGBSepia(&orig_pixels[0][0], 0, 0, 0, 16, 1);
- EXPECT_EQ(33u, orig_pixels[0][0]);
- EXPECT_EQ(43u, orig_pixels[0][1]);
- EXPECT_EQ(47u, orig_pixels[0][2]);
- EXPECT_EQ(128u, orig_pixels[0][3]);
- EXPECT_EQ(135u, orig_pixels[1][0]);
- EXPECT_EQ(175u, orig_pixels[1][1]);
- EXPECT_EQ(195u, orig_pixels[1][2]);
- EXPECT_EQ(0u, orig_pixels[1][3]);
- EXPECT_EQ(69u, orig_pixels[2][0]);
- EXPECT_EQ(89u, orig_pixels[2][1]);
- EXPECT_EQ(99u, orig_pixels[2][2]);
- EXPECT_EQ(255u, orig_pixels[2][3]);
- EXPECT_EQ(0u, orig_pixels[3][0]);
- EXPECT_EQ(0u, orig_pixels[3][1]);
- EXPECT_EQ(0u, orig_pixels[3][2]);
- EXPECT_EQ(255u, orig_pixels[3][3]);
- EXPECT_EQ(239u, orig_pixels[4][0]);
- EXPECT_EQ(255u, orig_pixels[4][1]);
- EXPECT_EQ(255u, orig_pixels[4][2]);
- EXPECT_EQ(255u, orig_pixels[4][3]);
- EXPECT_EQ(88u, orig_pixels[5][0]);
- EXPECT_EQ(114u, orig_pixels[5][1]);
- EXPECT_EQ(127u, orig_pixels[5][2]);
- EXPECT_EQ(224u, orig_pixels[5][3]);
+ ASSERT_EQ(33u, orig_pixels[0][0]);
+ ASSERT_EQ(43u, orig_pixels[0][1]);
+ ASSERT_EQ(47u, orig_pixels[0][2]);
+ ASSERT_EQ(128u, orig_pixels[0][3]);
+ ASSERT_EQ(135u, orig_pixels[1][0]);
+ ASSERT_EQ(175u, orig_pixels[1][1]);
+ ASSERT_EQ(195u, orig_pixels[1][2]);
+ ASSERT_EQ(0u, orig_pixels[1][3]);
+ ASSERT_EQ(69u, orig_pixels[2][0]);
+ ASSERT_EQ(89u, orig_pixels[2][1]);
+ ASSERT_EQ(99u, orig_pixels[2][2]);
+ ASSERT_EQ(255u, orig_pixels[2][3]);
+ ASSERT_EQ(0u, orig_pixels[3][0]);
+ ASSERT_EQ(0u, orig_pixels[3][1]);
+ ASSERT_EQ(0u, orig_pixels[3][2]);
+ ASSERT_EQ(255u, orig_pixels[3][3]);
+ ASSERT_EQ(239u, orig_pixels[4][0]);
+ ASSERT_EQ(255u, orig_pixels[4][1]);
+ ASSERT_EQ(255u, orig_pixels[4][2]);
+ ASSERT_EQ(255u, orig_pixels[4][3]);
+ ASSERT_EQ(88u, orig_pixels[5][0]);
+ ASSERT_EQ(114u, orig_pixels[5][1]);
+ ASSERT_EQ(127u, orig_pixels[5][2]);
+ ASSERT_EQ(224u, orig_pixels[5][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -531,22 +646,22 @@ TEST_F(LibYUVPlanarTest, TestARGBColorMatrix) {
// Do 16 to test asm version.
ARGBColorMatrix(&orig_pixels[0][0], 0, &dst_pixels_opt[0][0], 0,
&kRGBToSepia[0], 16, 1);
- EXPECT_EQ(31u, dst_pixels_opt[0][0]);
- EXPECT_EQ(43u, dst_pixels_opt[0][1]);
- EXPECT_EQ(47u, dst_pixels_opt[0][2]);
- EXPECT_EQ(128u, dst_pixels_opt[0][3]);
- EXPECT_EQ(135u, dst_pixels_opt[1][0]);
- EXPECT_EQ(175u, dst_pixels_opt[1][1]);
- EXPECT_EQ(195u, dst_pixels_opt[1][2]);
- EXPECT_EQ(0u, dst_pixels_opt[1][3]);
- EXPECT_EQ(67u, dst_pixels_opt[2][0]);
- EXPECT_EQ(87u, dst_pixels_opt[2][1]);
- EXPECT_EQ(99u, dst_pixels_opt[2][2]);
- EXPECT_EQ(255u, dst_pixels_opt[2][3]);
- EXPECT_EQ(87u, dst_pixels_opt[3][0]);
- EXPECT_EQ(112u, dst_pixels_opt[3][1]);
- EXPECT_EQ(127u, dst_pixels_opt[3][2]);
- EXPECT_EQ(224u, dst_pixels_opt[3][3]);
+ ASSERT_EQ(31u, dst_pixels_opt[0][0]);
+ ASSERT_EQ(43u, dst_pixels_opt[0][1]);
+ ASSERT_EQ(47u, dst_pixels_opt[0][2]);
+ ASSERT_EQ(128u, dst_pixels_opt[0][3]);
+ ASSERT_EQ(135u, dst_pixels_opt[1][0]);
+ ASSERT_EQ(175u, dst_pixels_opt[1][1]);
+ ASSERT_EQ(195u, dst_pixels_opt[1][2]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][3]);
+ ASSERT_EQ(67u, dst_pixels_opt[2][0]);
+ ASSERT_EQ(87u, dst_pixels_opt[2][1]);
+ ASSERT_EQ(99u, dst_pixels_opt[2][2]);
+ ASSERT_EQ(255u, dst_pixels_opt[2][3]);
+ ASSERT_EQ(87u, dst_pixels_opt[3][0]);
+ ASSERT_EQ(112u, dst_pixels_opt[3][1]);
+ ASSERT_EQ(127u, dst_pixels_opt[3][2]);
+ ASSERT_EQ(224u, dst_pixels_opt[3][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -565,10 +680,10 @@ TEST_F(LibYUVPlanarTest, TestARGBColorMatrix) {
}
for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
- EXPECT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
- EXPECT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
- EXPECT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
+ ASSERT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
+ ASSERT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
+ ASSERT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
+ ASSERT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
}
}
@@ -604,22 +719,22 @@ TEST_F(LibYUVPlanarTest, TestRGBColorMatrix) {
orig_pixels[3][3] = 224u;
// Do 16 to test asm version.
RGBColorMatrix(&orig_pixels[0][0], 0, &kRGBToSepia[0], 0, 0, 16, 1);
- EXPECT_EQ(31u, orig_pixels[0][0]);
- EXPECT_EQ(43u, orig_pixels[0][1]);
- EXPECT_EQ(47u, orig_pixels[0][2]);
- EXPECT_EQ(128u, orig_pixels[0][3]);
- EXPECT_EQ(135u, orig_pixels[1][0]);
- EXPECT_EQ(175u, orig_pixels[1][1]);
- EXPECT_EQ(195u, orig_pixels[1][2]);
- EXPECT_EQ(0u, orig_pixels[1][3]);
- EXPECT_EQ(67u, orig_pixels[2][0]);
- EXPECT_EQ(87u, orig_pixels[2][1]);
- EXPECT_EQ(99u, orig_pixels[2][2]);
- EXPECT_EQ(255u, orig_pixels[2][3]);
- EXPECT_EQ(87u, orig_pixels[3][0]);
- EXPECT_EQ(112u, orig_pixels[3][1]);
- EXPECT_EQ(127u, orig_pixels[3][2]);
- EXPECT_EQ(224u, orig_pixels[3][3]);
+ ASSERT_EQ(31u, orig_pixels[0][0]);
+ ASSERT_EQ(43u, orig_pixels[0][1]);
+ ASSERT_EQ(47u, orig_pixels[0][2]);
+ ASSERT_EQ(128u, orig_pixels[0][3]);
+ ASSERT_EQ(135u, orig_pixels[1][0]);
+ ASSERT_EQ(175u, orig_pixels[1][1]);
+ ASSERT_EQ(195u, orig_pixels[1][2]);
+ ASSERT_EQ(0u, orig_pixels[1][3]);
+ ASSERT_EQ(67u, orig_pixels[2][0]);
+ ASSERT_EQ(87u, orig_pixels[2][1]);
+ ASSERT_EQ(99u, orig_pixels[2][2]);
+ ASSERT_EQ(255u, orig_pixels[2][3]);
+ ASSERT_EQ(87u, orig_pixels[3][0]);
+ ASSERT_EQ(112u, orig_pixels[3][1]);
+ ASSERT_EQ(127u, orig_pixels[3][2]);
+ ASSERT_EQ(224u, orig_pixels[3][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -659,22 +774,22 @@ TEST_F(LibYUVPlanarTest, TestARGBColorTable) {
orig_pixels[3][3] = 3u;
// Do 16 to test asm version.
ARGBColorTable(&orig_pixels[0][0], 0, &kARGBTable[0], 0, 0, 16, 1);
- EXPECT_EQ(1u, orig_pixels[0][0]);
- EXPECT_EQ(2u, orig_pixels[0][1]);
- EXPECT_EQ(3u, orig_pixels[0][2]);
- EXPECT_EQ(4u, orig_pixels[0][3]);
- EXPECT_EQ(5u, orig_pixels[1][0]);
- EXPECT_EQ(6u, orig_pixels[1][1]);
- EXPECT_EQ(7u, orig_pixels[1][2]);
- EXPECT_EQ(8u, orig_pixels[1][3]);
- EXPECT_EQ(9u, orig_pixels[2][0]);
- EXPECT_EQ(10u, orig_pixels[2][1]);
- EXPECT_EQ(11u, orig_pixels[2][2]);
- EXPECT_EQ(12u, orig_pixels[2][3]);
- EXPECT_EQ(1u, orig_pixels[3][0]);
- EXPECT_EQ(6u, orig_pixels[3][1]);
- EXPECT_EQ(11u, orig_pixels[3][2]);
- EXPECT_EQ(16u, orig_pixels[3][3]);
+ ASSERT_EQ(1u, orig_pixels[0][0]);
+ ASSERT_EQ(2u, orig_pixels[0][1]);
+ ASSERT_EQ(3u, orig_pixels[0][2]);
+ ASSERT_EQ(4u, orig_pixels[0][3]);
+ ASSERT_EQ(5u, orig_pixels[1][0]);
+ ASSERT_EQ(6u, orig_pixels[1][1]);
+ ASSERT_EQ(7u, orig_pixels[1][2]);
+ ASSERT_EQ(8u, orig_pixels[1][3]);
+ ASSERT_EQ(9u, orig_pixels[2][0]);
+ ASSERT_EQ(10u, orig_pixels[2][1]);
+ ASSERT_EQ(11u, orig_pixels[2][2]);
+ ASSERT_EQ(12u, orig_pixels[2][3]);
+ ASSERT_EQ(1u, orig_pixels[3][0]);
+ ASSERT_EQ(6u, orig_pixels[3][1]);
+ ASSERT_EQ(11u, orig_pixels[3][2]);
+ ASSERT_EQ(16u, orig_pixels[3][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -715,22 +830,22 @@ TEST_F(LibYUVPlanarTest, TestRGBColorTable) {
orig_pixels[3][3] = 3u;
// Do 16 to test asm version.
RGBColorTable(&orig_pixels[0][0], 0, &kARGBTable[0], 0, 0, 16, 1);
- EXPECT_EQ(1u, orig_pixels[0][0]);
- EXPECT_EQ(2u, orig_pixels[0][1]);
- EXPECT_EQ(3u, orig_pixels[0][2]);
- EXPECT_EQ(0u, orig_pixels[0][3]); // Alpha unchanged.
- EXPECT_EQ(5u, orig_pixels[1][0]);
- EXPECT_EQ(6u, orig_pixels[1][1]);
- EXPECT_EQ(7u, orig_pixels[1][2]);
- EXPECT_EQ(1u, orig_pixels[1][3]); // Alpha unchanged.
- EXPECT_EQ(9u, orig_pixels[2][0]);
- EXPECT_EQ(10u, orig_pixels[2][1]);
- EXPECT_EQ(11u, orig_pixels[2][2]);
- EXPECT_EQ(2u, orig_pixels[2][3]); // Alpha unchanged.
- EXPECT_EQ(1u, orig_pixels[3][0]);
- EXPECT_EQ(6u, orig_pixels[3][1]);
- EXPECT_EQ(11u, orig_pixels[3][2]);
- EXPECT_EQ(3u, orig_pixels[3][3]); // Alpha unchanged.
+ ASSERT_EQ(1u, orig_pixels[0][0]);
+ ASSERT_EQ(2u, orig_pixels[0][1]);
+ ASSERT_EQ(3u, orig_pixels[0][2]);
+ ASSERT_EQ(0u, orig_pixels[0][3]); // Alpha unchanged.
+ ASSERT_EQ(5u, orig_pixels[1][0]);
+ ASSERT_EQ(6u, orig_pixels[1][1]);
+ ASSERT_EQ(7u, orig_pixels[1][2]);
+ ASSERT_EQ(1u, orig_pixels[1][3]); // Alpha unchanged.
+ ASSERT_EQ(9u, orig_pixels[2][0]);
+ ASSERT_EQ(10u, orig_pixels[2][1]);
+ ASSERT_EQ(11u, orig_pixels[2][2]);
+ ASSERT_EQ(2u, orig_pixels[2][3]); // Alpha unchanged.
+ ASSERT_EQ(1u, orig_pixels[3][0]);
+ ASSERT_EQ(6u, orig_pixels[3][1]);
+ ASSERT_EQ(11u, orig_pixels[3][2]);
+ ASSERT_EQ(3u, orig_pixels[3][3]); // Alpha unchanged.
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -756,10 +871,10 @@ TEST_F(LibYUVPlanarTest, TestARGBQuantize) {
1280, 1);
for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ((i / 8 * 8 + 8 / 2) & 255, orig_pixels[i][0]);
- EXPECT_EQ((i / 2 / 8 * 8 + 8 / 2) & 255, orig_pixels[i][1]);
- EXPECT_EQ((i / 3 / 8 * 8 + 8 / 2) & 255, orig_pixels[i][2]);
- EXPECT_EQ(i & 255, orig_pixels[i][3]);
+ ASSERT_EQ((i / 8 * 8 + 8 / 2) & 255, orig_pixels[i][0]);
+ ASSERT_EQ((i / 2 / 8 * 8 + 8 / 2) & 255, orig_pixels[i][1]);
+ ASSERT_EQ((i / 3 / 8 * 8 + 8 / 2) & 255, orig_pixels[i][2]);
+ ASSERT_EQ(i & 255, orig_pixels[i][3]);
}
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
ARGBQuantize(&orig_pixels[0][0], 0, (65536 + (8 / 2)) / 8, 8, 8 / 2, 0, 0,
@@ -767,27 +882,103 @@ TEST_F(LibYUVPlanarTest, TestARGBQuantize) {
}
}
-TEST_F(LibYUVPlanarTest, TestARGBMirror) {
- SIMD_ALIGNED(uint8_t orig_pixels[1280][4]);
- SIMD_ALIGNED(uint8_t dst_pixels[1280][4]);
+TEST_F(LibYUVPlanarTest, ARGBMirror_Opt) {
+ align_buffer_page_end(src_pixels, benchmark_width_ * benchmark_height_ * 4);
+ align_buffer_page_end(dst_pixels_opt,
+ benchmark_width_ * benchmark_height_ * 4);
+ align_buffer_page_end(dst_pixels_c, benchmark_width_ * benchmark_height_ * 4);
- for (int i = 0; i < 1280; ++i) {
- orig_pixels[i][0] = i;
- orig_pixels[i][1] = i / 2;
- orig_pixels[i][2] = i / 3;
- orig_pixels[i][3] = i / 4;
+ MemRandomize(src_pixels, benchmark_width_ * benchmark_height_ * 4);
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBMirror(src_pixels, benchmark_width_ * 4, dst_pixels_c,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ ARGBMirror(src_pixels, benchmark_width_ * 4, dst_pixels_opt,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
}
- ARGBMirror(&orig_pixels[0][0], 0, &dst_pixels[0][0], 0, 1280, 1);
+ for (int i = 0; i < benchmark_width_ * benchmark_height_ * 4; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
+}
- for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(i & 255, dst_pixels[1280 - 1 - i][0]);
- EXPECT_EQ((i / 2) & 255, dst_pixels[1280 - 1 - i][1]);
- EXPECT_EQ((i / 3) & 255, dst_pixels[1280 - 1 - i][2]);
- EXPECT_EQ((i / 4) & 255, dst_pixels[1280 - 1 - i][3]);
+TEST_F(LibYUVPlanarTest, MirrorPlane_16_Opt) {
+ align_buffer_page_end(src_pixels_8, benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_pixels_opt_8,
+ benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_pixels_c_8,
+ benchmark_width_ * benchmark_height_ * 2);
+ uint16_t* src_pixels = reinterpret_cast<uint16_t*>(src_pixels_8);
+ uint16_t* dst_pixels_opt = reinterpret_cast<uint16_t*>(dst_pixels_opt_8);
+ uint16_t* dst_pixels_c = reinterpret_cast<uint16_t*>(dst_pixels_c_8);
+
+ MemRandomize(src_pixels_8, benchmark_width_ * benchmark_height_ * 2);
+ MaskCpuFlags(disable_cpu_flags_);
+ MirrorPlane_16(src_pixels, benchmark_width_, dst_pixels_c, benchmark_width_,
+ benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MirrorPlane_16(src_pixels, benchmark_width_, dst_pixels_opt,
+ benchmark_width_, benchmark_width_, benchmark_height_);
}
- for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
- ARGBMirror(&orig_pixels[0][0], 0, &dst_pixels[0][0], 0, 1280, 1);
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels_8);
+ free_aligned_buffer_page_end(dst_pixels_opt_8);
+ free_aligned_buffer_page_end(dst_pixels_c_8);
+}
+
+TEST_F(LibYUVPlanarTest, MirrorPlane_Opt) {
+ align_buffer_page_end(src_pixels, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(dst_pixels_opt, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(dst_pixels_c, benchmark_width_ * benchmark_height_);
+
+ MemRandomize(src_pixels, benchmark_width_ * benchmark_height_);
+ MaskCpuFlags(disable_cpu_flags_);
+ MirrorPlane(src_pixels, benchmark_width_, dst_pixels_c, benchmark_width_,
+ benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MirrorPlane(src_pixels, benchmark_width_, dst_pixels_opt, benchmark_width_,
+ benchmark_width_, benchmark_height_);
+ }
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
+}
+
+TEST_F(LibYUVPlanarTest, MirrorUVPlane_Opt) {
+ align_buffer_page_end(src_pixels, benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_pixels_opt,
+ benchmark_width_ * benchmark_height_ * 2);
+ align_buffer_page_end(dst_pixels_c, benchmark_width_ * benchmark_height_ * 2);
+
+ MemRandomize(src_pixels, benchmark_width_ * benchmark_height_ * 2);
+ MaskCpuFlags(disable_cpu_flags_);
+ MirrorUVPlane(src_pixels, benchmark_width_ * 2, dst_pixels_c,
+ benchmark_width_ * 2, benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MirrorUVPlane(src_pixels, benchmark_width_ * 2, dst_pixels_opt,
+ benchmark_width_ * 2, benchmark_width_, benchmark_height_);
}
+ for (int i = 0; i < benchmark_width_ * benchmark_height_ * 2; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
}
TEST_F(LibYUVPlanarTest, TestShade) {
@@ -813,34 +1004,34 @@ TEST_F(LibYUVPlanarTest, TestShade) {
orig_pixels[3][3] = 0u;
// Do 8 pixels to allow opt version to be used.
ARGBShade(&orig_pixels[0][0], 0, &shade_pixels[0][0], 0, 8, 1, 0x80ffffff);
- EXPECT_EQ(10u, shade_pixels[0][0]);
- EXPECT_EQ(20u, shade_pixels[0][1]);
- EXPECT_EQ(40u, shade_pixels[0][2]);
- EXPECT_EQ(40u, shade_pixels[0][3]);
- EXPECT_EQ(0u, shade_pixels[1][0]);
- EXPECT_EQ(0u, shade_pixels[1][1]);
- EXPECT_EQ(0u, shade_pixels[1][2]);
- EXPECT_EQ(128u, shade_pixels[1][3]);
- EXPECT_EQ(0u, shade_pixels[2][0]);
- EXPECT_EQ(0u, shade_pixels[2][1]);
- EXPECT_EQ(0u, shade_pixels[2][2]);
- EXPECT_EQ(0u, shade_pixels[2][3]);
- EXPECT_EQ(0u, shade_pixels[3][0]);
- EXPECT_EQ(0u, shade_pixels[3][1]);
- EXPECT_EQ(0u, shade_pixels[3][2]);
- EXPECT_EQ(0u, shade_pixels[3][3]);
+ ASSERT_EQ(10u, shade_pixels[0][0]);
+ ASSERT_EQ(20u, shade_pixels[0][1]);
+ ASSERT_EQ(40u, shade_pixels[0][2]);
+ ASSERT_EQ(40u, shade_pixels[0][3]);
+ ASSERT_EQ(0u, shade_pixels[1][0]);
+ ASSERT_EQ(0u, shade_pixels[1][1]);
+ ASSERT_EQ(0u, shade_pixels[1][2]);
+ ASSERT_EQ(128u, shade_pixels[1][3]);
+ ASSERT_EQ(0u, shade_pixels[2][0]);
+ ASSERT_EQ(0u, shade_pixels[2][1]);
+ ASSERT_EQ(0u, shade_pixels[2][2]);
+ ASSERT_EQ(0u, shade_pixels[2][3]);
+ ASSERT_EQ(0u, shade_pixels[3][0]);
+ ASSERT_EQ(0u, shade_pixels[3][1]);
+ ASSERT_EQ(0u, shade_pixels[3][2]);
+ ASSERT_EQ(0u, shade_pixels[3][3]);
ARGBShade(&orig_pixels[0][0], 0, &shade_pixels[0][0], 0, 8, 1, 0x80808080);
- EXPECT_EQ(5u, shade_pixels[0][0]);
- EXPECT_EQ(10u, shade_pixels[0][1]);
- EXPECT_EQ(20u, shade_pixels[0][2]);
- EXPECT_EQ(40u, shade_pixels[0][3]);
+ ASSERT_EQ(5u, shade_pixels[0][0]);
+ ASSERT_EQ(10u, shade_pixels[0][1]);
+ ASSERT_EQ(20u, shade_pixels[0][2]);
+ ASSERT_EQ(40u, shade_pixels[0][3]);
ARGBShade(&orig_pixels[0][0], 0, &shade_pixels[0][0], 0, 8, 1, 0x10204080);
- EXPECT_EQ(5u, shade_pixels[0][0]);
- EXPECT_EQ(5u, shade_pixels[0][1]);
- EXPECT_EQ(5u, shade_pixels[0][2]);
- EXPECT_EQ(5u, shade_pixels[0][3]);
+ ASSERT_EQ(5u, shade_pixels[0][0]);
+ ASSERT_EQ(5u, shade_pixels[0][1]);
+ ASSERT_EQ(5u, shade_pixels[0][2]);
+ ASSERT_EQ(5u, shade_pixels[0][3]);
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
ARGBShade(&orig_pixels[0][0], 0, &shade_pixels[0][0], 0, 1280, 1,
@@ -891,37 +1082,37 @@ TEST_F(LibYUVPlanarTest, TestARGBInterpolate) {
ARGBInterpolate(&orig_pixels_0[0][0], 0, &orig_pixels_1[0][0], 0,
&interpolate_pixels[0][0], 0, 4, 1, 128);
- EXPECT_EQ(8u, interpolate_pixels[0][0]);
- EXPECT_EQ(16u, interpolate_pixels[0][1]);
- EXPECT_EQ(32u, interpolate_pixels[0][2]);
- EXPECT_EQ(64u, interpolate_pixels[0][3]);
- EXPECT_EQ(0u, interpolate_pixels[1][0]);
- EXPECT_EQ(0u, interpolate_pixels[1][1]);
- EXPECT_EQ(0u, interpolate_pixels[1][2]);
- EXPECT_EQ(128u, interpolate_pixels[1][3]);
- EXPECT_EQ(0u, interpolate_pixels[2][0]);
- EXPECT_EQ(0u, interpolate_pixels[2][1]);
- EXPECT_EQ(0u, interpolate_pixels[2][2]);
- EXPECT_EQ(0u, interpolate_pixels[2][3]);
- EXPECT_EQ(128u, interpolate_pixels[3][0]);
- EXPECT_EQ(128u, interpolate_pixels[3][1]);
- EXPECT_EQ(128u, interpolate_pixels[3][2]);
- EXPECT_EQ(128u, interpolate_pixels[3][3]);
+ ASSERT_EQ(8u, interpolate_pixels[0][0]);
+ ASSERT_EQ(16u, interpolate_pixels[0][1]);
+ ASSERT_EQ(32u, interpolate_pixels[0][2]);
+ ASSERT_EQ(64u, interpolate_pixels[0][3]);
+ ASSERT_EQ(0u, interpolate_pixels[1][0]);
+ ASSERT_EQ(0u, interpolate_pixels[1][1]);
+ ASSERT_EQ(0u, interpolate_pixels[1][2]);
+ ASSERT_EQ(128u, interpolate_pixels[1][3]);
+ ASSERT_EQ(0u, interpolate_pixels[2][0]);
+ ASSERT_EQ(0u, interpolate_pixels[2][1]);
+ ASSERT_EQ(0u, interpolate_pixels[2][2]);
+ ASSERT_EQ(0u, interpolate_pixels[2][3]);
+ ASSERT_EQ(128u, interpolate_pixels[3][0]);
+ ASSERT_EQ(128u, interpolate_pixels[3][1]);
+ ASSERT_EQ(128u, interpolate_pixels[3][2]);
+ ASSERT_EQ(128u, interpolate_pixels[3][3]);
ARGBInterpolate(&orig_pixels_0[0][0], 0, &orig_pixels_1[0][0], 0,
&interpolate_pixels[0][0], 0, 4, 1, 0);
- EXPECT_EQ(16u, interpolate_pixels[0][0]);
- EXPECT_EQ(32u, interpolate_pixels[0][1]);
- EXPECT_EQ(64u, interpolate_pixels[0][2]);
- EXPECT_EQ(128u, interpolate_pixels[0][3]);
+ ASSERT_EQ(16u, interpolate_pixels[0][0]);
+ ASSERT_EQ(32u, interpolate_pixels[0][1]);
+ ASSERT_EQ(64u, interpolate_pixels[0][2]);
+ ASSERT_EQ(128u, interpolate_pixels[0][3]);
ARGBInterpolate(&orig_pixels_0[0][0], 0, &orig_pixels_1[0][0], 0,
&interpolate_pixels[0][0], 0, 4, 1, 192);
- EXPECT_EQ(4u, interpolate_pixels[0][0]);
- EXPECT_EQ(8u, interpolate_pixels[0][1]);
- EXPECT_EQ(16u, interpolate_pixels[0][2]);
- EXPECT_EQ(32u, interpolate_pixels[0][3]);
+ ASSERT_EQ(4u, interpolate_pixels[0][0]);
+ ASSERT_EQ(8u, interpolate_pixels[0][1]);
+ ASSERT_EQ(16u, interpolate_pixels[0][2]);
+ ASSERT_EQ(32u, interpolate_pixels[0][3]);
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
ARGBInterpolate(&orig_pixels_0[0][0], 0, &orig_pixels_1[0][0], 0,
@@ -972,37 +1163,37 @@ TEST_F(LibYUVPlanarTest, TestInterpolatePlane) {
InterpolatePlane(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
&interpolate_pixels[0], 0, 16, 1, 128);
- EXPECT_EQ(8u, interpolate_pixels[0]);
- EXPECT_EQ(16u, interpolate_pixels[1]);
- EXPECT_EQ(32u, interpolate_pixels[2]);
- EXPECT_EQ(64u, interpolate_pixels[3]);
- EXPECT_EQ(0u, interpolate_pixels[4]);
- EXPECT_EQ(0u, interpolate_pixels[5]);
- EXPECT_EQ(0u, interpolate_pixels[6]);
- EXPECT_EQ(128u, interpolate_pixels[7]);
- EXPECT_EQ(0u, interpolate_pixels[8]);
- EXPECT_EQ(0u, interpolate_pixels[9]);
- EXPECT_EQ(0u, interpolate_pixels[10]);
- EXPECT_EQ(0u, interpolate_pixels[11]);
- EXPECT_EQ(128u, interpolate_pixels[12]);
- EXPECT_EQ(128u, interpolate_pixels[13]);
- EXPECT_EQ(128u, interpolate_pixels[14]);
- EXPECT_EQ(128u, interpolate_pixels[15]);
+ ASSERT_EQ(8u, interpolate_pixels[0]);
+ ASSERT_EQ(16u, interpolate_pixels[1]);
+ ASSERT_EQ(32u, interpolate_pixels[2]);
+ ASSERT_EQ(64u, interpolate_pixels[3]);
+ ASSERT_EQ(0u, interpolate_pixels[4]);
+ ASSERT_EQ(0u, interpolate_pixels[5]);
+ ASSERT_EQ(0u, interpolate_pixels[6]);
+ ASSERT_EQ(128u, interpolate_pixels[7]);
+ ASSERT_EQ(0u, interpolate_pixels[8]);
+ ASSERT_EQ(0u, interpolate_pixels[9]);
+ ASSERT_EQ(0u, interpolate_pixels[10]);
+ ASSERT_EQ(0u, interpolate_pixels[11]);
+ ASSERT_EQ(128u, interpolate_pixels[12]);
+ ASSERT_EQ(128u, interpolate_pixels[13]);
+ ASSERT_EQ(128u, interpolate_pixels[14]);
+ ASSERT_EQ(128u, interpolate_pixels[15]);
InterpolatePlane(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
&interpolate_pixels[0], 0, 16, 1, 0);
- EXPECT_EQ(16u, interpolate_pixels[0]);
- EXPECT_EQ(32u, interpolate_pixels[1]);
- EXPECT_EQ(64u, interpolate_pixels[2]);
- EXPECT_EQ(128u, interpolate_pixels[3]);
+ ASSERT_EQ(16u, interpolate_pixels[0]);
+ ASSERT_EQ(32u, interpolate_pixels[1]);
+ ASSERT_EQ(64u, interpolate_pixels[2]);
+ ASSERT_EQ(128u, interpolate_pixels[3]);
InterpolatePlane(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
&interpolate_pixels[0], 0, 16, 1, 192);
- EXPECT_EQ(4u, interpolate_pixels[0]);
- EXPECT_EQ(8u, interpolate_pixels[1]);
- EXPECT_EQ(16u, interpolate_pixels[2]);
- EXPECT_EQ(32u, interpolate_pixels[3]);
+ ASSERT_EQ(4u, interpolate_pixels[0]);
+ ASSERT_EQ(8u, interpolate_pixels[1]);
+ ASSERT_EQ(16u, interpolate_pixels[2]);
+ ASSERT_EQ(32u, interpolate_pixels[3]);
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
InterpolatePlane(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
@@ -1010,19 +1201,100 @@ TEST_F(LibYUVPlanarTest, TestInterpolatePlane) {
}
}
+TEST_F(LibYUVPlanarTest, TestInterpolatePlane_16) {
+ SIMD_ALIGNED(uint16_t orig_pixels_0[1280]);
+ SIMD_ALIGNED(uint16_t orig_pixels_1[1280]);
+ SIMD_ALIGNED(uint16_t interpolate_pixels[1280]);
+ memset(orig_pixels_0, 0, sizeof(orig_pixels_0));
+ memset(orig_pixels_1, 0, sizeof(orig_pixels_1));
+
+ orig_pixels_0[0] = 16u;
+ orig_pixels_0[1] = 32u;
+ orig_pixels_0[2] = 64u;
+ orig_pixels_0[3] = 128u;
+ orig_pixels_0[4] = 0u;
+ orig_pixels_0[5] = 0u;
+ orig_pixels_0[6] = 0u;
+ orig_pixels_0[7] = 255u;
+ orig_pixels_0[8] = 0u;
+ orig_pixels_0[9] = 0u;
+ orig_pixels_0[10] = 0u;
+ orig_pixels_0[11] = 0u;
+ orig_pixels_0[12] = 0u;
+ orig_pixels_0[13] = 0u;
+ orig_pixels_0[14] = 0u;
+ orig_pixels_0[15] = 0u;
+
+ orig_pixels_1[0] = 0u;
+ orig_pixels_1[1] = 0u;
+ orig_pixels_1[2] = 0u;
+ orig_pixels_1[3] = 0u;
+ orig_pixels_1[4] = 0u;
+ orig_pixels_1[5] = 0u;
+ orig_pixels_1[6] = 0u;
+ orig_pixels_1[7] = 0u;
+ orig_pixels_1[8] = 0u;
+ orig_pixels_1[9] = 0u;
+ orig_pixels_1[10] = 0u;
+ orig_pixels_1[11] = 0u;
+ orig_pixels_1[12] = 255u;
+ orig_pixels_1[13] = 255u;
+ orig_pixels_1[14] = 255u;
+ orig_pixels_1[15] = 255u;
+
+ InterpolatePlane_16(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
+ &interpolate_pixels[0], 0, 16, 1, 128);
+ ASSERT_EQ(8u, interpolate_pixels[0]);
+ ASSERT_EQ(16u, interpolate_pixels[1]);
+ ASSERT_EQ(32u, interpolate_pixels[2]);
+ ASSERT_EQ(64u, interpolate_pixels[3]);
+ ASSERT_EQ(0u, interpolate_pixels[4]);
+ ASSERT_EQ(0u, interpolate_pixels[5]);
+ ASSERT_EQ(0u, interpolate_pixels[6]);
+ ASSERT_EQ(128u, interpolate_pixels[7]);
+ ASSERT_EQ(0u, interpolate_pixels[8]);
+ ASSERT_EQ(0u, interpolate_pixels[9]);
+ ASSERT_EQ(0u, interpolate_pixels[10]);
+ ASSERT_EQ(0u, interpolate_pixels[11]);
+ ASSERT_EQ(128u, interpolate_pixels[12]);
+ ASSERT_EQ(128u, interpolate_pixels[13]);
+ ASSERT_EQ(128u, interpolate_pixels[14]);
+ ASSERT_EQ(128u, interpolate_pixels[15]);
+
+ InterpolatePlane_16(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
+ &interpolate_pixels[0], 0, 16, 1, 0);
+ ASSERT_EQ(16u, interpolate_pixels[0]);
+ ASSERT_EQ(32u, interpolate_pixels[1]);
+ ASSERT_EQ(64u, interpolate_pixels[2]);
+ ASSERT_EQ(128u, interpolate_pixels[3]);
+
+ InterpolatePlane_16(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
+ &interpolate_pixels[0], 0, 16, 1, 192);
+
+ ASSERT_EQ(4u, interpolate_pixels[0]);
+ ASSERT_EQ(8u, interpolate_pixels[1]);
+ ASSERT_EQ(16u, interpolate_pixels[2]);
+ ASSERT_EQ(32u, interpolate_pixels[3]);
+
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+ InterpolatePlane_16(&orig_pixels_0[0], 0, &orig_pixels_1[0], 0,
+ &interpolate_pixels[0], 0, 1280, 1, 123);
+ }
+}
+
#define TESTTERP(FMT_A, BPP_A, STRIDE_A, FMT_B, BPP_B, STRIDE_B, W1280, TERP, \
N, NEG, OFF) \
TEST_F(LibYUVPlanarTest, ARGBInterpolate##TERP##N) { \
- const int kWidth = ((W1280) > 0) ? (W1280) : 1; \
+ const int kWidth = W1280; \
const int kHeight = benchmark_height_; \
const int kStrideA = \
(kWidth * BPP_A + STRIDE_A - 1) / STRIDE_A * STRIDE_A; \
const int kStrideB = \
(kWidth * BPP_B + STRIDE_B - 1) / STRIDE_B * STRIDE_B; \
- align_buffer_page_end(src_argb_a, kStrideA* kHeight + OFF); \
- align_buffer_page_end(src_argb_b, kStrideA* kHeight + OFF); \
- align_buffer_page_end(dst_argb_c, kStrideB* kHeight); \
- align_buffer_page_end(dst_argb_opt, kStrideB* kHeight); \
+ align_buffer_page_end(src_argb_a, kStrideA * kHeight + OFF); \
+ align_buffer_page_end(src_argb_b, kStrideA * kHeight + OFF); \
+ align_buffer_page_end(dst_argb_c, kStrideB * kHeight); \
+ align_buffer_page_end(dst_argb_opt, kStrideB * kHeight); \
for (int i = 0; i < kStrideA * kHeight; ++i) { \
src_argb_a[i + OFF] = (fastrand() & 0xff); \
src_argb_b[i + OFF] = (fastrand() & 0xff); \
@@ -1036,7 +1308,7 @@ TEST_F(LibYUVPlanarTest, TestInterpolatePlane) {
dst_argb_opt, kStrideB, kWidth, NEG kHeight, TERP); \
} \
for (int i = 0; i < kStrideB * kHeight; ++i) { \
- EXPECT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]); \
} \
free_aligned_buffer_page_end(src_argb_a); \
free_aligned_buffer_page_end(src_argb_b); \
@@ -1045,7 +1317,7 @@ TEST_F(LibYUVPlanarTest, TestInterpolatePlane) {
}
#define TESTINTERPOLATE(TERP) \
- TESTTERP(ARGB, 4, 1, ARGB, 4, 1, benchmark_width_ - 1, TERP, _Any, +, 0) \
+ TESTTERP(ARGB, 4, 1, ARGB, 4, 1, benchmark_width_ + 1, TERP, _Any, +, 0) \
TESTTERP(ARGB, 4, 1, ARGB, 4, 1, benchmark_width_, TERP, _Unaligned, +, 1) \
TESTTERP(ARGB, 4, 1, ARGB, 4, 1, benchmark_width_, TERP, _Invert, -, 0) \
TESTTERP(ARGB, 4, 1, ARGB, 4, 1, benchmark_width_, TERP, _Opt, +, 0)
@@ -1062,7 +1334,8 @@ static int TestBlend(int width,
int disable_cpu_flags,
int benchmark_cpu_info,
int invert,
- int off) {
+ int off,
+ int attenuate) {
if (width < 1) {
width = 1;
}
@@ -1076,10 +1349,12 @@ static int TestBlend(int width,
src_argb_a[i + off] = (fastrand() & 0xff);
src_argb_b[i + off] = (fastrand() & 0xff);
}
- ARGBAttenuate(src_argb_a + off, kStride, src_argb_a + off, kStride, width,
- height);
- ARGBAttenuate(src_argb_b + off, kStride, src_argb_b + off, kStride, width,
- height);
+ MemRandomize(src_argb_a, kStride * height + off);
+ MemRandomize(src_argb_b, kStride * height + off);
+ if (attenuate) {
+ ARGBAttenuate(src_argb_a + off, kStride, src_argb_a + off, kStride, width,
+ height);
+ }
memset(dst_argb_c, 255, kStride * height);
memset(dst_argb_opt, 255, kStride * height);
@@ -1108,30 +1383,37 @@ static int TestBlend(int width,
TEST_F(LibYUVPlanarTest, ARGBBlend_Any) {
int max_diff =
- TestBlend(benchmark_width_ - 4, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ TestBlend(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 0, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBBlend_Unaligned) {
int max_diff =
TestBlend(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 1);
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 1, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBBlend_Invert) {
int max_diff =
TestBlend(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 1);
+ disable_cpu_flags_, benchmark_cpu_info_, -1, 0, 1);
+ ASSERT_LE(max_diff, 1);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBBlend_Unattenuated) {
+ int max_diff =
+ TestBlend(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 0, 0);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBBlend_Opt) {
int max_diff =
TestBlend(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 0, 1);
+ ASSERT_LE(max_diff, 1);
}
static void TestBlendPlane(int width,
@@ -1163,14 +1445,14 @@ static void TestBlendPlane(int width,
BlendPlane(src_argb_a + off, width, src_argb_b + off, width,
src_argb_alpha + off, width, dst_argb_opt + off, width, width, 1);
for (int i = 0; i < width; ++i) {
- EXPECT_EQ(src_argb_a[i + off], dst_argb_opt[i + off]);
+ ASSERT_EQ(src_argb_a[i + off], dst_argb_opt[i + off]);
}
// Test destination is maintained exactly if alpha is 0.
memset(src_argb_alpha + off, 0, width);
BlendPlane(src_argb_a + off, width, src_argb_b + off, width,
src_argb_alpha + off, width, dst_argb_opt + off, width, width, 1);
for (int i = 0; i < width; ++i) {
- EXPECT_EQ(src_argb_b[i + off], dst_argb_opt[i + off]);
+ ASSERT_EQ(src_argb_b[i + off], dst_argb_opt[i + off]);
}
for (int i = 0; i < kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
@@ -1189,7 +1471,7 @@ static void TestBlendPlane(int width,
invert * height);
}
for (int i = 0; i < kStride * height; ++i) {
- EXPECT_EQ(dst_argb_c[i + off], dst_argb_opt[i + off]);
+ ASSERT_EQ(dst_argb_c[i + off], dst_argb_opt[i + off]);
}
free_aligned_buffer_page_end(src_argb_a);
free_aligned_buffer_page_end(src_argb_b);
@@ -1207,7 +1489,7 @@ TEST_F(LibYUVPlanarTest, BlendPlane_Unaligned) {
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
}
TEST_F(LibYUVPlanarTest, BlendPlane_Any) {
- TestBlendPlane(benchmark_width_ - 4, benchmark_height_, benchmark_iterations_,
+ TestBlendPlane(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
}
TEST_F(LibYUVPlanarTest, BlendPlane_Invert) {
@@ -1215,7 +1497,7 @@ TEST_F(LibYUVPlanarTest, BlendPlane_Invert) {
disable_cpu_flags_, benchmark_cpu_info_, -1, 1);
}
-#define SUBSAMPLE(v, a) ((((v) + (a)-1)) / (a))
+#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))
static void TestI420Blend(int width,
int height,
@@ -1270,11 +1552,11 @@ static void TestI420Blend(int width,
width, invert * height);
}
for (int i = 0; i < width * height; ++i) {
- EXPECT_EQ(dst_y_c[i + off], dst_y_opt[i + off]);
+ ASSERT_EQ(dst_y_c[i + off], dst_y_opt[i + off]);
}
for (int i = 0; i < kSizeUV; ++i) {
- EXPECT_EQ(dst_u_c[i + off], dst_u_opt[i + off]);
- EXPECT_EQ(dst_v_c[i + off], dst_v_opt[i + off]);
+ ASSERT_EQ(dst_u_c[i + off], dst_u_opt[i + off]);
+ ASSERT_EQ(dst_v_c[i + off], dst_v_opt[i + off]);
}
free_aligned_buffer_page_end(src_y0);
free_aligned_buffer_page_end(src_u0);
@@ -1302,7 +1584,7 @@ TEST_F(LibYUVPlanarTest, I420Blend_Unaligned) {
// TODO(fbarchard): DISABLED because _Any uses C. Avoid C and re-enable.
TEST_F(LibYUVPlanarTest, DISABLED_I420Blend_Any) {
- TestI420Blend(benchmark_width_ - 4, benchmark_height_, benchmark_iterations_,
+ TestI420Blend(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
}
TEST_F(LibYUVPlanarTest, I420Blend_Invert) {
@@ -1310,90 +1592,105 @@ TEST_F(LibYUVPlanarTest, I420Blend_Invert) {
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
}
+#ifdef ENABLE_ROW_TESTS
TEST_F(LibYUVPlanarTest, TestAffine) {
- SIMD_ALIGNED(uint8_t orig_pixels_0[1280][4]);
+ SIMD_ALIGNED(uint8_t orig_pixels_0[2][1280][4]);
SIMD_ALIGNED(uint8_t interpolate_pixels_C[1280][4]);
- for (int i = 0; i < 1280; ++i) {
- for (int j = 0; j < 4; ++j) {
- orig_pixels_0[i][j] = i;
+ for (int r = 0; r < 2; ++r) {
+ for (int i = 0; i < 1280; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ orig_pixels_0[r][i][j] = i + r * 17;
+ }
}
}
- float uv_step[4] = {0.f, 0.f, 0.75f, 0.f};
+ float uv_step[4] = {0.f, 0.f, 0.75f, 1.f / 1280.f};
+ const int stride = 1280 * 4;
- ARGBAffineRow_C(&orig_pixels_0[0][0], 0, &interpolate_pixels_C[0][0], uv_step,
- 1280);
- EXPECT_EQ(0u, interpolate_pixels_C[0][0]);
- EXPECT_EQ(96u, interpolate_pixels_C[128][0]);
- EXPECT_EQ(191u, interpolate_pixels_C[255][3]);
+ ARGBAffineRow_C(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_C[0][0],
+ uv_step, 1280);
+ ASSERT_EQ(0u, interpolate_pixels_C[0][0]);
+ ASSERT_EQ(96u, interpolate_pixels_C[128][0]);
+ ASSERT_EQ(191u, interpolate_pixels_C[255][3]);
-#if defined(HAS_ARGBAFFINEROW_SSE2)
- SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
- ARGBAffineRow_SSE2(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
- uv_step, 1280);
- EXPECT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
+#if defined(HAS_ARGBAFFINEROW_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
+ ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1280);
+ ASSERT_EQ(0,
+ memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
+ ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1279);
+ ASSERT_EQ(0,
+ memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1279 * 4));
- int has_sse2 = TestCpuFlag(kCpuHasSSE2);
- if (has_sse2) {
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
- ARGBAffineRow_SSE2(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
- uv_step, 1280);
+ ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1280);
}
}
#endif
-}
-
-TEST_F(LibYUVPlanarTest, TestCopyPlane) {
- int err = 0;
- int yw = benchmark_width_;
- int yh = benchmark_height_;
- int b = 12;
- int i, j;
-
- int y_plane_size = (yw + b * 2) * (yh + b * 2);
- align_buffer_page_end(orig_y, y_plane_size);
- align_buffer_page_end(dst_c, y_plane_size);
- align_buffer_page_end(dst_opt, y_plane_size);
-
- memset(orig_y, 0, y_plane_size);
- memset(dst_c, 0, y_plane_size);
- memset(dst_opt, 0, y_plane_size);
+#if defined(HAS_ARGBAFFINEROW_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ if (has_neon) {
+ SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
+ ARGBAffineRow_NEON(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1280);
+ ASSERT_EQ(0,
+ memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
+ ARGBAffineRow_NEON(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1279);
+ ASSERT_EQ(0,
+ memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1279 * 4));
- // Fill image buffers with random data.
- for (i = b; i < (yh + b); ++i) {
- for (j = b; j < (yw + b); ++j) {
- orig_y[i * (yw + b * 2) + j] = fastrand() & 0xff;
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+ ARGBAffineRow_NEON(&orig_pixels_0[0][0][0], stride,
+ &interpolate_pixels_Opt[0][0], uv_step, 1280);
}
}
+#endif
+}
+#endif // ENABLE_ROW_TESTS
- // Fill destination buffers with random data.
- for (i = 0; i < y_plane_size; ++i) {
- uint8_t random_number = fastrand() & 0x7f;
- dst_c[i] = random_number;
- dst_opt[i] = dst_c[i];
- }
-
- int y_off = b * (yw + b * 2) + b;
+static int TestCopyPlane(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ const int y_plane_size = benchmark_width * benchmark_height;
+ align_buffer_page_end(orig_y, y_plane_size + off);
+ align_buffer_page_end(dst_c, y_plane_size);
+ align_buffer_page_end(dst_opt, y_plane_size);
- int y_st = yw + b * 2;
- int stride = 8;
+ MemRandomize(orig_y + off, y_plane_size);
+ memset(dst_c, 1, y_plane_size);
+ memset(dst_opt, 2, y_plane_size);
// Disable all optimizations.
- MaskCpuFlags(disable_cpu_flags_);
- for (j = 0; j < benchmark_iterations_; j++) {
- CopyPlane(orig_y + y_off, y_st, dst_c + y_off, stride, yw, yh);
+ MaskCpuFlags(disable_cpu_flags);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane(orig_y + off, benchmark_width, dst_c, benchmark_width,
+ benchmark_width, benchmark_height * invert);
}
// Enable optimizations.
- MaskCpuFlags(benchmark_cpu_info_);
- for (j = 0; j < benchmark_iterations_; j++) {
- CopyPlane(orig_y + y_off, y_st, dst_opt + y_off, stride, yw, yh);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane(orig_y + off, benchmark_width, dst_opt, benchmark_width,
+ benchmark_width, benchmark_height * invert);
}
- for (i = 0; i < y_plane_size; ++i) {
- if (dst_c[i] != dst_opt[i]) {
- ++err;
+ int max_diff = 0;
+ for (int i = 0; i < y_plane_size; ++i) {
+ int abs_diff =
+ abs(static_cast<int>(dst_c[i]) - static_cast<int>(dst_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
}
}
@@ -1401,81 +1698,395 @@ TEST_F(LibYUVPlanarTest, TestCopyPlane) {
free_aligned_buffer_page_end(dst_c);
free_aligned_buffer_page_end(dst_opt);
- EXPECT_EQ(0, err);
+ return max_diff;
}
-static int TestMultiply(int width,
- int height,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- int invert,
- int off) {
- if (width < 1) {
- width = 1;
- }
- const int kBpp = 4;
- const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(src_argb_b, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
- src_argb_a[i + off] = (fastrand() & 0xff);
- src_argb_b[i + off] = (fastrand() & 0xff);
- }
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+TEST_F(LibYUVPlanarTest, CopyPlane_Any) {
+ int max_diff = TestCopyPlane(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Unaligned) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Invert) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Opt) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, TestCopyPlaneZero) {
+ // Test to verify copying a rect with a zero height or width does
+ // not touch destination memory.
+ uint8_t src = 42;
+ uint8_t dst = 0;
+
+ // Disable all optimizations.
+ MaskCpuFlags(disable_cpu_flags_);
+ CopyPlane(&src, 0, &dst, 0, 0, 0);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+
+ CopyPlane(&src, 1, &dst, 1, 1, 0);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+ CopyPlane(&src, 1, &dst, 1, 0, 1);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+
+ // Enable optimizations.
+ MaskCpuFlags(benchmark_cpu_info_);
+ CopyPlane(&src, 0, &dst, 0, 0, 0);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+
+ CopyPlane(&src, 1, &dst, 1, 1, 0);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+
+ CopyPlane(&src, 1, &dst, 1, 0, 1);
+ ASSERT_EQ(src, 42);
+ ASSERT_EQ(dst, 0);
+}
+
+static int TestCopyPlane_16(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ const int y_plane_size = benchmark_width * benchmark_height;
+ align_buffer_page_end(orig_y, (y_plane_size + off) * 2);
+ align_buffer_page_end(dst_c, y_plane_size * 2);
+ align_buffer_page_end(dst_opt, y_plane_size * 2);
+ uint16_t* orig_y_16 = reinterpret_cast<uint16_t*>(orig_y) + off;
+ uint16_t* dst_c_16 = reinterpret_cast<uint16_t*>(dst_c);
+ uint16_t* dst_opt_16 = reinterpret_cast<uint16_t*>(dst_opt);
+
+ MemRandomize(orig_y, (y_plane_size + off) * 2);
+ memset(dst_c, 1, y_plane_size * 2);
+ memset(dst_opt, 2, y_plane_size * 2);
+
+ // Disable all optimizations.
MaskCpuFlags(disable_cpu_flags);
- ARGBMultiply(src_argb_a + off, kStride, src_argb_b + off, kStride, dst_argb_c,
- kStride, width, invert * height);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane_16(orig_y_16, benchmark_width, dst_c_16, benchmark_width,
+ benchmark_width, benchmark_height * invert);
+ }
+
+ // Enable optimizations.
MaskCpuFlags(benchmark_cpu_info);
- for (int i = 0; i < benchmark_iterations; ++i) {
- ARGBMultiply(src_argb_a + off, kStride, src_argb_b + off, kStride,
- dst_argb_opt, kStride, width, invert * height);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane_16(orig_y_16, benchmark_width, dst_opt_16, benchmark_width,
+ benchmark_width, benchmark_height * invert);
}
+
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
- int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
- static_cast<int>(dst_argb_opt[i]));
+ for (int i = 0; i < y_plane_size; ++i) {
+ int abs_diff =
+ abs(static_cast<int>(dst_c_16[i]) - static_cast<int>(dst_opt_16[i]));
if (abs_diff > max_diff) {
max_diff = abs_diff;
}
}
- free_aligned_buffer_page_end(src_argb_a);
- free_aligned_buffer_page_end(src_argb_b);
- free_aligned_buffer_page_end(dst_argb_c);
- free_aligned_buffer_page_end(dst_argb_opt);
+
+ free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+
return max_diff;
}
-TEST_F(LibYUVPlanarTest, ARGBMultiply_Any) {
- int max_diff = TestMultiply(benchmark_width_ - 1, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+TEST_F(LibYUVPlanarTest, CopyPlane_16_Any) {
+ int max_diff = TestCopyPlane_16(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_LE(max_diff, 0);
}
-TEST_F(LibYUVPlanarTest, ARGBMultiply_Unaligned) {
- int max_diff =
- TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 1);
-}
+TEST_F(LibYUVPlanarTest, CopyPlane_16_Unaligned) {
+ int max_diff = TestCopyPlane_16(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_16_Invert) {
+ int max_diff = TestCopyPlane_16(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_16_Opt) {
+ int max_diff = TestCopyPlane_16(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, TestDetilePlane) {
+ int i, j;
+
+ // orig is tiled. Allocate enough memory for tiles.
+ int tile_width = (benchmark_width_ + 15) & ~15;
+ int tile_height = (benchmark_height_ + 15) & ~15;
+ int tile_plane_size = tile_width * tile_height;
+ int y_plane_size = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(tile_y, tile_plane_size);
+ align_buffer_page_end(dst_c, y_plane_size);
+ align_buffer_page_end(dst_opt, y_plane_size);
+
+ MemRandomize(tile_y, tile_plane_size);
+ memset(dst_c, 0, y_plane_size);
+ memset(dst_opt, 0, y_plane_size);
+
+ // Disable all optimizations.
+ MaskCpuFlags(disable_cpu_flags_);
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetilePlane(tile_y, tile_width, dst_c, benchmark_width_, benchmark_width_,
+ benchmark_height_, 16);
+ }
+
+ // Enable optimizations.
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetilePlane(tile_y, tile_width, dst_opt, benchmark_width_, benchmark_width_,
+ benchmark_height_, 16);
+ }
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(tile_y);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+}
+
+TEST_F(LibYUVPlanarTest, TestDetilePlane_16) {
+ int i, j;
+
+ // orig is tiled. Allocate enough memory for tiles.
+ int tile_width = (benchmark_width_ + 15) & ~15;
+ int tile_height = (benchmark_height_ + 15) & ~15;
+ int tile_plane_size = tile_width * tile_height * 2;
+ int y_plane_size = benchmark_width_ * benchmark_height_ * 2;
+ align_buffer_page_end(tile_y, tile_plane_size);
+ align_buffer_page_end(dst_c, y_plane_size);
+ align_buffer_page_end(dst_opt, y_plane_size);
+
+ MemRandomize(tile_y, tile_plane_size);
+ memset(dst_c, 0, y_plane_size);
+ memset(dst_opt, 0, y_plane_size);
+
+ // Disable all optimizations.
+ MaskCpuFlags(disable_cpu_flags_);
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetilePlane_16((const uint16_t*)tile_y, tile_width, (uint16_t*)dst_c,
+ benchmark_width_, benchmark_width_, benchmark_height_, 16);
+ }
+
+ // Enable optimizations.
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetilePlane_16((const uint16_t*)tile_y, tile_width, (uint16_t*)dst_opt,
+ benchmark_width_, benchmark_width_, benchmark_height_, 16);
+ }
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(tile_y);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+}
+
+// Compares DetileSplitUV to 2 step Detile + SplitUV
+TEST_F(LibYUVPlanarTest, TestDetileSplitUVPlane_Correctness) {
+ int i, j;
+
+ // orig is tiled. Allocate enough memory for tiles.
+ int uv_width = (benchmark_width_ + 1) & ~1;
+ int tile_width = (uv_width + 15) & ~15;
+ int tile_height = (benchmark_height_ + 15) & ~15;
+ int tile_plane_size = tile_width * tile_height;
+ int uv_plane_size = (uv_width / 2) * benchmark_height_;
+ align_buffer_page_end(tile_uv, tile_plane_size);
+ align_buffer_page_end(detiled_uv, tile_plane_size);
+ align_buffer_page_end(dst_u_two_stage, uv_plane_size);
+ align_buffer_page_end(dst_u_opt, uv_plane_size);
+ align_buffer_page_end(dst_v_two_stage, uv_plane_size);
+ align_buffer_page_end(dst_v_opt, uv_plane_size);
+
+ MemRandomize(tile_uv, tile_plane_size);
+ memset(detiled_uv, 0, tile_plane_size);
+ memset(dst_u_two_stage, 0, uv_plane_size);
+ memset(dst_u_opt, 0, uv_plane_size);
+ memset(dst_v_two_stage, 0, uv_plane_size);
+ memset(dst_v_opt, 0, uv_plane_size);
+
+ DetileSplitUVPlane(tile_uv, tile_width, dst_u_opt, uv_width / 2, dst_v_opt,
+ uv_width / 2, uv_width, benchmark_height_, 16);
+
+ // Benchmark 2 step conversion for comparison.
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetilePlane(tile_uv, tile_width, detiled_uv, uv_width, uv_width,
+ benchmark_height_, 16);
+ SplitUVPlane(detiled_uv, uv_width, dst_u_two_stage, uv_width / 2,
+ dst_v_two_stage, uv_width / 2, uv_width / 2,
+ benchmark_height_);
+ }
+
+ for (i = 0; i < uv_plane_size; ++i) {
+ ASSERT_EQ(dst_u_two_stage[i], dst_u_opt[i]);
+ ASSERT_EQ(dst_v_two_stage[i], dst_v_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(tile_uv);
+ free_aligned_buffer_page_end(detiled_uv);
+ free_aligned_buffer_page_end(dst_u_two_stage);
+ free_aligned_buffer_page_end(dst_u_opt);
+ free_aligned_buffer_page_end(dst_v_two_stage);
+ free_aligned_buffer_page_end(dst_v_opt);
+}
+
+TEST_F(LibYUVPlanarTest, TestDetileSplitUVPlane_Benchmark) {
+ int i, j;
+
+ // orig is tiled. Allocate enough memory for tiles.
+ int uv_width = (benchmark_width_ + 1) & ~1;
+ int tile_width = (uv_width + 15) & ~15;
+ int tile_height = (benchmark_height_ + 15) & ~15;
+ int tile_plane_size = tile_width * tile_height;
+ int uv_plane_size = (uv_width / 2) * benchmark_height_;
+ align_buffer_page_end(tile_uv, tile_plane_size);
+ align_buffer_page_end(dst_u_c, uv_plane_size);
+ align_buffer_page_end(dst_u_opt, uv_plane_size);
+ align_buffer_page_end(dst_v_c, uv_plane_size);
+ align_buffer_page_end(dst_v_opt, uv_plane_size);
+
+ MemRandomize(tile_uv, tile_plane_size);
+ memset(dst_u_c, 0, uv_plane_size);
+ memset(dst_u_opt, 0, uv_plane_size);
+ memset(dst_v_c, 0, uv_plane_size);
+ memset(dst_v_opt, 0, uv_plane_size);
+
+ // Disable all optimizations.
+ MaskCpuFlags(disable_cpu_flags_);
+
+ DetileSplitUVPlane(tile_uv, tile_width, dst_u_c, uv_width / 2, dst_v_c,
+ uv_width / 2, uv_width, benchmark_height_, 16);
+
+ // Enable optimizations.
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (j = 0; j < benchmark_iterations_; j++) {
+ DetileSplitUVPlane(tile_uv, tile_width, dst_u_opt, uv_width / 2, dst_v_opt,
+ uv_width / 2, uv_width, benchmark_height_, 16);
+ }
+
+ for (i = 0; i < uv_plane_size; ++i) {
+ ASSERT_EQ(dst_u_c[i], dst_u_opt[i]);
+ ASSERT_EQ(dst_v_c[i], dst_v_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(tile_uv);
+ free_aligned_buffer_page_end(dst_u_c);
+ free_aligned_buffer_page_end(dst_u_opt);
+ free_aligned_buffer_page_end(dst_v_c);
+ free_aligned_buffer_page_end(dst_v_opt);
+}
+
+static int TestMultiply(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kBpp = 4;
+ const int kStride = width * kBpp;
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(src_argb_b, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
+ src_argb_a[i + off] = (fastrand() & 0xff);
+ src_argb_b[i + off] = (fastrand() & 0xff);
+ }
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
+
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBMultiply(src_argb_a + off, kStride, src_argb_b + off, kStride, dst_argb_c,
+ kStride, width, invert * height);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBMultiply(src_argb_a + off, kStride, src_argb_b + off, kStride,
+ dst_argb_opt, kStride, width, invert * height);
+ }
+ int max_diff = 0;
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
+ int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
+ static_cast<int>(dst_argb_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ free_aligned_buffer_page_end(src_argb_a);
+ free_aligned_buffer_page_end(src_argb_b);
+ free_aligned_buffer_page_end(dst_argb_c);
+ free_aligned_buffer_page_end(dst_argb_opt);
+ return max_diff;
+}
+
+TEST_F(LibYUVPlanarTest, ARGBMultiply_Any) {
+ int max_diff = TestMultiply(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBMultiply_Unaligned) {
+ int max_diff =
+ TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
+ ASSERT_LE(max_diff, 0);
+}
TEST_F(LibYUVPlanarTest, ARGBMultiply_Invert) {
int max_diff =
TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBMultiply_Opt) {
int max_diff =
TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 0);
}
static int TestAdd(int width,
@@ -1490,16 +2101,16 @@ static int TestAdd(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(src_argb_b, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(src_argb_b, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
src_argb_b[i + off] = (fastrand() & 0xff);
}
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBAdd(src_argb_a + off, kStride, src_argb_b + off, kStride, dst_argb_c,
@@ -1510,7 +2121,7 @@ static int TestAdd(int width,
kStride, width, invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -1526,30 +2137,30 @@ static int TestAdd(int width,
TEST_F(LibYUVPlanarTest, ARGBAdd_Any) {
int max_diff =
- TestAdd(benchmark_width_ - 1, benchmark_height_, benchmark_iterations_,
+ TestAdd(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBAdd_Unaligned) {
int max_diff =
TestAdd(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBAdd_Invert) {
int max_diff =
TestAdd(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBAdd_Opt) {
int max_diff =
TestAdd(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
static int TestSubtract(int width,
@@ -1564,16 +2175,16 @@ static int TestSubtract(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(src_argb_b, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(src_argb_b, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
src_argb_b[i + off] = (fastrand() & 0xff);
}
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBSubtract(src_argb_a + off, kStride, src_argb_b + off, kStride, dst_argb_c,
@@ -1584,7 +2195,7 @@ static int TestSubtract(int width,
dst_argb_opt, kStride, width, invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -1599,31 +2210,31 @@ static int TestSubtract(int width,
}
TEST_F(LibYUVPlanarTest, ARGBSubtract_Any) {
- int max_diff = TestSubtract(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestSubtract(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBSubtract_Unaligned) {
int max_diff =
TestSubtract(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBSubtract_Invert) {
int max_diff =
TestSubtract(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, ARGBSubtract_Opt) {
int max_diff =
TestSubtract(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
static int TestSobel(int width,
@@ -1638,15 +2249,15 @@ static int TestSobel(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- memset(src_argb_a, 0, kStride * height + off);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ memset(src_argb_a, 0, (size_t)kStride * height + off);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
}
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBSobel(src_argb_a + off, kStride, dst_argb_c, kStride, width,
@@ -1657,7 +2268,7 @@ static int TestSobel(int width,
invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -1672,30 +2283,30 @@ static int TestSobel(int width,
TEST_F(LibYUVPlanarTest, ARGBSobel_Any) {
int max_diff =
- TestSobel(benchmark_width_ - 1, benchmark_height_, benchmark_iterations_,
+ TestSobel(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobel_Unaligned) {
int max_diff =
TestSobel(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobel_Invert) {
int max_diff =
TestSobel(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobel_Opt) {
int max_diff =
TestSobel(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
static int TestSobelToPlane(int width,
@@ -1745,31 +2356,31 @@ static int TestSobelToPlane(int width,
}
TEST_F(LibYUVPlanarTest, ARGBSobelToPlane_Any) {
- int max_diff = TestSobelToPlane(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestSobelToPlane(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelToPlane_Unaligned) {
int max_diff = TestSobelToPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelToPlane_Invert) {
int max_diff = TestSobelToPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, -1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelToPlane_Opt) {
int max_diff = TestSobelToPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
static int TestSobelXY(int width,
@@ -1784,15 +2395,15 @@ static int TestSobelXY(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- memset(src_argb_a, 0, kStride * height + off);
- for (int i = 0; i < kStride * height; ++i) {
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ memset(src_argb_a, 0, (size_t)kStride * height + off);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
}
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBSobelXY(src_argb_a + off, kStride, dst_argb_c, kStride, width,
@@ -1803,7 +2414,7 @@ static int TestSobelXY(int width,
invert * height);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -1817,31 +2428,31 @@ static int TestSobelXY(int width,
}
TEST_F(LibYUVPlanarTest, ARGBSobelXY_Any) {
- int max_diff = TestSobelXY(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestSobelXY(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelXY_Unaligned) {
int max_diff =
TestSobelXY(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelXY_Invert) {
int max_diff =
TestSobelXY(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBSobelXY_Opt) {
int max_diff =
TestSobelXY(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
static int TestBlur(int width,
@@ -1857,16 +2468,16 @@ static int TestBlur(int width,
}
const int kBpp = 4;
const int kStride = width * kBpp;
- align_buffer_page_end(src_argb_a, kStride * height + off);
- align_buffer_page_end(dst_cumsum, width * height * 16);
- align_buffer_page_end(dst_argb_c, kStride * height);
- align_buffer_page_end(dst_argb_opt, kStride * height);
- for (int i = 0; i < kStride * height; ++i) {
+ const int kCumsumHeight = radius * 2 + 2;
+ align_buffer_page_end(src_argb_a, (size_t)kStride * height + off);
+ align_buffer_page_end(dst_cumsum, (size_t)width * kCumsumHeight * 16);
+ align_buffer_page_end(dst_argb_c, (size_t)kStride * height);
+ align_buffer_page_end(dst_argb_opt, (size_t)kStride * height);
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
src_argb_a[i + off] = (fastrand() & 0xff);
}
- memset(dst_cumsum, 0, width * height * 16);
- memset(dst_argb_c, 0, kStride * height);
- memset(dst_argb_opt, 0, kStride * height);
+ memset(dst_argb_c, 0, (size_t)kStride * height);
+ memset(dst_argb_opt, 0, (size_t)kStride * height);
MaskCpuFlags(disable_cpu_flags);
ARGBBlur(src_argb_a + off, kStride, dst_argb_c, kStride,
@@ -1879,7 +2490,7 @@ static int TestBlur(int width,
invert * height, radius);
}
int max_diff = 0;
- for (int i = 0; i < kStride * height; ++i) {
+ for (size_t i = 0; i < (size_t)kStride * height; ++i) {
int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
static_cast<int>(dst_argb_opt[i]));
if (abs_diff > max_diff) {
@@ -1893,62 +2504,72 @@ static int TestBlur(int width,
return max_diff;
}
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+#define DISABLED_ARM(name) name
+#else
+#define DISABLED_ARM(name) DISABLED_##name
+#endif
+
static const int kBlurSize = 55;
-TEST_F(LibYUVPlanarTest, ARGBBlur_Any) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlur_Any)) {
int max_diff =
- TestBlur(benchmark_width_ - 1, benchmark_height_, benchmark_iterations_,
+ TestBlur(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSize);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlur_Unaligned) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlur_Unaligned)) {
int max_diff =
TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1, kBlurSize);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlur_Invert) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlur_Invert)) {
int max_diff =
TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0, kBlurSize);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlur_Opt) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlur_Opt)) {
int max_diff =
TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSize);
- EXPECT_LE(max_diff, 1);
+ ASSERT_LE(max_diff, 1);
}
-static const int kBlurSmallSize = 5;
-TEST_F(LibYUVPlanarTest, ARGBBlurSmall_Any) {
+static const int kBlurSmallSize = 1;
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Any)) {
int max_diff =
- TestBlur(benchmark_width_ - 1, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSmallSize);
- EXPECT_LE(max_diff, 1);
+ TestBlur((benchmark_width_ + 3) / 4 + 1, (benchmark_height_ + 3) / 4,
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_,
+ +1, 0, kBlurSmallSize);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlurSmall_Unaligned) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Unaligned)) {
int max_diff =
- TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 1, kBlurSmallSize);
- EXPECT_LE(max_diff, 1);
+ TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4,
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_,
+ +1, 1, kBlurSmallSize);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlurSmall_Invert) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Invert)) {
int max_diff =
- TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, -1, 0, kBlurSmallSize);
- EXPECT_LE(max_diff, 1);
+ TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4,
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_,
+ -1, 0, kBlurSmallSize);
+ ASSERT_LE(max_diff, 1);
}
-TEST_F(LibYUVPlanarTest, ARGBBlurSmall_Opt) {
+TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Opt)) {
int max_diff =
- TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_,
- disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSmallSize);
- EXPECT_LE(max_diff, 1);
+ TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4,
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_,
+ +1, 0, kBlurSmallSize);
+ ASSERT_LE(max_diff, 1);
}
TEST_F(LibYUVPlanarTest, TestARGBPolynomial) {
@@ -1992,26 +2613,26 @@ TEST_F(LibYUVPlanarTest, TestARGBPolynomial) {
// Do 16 to test asm version.
ARGBPolynomial(&orig_pixels[0][0], 0, &dst_pixels_opt[0][0], 0,
&kWarmifyPolynomial[0], 16, 1);
- EXPECT_EQ(235u, dst_pixels_opt[0][0]);
- EXPECT_EQ(0u, dst_pixels_opt[0][1]);
- EXPECT_EQ(0u, dst_pixels_opt[0][2]);
- EXPECT_EQ(128u, dst_pixels_opt[0][3]);
- EXPECT_EQ(0u, dst_pixels_opt[1][0]);
- EXPECT_EQ(233u, dst_pixels_opt[1][1]);
- EXPECT_EQ(0u, dst_pixels_opt[1][2]);
- EXPECT_EQ(0u, dst_pixels_opt[1][3]);
- EXPECT_EQ(0u, dst_pixels_opt[2][0]);
- EXPECT_EQ(0u, dst_pixels_opt[2][1]);
- EXPECT_EQ(241u, dst_pixels_opt[2][2]);
- EXPECT_EQ(255u, dst_pixels_opt[2][3]);
- EXPECT_EQ(235u, dst_pixels_opt[3][0]);
- EXPECT_EQ(233u, dst_pixels_opt[3][1]);
- EXPECT_EQ(241u, dst_pixels_opt[3][2]);
- EXPECT_EQ(255u, dst_pixels_opt[3][3]);
- EXPECT_EQ(10u, dst_pixels_opt[4][0]);
- EXPECT_EQ(59u, dst_pixels_opt[4][1]);
- EXPECT_EQ(188u, dst_pixels_opt[4][2]);
- EXPECT_EQ(224u, dst_pixels_opt[4][3]);
+ ASSERT_EQ(235u, dst_pixels_opt[0][0]);
+ ASSERT_EQ(0u, dst_pixels_opt[0][1]);
+ ASSERT_EQ(0u, dst_pixels_opt[0][2]);
+ ASSERT_EQ(128u, dst_pixels_opt[0][3]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][0]);
+ ASSERT_EQ(233u, dst_pixels_opt[1][1]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][2]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][3]);
+ ASSERT_EQ(0u, dst_pixels_opt[2][0]);
+ ASSERT_EQ(0u, dst_pixels_opt[2][1]);
+ ASSERT_EQ(241u, dst_pixels_opt[2][2]);
+ ASSERT_EQ(255u, dst_pixels_opt[2][3]);
+ ASSERT_EQ(235u, dst_pixels_opt[3][0]);
+ ASSERT_EQ(233u, dst_pixels_opt[3][1]);
+ ASSERT_EQ(241u, dst_pixels_opt[3][2]);
+ ASSERT_EQ(255u, dst_pixels_opt[3][3]);
+ ASSERT_EQ(10u, dst_pixels_opt[4][0]);
+ ASSERT_EQ(59u, dst_pixels_opt[4][1]);
+ ASSERT_EQ(188u, dst_pixels_opt[4][2]);
+ ASSERT_EQ(224u, dst_pixels_opt[4][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -2031,53 +2652,59 @@ TEST_F(LibYUVPlanarTest, TestARGBPolynomial) {
}
for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
- EXPECT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
- EXPECT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
- EXPECT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
- }
-}
-
-int TestHalfFloatPlane(int benchmark_width,
- int benchmark_height,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- float scale,
- int mask) {
- int i, j;
- const int y_plane_size = benchmark_width * benchmark_height * 2;
-
- align_buffer_page_end(orig_y, y_plane_size * 3);
- uint8_t* dst_opt = orig_y + y_plane_size;
- uint8_t* dst_c = orig_y + y_plane_size * 2;
+ ASSERT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
+ ASSERT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
+ ASSERT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
+ ASSERT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
+ }
+}
+
+static int TestHalfFloatPlane(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ float scale,
+ int mask,
+ int invert,
+ int off) {
+ if (benchmark_width <= 0 || benchmark_height <= 0) {
+ return 0;
+ }
+ const size_t y_plane_size = static_cast<size_t>(benchmark_width) *
+ static_cast<size_t>(benchmark_height) * 2;
+ align_buffer_page_end(orig_y, y_plane_size + off);
+ align_buffer_page_end(dst_c, y_plane_size);
+ align_buffer_page_end(dst_opt, y_plane_size);
- MemRandomize(orig_y, y_plane_size);
- memset(dst_c, 0, y_plane_size);
- memset(dst_opt, 1, y_plane_size);
+ MemRandomize(orig_y + off, y_plane_size);
+ memset(dst_c, 1, y_plane_size);
+ memset(dst_opt, 2, y_plane_size);
- for (i = 0; i < y_plane_size / 2; ++i) {
- reinterpret_cast<uint16_t*>(orig_y)[i] &= mask;
+ for (size_t i = 0; i < y_plane_size / 2; ++i) {
+ reinterpret_cast<uint16_t*>(orig_y + off)[i] &= mask;
}
// Disable all optimizations.
MaskCpuFlags(disable_cpu_flags);
- for (j = 0; j < benchmark_iterations; j++) {
- HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y), benchmark_width * 2,
- reinterpret_cast<uint16_t*>(dst_c), benchmark_width * 2,
- scale, benchmark_width, benchmark_height);
+ for (int j = 0; j < benchmark_iterations; j++) {
+ HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
+ benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_c),
+ benchmark_width * 2, scale, benchmark_width,
+ benchmark_height * invert);
}
// Enable optimizations.
MaskCpuFlags(benchmark_cpu_info);
- for (j = 0; j < benchmark_iterations; j++) {
- HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y), benchmark_width * 2,
- reinterpret_cast<uint16_t*>(dst_opt), benchmark_width * 2,
- scale, benchmark_width, benchmark_height);
+ for (int j = 0; j < benchmark_iterations; j++) {
+ HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
+ benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_opt),
+ benchmark_width * 2, scale, benchmark_width,
+ benchmark_height * invert);
}
int max_diff = 0;
- for (i = 0; i < y_plane_size / 2; ++i) {
+ for (size_t i = 0; i < y_plane_size / 2; ++i) {
int abs_diff =
abs(static_cast<int>(reinterpret_cast<uint16_t*>(dst_c)[i]) -
static_cast<int>(reinterpret_cast<uint16_t*>(dst_opt)[i]));
@@ -2087,107 +2714,142 @@ int TestHalfFloatPlane(int benchmark_width,
}
free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
return max_diff;
}
-#if defined(__arm__)
-static void EnableFlushDenormalToZero(void) {
- uint32_t cw;
- __asm__ __volatile__(
- "vmrs %0, fpscr \n"
- "orr %0, %0, #0x1000000 \n"
- "vmsr fpscr, %0 \n"
- : "=r"(cw)::"memory");
-}
-#endif
-
-// 5 bit exponent with bias of 15 will underflow to a denormal if scale causes
-// exponent to be less than 0. 15 - log2(65536) = -1/ This shouldnt normally
-// happen since scale is 1/(1<<bits) where bits is 9, 10 or 12.
-
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_denormal) {
-// 32 bit arm rounding on denormal case is off by 1 compared to C.
-#if defined(__arm__)
- EnableFlushDenormalToZero();
-#endif
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 65536.0f, 65535);
- EXPECT_EQ(0, diff);
-}
-
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_One) {
int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 65535);
- EXPECT_LE(diff, 1);
+ benchmark_cpu_info_, 1.0f, 65535, +1, 0);
+ ASSERT_LE(diff, 1);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4096.0f, 65535);
- EXPECT_EQ(0, diff);
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
+ ASSERT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_12bit_Opt) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4095.0f, 4095, +1, 0);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_10bit_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 1024.0f, 1023);
- EXPECT_EQ(0, diff);
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 1023.0f, 1023, +1, 0);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_9bit_Opt) {
int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 512.0f, 511);
- EXPECT_EQ(0, diff);
+ benchmark_cpu_info_, 1.0f / 511.0f, 511, +1, 0);
+ ASSERT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4096.0f, 4095);
- EXPECT_EQ(0, diff);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Any) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 0);
+ ASSERT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Offby1) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4095.0f, 4095);
- EXPECT_EQ(0, diff);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Unaligned) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 2);
+ ASSERT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_One) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 2047);
- EXPECT_EQ(0, diff);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Invert) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, -1, 0);
+ ASSERT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_12bit_One) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 4095);
- EXPECT_LE(diff, 1);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Opt) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 0);
+ ASSERT_EQ(0, diff);
}
-float TestByteToFloat(int benchmark_width,
- int benchmark_height,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- float scale) {
- int i, j;
- const int y_plane_size = benchmark_width * benchmark_height;
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_denormal) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
+ ASSERT_EQ(0, diff);
+}
- align_buffer_page_end(orig_y, y_plane_size * (1 + 4 + 4));
- float* dst_opt = reinterpret_cast<float*>(orig_y + y_plane_size);
- float* dst_c = reinterpret_cast<float*>(orig_y + y_plane_size * 5);
+#if defined(__arm__) && !defined(__SOFTFP__)
+static void EnableFlushDenormalToZero(void) {
+ uint32_t cw;
+ asm volatile(
+ "vmrs %0, fpscr \n"
+ "orr %0, %0, #0x1000000 \n"
+ "vmsr fpscr, %0 \n"
+ : "=r"(cw)::"memory", "cc"); // Clobber List
+}
- MemRandomize(orig_y, y_plane_size);
- memset(dst_c, 0, y_plane_size * 4);
- memset(dst_opt, 1, y_plane_size * 4);
+static void DisableFlushDenormalToZero(void) {
+ uint32_t cw;
+ asm volatile(
+ "vmrs %0, fpscr \n"
+ "bic %0, %0, #0x1000000 \n"
+ "vmsr fpscr, %0 \n"
+ : "=r"(cw)::"memory", "cc"); // Clobber List
+}
+
+// 5 bit exponent with bias of 15 will underflow to a denormal if scale causes
+// exponent to be less than 0. 15 - log2(65536) = -1/ This shouldnt normally
+// happen since scale is 1/(1<<bits) where bits is 9, 10 or 12.
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_flush_denormal) {
+ // 32 bit arm rounding on denormal case is off by 1 compared to C.
+ EnableFlushDenormalToZero();
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
+ DisableFlushDenormalToZero();
+ ASSERT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_10bit_flush_denormal) {
+ EnableFlushDenormalToZero();
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 1023.0f, 1023, +1, 0);
+ DisableFlushDenormalToZero();
+ ASSERT_EQ(0, diff);
+}
+#endif // defined(__arm__) && !defined(__SOFTFP__)
+
+static float TestByteToFloat(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ float scale) {
+ int i, j;
+ const int y_plane_size = benchmark_width * benchmark_height;
+
+ align_buffer_page_end(orig_y, y_plane_size);
+ align_buffer_page_end(dst_opt_mem, y_plane_size * 4);
+ align_buffer_page_end(dst_c_mem, y_plane_size * 4);
+ float* dst_opt = reinterpret_cast<float*>(dst_opt_mem);
+ float* dst_c = reinterpret_cast<float*>(dst_c_mem);
+
+ MemRandomize(orig_y, y_plane_size);
+ memset(dst_c, 0, y_plane_size * 4);
+ memset(dst_opt, 1, y_plane_size * 4);
// Disable all optimizations.
MaskCpuFlags(disable_cpu_flags);
@@ -2208,6 +2870,8 @@ float TestByteToFloat(int benchmark_width,
}
free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_opt_mem);
+ free_aligned_buffer_page_end(dst_c_mem);
return max_diff;
}
@@ -2215,7 +2879,7 @@ TEST_F(LibYUVPlanarTest, TestByteToFloat) {
float diff = TestByteToFloat(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, 1.0f);
- EXPECT_EQ(0.f, diff);
+ ASSERT_EQ(0.f, diff);
}
TEST_F(LibYUVPlanarTest, TestARGBLumaColorTable) {
@@ -2253,22 +2917,22 @@ TEST_F(LibYUVPlanarTest, TestARGBLumaColorTable) {
// Do 16 to test asm version.
ARGBLumaColorTable(&orig_pixels[0][0], 0, &dst_pixels_opt[0][0], 0,
&lumacolortable[0], 16, 1);
- EXPECT_EQ(253u, dst_pixels_opt[0][0]);
- EXPECT_EQ(0u, dst_pixels_opt[0][1]);
- EXPECT_EQ(0u, dst_pixels_opt[0][2]);
- EXPECT_EQ(128u, dst_pixels_opt[0][3]);
- EXPECT_EQ(0u, dst_pixels_opt[1][0]);
- EXPECT_EQ(253u, dst_pixels_opt[1][1]);
- EXPECT_EQ(0u, dst_pixels_opt[1][2]);
- EXPECT_EQ(0u, dst_pixels_opt[1][3]);
- EXPECT_EQ(0u, dst_pixels_opt[2][0]);
- EXPECT_EQ(0u, dst_pixels_opt[2][1]);
- EXPECT_EQ(253u, dst_pixels_opt[2][2]);
- EXPECT_EQ(255u, dst_pixels_opt[2][3]);
- EXPECT_EQ(48u, dst_pixels_opt[3][0]);
- EXPECT_EQ(192u, dst_pixels_opt[3][1]);
- EXPECT_EQ(64u, dst_pixels_opt[3][2]);
- EXPECT_EQ(224u, dst_pixels_opt[3][3]);
+ ASSERT_EQ(253u, dst_pixels_opt[0][0]);
+ ASSERT_EQ(0u, dst_pixels_opt[0][1]);
+ ASSERT_EQ(0u, dst_pixels_opt[0][2]);
+ ASSERT_EQ(128u, dst_pixels_opt[0][3]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][0]);
+ ASSERT_EQ(253u, dst_pixels_opt[1][1]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][2]);
+ ASSERT_EQ(0u, dst_pixels_opt[1][3]);
+ ASSERT_EQ(0u, dst_pixels_opt[2][0]);
+ ASSERT_EQ(0u, dst_pixels_opt[2][1]);
+ ASSERT_EQ(253u, dst_pixels_opt[2][2]);
+ ASSERT_EQ(255u, dst_pixels_opt[2][3]);
+ ASSERT_EQ(48u, dst_pixels_opt[3][0]);
+ ASSERT_EQ(192u, dst_pixels_opt[3][1]);
+ ASSERT_EQ(64u, dst_pixels_opt[3][2]);
+ ASSERT_EQ(224u, dst_pixels_opt[3][3]);
for (int i = 0; i < 1280; ++i) {
orig_pixels[i][0] = i;
@@ -2287,99 +2951,253 @@ TEST_F(LibYUVPlanarTest, TestARGBLumaColorTable) {
lumacolortable, 1280, 1);
}
for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
- EXPECT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
- EXPECT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
- EXPECT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
+ ASSERT_EQ(dst_pixels_c[i][0], dst_pixels_opt[i][0]);
+ ASSERT_EQ(dst_pixels_c[i][1], dst_pixels_opt[i][1]);
+ ASSERT_EQ(dst_pixels_c[i][2], dst_pixels_opt[i][2]);
+ ASSERT_EQ(dst_pixels_c[i][3], dst_pixels_opt[i][3]);
}
free_aligned_buffer_page_end(lumacolortable);
}
-TEST_F(LibYUVPlanarTest, TestARGBCopyAlpha) {
- const int kSize = benchmark_width_ * benchmark_height_ * 4;
- align_buffer_page_end(orig_pixels, kSize);
+static int TestARGBCopyAlpha(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kStride = width * 4;
+ const int kSize = kStride * height;
+ align_buffer_page_end(orig_pixels, kSize + off);
align_buffer_page_end(dst_pixels_opt, kSize);
align_buffer_page_end(dst_pixels_c, kSize);
- MemRandomize(orig_pixels, kSize);
- MemRandomize(dst_pixels_opt, kSize);
- memcpy(dst_pixels_c, dst_pixels_opt, kSize);
+ MemRandomize(orig_pixels + off, kSize);
+ MemRandomize(dst_pixels_c, kSize);
+ // Copy RGB bytes only, leaving alpha bytes in dst_pixels_opt uninitialized.
+ MSAN_POISON(dst_pixels_opt, kSize);
+ for (int i = 0; i < width * height; ++i) {
+ dst_pixels_opt[i * 4 + 0] = dst_pixels_c[i * 4 + 0];
+ dst_pixels_opt[i * 4 + 1] = dst_pixels_c[i * 4 + 1];
+ dst_pixels_opt[i * 4 + 2] = dst_pixels_c[i * 4 + 2];
+ }
- MaskCpuFlags(disable_cpu_flags_);
- ARGBCopyAlpha(orig_pixels, benchmark_width_ * 4, dst_pixels_c,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
- MaskCpuFlags(benchmark_cpu_info_);
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBCopyAlpha(orig_pixels + off, kStride, dst_pixels_c, kStride, width,
+ invert * height);
+ MaskCpuFlags(benchmark_cpu_info);
- for (int i = 0; i < benchmark_iterations_; ++i) {
- ARGBCopyAlpha(orig_pixels, benchmark_width_ * 4, dst_pixels_opt,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBCopyAlpha(orig_pixels + off, kStride, dst_pixels_opt, kStride, width,
+ invert * height);
}
+ int max_diff = 0;
for (int i = 0; i < kSize; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ int abs_diff = abs(static_cast<int>(dst_pixels_c[i]) -
+ static_cast<int>(dst_pixels_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
free_aligned_buffer_page_end(dst_pixels_c);
free_aligned_buffer_page_end(dst_pixels_opt);
free_aligned_buffer_page_end(orig_pixels);
+ return max_diff;
}
-TEST_F(LibYUVPlanarTest, TestARGBExtractAlpha) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(src_pixels, kPixels * 4);
+TEST_F(LibYUVPlanarTest, ARGBCopyAlpha_Any) {
+ int max_diff = TestARGBCopyAlpha(benchmark_width_ - 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyAlpha_Unaligned) {
+ int max_diff = TestARGBCopyAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyAlpha_Invert) {
+ int max_diff = TestARGBCopyAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyAlpha_Opt) {
+ int max_diff = TestARGBCopyAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+static int TestARGBExtractAlpha(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kPixels = width * height;
+ align_buffer_page_end(src_pixels, kPixels * 4 + off);
align_buffer_page_end(dst_pixels_opt, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels);
- MemRandomize(src_pixels, kPixels * 4);
- MemRandomize(dst_pixels_opt, kPixels);
- memcpy(dst_pixels_c, dst_pixels_opt, kPixels);
+ MemRandomize(src_pixels + off, kPixels * 4);
+ // Leave dst_pixels_opt uninitialized to verify MSan unpoisons all written
+ // alpha bytes (including masked tail stores).
+ MSAN_POISON(dst_pixels_opt, kPixels);
- MaskCpuFlags(disable_cpu_flags_);
- ARGBExtractAlpha(src_pixels, benchmark_width_ * 4, dst_pixels_c,
- benchmark_width_, benchmark_width_, benchmark_height_);
- MaskCpuFlags(benchmark_cpu_info_);
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBExtractAlpha(src_pixels + off, width * 4, dst_pixels_c, width, width,
+ invert * height);
- for (int i = 0; i < benchmark_iterations_; ++i) {
- ARGBExtractAlpha(src_pixels, benchmark_width_ * 4, dst_pixels_opt,
- benchmark_width_, benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBExtractAlpha(src_pixels + off, width * 4, dst_pixels_opt, width, width,
+ invert * height);
}
+ int max_diff = 0;
for (int i = 0; i < kPixels; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ int abs_diff = abs(static_cast<int>(dst_pixels_c[i]) -
+ static_cast<int>(dst_pixels_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
free_aligned_buffer_page_end(dst_pixels_c);
free_aligned_buffer_page_end(dst_pixels_opt);
free_aligned_buffer_page_end(src_pixels);
+ return max_diff;
}
-TEST_F(LibYUVPlanarTest, TestARGBCopyYToAlpha) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(orig_pixels, kPixels);
+TEST_F(LibYUVPlanarTest, ARGBExtractAlpha_Any) {
+ int max_diff = TestARGBExtractAlpha(benchmark_width_ - 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBExtractAlpha_Unaligned) {
+ int max_diff = TestARGBExtractAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBExtractAlpha_Invert) {
+ int max_diff = TestARGBExtractAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBExtractAlpha_Opt) {
+ int max_diff = TestARGBExtractAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+static int TestARGBCopyYToAlpha(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kPixels = width * height;
+ align_buffer_page_end(orig_pixels, kPixels + off);
align_buffer_page_end(dst_pixels_opt, kPixels * 4);
align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_uninit_opt, kPixels * 4);
- MemRandomize(orig_pixels, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 4);
- memcpy(dst_pixels_c, dst_pixels_opt, kPixels * 4);
+ MemRandomize(orig_pixels + off, kPixels);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+ // Copy RGB bytes only, leaving alpha bytes in dst_pixels_opt uninitialized.
+ MSAN_POISON(dst_pixels_opt, kPixels * 4);
+ for (int i = 0; i < kPixels; ++i) {
+ dst_pixels_opt[i * 4 + 0] = dst_pixels_c[i * 4 + 0];
+ dst_pixels_opt[i * 4 + 1] = dst_pixels_c[i * 4 + 1];
+ dst_pixels_opt[i * 4 + 2] = dst_pixels_c[i * 4 + 2];
+ }
+ // Also test with completely uninitialized destination buffer for MSan.
+ MSAN_POISON(dst_uninit_opt, kPixels * 4);
- MaskCpuFlags(disable_cpu_flags_);
- ARGBCopyYToAlpha(orig_pixels, benchmark_width_, dst_pixels_c,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
- MaskCpuFlags(benchmark_cpu_info_);
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBCopyYToAlpha(orig_pixels + off, width, dst_pixels_c, width * 4, width,
+ invert * height);
- for (int i = 0; i < benchmark_iterations_; ++i) {
- ARGBCopyYToAlpha(orig_pixels, benchmark_width_, dst_pixels_opt,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info);
+ ARGBCopyYToAlpha(orig_pixels + off, width, dst_uninit_opt, width * 4, width,
+ invert * height);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBCopyYToAlpha(orig_pixels + off, width, dst_pixels_opt, width * 4, width,
+ invert * height);
}
+ int max_diff = 0;
for (int i = 0; i < kPixels * 4; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ int abs_diff = abs(static_cast<int>(dst_pixels_c[i]) -
+ static_cast<int>(dst_pixels_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ for (int i = 0; i < kPixels; ++i) {
+ int abs_diff = abs(static_cast<int>(dst_pixels_c[i * 4 + 3]) -
+ static_cast<int>(dst_uninit_opt[i * 4 + 3]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
+ free_aligned_buffer_page_end(dst_uninit_opt);
free_aligned_buffer_page_end(dst_pixels_c);
free_aligned_buffer_page_end(dst_pixels_opt);
free_aligned_buffer_page_end(orig_pixels);
+ return max_diff;
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyYToAlpha_Any) {
+ int max_diff = TestARGBCopyYToAlpha(benchmark_width_ - 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyYToAlpha_Unaligned) {
+ int max_diff = TestARGBCopyYToAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyYToAlpha_Invert) {
+ int max_diff = TestARGBCopyYToAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0);
+ ASSERT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBCopyYToAlpha_Opt) {
+ int max_diff = TestARGBCopyYToAlpha(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ ASSERT_EQ(0, max_diff);
}
static int TestARGBRect(int width,
@@ -2432,153 +3250,210 @@ static int TestARGBRect(int width,
}
TEST_F(LibYUVPlanarTest, ARGBRect_Any) {
- int max_diff = TestARGBRect(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestARGBRect(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0, 4);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBRect_Unaligned) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1, 4);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBRect_Invert) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0, 4);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, ARGBRect_Opt) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0, 4);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, SetPlane_Any) {
- int max_diff = TestARGBRect(benchmark_width_ - 1, benchmark_height_,
+ int max_diff = TestARGBRect(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, SetPlane_Unaligned) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, SetPlane_Invert) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, SetPlane_Opt) {
int max_diff =
TestARGBRect(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0, 1);
- EXPECT_EQ(0, max_diff);
+ ASSERT_EQ(0, max_diff);
}
TEST_F(LibYUVPlanarTest, MergeUVPlane_Opt) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(src_pixels, kPixels * 2);
- align_buffer_page_end(tmp_pixels_u, kPixels);
- align_buffer_page_end(tmp_pixels_v, kPixels);
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_u, kPixels);
+ align_buffer_page_end(src_pixels_v, kPixels);
align_buffer_page_end(dst_pixels_opt, kPixels * 2);
align_buffer_page_end(dst_pixels_c, kPixels * 2);
- MemRandomize(src_pixels, kPixels * 2);
- MemRandomize(tmp_pixels_u, kPixels);
- MemRandomize(tmp_pixels_v, kPixels);
+ MemRandomize(src_pixels_u, kPixels);
+ MemRandomize(src_pixels_v, kPixels);
MemRandomize(dst_pixels_opt, kPixels * 2);
MemRandomize(dst_pixels_c, kPixels * 2);
MaskCpuFlags(disable_cpu_flags_);
- SplitUVPlane(src_pixels, benchmark_width_ * 2, tmp_pixels_u, benchmark_width_,
- tmp_pixels_v, benchmark_width_, benchmark_width_,
- benchmark_height_);
- MergeUVPlane(tmp_pixels_u, benchmark_width_, tmp_pixels_v, benchmark_width_,
+ MergeUVPlane(src_pixels_u, benchmark_width_, src_pixels_v, benchmark_width_,
dst_pixels_c, benchmark_width_ * 2, benchmark_width_,
benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
- SplitUVPlane(src_pixels, benchmark_width_ * 2, tmp_pixels_u, benchmark_width_,
- tmp_pixels_v, benchmark_width_, benchmark_width_,
- benchmark_height_);
-
for (int i = 0; i < benchmark_iterations_; ++i) {
- MergeUVPlane(tmp_pixels_u, benchmark_width_, tmp_pixels_v, benchmark_width_,
+ MergeUVPlane(src_pixels_u, benchmark_width_, src_pixels_v, benchmark_width_,
dst_pixels_opt, benchmark_width_ * 2, benchmark_width_,
benchmark_height_);
}
for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
- free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_u);
- free_aligned_buffer_page_end(tmp_pixels_v);
+ free_aligned_buffer_page_end(src_pixels_u);
+ free_aligned_buffer_page_end(src_pixels_v);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
+}
+
+// 16 bit channel split and merge
+TEST_F(LibYUVPlanarTest, MergeUVPlane_16_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_u, kPixels * 2);
+ align_buffer_page_end(src_pixels_v, kPixels * 2);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 2 * 2);
+ align_buffer_page_end(dst_pixels_c, kPixels * 2 * 2);
+ MemRandomize(src_pixels_u, kPixels * 2);
+ MemRandomize(src_pixels_v, kPixels * 2);
+ MemRandomize(dst_pixels_opt, kPixels * 2 * 2);
+ MemRandomize(dst_pixels_c, kPixels * 2 * 2);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ MergeUVPlane_16((const uint16_t*)src_pixels_u, benchmark_width_,
+ (const uint16_t*)src_pixels_v, benchmark_width_,
+ (uint16_t*)dst_pixels_c, benchmark_width_ * 2,
+ benchmark_width_, benchmark_height_, 12);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MergeUVPlane_16((const uint16_t*)src_pixels_u, benchmark_width_,
+ (const uint16_t*)src_pixels_v, benchmark_width_,
+ (uint16_t*)dst_pixels_opt, benchmark_width_ * 2,
+ benchmark_width_, benchmark_height_, 12);
+ }
+
+ for (int i = 0; i < kPixels * 2 * 2; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels_u);
+ free_aligned_buffer_page_end(src_pixels_v);
free_aligned_buffer_page_end(dst_pixels_opt);
free_aligned_buffer_page_end(dst_pixels_c);
}
TEST_F(LibYUVPlanarTest, SplitUVPlane_Opt) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+ const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 2);
- align_buffer_page_end(tmp_pixels_u, kPixels);
- align_buffer_page_end(tmp_pixels_v, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 2);
- align_buffer_page_end(dst_pixels_c, kPixels * 2);
+ align_buffer_page_end(dst_pixels_u_c, kPixels);
+ align_buffer_page_end(dst_pixels_v_c, kPixels);
+ align_buffer_page_end(dst_pixels_u_opt, kPixels);
+ align_buffer_page_end(dst_pixels_v_opt, kPixels);
MemRandomize(src_pixels, kPixels * 2);
- MemRandomize(tmp_pixels_u, kPixels);
- MemRandomize(tmp_pixels_v, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 2);
- MemRandomize(dst_pixels_c, kPixels * 2);
+ MemRandomize(dst_pixels_u_c, kPixels);
+ MemRandomize(dst_pixels_v_c, kPixels);
+ MemRandomize(dst_pixels_u_opt, kPixels);
+ MemRandomize(dst_pixels_v_opt, kPixels);
MaskCpuFlags(disable_cpu_flags_);
- SplitUVPlane(src_pixels, benchmark_width_ * 2, tmp_pixels_u, benchmark_width_,
- tmp_pixels_v, benchmark_width_, benchmark_width_,
- benchmark_height_);
- MergeUVPlane(tmp_pixels_u, benchmark_width_, tmp_pixels_v, benchmark_width_,
- dst_pixels_c, benchmark_width_ * 2, benchmark_width_,
- benchmark_height_);
+ SplitUVPlane(src_pixels, benchmark_width_ * 2, dst_pixels_u_c,
+ benchmark_width_, dst_pixels_v_c, benchmark_width_,
+ benchmark_width_, benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- SplitUVPlane(src_pixels, benchmark_width_ * 2, tmp_pixels_u,
- benchmark_width_, tmp_pixels_v, benchmark_width_,
+ SplitUVPlane(src_pixels, benchmark_width_ * 2, dst_pixels_u_opt,
+ benchmark_width_, dst_pixels_v_opt, benchmark_width_,
benchmark_width_, benchmark_height_);
}
- MergeUVPlane(tmp_pixels_u, benchmark_width_, tmp_pixels_v, benchmark_width_,
- dst_pixels_opt, benchmark_width_ * 2, benchmark_width_,
- benchmark_height_);
- for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ for (int i = 0; i < kPixels; ++i) {
+ ASSERT_EQ(dst_pixels_u_c[i], dst_pixels_u_opt[i]);
+ ASSERT_EQ(dst_pixels_v_c[i], dst_pixels_v_opt[i]);
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_u);
- free_aligned_buffer_page_end(tmp_pixels_v);
- free_aligned_buffer_page_end(dst_pixels_opt);
- free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_u_c);
+ free_aligned_buffer_page_end(dst_pixels_v_c);
+ free_aligned_buffer_page_end(dst_pixels_u_opt);
+ free_aligned_buffer_page_end(dst_pixels_v_opt);
+}
+
+// 16 bit channel split
+TEST_F(LibYUVPlanarTest, SplitUVPlane_16_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 2 * 2);
+ align_buffer_page_end(dst_pixels_u_c, kPixels * 2);
+ align_buffer_page_end(dst_pixels_v_c, kPixels * 2);
+ align_buffer_page_end(dst_pixels_u_opt, kPixels * 2);
+ align_buffer_page_end(dst_pixels_v_opt, kPixels * 2);
+ MemRandomize(src_pixels, kPixels * 2 * 2);
+ MemRandomize(dst_pixels_u_c, kPixels * 2);
+ MemRandomize(dst_pixels_v_c, kPixels * 2);
+ MemRandomize(dst_pixels_u_opt, kPixels * 2);
+ MemRandomize(dst_pixels_v_opt, kPixels * 2);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitUVPlane_16((const uint16_t*)src_pixels, benchmark_width_ * 2,
+ (uint16_t*)dst_pixels_u_c, benchmark_width_,
+ (uint16_t*)dst_pixels_v_c, benchmark_width_, benchmark_width_,
+ benchmark_height_, 10);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ SplitUVPlane_16((const uint16_t*)src_pixels, benchmark_width_ * 2,
+ (uint16_t*)dst_pixels_u_opt, benchmark_width_,
+ (uint16_t*)dst_pixels_v_opt, benchmark_width_,
+ benchmark_width_, benchmark_height_, 10);
+ }
+
+ for (int i = 0; i < kPixels * 2; ++i) {
+ ASSERT_EQ(dst_pixels_u_c[i], dst_pixels_u_opt[i]);
+ ASSERT_EQ(dst_pixels_v_c[i], dst_pixels_v_opt[i]);
+ }
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_u_c);
+ free_aligned_buffer_page_end(dst_pixels_v_c);
+ free_aligned_buffer_page_end(dst_pixels_u_opt);
+ free_aligned_buffer_page_end(dst_pixels_v_opt);
}
TEST_F(LibYUVPlanarTest, SwapUVPlane_Opt) {
// Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+ const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 2);
align_buffer_page_end(dst_pixels_opt, kPixels * 2);
align_buffer_page_end(dst_pixels_c, kPixels * 2);
@@ -2598,7 +3473,7 @@ TEST_F(LibYUVPlanarTest, SwapUVPlane_Opt) {
}
for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
free_aligned_buffer_page_end(src_pixels);
@@ -2608,169 +3483,712 @@ TEST_F(LibYUVPlanarTest, SwapUVPlane_Opt) {
TEST_F(LibYUVPlanarTest, MergeRGBPlane_Opt) {
// Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+ const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 3);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 3);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels * 3);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 3);
MemRandomize(src_pixels, kPixels * 3);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 3);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
MemRandomize(dst_pixels_c, kPixels * 3);
+ MemRandomize(dst_pixels_opt, kPixels * 3);
MaskCpuFlags(disable_cpu_flags_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_c,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ MergeRGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ dst_pixels_c, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g,
- benchmark_width_, tmp_pixels_b, benchmark_width_,
+ MergeRGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
dst_pixels_opt, benchmark_width_ * 3, benchmark_width_,
benchmark_height_);
}
for (int i = 0; i < kPixels * 3; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
}
TEST_F(LibYUVPlanarTest, SplitRGBPlane_Opt) {
// Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+ const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 3);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 3);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels * 3);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 3);
MemRandomize(src_pixels, kPixels * 3);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 3);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
MemRandomize(dst_pixels_c, kPixels * 3);
+ MemRandomize(dst_pixels_opt, kPixels * 3);
MaskCpuFlags(disable_cpu_flags_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_c,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ MergeRGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ dst_pixels_c, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, benchmark_width_,
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, benchmark_width_,
benchmark_height_);
}
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_opt,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ MergeRGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ dst_pixels_opt, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
for (int i = 0; i < kPixels * 3; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
}
-// TODO(fbarchard): improve test for platforms and cpu detect
-#ifdef HAS_MERGEUVROW_16_AVX2
-TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(src_pixels_u, kPixels * 2);
- align_buffer_page_end(src_pixels_v, kPixels * 2);
- align_buffer_page_end(dst_pixels_uv_opt, kPixels * 2 * 2);
- align_buffer_page_end(dst_pixels_uv_c, kPixels * 2 * 2);
+// Call SplitRGBPlane() with a width that is not a multiple of 32. Verify there
+// is no stack buffer overflow in SplitRGBRow_Any_AVX2().
+TEST_F(LibYUVPlanarTest, SplitRGBPlaneOverflow) {
+ int width = 1024 + 31; // 1055
+ const int height = 2;
+ align_buffer_page_end(src_rgb, width * 3 * height);
+ align_buffer_page_end(dst_r, width * height);
+ align_buffer_page_end(dst_g, width * height);
+ align_buffer_page_end(dst_b, width * height);
+ memset(src_rgb, 0x5A, width * 3 * height);
+ SplitRGBPlane(src_rgb, width * 3, dst_r, width, dst_g, width, dst_b, width,
+ width, height);
+ free_aligned_buffer_page_end(src_rgb);
+ free_aligned_buffer_page_end(dst_r);
+ free_aligned_buffer_page_end(dst_g);
+ free_aligned_buffer_page_end(dst_b);
+}
+
+TEST_F(LibYUVPlanarTest, MergeARGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(tmp_pixels_c_a, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_a, kPixels);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
- MemRandomize(src_pixels_u, kPixels * 2);
- MemRandomize(src_pixels_v, kPixels * 2);
- memset(dst_pixels_uv_opt, 0, kPixels * 2 * 2);
- memset(dst_pixels_uv_c, 1, kPixels * 2 * 2);
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(tmp_pixels_c_a, kPixels);
+ MemRandomize(tmp_pixels_opt_a, kPixels);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+ MemRandomize(dst_pixels_c, kPixels * 4);
- MergeUVRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_u),
- reinterpret_cast<const uint16_t*>(src_pixels_v),
- reinterpret_cast<uint16_t*>(dst_pixels_uv_c), 64, kPixels);
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, tmp_pixels_c_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ tmp_pixels_c_a, benchmark_width_, dst_pixels_c,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, tmp_pixels_opt_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
- MergeUVRow_16_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_u),
- reinterpret_cast<const uint16_t*>(src_pixels_v),
- reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 64,
- kPixels);
- } else {
- MergeUVRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_u),
- reinterpret_cast<const uint16_t*>(src_pixels_v),
- reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 64,
- kPixels);
- }
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
}
- for (int i = 0; i < kPixels * 2 * 2; ++i) {
- EXPECT_EQ(dst_pixels_uv_opt[i], dst_pixels_uv_c[i]);
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
- free_aligned_buffer_page_end(src_pixels_u);
- free_aligned_buffer_page_end(src_pixels_v);
- free_aligned_buffer_page_end(dst_pixels_uv_opt);
- free_aligned_buffer_page_end(dst_pixels_uv_c);
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(tmp_pixels_c_a);
+ free_aligned_buffer_page_end(tmp_pixels_opt_a);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
}
-#endif
-
-// TODO(fbarchard): Improve test for more platforms.
-#ifdef HAS_MULTIPLYROW_16_AVX2
-TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(src_pixels_y, kPixels * 2);
- align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
- align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
- MemRandomize(src_pixels_y, kPixels * 2);
- memset(dst_pixels_y_opt, 0, kPixels * 2);
+TEST_F(LibYUVPlanarTest, SplitARGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(tmp_pixels_c_a, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_a, kPixels);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(tmp_pixels_c_a, kPixels);
+ MemRandomize(tmp_pixels_opt_a, kPixels);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, tmp_pixels_c_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ tmp_pixels_c_a, benchmark_width_, dst_pixels_c,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, tmp_pixels_opt_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ }
+
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(tmp_pixels_c_a);
+ free_aligned_buffer_page_end(tmp_pixels_opt_a);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+// Call SplitARGBPlane() with a width that is not a multiple of 16. Verify there
+// is no stack buffer overflow in SplitXRGBRow_Any_AVX2().
+TEST_F(LibYUVPlanarTest, SplitARGBPlaneOverflow) {
+ int width = 1024 + 15; // 1039
+ int height = 2;
+ align_buffer_page_end(src_argb, width * 4 * height);
+ align_buffer_page_end(dst_r, width * height);
+ align_buffer_page_end(dst_g, width * height);
+ align_buffer_page_end(dst_b, width * height);
+ memset(src_argb, 0x5A, width * 4 * height);
+ // dst_a == NULL selects SplitXRGBRow
+ SplitARGBPlane(src_argb, width * 4, dst_r, width, dst_g, width, dst_b, width,
+ NULL, 0, width, height);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_r);
+ free_aligned_buffer_page_end(dst_g);
+ free_aligned_buffer_page_end(dst_b);
+}
+
+TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_, NULL, 0,
+ dst_pixels_c, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_, NULL,
+ 0, dst_pixels_opt, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+TEST_F(LibYUVPlanarTest, SplitXRGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_, NULL, 0,
+ dst_pixels_c, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, NULL, 0,
+ benchmark_width_, benchmark_height_);
+ }
+
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_, NULL, 0,
+ dst_pixels_opt, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+// Merge 4 channels
+#define TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVPlanarTest, FUNC##Plane_##DEPTH##N) { \
+ const int kWidth = W1280; \
+ const int kPixels = kWidth * benchmark_height_; \
+ align_buffer_page_end(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_a, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(dst_memory_c, kPixels * 4 * sizeof(DTYPE)); \
+ align_buffer_page_end(dst_memory_opt, kPixels * 4 * sizeof(DTYPE)); \
+ MemRandomize(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_a, kPixels * sizeof(STYPE) + OFF); \
+ memset(dst_memory_c, 0, kPixels * 4 * sizeof(DTYPE)); \
+ memset(dst_memory_opt, 0, kPixels * 4 * sizeof(DTYPE)); \
+ STYPE* src_pixels_r = reinterpret_cast<STYPE*>(src_memory_r + OFF); \
+ STYPE* src_pixels_g = reinterpret_cast<STYPE*>(src_memory_g + OFF); \
+ STYPE* src_pixels_b = reinterpret_cast<STYPE*>(src_memory_b + OFF); \
+ STYPE* src_pixels_a = reinterpret_cast<STYPE*>(src_memory_a + OFF); \
+ DTYPE* dst_pixels_c = reinterpret_cast<DTYPE*>(dst_memory_c); \
+ DTYPE* dst_pixels_opt = reinterpret_cast<DTYPE*>(dst_memory_opt); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, src_pixels_a, kWidth, dst_pixels_c, kWidth * 4, \
+ kWidth, NEG benchmark_height_, DEPTH); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, src_pixels_a, kWidth, dst_pixels_opt, kWidth * 4, \
+ kWidth, NEG benchmark_height_, DEPTH); \
+ } \
+ for (int i = 0; i < kPixels * 4; ++i) { \
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_memory_r); \
+ free_aligned_buffer_page_end(src_memory_g); \
+ free_aligned_buffer_page_end(src_memory_b); \
+ free_aligned_buffer_page_end(src_memory_a); \
+ free_aligned_buffer_page_end(dst_memory_c); \
+ free_aligned_buffer_page_end(dst_memory_opt); \
+ }
+
+// Merge 3 channel RGB into 4 channel XRGB with opaque alpha
+#define TESTQPLANAROTOPI(FUNC, STYPE, DTYPE, DEPTH, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVPlanarTest, FUNC##Plane_Opaque_##DEPTH##N) { \
+ const int kWidth = W1280; \
+ const int kPixels = kWidth * benchmark_height_; \
+ align_buffer_page_end(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(dst_memory_c, kPixels * 4 * sizeof(DTYPE)); \
+ align_buffer_page_end(dst_memory_opt, kPixels * 4 * sizeof(DTYPE)); \
+ MemRandomize(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ memset(dst_memory_c, 0, kPixels * 4 * sizeof(DTYPE)); \
+ memset(dst_memory_opt, 0, kPixels * 4 * sizeof(DTYPE)); \
+ STYPE* src_pixels_r = reinterpret_cast<STYPE*>(src_memory_r + OFF); \
+ STYPE* src_pixels_g = reinterpret_cast<STYPE*>(src_memory_g + OFF); \
+ STYPE* src_pixels_b = reinterpret_cast<STYPE*>(src_memory_b + OFF); \
+ DTYPE* dst_pixels_c = reinterpret_cast<DTYPE*>(dst_memory_c); \
+ DTYPE* dst_pixels_opt = reinterpret_cast<DTYPE*>(dst_memory_opt); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, NULL, 0, dst_pixels_c, kWidth * 4, kWidth, \
+ NEG benchmark_height_, DEPTH); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, NULL, 0, dst_pixels_opt, kWidth * 4, kWidth, \
+ NEG benchmark_height_, DEPTH); \
+ } \
+ for (int i = 0; i < kPixels * 4; ++i) { \
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_memory_r); \
+ free_aligned_buffer_page_end(src_memory_g); \
+ free_aligned_buffer_page_end(src_memory_b); \
+ free_aligned_buffer_page_end(dst_memory_c); \
+ free_aligned_buffer_page_end(dst_memory_opt); \
+ }
+
+#define TESTQPLANARTOP(FUNC, STYPE, DTYPE, DEPTH) \
+ TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_ + 1, _Any, +, 0) \
+ TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Unaligned, +, \
+ 2) \
+ TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Invert, -, 0) \
+ TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Opt, +, 0) \
+ TESTQPLANAROTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_ + 1, _Any, +, \
+ 0) \
+ TESTQPLANAROTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Unaligned, +, \
+ 2) \
+ TESTQPLANAROTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Invert, -, 0) \
+ TESTQPLANAROTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Opt, +, 0)
+
+TESTQPLANARTOP(MergeAR64, uint16_t, uint16_t, 10)
+TESTQPLANARTOP(MergeAR64, uint16_t, uint16_t, 12)
+TESTQPLANARTOP(MergeAR64, uint16_t, uint16_t, 16)
+TESTQPLANARTOP(MergeARGB16To8, uint16_t, uint8_t, 10)
+TESTQPLANARTOP(MergeARGB16To8, uint16_t, uint8_t, 12)
+TESTQPLANARTOP(MergeARGB16To8, uint16_t, uint8_t, 16)
+
+#define TESTTPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, W1280, N, NEG, OFF) \
+ TEST_F(LibYUVPlanarTest, FUNC##Plane_##DEPTH##N) { \
+ const int kWidth = W1280; \
+ const int kPixels = kWidth * benchmark_height_; \
+ align_buffer_page_end(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ align_buffer_page_end(dst_memory_c, kPixels * 4 * sizeof(DTYPE)); \
+ align_buffer_page_end(dst_memory_opt, kPixels * 4 * sizeof(DTYPE)); \
+ MemRandomize(src_memory_r, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_g, kPixels * sizeof(STYPE) + OFF); \
+ MemRandomize(src_memory_b, kPixels * sizeof(STYPE) + OFF); \
+ STYPE* src_pixels_r = reinterpret_cast<STYPE*>(src_memory_r + OFF); \
+ STYPE* src_pixels_g = reinterpret_cast<STYPE*>(src_memory_g + OFF); \
+ STYPE* src_pixels_b = reinterpret_cast<STYPE*>(src_memory_b + OFF); \
+ DTYPE* dst_pixels_c = reinterpret_cast<DTYPE*>(dst_memory_c); \
+ DTYPE* dst_pixels_opt = reinterpret_cast<DTYPE*>(dst_memory_opt); \
+ memset(dst_pixels_c, 1, kPixels * 4 * sizeof(DTYPE)); \
+ memset(dst_pixels_opt, 2, kPixels * 4 * sizeof(DTYPE)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, dst_pixels_c, kWidth * 4, kWidth, \
+ NEG benchmark_height_, DEPTH); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ FUNC##Plane(src_pixels_r, kWidth, src_pixels_g, kWidth, src_pixels_b, \
+ kWidth, dst_pixels_opt, kWidth * 4, kWidth, \
+ NEG benchmark_height_, DEPTH); \
+ } \
+ for (int i = 0; i < kPixels * 4; ++i) { \
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); \
+ } \
+ free_aligned_buffer_page_end(src_memory_r); \
+ free_aligned_buffer_page_end(src_memory_g); \
+ free_aligned_buffer_page_end(src_memory_b); \
+ free_aligned_buffer_page_end(dst_memory_c); \
+ free_aligned_buffer_page_end(dst_memory_opt); \
+ }
+
+#define TESTTPLANARTOP(FUNC, STYPE, DTYPE, DEPTH) \
+ TESTTPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_ + 1, _Any, +, 0) \
+ TESTTPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Unaligned, +, \
+ 2) \
+ TESTTPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Invert, -, 0) \
+ TESTTPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, benchmark_width_, _Opt, +, 0)
+
+TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 10)
+TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 12)
+TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 16)
+
+// TODO(fbarchard): improve test for platforms and cpu detect
+#if defined(HAS_MERGEUVROW_16_AVX512BW) || defined(HAS_MERGEUVROW_16_AVX2)
+TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) {
+ // Round count up to multiple of 16
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+
+ align_buffer_page_end(src_pixels_u, kPixels * 2);
+ align_buffer_page_end(src_pixels_v, kPixels * 2);
+ align_buffer_page_end(dst_pixels_uv_opt, kPixels * 2 * 2);
+ align_buffer_page_end(dst_pixels_uv_c, kPixels * 2 * 2);
+
+ MemRandomize(src_pixels_u, kPixels * 2);
+ MemRandomize(src_pixels_v, kPixels * 2);
+ memset(dst_pixels_uv_opt, 0, kPixels * 2 * 2);
+ memset(dst_pixels_uv_c, 1, kPixels * 2 * 2);
+
+ MergeUVRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_u),
+ reinterpret_cast<const uint16_t*>(src_pixels_v),
+ reinterpret_cast<uint16_t*>(dst_pixels_uv_c), 16, kPixels);
+
+#if defined(HAS_MERGEUVROW_16_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_MERGEUVROW_16_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+#if defined(HAS_MERGEUVROW_16_AVX512BW)
+ if (has_avx512) {
+ MergeUVRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_u),
+ reinterpret_cast<const uint16_t*>(src_pixels_v),
+ reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
+ kPixels);
+ } else
+#endif
+#if defined(HAS_MERGEUVROW_16_AVX2)
+ if (has_avx2) {
+ MergeUVRow_16_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_u),
+ reinterpret_cast<const uint16_t*>(src_pixels_v),
+ reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
+ kPixels);
+ } else
+#endif
+ {
+ MergeUVRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_u),
+ reinterpret_cast<const uint16_t*>(src_pixels_v),
+ reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
+ kPixels);
+ }
+ }
+
+ for (int i = 0; i < kPixels * 2 * 2; ++i) {
+ ASSERT_EQ(dst_pixels_uv_opt[i], dst_pixels_uv_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_u);
+ free_aligned_buffer_page_end(src_pixels_v);
+ free_aligned_buffer_page_end(dst_pixels_uv_opt);
+ free_aligned_buffer_page_end(dst_pixels_uv_c);
+}
+#endif
+
+#if defined(HAS_MULTIPLYROW_16_AVX512BW) || \
+ defined(HAS_MULTIPLYROW_16_AVX2) || defined(HAS_MULTIPLYROW_16_NEON) || \
+ defined(HAS_MULTIPLYROW_16_SME) || defined(HAS_MULTIPLYROW_16_RVV) || \
+ defined(HAS_MULTIPLYROW_16_HVX)
+TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
+ // Round count up to multiple of 64
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
+
+ align_buffer_page_end(src_pixels_y, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
+
+ MemRandomize(src_pixels_y, kPixels * 2);
+ memset(dst_pixels_y_opt, 0, kPixels * 2);
memset(dst_pixels_y_c, 1, kPixels * 2);
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_c), 64, kPixels);
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ int has_sme = TestCpuFlag(kCpuHasSME);
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ int has_rvv = TestCpuFlag(kCpuHasRVV);
+#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ int has_hvx = TestCpuFlag(kCpuHasHVX);
+#endif
+
for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (has_avx512) {
+ MultiplyRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ kPixels);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (has_avx2) {
MultiplyRow_16_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
- } else {
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (has_sme) {
+ MultiplyRow_16_SME(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ kPixels);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ if (has_neon) {
+ MultiplyRow_16_NEON(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ kPixels);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_RVV)
+ if (has_rvv) {
+ MultiplyRow_16_RVV(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ kPixels);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_HVX)
+ if (has_hvx) {
+ MultiplyRow_16_HVX(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ kPixels);
+ } else
+#endif
+ {
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
@@ -2778,18 +4196,85 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
}
for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+
+#if defined(HAS_MULTIPLYROW_16_AVX512BW) || \
+ defined(HAS_MULTIPLYROW_16_AVX2) || defined(HAS_MULTIPLYROW_16_NEON)
+TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) {
+ const int kMaxPixels = 256;
+ align_buffer_page_end(src_pixels_y, kMaxPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kMaxPixels * 2);
+ align_buffer_page_end(dst_pixels_y_c, kMaxPixels * 2);
+
+ for (int i = 0; i < kMaxPixels; ++i) {
+ reinterpret_cast<uint16_t*>(src_pixels_y)[i] = fastrand() & 1023;
+ }
+
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+#endif
+
+ for (int width = 1; width <= 129; ++width) {
+ memset(dst_pixels_y_opt, 0, kMaxPixels * 2);
+ memset(dst_pixels_y_c, 1, kMaxPixels * 2);
+
+ MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_c), 64, width);
+
+#if defined(HAS_MULTIPLYROW_16_AVX512BW)
+ if (has_avx512) {
+ MultiplyRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ width);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_AVX2)
+ if (has_avx2) {
+ MultiplyRow_16_Any_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ width);
+ } else
+#endif
+#if defined(HAS_MULTIPLYROW_16_NEON)
+ if (has_neon) {
+ MultiplyRow_16_Any_NEON(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ width);
+ } else
+#endif
+ {
+ MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
+ width);
+ }
+
+ for (int i = 0; i < width * 2; ++i) {
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
-#endif // HAS_MULTIPLYROW_16_AVX2
+#endif // HAS_MULTIPLYROW_16_AVX512BW || HAS_MULTIPLYROW_16_AVX2 ||
+ // HAS_MULTIPLYROW_16_NEON
+#endif // HAS_MULTIPLYROW_16_...
TEST_F(LibYUVPlanarTest, Convert16To8Plane) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
+ const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels_y, kPixels * 2);
align_buffer_page_end(dst_pixels_y_opt, kPixels);
align_buffer_page_end(dst_pixels_y_c, kPixels);
@@ -2811,7 +4296,7 @@ TEST_F(LibYUVPlanarTest, Convert16To8Plane) {
}
for (int i = 0; i < kPixels; ++i) {
- EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
free_aligned_buffer_page_end(src_pixels_y);
@@ -2819,81 +4304,88 @@ TEST_F(LibYUVPlanarTest, Convert16To8Plane) {
free_aligned_buffer_page_end(dst_pixels_y_c);
}
-#ifdef ENABLE_ROW_TESTS
-// TODO(fbarchard): Improve test for more platforms.
-#ifdef HAS_CONVERT16TO8ROW_AVX2
-TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
- // AVX2 does multiple of 32, so round count up
- const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
- align_buffer_page_end(src_pixels_y, kPixels * 2);
+TEST_F(LibYUVPlanarTest, Convert8To8Plane) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_y, kPixels);
align_buffer_page_end(dst_pixels_y_opt, kPixels);
align_buffer_page_end(dst_pixels_y_c, kPixels);
- MemRandomize(src_pixels_y, kPixels * 2);
- // clamp source range to 10 bits.
+ MemRandomize(src_pixels_y, kPixels);
+ memset(dst_pixels_y_opt, 0, kPixels);
+ memset(dst_pixels_y_c, 1, kPixels);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ Convert8To8Plane(src_pixels_y, benchmark_width_, dst_pixels_y_c,
+ benchmark_width_, 220, 16, benchmark_width_,
+ benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ Convert8To8Plane(src_pixels_y, benchmark_width_, dst_pixels_y_opt,
+ benchmark_width_, 220, 16, benchmark_width_,
+ benchmark_height_);
+ }
+
for (int i = 0; i < kPixels; ++i) {
- reinterpret_cast<uint16_t*>(src_pixels_y)[i] &= 1023;
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+
+TEST_F(LibYUVPlanarTest, YUY2ToY) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_y, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels);
+ align_buffer_page_end(dst_pixels_y_c, kPixels);
+
+ MemRandomize(src_pixels_y, kPixels * 2);
memset(dst_pixels_y_opt, 0, kPixels);
memset(dst_pixels_y_c, 1, kPixels);
- Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
- dst_pixels_y_c, 16384, kPixels);
+ MaskCpuFlags(disable_cpu_flags_);
+ YUY2ToY(src_pixels_y, benchmark_width_ * 2, dst_pixels_y_c, benchmark_width_,
+ benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
- Convert16To8Row_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
- dst_pixels_y_opt, 16384, kPixels);
- } else if (has_ssse3) {
- Convert16To8Row_SSSE3(reinterpret_cast<const uint16_t*>(src_pixels_y),
- dst_pixels_y_opt, 16384, kPixels);
- } else {
- Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
- dst_pixels_y_opt, 16384, kPixels);
- }
+ YUY2ToY(src_pixels_y, benchmark_width_ * 2, dst_pixels_y_opt,
+ benchmark_width_, benchmark_width_, benchmark_height_);
}
for (int i = 0; i < kPixels; ++i) {
- EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
-#endif // HAS_CONVERT16TO8ROW_AVX2
-#endif // ENABLE_ROW_TESTS
-TEST_F(LibYUVPlanarTest, Convert8To16Plane) {
- // Round count up to multiple of 16
- const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
- align_buffer_page_end(src_pixels_y, kPixels);
- align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
- align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
+TEST_F(LibYUVPlanarTest, UYVYToY) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_y, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels);
+ align_buffer_page_end(dst_pixels_y_c, kPixels);
- MemRandomize(src_pixels_y, kPixels);
- memset(dst_pixels_y_opt, 0, kPixels * 2);
- memset(dst_pixels_y_c, 1, kPixels * 2);
+ MemRandomize(src_pixels_y, kPixels * 2);
+ memset(dst_pixels_y_opt, 0, kPixels);
+ memset(dst_pixels_y_c, 1, kPixels);
MaskCpuFlags(disable_cpu_flags_);
- Convert8To16Plane(src_pixels_y, benchmark_width_,
- reinterpret_cast<uint16_t*>(dst_pixels_y_c),
- benchmark_width_, 1024, benchmark_width_,
- benchmark_height_);
+ UYVYToY(src_pixels_y, benchmark_width_ * 2, dst_pixels_y_c, benchmark_width_,
+ benchmark_width_, benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- Convert8To16Plane(src_pixels_y, benchmark_width_,
- reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
- benchmark_width_, 1024, benchmark_width_,
- benchmark_height_);
+ UYVYToY(src_pixels_y, benchmark_width_ * 2, dst_pixels_y_opt,
+ benchmark_width_, benchmark_width_, benchmark_height_);
}
- for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ for (int i = 0; i < kPixels; ++i) {
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
free_aligned_buffer_page_end(src_pixels_y);
@@ -2903,47 +4395,433 @@ TEST_F(LibYUVPlanarTest, Convert8To16Plane) {
#ifdef ENABLE_ROW_TESTS
// TODO(fbarchard): Improve test for more platforms.
-#ifdef HAS_CONVERT8TO16ROW_AVX2
-TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
- const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
- align_buffer_page_end(src_pixels_y, kPixels);
- align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
- align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
-
- MemRandomize(src_pixels_y, kPixels);
- memset(dst_pixels_y_opt, 0, kPixels * 2);
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW) || defined(HAS_CONVERT16TO8ROW_AVX2)
+TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
+ // AVX512 does multiple of 64, so round count up
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
+ align_buffer_page_end(src_pixels_y, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels);
+ align_buffer_page_end(dst_pixels_y_c, kPixels);
+
+ MemRandomize(src_pixels_y, kPixels * 2);
+ // clamp source range to 10 bits.
+ for (int i = 0; i < kPixels; ++i) {
+ reinterpret_cast<uint16_t*>(src_pixels_y)[i] &= 1023;
+ }
+
+ memset(dst_pixels_y_opt, 0, kPixels);
+ memset(dst_pixels_y_c, 1, kPixels);
+
+ Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ dst_pixels_y_c, 16384, kPixels);
+
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (has_avx512) {
+ Convert16To8Row_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ dst_pixels_y_opt, 16384, kPixels);
+ } else
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ if (has_avx2) {
+ Convert16To8Row_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ dst_pixels_y_opt, 16384, kPixels);
+ } else
+#endif
+ {
+ Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
+ dst_pixels_y_opt, 16384, kPixels);
+ }
+ }
+
+ for (int i = 0; i < kPixels; ++i) {
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+#endif // HAS_CONVERT16TO8ROW_AVX512BW || HAS_CONVERT16TO8ROW_AVX2
+
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \
+ defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || \
+ defined(HAS_HALFWIDTHROW_16TO8_NEON)
+TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Opt) {
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
+ align_buffer_page_end_16(src_pixels_uv, kPixels * 4);
+ align_buffer_page_end(dst_pixels_uv_opt, kPixels);
+ align_buffer_page_end(dst_pixels_uv_c, kPixels);
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ src_pixels_uv[i] = fastrand() & 1023;
+ }
+
+ memset(dst_pixels_uv_opt, 0, kPixels);
+ memset(dst_pixels_uv_c, 1, kPixels);
+
+ HalfWidthRow_16To8_C(src_pixels_uv, kPixels * 2, dst_pixels_uv_c, 16384,
+ kPixels);
+
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+#endif
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ if (has_avx512) {
+ HalfWidthRow_16To8_AVX512BW(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt,
+ 16384, kPixels);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (has_avx2) {
+ HalfWidthRow_16To8_AVX2(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt,
+ 16384, kPixels);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ if (has_ssse3) {
+ HalfWidthRow_16To8_SSSE3(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt,
+ 16384, kPixels);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ if (has_neon) {
+ HalfWidthRow_16To8_NEON(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt,
+ 16384, kPixels);
+ } else
+#endif
+ {
+ HalfWidthRow_16To8_C(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt, 16384,
+ kPixels);
+ }
+ }
+
+ for (int i = 0; i < kPixels; ++i) {
+ ASSERT_EQ(dst_pixels_uv_opt[i], dst_pixels_uv_c[i]);
+ }
+
+ free_aligned_buffer_page_end_16(src_pixels_uv);
+ free_aligned_buffer_page_end(dst_pixels_uv_opt);
+ free_aligned_buffer_page_end(dst_pixels_uv_c);
+}
+
+TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) {
+ const int kMaxPixels = 256;
+ align_buffer_page_end_16(src_pixels_uv, kMaxPixels * 4);
+ align_buffer_page_end(dst_pixels_uv_opt, kMaxPixels);
+ align_buffer_page_end(dst_pixels_uv_c, kMaxPixels);
+
+ for (int i = 0; i < kMaxPixels * 4; ++i) {
+ src_pixels_uv[i] = fastrand() & 1023;
+ }
+
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+#endif
+
+ for (int width = 1; width <= 129; ++width) {
+ memset(dst_pixels_uv_opt, 0, kMaxPixels);
+ memset(dst_pixels_uv_c, 1, kMaxPixels);
+
+ HalfWidthRow_16To8_C(src_pixels_uv, kMaxPixels * 2, dst_pixels_uv_c, 16384,
+ width);
+
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ if (has_avx512) {
+ HalfWidthRow_16To8_AVX512BW(src_pixels_uv, kMaxPixels * 2,
+ dst_pixels_uv_opt, 16384, width);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (has_avx2) {
+ HalfWidthRow_16To8_Any_AVX2(src_pixels_uv, kMaxPixels * 2,
+ dst_pixels_uv_opt, 16384, width);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ if (has_ssse3) {
+ HalfWidthRow_16To8_Any_SSSE3(src_pixels_uv, kMaxPixels * 2,
+ dst_pixels_uv_opt, 16384, width);
+ } else
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
+ if (has_neon) {
+ HalfWidthRow_16To8_Any_NEON(src_pixels_uv, kMaxPixels * 2,
+ dst_pixels_uv_opt, 16384, width);
+ } else
+#endif
+ {
+ HalfWidthRow_16To8_C(src_pixels_uv, kMaxPixels * 2, dst_pixels_uv_opt,
+ 16384, width);
+ }
+
+ for (int i = 0; i < width; ++i) {
+ ASSERT_EQ(dst_pixels_uv_opt[i], dst_pixels_uv_c[i]);
+ }
+ }
+
+ free_aligned_buffer_page_end_16(src_pixels_uv);
+ free_aligned_buffer_page_end(dst_pixels_uv_opt);
+ free_aligned_buffer_page_end(dst_pixels_uv_c);
+}
+#endif
+
+#if defined(HAS_UYVYTOYROW_AVX512BW) || defined(HAS_UYVYTOYROW_AVX2) || \
+ defined(HAS_UYVYTOYROW_NEON)
+TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) {
+ // AVX2/AVX512 does multiple of 32, so round count up
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
+ align_buffer_page_end(src_pixels_y, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels);
+ align_buffer_page_end(dst_pixels_y_c, kPixels);
+
+ MemRandomize(src_pixels_y, kPixels * 2);
+ memset(dst_pixels_y_opt, 0, kPixels);
+ memset(dst_pixels_y_c, 1, kPixels);
+
+ UYVYToYRow_C(src_pixels_y, dst_pixels_y_c, kPixels);
+
+#if defined(HAS_UYVYTOYROW_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
+#if defined(HAS_UYVYTOYROW_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+#endif
+#if defined(HAS_UYVYTOYROW_NEON)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+#endif
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+#if defined(HAS_UYVYTOYROW_AVX512BW)
+ if (has_avx512) {
+ UYVYToYRow_AVX512BW(src_pixels_y, dst_pixels_y_opt, kPixels);
+ } else
+#endif
+#if defined(HAS_UYVYTOYROW_AVX2)
+ if (has_avx2) {
+ UYVYToYRow_AVX2(src_pixels_y, dst_pixels_y_opt, kPixels);
+ } else
+#endif
+#if defined(HAS_UYVYTOYROW_NEON)
+ if (has_neon) {
+ UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels);
+ } else
+#endif
+ {
+ UYVYToYRow_C(src_pixels_y, dst_pixels_y_opt, kPixels);
+ }
+ }
+
+ for (int i = 0; i < kPixels; ++i) {
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+#endif // HAS_UYVYTOYROW_...
+
+#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_ARGB1555TOYROW_AVX2) && \
+ defined(HAS_ARGB4444TOYROW_AVX2)
+TEST_F(LibYUVPlanarTest, RGB16ToYUVRow_Opt) {
+ if (!TestCpuFlag(kCpuHasAVX2)) {
+ return;
+ }
+ const int kMaxWidth = 128;
+ SIMD_ALIGNED(uint8_t src0[kMaxWidth * 2]);
+ SIMD_ALIGNED(uint8_t src1[kMaxWidth * 2]);
+ SIMD_ALIGNED(uint8_t dst_y_c[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dst_y_opt[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dst_u_c[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dst_v_c[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dst_u_opt[kMaxWidth]);
+ SIMD_ALIGNED(uint8_t dst_v_opt[kMaxWidth]);
+ MemRandomize(src0, sizeof(src0));
+ MemRandomize(src1, sizeof(src1));
+
+ for (int width : {1, 2, 7, 15, 16, 17, 31, 32, 33, 64, 127, 128}) {
+ const int halfwidth = (width + 1) / 2;
+ const int stride = kMaxWidth * 2;
+
+ RGB565ToYRow_C(src0, dst_y_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ RGB565ToYRow_AVX2(src0, dst_y_opt, width);
+ } else {
+ RGB565ToYRow_Any_AVX2(src0, dst_y_opt, width);
+ }
+ for (int i = 0; i < width; ++i) {
+ ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
+ }
+
+ ARGB1555ToYRow_C(src0, dst_y_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ ARGB1555ToYRow_AVX2(src0, dst_y_opt, width);
+ } else {
+ ARGB1555ToYRow_Any_AVX2(src0, dst_y_opt, width);
+ }
+ for (int i = 0; i < width; ++i) {
+ ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
+ }
+
+ ARGB4444ToYRow_C(src0, dst_y_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ ARGB4444ToYRow_AVX2(src0, dst_y_opt, width);
+ } else {
+ ARGB4444ToYRow_Any_AVX2(src0, dst_y_opt, width);
+ }
+ for (int i = 0; i < width; ++i) {
+ ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
+ }
+
+ SIMD_ALIGNED(uint8_t src2rows[kMaxWidth * 2 * 2]);
+ memcpy(src2rows, src0, stride);
+ memcpy(src2rows + stride, src1, stride);
+
+ RGB565ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ RGB565ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ } else {
+ RGB565ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ }
+ for (int i = 0; i < halfwidth; ++i) {
+ ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
+ ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
+ }
+
+ ARGB1555ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ ARGB1555ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ } else {
+ ARGB1555ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ }
+ for (int i = 0; i < halfwidth; ++i) {
+ ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
+ ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
+ }
+
+ ARGB4444ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
+ if (IS_ALIGNED(width, 32)) {
+ ARGB4444ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ } else {
+ ARGB4444ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
+ }
+ for (int i = 0; i < halfwidth; ++i) {
+ ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
+ ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
+ }
+ }
+}
+#endif
+
+#endif // ENABLE_ROW_TESTS
+
+TEST_F(LibYUVPlanarTest, Convert8To16Plane) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_y, kPixels);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
+
+ MemRandomize(src_pixels_y, kPixels);
+ memset(dst_pixels_y_opt, 0, kPixels * 2);
+ memset(dst_pixels_y_c, 1, kPixels * 2);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ Convert8To16Plane(src_pixels_y, benchmark_width_,
+ reinterpret_cast<uint16_t*>(dst_pixels_y_c),
+ benchmark_width_, 10, benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ Convert8To16Plane(src_pixels_y, benchmark_width_,
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
+ benchmark_width_, 1024, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < kPixels * 2; ++i) {
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+
+#ifdef ENABLE_ROW_TESTS
+// TODO(fbarchard): Improve test for more platforms.
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW) || defined(HAS_CONVERT8TO16ROW_AVX2)
+TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
+ const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
+ align_buffer_page_end(src_pixels_y, kPixels);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
+ align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
+
+ MemRandomize(src_pixels_y, kPixels);
+ memset(dst_pixels_y_opt, 0, kPixels * 2);
memset(dst_pixels_y_c, 1, kPixels * 2);
Convert8To16Row_C(src_pixels_y, reinterpret_cast<uint16_t*>(dst_pixels_y_c),
- 1024, kPixels);
+ 10, kPixels);
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
+#endif
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_sse2 = TestCpuFlag(kCpuHasSSE2);
for (int i = 0; i < benchmark_iterations_; ++i) {
- if (has_avx2) {
+#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
+ if (has_avx512) {
+ Convert8To16Row_AVX512BW(src_pixels_y,
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
+ 10, kPixels);
+ } else
+#endif
+ if (has_avx2) {
Convert8To16Row_AVX2(src_pixels_y,
- reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 1024,
- kPixels);
- } else if (has_sse2) {
- Convert8To16Row_SSE2(src_pixels_y,
- reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 1024,
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 10,
kPixels);
} else {
Convert8To16Row_C(src_pixels_y,
- reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 1024,
+ reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 10,
kPixels);
}
}
for (int i = 0; i < kPixels * 2; ++i) {
- EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
-#endif // HAS_CONVERT8TO16ROW_AVX2
+#endif // HAS_CONVERT8TO16ROW_AVX512BW || HAS_CONVERT8TO16ROW_AVX2
float TestScaleMaxSamples(int benchmark_width,
int benchmark_height,
@@ -2976,6 +4854,16 @@ float TestScaleMaxSamples(int benchmark_width,
max_opt = ScaleMaxSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
+#elif defined(HAS_SCALEMAXSAMPLES_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ max_opt = ScaleMaxSamples_AVX2(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale,
+ kPixels);
+ } else {
+ max_opt = ScaleMaxSamples_C(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale,
+ kPixels);
+ }
#else
max_opt =
ScaleMaxSamples_C(reinterpret_cast<float*>(orig_y),
@@ -3004,13 +4892,13 @@ float TestScaleMaxSamples(int benchmark_width,
TEST_F(LibYUVPlanarTest, TestScaleMaxSamples_C) {
float diff = TestScaleMaxSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, false);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestScaleMaxSamples_Opt) {
float diff = TestScaleMaxSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, true);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
float TestScaleSumSamples(int benchmark_width,
@@ -3044,6 +4932,16 @@ float TestScaleSumSamples(int benchmark_width,
sum_opt = ScaleSumSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
+#elif defined(HAS_SCALESUMSAMPLES_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ sum_opt = ScaleSumSamples_AVX2(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale,
+ kPixels);
+ } else {
+ sum_opt = ScaleSumSamples_C(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale,
+ kPixels);
+ }
#else
sum_opt =
ScaleSumSamples_C(reinterpret_cast<float*>(orig_y),
@@ -3083,13 +4981,13 @@ float TestScaleSumSamples(int benchmark_width,
TEST_F(LibYUVPlanarTest, TestScaleSumSamples_C) {
float diff = TestScaleSumSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, false);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestScaleSumSamples_Opt) {
float diff = TestScaleSumSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, true);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
float TestScaleSamples(int benchmark_width,
@@ -3121,6 +5019,14 @@ float TestScaleSamples(int benchmark_width,
#ifdef HAS_SCALESUMSAMPLES_NEON
ScaleSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale, kPixels);
+#elif defined(HAS_SCALESAMPLES_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ScaleSamples_AVX2(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale, kPixels);
+ } else {
+ ScaleSamples_C(reinterpret_cast<float*>(orig_y),
+ reinterpret_cast<float*>(dst_opt), scale, kPixels);
+ }
#else
ScaleSamples_C(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale, kPixels);
@@ -3147,13 +5053,13 @@ float TestScaleSamples(int benchmark_width,
TEST_F(LibYUVPlanarTest, TestScaleSamples_C) {
float diff = TestScaleSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, false);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestScaleSamples_Opt) {
float diff = TestScaleSamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, 1.2f, true);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
float TestCopySamples(int benchmark_width,
@@ -3207,52 +5113,59 @@ float TestCopySamples(int benchmark_width,
TEST_F(LibYUVPlanarTest, TestCopySamples_C) {
float diff = TestCopySamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, false);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestCopySamples_Opt) {
float diff = TestCopySamples(benchmark_width_, benchmark_height_,
benchmark_iterations_, true);
- EXPECT_EQ(0, diff);
+ ASSERT_EQ(0, diff);
}
extern "C" void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width);
extern "C" void GaussRow_C(const uint32_t* src, uint16_t* dst, int width);
TEST_F(LibYUVPlanarTest, TestGaussRow_Opt) {
- SIMD_ALIGNED(uint32_t orig_pixels[640 + 4]);
- SIMD_ALIGNED(uint16_t dst_pixels_c[640]);
- SIMD_ALIGNED(uint16_t dst_pixels_opt[640]);
+ SIMD_ALIGNED(uint32_t orig_pixels[1280 + 8]);
+ SIMD_ALIGNED(uint16_t dst_pixels_c[1280]);
+ SIMD_ALIGNED(uint16_t dst_pixels_opt[1280]);
memset(orig_pixels, 0, sizeof(orig_pixels));
memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
- for (int i = 0; i < 640 + 4; ++i) {
+ for (int i = 0; i < 1280 + 8; ++i) {
orig_pixels[i] = i * 256;
}
- GaussRow_C(&orig_pixels[0], &dst_pixels_c[0], 640);
- for (int i = 0; i < benchmark_pixels_div1280_ * 2; ++i) {
-#if !defined(LIBYUV_DISABLE_NEON) && \
+ GaussRow_C(&orig_pixels[0], &dst_pixels_c[0], 1280);
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+#if defined(HAS_GAUSSROW_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ GaussRow_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ } else {
+ GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ }
+#elif !defined(LIBYUV_DISABLE_NEON) && \
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
int has_neon = TestCpuFlag(kCpuHasNEON);
if (has_neon) {
- GaussRow_NEON(&orig_pixels[0], &dst_pixels_opt[0], 640);
+ GaussRow_NEON(&orig_pixels[0], &dst_pixels_opt[0], 1280);
} else {
- GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 640);
+ GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
}
#else
- GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 640);
+ GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
#endif
}
- for (int i = 0; i < 640; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ for (int i = 0; i < 1280; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
- EXPECT_EQ(dst_pixels_c[0],
+ ASSERT_EQ(dst_pixels_c[0],
static_cast<uint16_t>(0 * 1 + 1 * 4 + 2 * 6 + 3 * 4 + 4 * 1));
- EXPECT_EQ(dst_pixels_c[639], static_cast<uint16_t>(10256));
+ ASSERT_EQ(dst_pixels_c[639], static_cast<uint16_t>(10256));
}
extern "C" void GaussCol_NEON(const uint16_t* src0,
@@ -3272,48 +5185,144 @@ extern "C" void GaussCol_C(const uint16_t* src0,
int width);
TEST_F(LibYUVPlanarTest, TestGaussCol_Opt) {
- SIMD_ALIGNED(uint16_t orig_pixels[640 * 5]);
- SIMD_ALIGNED(uint32_t dst_pixels_c[640]);
- SIMD_ALIGNED(uint32_t dst_pixels_opt[640]);
+ SIMD_ALIGNED(uint16_t orig_pixels[1280 * 5]);
+ SIMD_ALIGNED(uint32_t dst_pixels_c[1280]);
+ SIMD_ALIGNED(uint32_t dst_pixels_opt[1280]);
memset(orig_pixels, 0, sizeof(orig_pixels));
memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
- for (int i = 0; i < 640 * 5; ++i) {
- orig_pixels[i] = i;
+ for (int i = 0; i < 1280 * 5; ++i) {
+ orig_pixels[i] = static_cast<float>(i);
}
- GaussCol_C(&orig_pixels[0], &orig_pixels[640], &orig_pixels[640 * 2],
- &orig_pixels[640 * 3], &orig_pixels[640 * 4], &dst_pixels_c[0],
- 640);
- for (int i = 0; i < benchmark_pixels_div1280_ * 2; ++i) {
-#if !defined(LIBYUV_DISABLE_NEON) && \
+ GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0],
+ 1280);
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+#if defined(HAS_GAUSSCOL_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ GaussCol_AVX2(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
+ } else {
+ GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
+ }
+#elif !defined(LIBYUV_DISABLE_NEON) && \
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
int has_neon = TestCpuFlag(kCpuHasNEON);
if (has_neon) {
- GaussCol_NEON(&orig_pixels[0], &orig_pixels[640], &orig_pixels[640 * 2],
- &orig_pixels[640 * 3], &orig_pixels[640 * 4],
- &dst_pixels_opt[0], 640);
+ GaussCol_NEON(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
} else {
- GaussCol_C(&orig_pixels[0], &orig_pixels[640], &orig_pixels[640 * 2],
- &orig_pixels[640 * 3], &orig_pixels[640 * 4],
- &dst_pixels_opt[0], 640);
+ GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
}
#else
- GaussCol_C(&orig_pixels[0], &orig_pixels[640], &orig_pixels[640 * 2],
- &orig_pixels[640 * 3], &orig_pixels[640 * 4], &dst_pixels_opt[0],
- 640);
+ GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
#endif
}
- for (int i = 0; i < 640; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ for (int i = 0; i < 1280; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+}
+
+TEST_F(LibYUVPlanarTest, TestGaussRow_F32_Opt) {
+ SIMD_ALIGNED(float orig_pixels[1280 + 4]);
+ SIMD_ALIGNED(float dst_pixels_c[1280]);
+ SIMD_ALIGNED(float dst_pixels_opt[1280]);
+
+ memset(orig_pixels, 0, sizeof(orig_pixels));
+ memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
+ memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
+
+ for (int i = 0; i < 1280 + 4; ++i) {
+ orig_pixels[i] = static_cast<float>(i);
+ }
+ GaussRow_F32_C(&orig_pixels[0], &dst_pixels_c[0], 1280);
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+#if defined(HAS_GAUSSROW_F32_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ GaussRow_F32_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ } else {
+ GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ }
+#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ if (has_neon) {
+ GaussRow_F32_NEON(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ } else {
+ GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+ }
+#else
+ GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
+#endif
+ }
+
+ for (int i = 0; i < 1280; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+}
+
+TEST_F(LibYUVPlanarTest, TestGaussCol_F32_Opt) {
+ SIMD_ALIGNED(float dst_pixels_c[1280]);
+ SIMD_ALIGNED(float dst_pixels_opt[1280]);
+ align_buffer_page_end(orig_pixels_buf, 1280 * 5 * 4); // 5 rows
+ float* orig_pixels = reinterpret_cast<float*>(orig_pixels_buf);
+
+ memset(orig_pixels, 0, 1280 * 5 * 4);
+ memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
+ memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
+
+ for (int i = 0; i < 1280 * 5; ++i) {
+ orig_pixels[i] = static_cast<float>(i);
+ }
+ GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_c[0], 1280);
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+#if defined(HAS_GAUSSCOL_F32_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ GaussCol_F32_AVX2(&orig_pixels[0], &orig_pixels[1280],
+ &orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
+ &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
+ } else {
+ GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280],
+ &orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
+ &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
+ }
+#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ if (has_neon) {
+ GaussCol_F32_NEON(&orig_pixels[0], &orig_pixels[1280],
+ &orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
+ &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
+ } else {
+ GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280],
+ &orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
+ &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
+ }
+#else
+ GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
+ &orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
+ &dst_pixels_opt[0], 1280);
+#endif
}
- EXPECT_EQ(dst_pixels_c[0],
- static_cast<uint32_t>(0 * 1 + 640 * 4 + 640 * 2 * 6 + 640 * 3 * 4 +
- 640 * 4 * 1));
- EXPECT_EQ(dst_pixels_c[639], static_cast<uint32_t>(30704));
+ for (int i = 0; i < 1280; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ free_aligned_buffer_page_end(orig_pixels_buf);
}
TEST_F(LibYUVPlanarTest, SwapUVRow) {
@@ -3326,6 +5335,22 @@ TEST_F(LibYUVPlanarTest, SwapUVRow) {
MemRandomize(src_pixels_vu, kPixels * 2);
memset(dst_pixels_uv, 1, kPixels * 2);
+#if defined(HAS_SWAPUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ SwapUVRow = SwapUVRow_Any_SSSE3;
+ if (IS_ALIGNED(kPixels, 16)) {
+ SwapUVRow = SwapUVRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_SWAPUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SwapUVRow = SwapUVRow_Any_AVX2;
+ if (IS_ALIGNED(kPixels, 32)) {
+ SwapUVRow = SwapUVRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_SWAPUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SwapUVRow = SwapUVRow_Any_NEON;
@@ -3334,18 +5359,1274 @@ TEST_F(LibYUVPlanarTest, SwapUVRow) {
}
}
#endif
+#if defined(HAS_SWAPUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ SwapUVRow = SwapUVRow_RVV;
+ }
+#endif
+#if defined(HAS_SWAPUVROW_HVX)
+ if (TestCpuFlag(kCpuHasHVX)) {
+ SwapUVRow = SwapUVRow_HVX;
+ }
+#endif
for (int j = 0; j < benchmark_iterations_; j++) {
SwapUVRow(src_pixels_vu, dst_pixels_uv, kPixels);
}
for (int i = 0; i < kPixels; ++i) {
- EXPECT_EQ(dst_pixels_uv[i * 2 + 0], src_pixels_vu[i * 2 + 1]);
- EXPECT_EQ(dst_pixels_uv[i * 2 + 1], src_pixels_vu[i * 2 + 0]);
+ ASSERT_EQ(dst_pixels_uv[i * 2 + 0], src_pixels_vu[i * 2 + 1]);
+ ASSERT_EQ(dst_pixels_uv[i * 2 + 1], src_pixels_vu[i * 2 + 0]);
}
free_aligned_buffer_page_end(src_pixels_vu);
free_aligned_buffer_page_end(dst_pixels_uv);
}
+
+TEST_F(LibYUVPlanarTest, HalfRow_Opt) {
+ const int kMaxPixels = 160;
+ align_buffer_page_end(src_pixels, kMaxPixels * 2);
+ align_buffer_page_end(dst_pixels_opt, kMaxPixels);
+ align_buffer_page_end(dst_pixels_c, kMaxPixels);
+ MemRandomize(src_pixels, kMaxPixels * 2);
+
+ for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) {
+ for (int width = 1; width <= 129; ++width) {
+ memset(dst_pixels_opt, 0, kMaxPixels);
+ memset(dst_pixels_c, 1, kMaxPixels);
+ HalfRow_C(src_pixels, stride, dst_pixels_c, width);
+#if defined(HAS_HALFROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ if (IS_ALIGNED(width, 32)) {
+ HalfRow_AVX2(src_pixels, stride, dst_pixels_opt, width);
+ } else {
+ HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
+ }
+ } else
+#endif
+ {
+ HalfRow_C(src_pixels, stride, dst_pixels_opt, width);
+ }
+ ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width))
+ << "stride=" << stride << " width=" << width;
+ }
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
+}
+
+TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) {
+ const int kMaxPixels = 160;
+ align_buffer_page_end_16(src_pixels, kMaxPixels * 2);
+ align_buffer_page_end_16(dst_pixels_opt, kMaxPixels);
+ align_buffer_page_end_16(dst_pixels_c, kMaxPixels);
+ for (int i = 0; i < kMaxPixels * 2; ++i) {
+ src_pixels[i] = static_cast<uint16_t>(fastrand());
+ }
+
+ for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) {
+ for (int width = 1; width <= 129; ++width) {
+ memset(dst_pixels_opt, 0, kMaxPixels * 2);
+ memset(dst_pixels_c, 1, kMaxPixels * 2);
+ HalfRow_16_C(src_pixels, stride, dst_pixels_c, width);
+#if defined(HAS_HALFROW_16_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ if (IS_ALIGNED(width, 16)) {
+ HalfRow_16_AVX2(src_pixels, stride, dst_pixels_opt, width);
+ } else {
+ HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
+ }
+ } else
+#endif
+ {
+ HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width);
+ }
+ ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * 2))
+ << "stride=" << stride << " width=" << width;
+ }
+ }
+
+ free_aligned_buffer_page_end_16(src_pixels);
+ free_aligned_buffer_page_end_16(dst_pixels_opt);
+ free_aligned_buffer_page_end_16(dst_pixels_c);
+}
+
+TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) {
+ const int kMaxPixels = 160;
+ align_buffer_page_end_16(src_pixels, kMaxPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kMaxPixels);
+ align_buffer_page_end(dst_pixels_c, kMaxPixels);
+ for (int i = 0; i < kMaxPixels * 4; ++i) {
+ src_pixels[i] = fastrand() & 1023;
+ }
+
+ for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels * 2}}) {
+ for (int width = 1; width <= 129; ++width) {
+ memset(dst_pixels_opt, 0, kMaxPixels);
+ memset(dst_pixels_c, 1, kMaxPixels);
+ HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_c, 16384, width);
+#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384,
+ width);
+ } else
#endif
+ {
+ HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384,
+ width);
+ }
+ ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width))
+ << "stride=" << stride << " width=" << width;
+ }
+ }
+
+ free_aligned_buffer_page_end_16(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(dst_pixels_c);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBChannelReorderRow_C_Test) {
+ const int kPixels = 65;
+ align_buffer_page_end(src_argb, kPixels * 4);
+ align_buffer_page_end(dst_row, kPixels * 4);
+ align_buffer_page_end(dst_api, kPixels * 4);
+ MemRandomize(src_argb, kPixels * 4);
+
+ ARGBToABGRRow_C(src_argb, dst_row, kPixels);
+ ARGBToABGR(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
+ ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
+
+ ARGBToBGRARow_C(src_argb, dst_row, kPixels);
+ ARGBToBGRA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
+ ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
+
+ ARGBToRGBARow_C(src_argb, dst_row, kPixels);
+ ARGBToRGBA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
+ ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
+
+ RGBAToARGBRow_C(src_argb, dst_row, kPixels);
+ RGBAToARGB(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
+ ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
+
+ align_buffer_page_end_16(src_ar64, kPixels * 4);
+ align_buffer_page_end_16(dst_ab64_row, kPixels * 4);
+ align_buffer_page_end_16(dst_ab64_api, kPixels * 4);
+ for (int i = 0; i < kPixels * 4; ++i) {
+ src_ar64[i] = static_cast<uint16_t>(fastrand());
+ }
+ AR64ToAB64Row_C(src_ar64, dst_ab64_row, kPixels);
+ AR64ToAB64(src_ar64, kPixels * 4, dst_ab64_api, kPixels * 4, kPixels, 1);
+ ASSERT_EQ(0, memcmp(dst_ab64_row, dst_ab64_api, kPixels * 8));
+
+ free_aligned_buffer_page_end_16(src_ar64);
+ free_aligned_buffer_page_end_16(dst_ab64_row);
+ free_aligned_buffer_page_end_16(dst_ab64_api);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_row);
+ free_aligned_buffer_page_end(dst_api);
+}
+#endif // ENABLE_ROW_TESTS
+
+TEST_F(LibYUVPlanarTest, TestGaussPlane_F32) {
+ const int kSize = benchmark_width_ * benchmark_height_ * 4;
+ align_buffer_page_end(orig_pixels, kSize);
+ align_buffer_page_end(dst_pixels_opt, kSize);
+ align_buffer_page_end(dst_pixels_c, kSize);
+
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ ((float*)(orig_pixels))[i] = (i & 1023) * 3.14f;
+ }
+ memset(dst_pixels_opt, 1, kSize);
+ memset(dst_pixels_c, 2, kSize);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ GaussPlane_F32((const float*)(orig_pixels), benchmark_width_,
+ (float*)(dst_pixels_c), benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ GaussPlane_F32((const float*)(orig_pixels), benchmark_width_,
+ (float*)(dst_pixels_opt), benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ }
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ ASSERT_NEAR(((float*)(dst_pixels_c))[i], ((float*)(dst_pixels_opt))[i], 1.f)
+ << i;
+ }
+
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVPlanarTest, HalfMergeUVPlane_Opt) {
+ int dst_width = (benchmark_width_ + 1) / 2;
+ int dst_height = (benchmark_height_ + 1) / 2;
+ align_buffer_page_end(src_pixels_u, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(src_pixels_v, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(dst_pixels_uv_opt, dst_width * 2 * dst_height);
+ align_buffer_page_end(dst_pixels_uv_c, dst_width * 2 * dst_height);
+
+ MemRandomize(src_pixels_u, benchmark_width_ * benchmark_height_);
+ MemRandomize(src_pixels_v, benchmark_width_ * benchmark_height_);
+ MemRandomize(dst_pixels_uv_opt, dst_width * 2 * dst_height);
+ MemRandomize(dst_pixels_uv_c, dst_width * 2 * dst_height);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ HalfMergeUVPlane(src_pixels_u, benchmark_width_, src_pixels_v,
+ benchmark_width_, dst_pixels_uv_c, dst_width * 2,
+ benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ HalfMergeUVPlane(src_pixels_u, benchmark_width_, src_pixels_v,
+ benchmark_width_, dst_pixels_uv_opt, dst_width * 2,
+ benchmark_width_, benchmark_height_);
+ }
+
+ for (int i = 0; i < dst_width * 2 * dst_height; ++i) {
+ ASSERT_EQ(dst_pixels_uv_c[i], dst_pixels_uv_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_u);
+ free_aligned_buffer_page_end(src_pixels_v);
+ free_aligned_buffer_page_end(dst_pixels_uv_opt);
+ free_aligned_buffer_page_end(dst_pixels_uv_c);
+}
+
+TEST_F(LibYUVPlanarTest, NV12Copy) {
+ const int halfwidth = (benchmark_width_ + 1) >> 1;
+ const int halfheight = (benchmark_height_ + 1) >> 1;
+ align_buffer_page_end(src_y, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(src_uv, halfwidth * 2 * halfheight);
+ align_buffer_page_end(dst_y, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(dst_uv, halfwidth * 2 * halfheight);
+
+ MemRandomize(src_y, benchmark_width_ * benchmark_height_);
+ MemRandomize(src_uv, halfwidth * 2 * halfheight);
+ MemRandomize(dst_y, benchmark_width_ * benchmark_height_);
+ MemRandomize(dst_uv, halfwidth * 2 * halfheight);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ NV12Copy(src_y, benchmark_width_, src_uv, halfwidth * 2, dst_y,
+ benchmark_width_, dst_uv, halfwidth * 2, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ ASSERT_EQ(src_y[i], dst_y[i]);
+ }
+ for (int i = 0; i < halfwidth * 2 * halfheight; ++i) {
+ ASSERT_EQ(src_uv[i], dst_uv[i]);
+ }
+
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_uv);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_uv);
+}
+
+TEST_F(LibYUVPlanarTest, NV21Copy) {
+ const int halfwidth = (benchmark_width_ + 1) >> 1;
+ const int halfheight = (benchmark_height_ + 1) >> 1;
+ align_buffer_page_end(src_y, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(src_vu, halfwidth * 2 * halfheight);
+ align_buffer_page_end(dst_y, benchmark_width_ * benchmark_height_);
+ align_buffer_page_end(dst_vu, halfwidth * 2 * halfheight);
+
+ MemRandomize(src_y, benchmark_width_ * benchmark_height_);
+ MemRandomize(src_vu, halfwidth * 2 * halfheight);
+ MemRandomize(dst_y, benchmark_width_ * benchmark_height_);
+ MemRandomize(dst_vu, halfwidth * 2 * halfheight);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ NV21Copy(src_y, benchmark_width_, src_vu, halfwidth * 2, dst_y,
+ benchmark_width_, dst_vu, halfwidth * 2, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
+ ASSERT_EQ(src_y[i], dst_y[i]);
+ }
+ for (int i = 0; i < halfwidth * 2 * halfheight; ++i) {
+ ASSERT_EQ(src_vu[i], dst_vu[i]);
+ }
+
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_vu);
+ free_aligned_buffer_page_end(dst_y);
+ free_aligned_buffer_page_end(dst_vu);
+}
+
+#if defined(ENABLE_ROW_TESTS) && !defined(LIBYUV_DISABLE_NEON) && \
+ defined(__aarch64__)
+
+TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) {
+ int i, j;
+ const int y_plane_size = benchmark_width_ * benchmark_height_;
+
+ align_buffer_page_end(orig_f, y_plane_size * 4);
+ align_buffer_page_end(orig_y, y_plane_size * 2);
+ align_buffer_page_end(dst_opt, y_plane_size * 4);
+ align_buffer_page_end(rec_opt, y_plane_size * 2);
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ((float*)orig_f)[i] = (float)(i % 10000) * 3.14f;
+ }
+ memset(orig_y, 1, y_plane_size * 2);
+ memset(dst_opt, 2, y_plane_size * 4);
+ memset(rec_opt, 3, y_plane_size * 2);
+
+ ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y,
+ y_plane_size);
+
+ for (j = 0; j < benchmark_iterations_; j++) {
+ ConvertFP16ToFP32Row_NEON((const uint16_t*)orig_y, (float*)dst_opt,
+ y_plane_size);
+ }
+
+ ConvertFP32ToFP16Row_NEON((const float*)dst_opt, (uint16_t*)rec_opt,
+ y_plane_size);
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ASSERT_EQ(((const uint16_t*)orig_y)[i], ((const uint16_t*)rec_opt)[i]);
+ }
+
+ free_aligned_buffer_page_end(orig_f);
+ free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(rec_opt);
+}
+
+TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) {
+ int i, j;
+ const int y_plane_size = benchmark_width_ * benchmark_height_;
+
+ align_buffer_page_end(orig_f, y_plane_size * 4);
+ align_buffer_page_end(orig_y, y_plane_size * 2);
+ align_buffer_page_end(dst_opt, y_plane_size * 4);
+ align_buffer_page_end(rec_opt, y_plane_size * 2);
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ((float*)orig_f)[i] = (float)(i % 10000) * 3.14f;
+ }
+ memset(orig_y, 1, y_plane_size * 2);
+ memset(dst_opt, 2, y_plane_size * 4);
+ memset(rec_opt, 3, y_plane_size * 2);
+
+ ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y,
+ y_plane_size);
+
+ for (j = 0; j < benchmark_iterations_; j++) {
+ ConvertFP16ToFP32Column_NEON((const uint16_t*)orig_y, 1, (float*)dst_opt,
+ y_plane_size);
+ }
+
+ ConvertFP32ToFP16Row_NEON((const float*)dst_opt, (uint16_t*)rec_opt,
+ y_plane_size);
+
+ for (i = 0; i < y_plane_size; ++i) {
+ ASSERT_EQ(((const uint16_t*)orig_y)[i], ((const uint16_t*)rec_opt)[i]);
+ }
+
+ free_aligned_buffer_page_end(orig_f);
+ free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(rec_opt);
+}
+
+#endif // defined(ENABLE_ROW_TESTS) && defined(__aarch64__)
+
+static int TestARGBOr(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off,
+ uint32_t value) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kBpp = 4;
+ const int kStride = width * kBpp;
+ align_buffer_page_end(src_argb, kStride * abs(height) + off);
+ align_buffer_page_end(dst_argb_c, kStride * abs(height));
+ align_buffer_page_end(dst_argb_opt, kStride * abs(height));
+ MemRandomize(src_argb, kStride * abs(height) + off);
+ memset(dst_argb_c, 0, kStride * abs(height));
+ memset(dst_argb_opt, 1, kStride * abs(height));
+
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBOr(src_argb + off, kStride, dst_argb_c, kStride, width, invert * height,
+ value);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBOr(src_argb + off, kStride, dst_argb_opt, kStride, width,
+ invert * height, value);
+ }
+ int max_diff = 0;
+ for (int i = 0; i < kStride * abs(height); ++i) {
+ int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
+ static_cast<int>(dst_argb_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_argb_c);
+ free_aligned_buffer_page_end(dst_argb_opt);
+ return max_diff;
+}
+
+TEST_F(LibYUVPlanarTest, ARGBOr_Any) {
+ int max_diff = TestARGBOr(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0, 0xff000000u);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBOr_Unaligned) {
+ int max_diff = TestARGBOr(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1, 0xff000000u);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBOr_Invert) {
+ int max_diff = TestARGBOr(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0, 0xff000000u);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBOr_Opt) {
+ int max_diff = TestARGBOr(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0, 0xff000000u);
+ EXPECT_EQ(0, max_diff);
+}
+
+static int TestARGBAlphaBlend(int width,
+ int height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ if (width < 1) {
+ width = 1;
+ }
+ const int kBpp = 4;
+ const int kStride = width * kBpp;
+ align_buffer_page_end(src_argb0, kStride * abs(height) + off);
+ align_buffer_page_end(src_argb1, kStride * abs(height) + off);
+ align_buffer_page_end(dst_argb_c, kStride * abs(height));
+ align_buffer_page_end(dst_argb_opt, kStride * abs(height));
+ MemRandomize(src_argb0, kStride * abs(height) + off);
+ MemRandomize(src_argb1, kStride * abs(height) + off);
+ memset(dst_argb_c, 0, kStride * abs(height));
+ memset(dst_argb_opt, 1, kStride * abs(height));
+
+ MaskCpuFlags(disable_cpu_flags);
+ ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride, dst_argb_c,
+ kStride, width, invert * height);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride,
+ dst_argb_opt, kStride, width, invert * height);
+ }
+ int max_diff = 0;
+ for (int i = 0; i < kStride * abs(height); ++i) {
+ int abs_diff = abs(static_cast<int>(dst_argb_c[i]) -
+ static_cast<int>(dst_argb_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ free_aligned_buffer_page_end(src_argb0);
+ free_aligned_buffer_page_end(src_argb1);
+ free_aligned_buffer_page_end(dst_argb_c);
+ free_aligned_buffer_page_end(dst_argb_opt);
+ return max_diff;
+}
+
+TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Any) {
+ int max_diff = TestARGBAlphaBlend(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Unaligned) {
+ int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 1);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Invert) {
+ int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, -1, 0);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Opt) {
+ int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ EXPECT_EQ(0, max_diff);
+}
+
+TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) {
+ const int kWidth = 256;
+ const int kHeight = 256;
+ const int kStride = kWidth * 4;
+ align_buffer_page_end(src_argb0, kStride * kHeight);
+ align_buffer_page_end(src_argb1, kStride * kHeight);
+ align_buffer_page_end(dst_argb_c, kStride * kHeight);
+ align_buffer_page_end(dst_argb_opt, kStride * kHeight);
+ for (int da = 0; da < 256; ++da) {
+ for (int sa = 0; sa < 256; ++sa) {
+ int idx = (da * 256 + sa) * 4;
+ src_argb0[idx + 0] = (sa * 7 + da * 3 + 13) & 255;
+ src_argb0[idx + 1] = (sa * 13 + da * 5 + 29) & 255;
+ src_argb0[idx + 2] = (sa * 29 + da * 11 + 47) & 255;
+ src_argb0[idx + 3] = sa;
+ src_argb1[idx + 0] = (da * 17 + sa * 19 + 53) & 255;
+ src_argb1[idx + 1] = (da * 23 + sa * 31 + 71) & 255;
+ src_argb1[idx + 2] = (da * 37 + sa * 41 + 89) & 255;
+ src_argb1[idx + 3] = da;
+ }
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_c, kStride,
+ kWidth, kHeight);
+ MaskCpuFlags(benchmark_cpu_info_);
+ ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_opt, kStride,
+ kWidth, kHeight);
+ for (int i = 0; i < kStride * kHeight; ++i) {
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]) << "Mismatch at byte " << i;
+ }
+ free_aligned_buffer_page_end(src_argb0);
+ free_aligned_buffer_page_end(src_argb1);
+ free_aligned_buffer_page_end(dst_argb_c);
+ free_aligned_buffer_page_end(dst_argb_opt);
+}
+
+TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) {
+ static const int kWidths[] = {1, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 99};
+ const int kMaxW = 128;
+ const int kMaxSrcUV = kMaxW * 3 * 2 + 128;
+ align_buffer_page_end(src0, kMaxW * 4 + 128);
+ align_buffer_page_end(src1, kMaxW * 4 + 128);
+ align_buffer_page_end(src2, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_c0, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_c1, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_c2, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_opt0, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_opt1, kMaxW * 4 + 128);
+ align_buffer_page_end(dst_opt2, kMaxW * 4 + 128);
+ align_buffer_page_end(uv_src, kMaxSrcUV * 3);
+
+ for (size_t wi = 0; wi < sizeof(kWidths) / sizeof(kWidths[0]); ++wi) {
+ const int w = kWidths[wi];
+ const int uv_w = (w + 1) / 2;
+
+ // 1. ARGBToI444 and ARGBToI420 with poisoned RGBX alpha and poisoned tail.
+ for (int i = 0; i < w; ++i) {
+ src0[i * 4 + 0] = static_cast<uint8_t>((i * 11 + 3) & 255);
+ src0[i * 4 + 1] = static_cast<uint8_t>((i * 17 + 7) & 255);
+ src0[i * 4 + 2] = static_cast<uint8_t>((i * 29 + 13) & 255);
+ src0[i * 4 + 3] = 0u;
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBToI444(src0, w * 4, dst_c0, w, dst_c1, w, dst_c2, w, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < w; ++i) {
+ MSAN_POISON(&src0[i * 4 + 3], 1);
+ }
+ MSAN_POISON(src0 + w * 4, 128);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ MSAN_POISON(dst_opt2, kMaxW * 4);
+ ARGBToI444(src0, w * 4, dst_opt0, w, dst_opt1, w, dst_opt2, w, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w);
+ MSAN_CHECK_INITIALIZED(dst_opt1, w);
+ MSAN_CHECK_INITIALIZED(dst_opt2, w);
+ for (int i = 0; i < w; ++i) {
+ EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI444 Y w=" << w;
+ EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI444 U w=" << w;
+ EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI444 V w=" << w;
+ }
+
+ for (int i = 0; i < w; ++i) {
+ src0[i * 4 + 3] = 0u;
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBToI420(src0, w * 4, dst_c0, w, dst_c1, uv_w, dst_c2, uv_w, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < w; ++i) {
+ MSAN_POISON(&src0[i * 4 + 3], 1);
+ }
+ MSAN_POISON(src0 + w * 4, 128);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ MSAN_POISON(dst_opt2, kMaxW * 4);
+ ARGBToI420(src0, w * 4, dst_opt0, w, dst_opt1, uv_w, dst_opt2, uv_w, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w);
+ MSAN_CHECK_INITIALIZED(dst_opt1, uv_w);
+ MSAN_CHECK_INITIALIZED(dst_opt2, uv_w);
+ for (int i = 0; i < w; ++i) {
+ EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI420 Y w=" << w;
+ }
+ for (int i = 0; i < uv_w; ++i) {
+ EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI420 U w=" << w;
+ EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI420 V w=" << w;
+ }
+
+ // 2. ARGBMirror with poisoned alpha on even pixels (verifying per-lane
+ // shadow permutation) and poisoned tail.
+ for (int i = 0; i < w; ++i) {
+ src0[i * 4 + 0] = static_cast<uint8_t>((i * 5 + 1) & 255);
+ src0[i * 4 + 1] = static_cast<uint8_t>((i * 9 + 2) & 255);
+ src0[i * 4 + 2] = static_cast<uint8_t>((i * 15 + 3) & 255);
+ src0[i * 4 + 3] = static_cast<uint8_t>((i * 21 + 4) & 255);
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBMirror(src0, w * 4, dst_c0, w * 4, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < w; i += 2) {
+ MSAN_POISON(&src0[i * 4 + 3], 1);
+ }
+ MSAN_POISON(src0 + w * 4, 128);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ ARGBMirror(src0, w * 4, dst_opt0, w * 4, w, 1);
+ for (int i = 0; i < w; ++i) {
+ const int src_i = w - 1 - i;
+ MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4], 3);
+ EXPECT_EQ(dst_c0[i * 4 + 0], dst_opt0[i * 4 + 0]);
+ EXPECT_EQ(dst_c0[i * 4 + 1], dst_opt0[i * 4 + 1]);
+ EXPECT_EQ(dst_c0[i * 4 + 2], dst_opt0[i * 4 + 2]);
+ if (src_i & 1) {
+ MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4 + 3], 1);
+ EXPECT_EQ(dst_c0[i * 4 + 3], dst_opt0[i * 4 + 3]);
+ }
+ }
+
+ // 3. RGB24Mirror, RAWToRGBA, RAWToRGB24, MergeRGBPlane with poisoned tails.
+ for (int i = 0; i < w * 3; ++i) {
+ src0[i] = static_cast<uint8_t>((i * 19 + 5) & 255);
+ }
+ MSAN_POISON(src0 + w * 3, 128);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ RGB24Mirror(src0, w * 3, dst_c0, w * 3, w, 1);
+ RAWToRGBA(src0, w * 3, dst_c1, w * 4, w, 1);
+ RAWToRGB24(src0, w * 3, dst_c2, w * 3, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ MSAN_POISON(dst_opt2, kMaxW * 4);
+ RGB24Mirror(src0, w * 3, dst_opt0, w * 3, w, 1);
+ RAWToRGBA(src0, w * 3, dst_opt1, w * 4, w, 1);
+ RAWToRGB24(src0, w * 3, dst_opt2, w * 3, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 3);
+ MSAN_CHECK_INITIALIZED(dst_opt1, w * 4);
+ MSAN_CHECK_INITIALIZED(dst_opt2, w * 3);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3));
+ EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 4));
+ EXPECT_EQ(0, memcmp(dst_c2, dst_opt2, w * 3));
+
+ for (int i = 0; i < w; ++i) {
+ src0[i] = static_cast<uint8_t>((i * 3 + 11) & 255);
+ src1[i] = static_cast<uint8_t>((i * 7 + 23) & 255);
+ src2[i] = static_cast<uint8_t>((i * 13 + 37) & 255);
+ }
+ MSAN_POISON(src0 + w, 128);
+ MSAN_POISON(src1 + w, 128);
+ MSAN_POISON(src2 + w, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ MergeRGBPlane(src0, w, src1, w, src2, w, dst_c0, w * 3, w, 1);
+ MirrorPlane(src0, w, dst_c1, w, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ MergeRGBPlane(src0, w, src1, w, src2, w, dst_opt0, w * 3, w, 1);
+ MirrorPlane(src0, w, dst_opt1, w, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 3);
+ MSAN_CHECK_INITIALIZED(dst_opt1, w);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3));
+ EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w));
+
+ // 4. MirrorUVPlane and MirrorPlane_16 with poisoned tails.
+ for (int i = 0; i < w * 2; ++i) {
+ src0[i] = static_cast<uint8_t>((i * 23 + 9) & 255);
+ }
+ MSAN_POISON(src0 + w * 2, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ MirrorUVPlane(src0, w * 2, dst_c0, w * 2, w, 1);
+ MirrorPlane_16(reinterpret_cast<const uint16_t*>(src0), w,
+ reinterpret_cast<uint16_t*>(dst_c1), w, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ MirrorUVPlane(src0, w * 2, dst_opt0, w * 2, w, 1);
+ MirrorPlane_16(reinterpret_cast<const uint16_t*>(src0), w,
+ reinterpret_cast<uint16_t*>(dst_opt1), w, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 2);
+ MSAN_CHECK_INITIALIZED(dst_opt1, w * 2);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2));
+ EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 2));
+
+ // 5. I420ToAR30, NV12ToARGB, NV12ToRGB24, I444ToARGB, I210ToARGB.
+ for (int i = 0; i < w; ++i) {
+ src0[i] = static_cast<uint8_t>((i * 5 + 16) & 255);
+ src1[i] = static_cast<uint8_t>((i * 9 + 64) & 255);
+ src2[i] = static_cast<uint8_t>((i * 11 + 128) & 255);
+ }
+ MSAN_POISON(src0 + w, 128);
+ MSAN_POISON(src1 + uv_w, 128);
+ MSAN_POISON(src2 + uv_w, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_c0, w * 4, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_opt0, w * 4, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
+
+ for (int i = 0; i < uv_w * 2; ++i) {
+ src1[i] = static_cast<uint8_t>((i * 13 + 80) & 255);
+ }
+ MSAN_POISON(src1 + uv_w * 2, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ NV12ToARGB(src0, w, src1, uv_w * 2, dst_c0, w * 4, w, 1);
+ NV12ToRGB24(src0, w, src1, uv_w * 2, dst_c1, w * 3, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ MSAN_POISON(dst_opt1, kMaxW * 4);
+ NV12ToARGB(src0, w, src1, uv_w * 2, dst_opt0, w * 4, w, 1);
+ NV12ToRGB24(src0, w, src1, uv_w * 2, dst_opt1, w * 3, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
+ MSAN_CHECK_INITIALIZED(dst_opt1, w * 3);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
+ EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 3));
+
+ for (int i = 0; i < w; ++i) {
+ reinterpret_cast<uint16_t*>(src0)[i] =
+ static_cast<uint16_t>((i * 17 + 64) & 1023);
+ }
+ for (int i = 0; i < uv_w; ++i) {
+ reinterpret_cast<uint16_t*>(src1)[i] =
+ static_cast<uint16_t>((i * 23 + 256) & 1023);
+ reinterpret_cast<uint16_t*>(src2)[i] =
+ static_cast<uint16_t>((i * 31 + 512) & 1023);
+ }
+ MSAN_POISON(src0 + w * 2, 128);
+ MSAN_POISON(src1 + uv_w * 2, 128);
+ MSAN_POISON(src2 + uv_w * 2, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ I210ToARGB(reinterpret_cast<const uint16_t*>(src0), w,
+ reinterpret_cast<const uint16_t*>(src1), uv_w,
+ reinterpret_cast<const uint16_t*>(src2), uv_w, dst_c0, w * 4, w,
+ 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ I210ToARGB(reinterpret_cast<const uint16_t*>(src0), w,
+ reinterpret_cast<const uint16_t*>(src1), uv_w,
+ reinterpret_cast<const uint16_t*>(src2), uv_w, dst_opt0, w * 4,
+ w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
+
+ // 6. ARGBAlphaBlend with poisoned tails and poisoned destination.
+ for (int i = 0; i < w * 4; ++i) {
+ src0[i] = static_cast<uint8_t>((i * 29 + 3) & 255);
+ src1[i] = static_cast<uint8_t>((i * 37 + 11) & 255);
+ }
+ MSAN_POISON(src0 + w * 4, 128);
+ MSAN_POISON(src1 + w * 4, 128);
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_c0, w * 4, w, 1);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_opt0, w * 4, w, 1);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
+
+ // 7. UVScale 3x box downscale (hits ScaleUVAddCols3_AVX512VBMI).
+ const int src_uv_w = w * 3;
+ const int src_uv_stride = src_uv_w * 2 + 64;
+ MSAN_POISON(uv_src, kMaxSrcUV * 3);
+ for (int row = 0; row < 3; ++row) {
+ for (int i = 0; i < src_uv_w * 2; ++i) {
+ uv_src[row * src_uv_stride + i] =
+ static_cast<uint8_t>((row * 53 + i * 17 + 7) & 255);
+ }
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_c0, w * 2, w, 1,
+ kFilterBox);
+ MaskCpuFlags(benchmark_cpu_info_);
+ MSAN_POISON(dst_opt0, kMaxW * 4);
+ UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_opt0, w * 2, w, 1,
+ kFilterBox);
+ MSAN_CHECK_INITIALIZED(dst_opt0, w * 2);
+ EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2));
+ }
+
+ free_aligned_buffer_page_end(src0);
+ free_aligned_buffer_page_end(src1);
+ free_aligned_buffer_page_end(src2);
+ free_aligned_buffer_page_end(dst_c0);
+ free_aligned_buffer_page_end(dst_c1);
+ free_aligned_buffer_page_end(dst_c2);
+ free_aligned_buffer_page_end(dst_opt0);
+ free_aligned_buffer_page_end(dst_opt1);
+ free_aligned_buffer_page_end(dst_opt2);
+ free_aligned_buffer_page_end(uv_src);
+}
+
+// Regression test for RGBToUVMatrixRow_Any_SSSE3 MASK=15 (previously MASK=7
+// overread 24 bytes past src_rgb24 at width=24).
+TEST_F(LibYUVPlanarTest, RGB24ToI420_Width24_OOB) {
+ const int kWidth = 24;
+ const int kHeight = 2;
+ align_buffer_page_end(src_rgb24, kWidth * 3 * kHeight);
+ align_buffer_page_end(dst_y_c, kWidth * kHeight);
+ align_buffer_page_end(dst_u_c, (kWidth / 2) * (kHeight / 2));
+ align_buffer_page_end(dst_v_c, (kWidth / 2) * (kHeight / 2));
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight);
+ align_buffer_page_end(dst_u_opt, (kWidth / 2) * (kHeight / 2));
+ align_buffer_page_end(dst_v_opt, (kWidth / 2) * (kHeight / 2));
+ MemRandomize(src_rgb24, kWidth * 3 * kHeight);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ RGB24ToI420(src_rgb24, kWidth * 3, dst_y_c, kWidth, dst_u_c, kWidth / 2,
+ dst_v_c, kWidth / 2, kWidth, kHeight);
+
+ const int kNoAvx512 =
+ benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL |
+ kCpuHasAVX512VBMI | kCpuHasAVX512VNNI);
+ const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2;
+ const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only};
+ for (int m = 0; m < 3; ++m) {
+ MaskCpuFlags(kModes[m]);
+ memset(dst_y_opt, 0, kWidth * kHeight);
+ memset(dst_u_opt, 0, (kWidth / 2) * (kHeight / 2));
+ memset(dst_v_opt, 0, (kWidth / 2) * (kHeight / 2));
+ RGB24ToI420(src_rgb24, kWidth * 3, dst_y_opt, kWidth, dst_u_opt, kWidth / 2,
+ dst_v_opt, kWidth / 2, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_y_c[i], dst_y_opt[i]);
+ }
+ for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) {
+ ASSERT_EQ(dst_u_c[i], dst_u_opt[i]);
+ ASSERT_EQ(dst_v_c[i], dst_v_opt[i]);
+ }
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ free_aligned_buffer_page_end(src_rgb24);
+ free_aligned_buffer_page_end(dst_y_c);
+ free_aligned_buffer_page_end(dst_u_c);
+ free_aligned_buffer_page_end(dst_v_c);
+ free_aligned_buffer_page_end(dst_y_opt);
+ free_aligned_buffer_page_end(dst_u_opt);
+ free_aligned_buffer_page_end(dst_v_opt);
+}
+
+#ifdef ENABLE_ROW_TESTS
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \
+ defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || \
+ defined(HAS_HALFROW_16TO8_AVX512BW)
+// Regression test for HalfWidthRow_16To8 (>= 0x8000 unsigned MSB samples and
+// exact page-end tail bounds) and HalfRow_16To8 (non-power-of-two scale).
+TEST_F(LibYUVPlanarTest, Half16To8_HighBitAndNonPow2Scale) {
+ const int kWidths[] = {1, 7, 15, 24, 31, 33, 63, 65};
+ for (size_t w_idx = 0; w_idx < sizeof(kWidths) / sizeof(kWidths[0]);
+ ++w_idx) {
+ const int w = kWidths[w_idx];
+ align_buffer_page_end_16(src_uv_hw, w * 4);
+ align_buffer_page_end(dst_c, w);
+ align_buffer_page_end(dst_opt, w);
+ for (int i = 0; i < w * 4; ++i) {
+ src_uv_hw[i] = static_cast<uint16_t>(0x8000u + ((i * 257u) & 0x7fffu));
+ }
+ HalfWidthRow_16To8_C(src_uv_hw, w * 2, dst_c, 256, w);
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ memset(dst_opt, 0x5a, w);
+ HalfWidthRow_16To8_AVX512BW(src_uv_hw, w * 2, dst_opt, 256, w);
+ for (int i = 0; i < w; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX512BW w=" << w << " i=" << i;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ memset(dst_opt, 0x5a, w);
+ HalfWidthRow_16To8_Any_AVX2(src_uv_hw, w * 2, dst_opt, 256, w);
+ for (int i = 0; i < w; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX2 w=" << w << " i=" << i;
+ }
+ }
+#endif
+#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ memset(dst_opt, 0x5a, w);
+ HalfWidthRow_16To8_Any_SSSE3(src_uv_hw, w * 2, dst_opt, 256, w);
+ for (int i = 0; i < w; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "SSSE3 w=" << w << " i=" << i;
+ }
+ }
+#endif
+
+ // HalfRow_16To8 with non-power-of-two scale (e.g. 16320).
+ align_buffer_page_end_16(src_uv_hr, w * 2);
+ for (int i = 0; i < w * 2; ++i) {
+ src_uv_hr[i] = static_cast<uint16_t>((i * 73u) & 1023u);
+ }
+ HalfRow_16To8_C(src_uv_hr, w, dst_c, 16320, w);
+#if defined(HAS_HALFROW_16TO8_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ memset(dst_opt, 0x5a, w);
+ HalfRow_16To8_AVX512BW(src_uv_hr, w, dst_opt, 16320, w);
+ for (int i = 0; i < w; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i])
+ << "HalfRow_16To8_AVX512BW w=" << w << " i=" << i;
+ }
+ }
+#endif
+ free_aligned_buffer_page_end_16(src_uv_hr);
+ free_aligned_buffer_page_end_16(src_uv_hw);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+ }
+}
+#endif // HAS_HALFWIDTHROW_16TO8_* || HAS_HALFROW_16TO8_AVX512BW
+
+#if defined(HAS_UNPACKMT2T_AVX2)
+// Regression test for UnpackMT2T_C and UnpackMT2T_AVX2 when size is not a
+// multiple of the 80-byte (64-pixel) MT2T block size.
+TEST_F(LibYUVPlanarTest, UnpackMT2T_PartialTail) {
+ if (!TestCpuFlag(kCpuHasAVX2)) {
+ return;
+ }
+ const size_t kSrcSize = 120; // 1 full 80-byte block + 40 trailing bytes
+ const size_t kDstCount = (kSrcSize / 80) * 64;
+ align_buffer_page_end(src, kSrcSize);
+ align_buffer_page_end_16(dst_c, kDstCount);
+ align_buffer_page_end_16(dst_opt, kDstCount);
+ MemRandomize(src, kSrcSize);
+ UnpackMT2T_C(src, dst_c, kSrcSize);
+ UnpackMT2T_AVX2(src, dst_opt, kSrcSize);
+ for (size_t i = 0; i < kDstCount; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
+ }
+ // Verify size < 80 performs zero out-of-bounds reads or writes.
+ UnpackMT2T_C(src + 80, dst_c, 40);
+ UnpackMT2T_AVX2(src + 80, dst_opt, 40);
+ for (size_t i = 0; i < kDstCount; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
+ }
+ free_aligned_buffer_page_end(src);
+ free_aligned_buffer_page_end_16(dst_c);
+ free_aligned_buffer_page_end_16(dst_opt);
+}
+#endif // HAS_UNPACKMT2T_AVX2
+#endif // ENABLE_ROW_TESTS
+
+// Regression test for ConvertToLSBPlane_16 with depth=16 (scale=65536).
+TEST_F(LibYUVPlanarTest, ConvertToLSBPlane_16_Depth16) {
+ const int kWidth = 37;
+ const int kHeight = 4;
+ align_buffer_page_end_16(src_y, kWidth * kHeight);
+ align_buffer_page_end_16(dst_c, kWidth * kHeight);
+ align_buffer_page_end_16(dst_opt, kWidth * kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ src_y[i] = static_cast<uint16_t>(0x1234u + i * 251u);
+ }
+ MaskCpuFlags(disable_cpu_flags_);
+ ConvertToLSBPlane_16(src_y, kWidth, dst_c, kWidth, kWidth, kHeight, 16);
+
+ const int kNoAvx512 =
+ benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL);
+ MaskCpuFlags(benchmark_cpu_info_);
+ memset(dst_opt, 0, kWidth * kHeight * 2);
+ ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]);
+ }
+
+ MaskCpuFlags(kNoAvx512);
+ memset(dst_opt, 0, kWidth * kHeight * 2);
+ ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]);
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ free_aligned_buffer_page_end_16(src_y);
+ free_aligned_buffer_page_end_16(dst_c);
+ free_aligned_buffer_page_end_16(dst_opt);
+}
+
+// Regression test for ARGBPolynomial saturation on outputs >= 32768 / 65536,
+// odd widths, and 4-byte-aligned (non-16-byte-aligned) polynomial arrays.
+TEST_F(LibYUVPlanarTest, ARGBPolynomial_LargeOutputAndUnalignedPoly) {
+ const int kWidth = 37;
+ const int kHeight = 3;
+ align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
+ align_buffer_page_end(poly_buf, 17 * sizeof(float));
+ // Offset by +1 float so poly is 4-byte aligned, not 16-byte aligned.
+ float* poly = reinterpret_cast<float*>(poly_buf) + 1;
+ for (int c = 0; c < 4; ++c) {
+ poly[0 + c] = 0.0f;
+ poly[4 + c] = 300.0f; // 255 * 300 = 76500 >= 65536 -> must clamp to 255
+ poly[8 + c] = 1.0f; // 200^2 = 40000 in [32768, 65535] -> clamp to 255
+ poly[12 + c] = 0.0f;
+ }
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ src_argb[i] = static_cast<uint8_t>(i & 255);
+ }
+
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBPolynomial(src_argb, kWidth * 4, dst_c, kWidth * 4, poly, kWidth,
+ kHeight);
+
+ const int kSse2Only =
+ benchmark_cpu_info_ & ~(kCpuHasAVX2 | kCpuHasFMA3 | kCpuHasAVX512BW);
+ const int kModes[2] = {benchmark_cpu_info_, kSse2Only};
+ for (int m = 0; m < 2; ++m) {
+ MaskCpuFlags(kModes[m]);
+ memset(dst_opt, 0, kWidth * 4 * kHeight);
+ ARGBPolynomial(src_argb, kWidth * 4, dst_opt, kWidth * 4, poly, kWidth,
+ kHeight);
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "mode=" << m << " i=" << i;
+ }
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(poly_buf);
+}
+
+// Regression test for ARGBBlur / CumulativeSumToAverageRow_AVX2 when radius=0
+// or radius=1 produces small window areas (area == 1 or area == 2) on 0xff.
+TEST_F(LibYUVPlanarTest, ARGBBlur_SmallRadiusWhite) {
+ const int kWidth = 37;
+ const int kHeight = 4;
+ const int kCumSumStride32 = (kWidth + 16) * 4;
+ align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
+ align_buffer_page_end(cumsum, kCumSumStride32 * (kHeight + 2) * 4);
+ memset(src_argb, 0xff, kWidth * 4 * kHeight);
+
+ for (int radius = 1; radius <= 2; ++radius) {
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBBlur(src_argb, kWidth * 4, dst_c, kWidth * 4,
+ reinterpret_cast<int32_t*>(cumsum), kCumSumStride32, kWidth,
+ kHeight, radius);
+ MaskCpuFlags(benchmark_cpu_info_);
+ memset(dst_opt, 0, kWidth * 4 * kHeight);
+ ARGBBlur(src_argb, kWidth * 4, dst_opt, kWidth * 4,
+ reinterpret_cast<int32_t*>(cumsum), kCumSumStride32, kWidth,
+ kHeight, radius);
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "radius=" << radius << " i=" << i;
+ }
+ }
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(cumsum);
+}
+
+// Regression test for ARGBLumaColorTable with standard BT.601 green weight
+// 150 (> 127) and bright pixels where R*77 + G*150 + B*29 > 32767.
+TEST_F(LibYUVPlanarTest, ARGBLumaColorTable_BrightPixels) {
+ const int kWidth = 37;
+ const int kHeight = 3;
+ align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
+ align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
+ align_buffer_page_end(luma, 32768);
+ for (int i = 0; i < 32768; ++i) {
+ luma[i] = static_cast<uint8_t>(((i >> 8) * 3 + (i & 255)) & 255);
+ }
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ src_argb[i * 4 + 0] = static_cast<uint8_t>(200 + (i % 56));
+ src_argb[i * 4 + 1] = static_cast<uint8_t>(200 + ((i * 3) % 56));
+ src_argb[i * 4 + 2] = static_cast<uint8_t>(200 + ((i * 7) % 56));
+ src_argb[i * 4 + 3] = static_cast<uint8_t>(i & 255);
+ }
+
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBLumaColorTable(src_argb, kWidth * 4, dst_c, kWidth * 4, luma, kWidth,
+ kHeight);
+ MaskCpuFlags(benchmark_cpu_info_);
+ ARGBLumaColorTable(src_argb, kWidth * 4, dst_opt, kWidth * 4, luma, kWidth,
+ kHeight);
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
+ }
+
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
+ free_aligned_buffer_page_end(luma);
+}
+
+#if defined(ENABLE_ROW_TESTS) && !defined(LIBYUV_DISABLE_X86) && \
+ (defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) || \
+ defined(__i386__))
+// Regression test for 2-byte-aligned (non-16/32-byte-aligned) Intel
+// YuvConstants and ArgbConstants structs and custom non-0x8000 kAddUV offsets.
+TEST_F(LibYUVPlanarTest, UnalignedMatrixConstantsAndCustomKAddUV) {
+ const int kWidth = 38;
+ const int kHeight = 4;
+ align_buffer_page_end(yuvc_buf, sizeof(YuvConstants) + 16);
+ align_buffer_page_end(argbc_buf, sizeof(ArgbConstants) + 16);
+ // Offset by +2 bytes (matches alignof == 2, misaligned for movdqa/vmovdqa).
+ YuvConstants* yuvc = reinterpret_cast<YuvConstants*>(yuvc_buf + 2);
+ ArgbConstants* argbc = reinterpret_cast<ArgbConstants*>(argbc_buf + 2);
+ memcpy(yuvc, &kYuvI601Constants, sizeof(YuvConstants));
+ memcpy(argbc, &kArgbI601Constants, sizeof(ArgbConstants));
+ for (int i = 0; i < 16; ++i) {
+ argbc->kAddUV[i] = 0x7200u; // Non-default UV bias (!= 0x8000)
+ }
+
+ align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
+ align_buffer_page_end(y_c, kWidth * kHeight);
+ align_buffer_page_end(u_c, kWidth * kHeight);
+ align_buffer_page_end(v_c, kWidth * kHeight);
+ align_buffer_page_end(y_opt, kWidth * kHeight);
+ align_buffer_page_end(u_opt, kWidth * kHeight);
+ align_buffer_page_end(v_opt, kWidth * kHeight);
+ align_buffer_page_end(rgb_c, kWidth * 4 * kHeight);
+ align_buffer_page_end(rgb_opt, kWidth * 4 * kHeight);
+ MemRandomize(src_argb, kWidth * 4 * kHeight);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBToI444Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth, v_c, kWidth,
+ argbc, kWidth, kHeight);
+ I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_c,
+ kWidth * 4, yuvc, kWidth, kHeight);
+
+ const int kNoAvx512 =
+ benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL |
+ kCpuHasAVX512VBMI | kCpuHasAVX512VNNI);
+ const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2;
+ const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only};
+ for (int m = 0; m < 3; ++m) {
+ MaskCpuFlags(kModes[m]);
+ ARGBToI444Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth, v_opt,
+ kWidth, argbc, kWidth, kHeight);
+ I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_opt,
+ kWidth * 4, yuvc, kWidth, kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ ASSERT_EQ(y_c[i], y_opt[i]);
+ ASSERT_EQ(u_c[i], u_opt[i]) << "mode=" << m << " i=" << i;
+ ASSERT_EQ(v_c[i], v_opt[i]) << "mode=" << m << " i=" << i;
+ }
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i;
+ }
+ }
+
+ // Also test ARGBToI420Matrix with custom kAddUV and I400ToARGBMatrix with
+ // unaligned YuvConstants.
+ MaskCpuFlags(disable_cpu_flags_);
+ ARGBToI420Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth / 2, v_c,
+ kWidth / 2, argbc, kWidth, kHeight);
+ I400ToARGBMatrix(y_c, kWidth, rgb_c, kWidth * 4, yuvc, kWidth, kHeight);
+ for (int m = 0; m < 3; ++m) {
+ MaskCpuFlags(kModes[m]);
+ ARGBToI420Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth / 2,
+ v_opt, kWidth / 2, argbc, kWidth, kHeight);
+ I400ToARGBMatrix(y_c, kWidth, rgb_opt, kWidth * 4, yuvc, kWidth, kHeight);
+ for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) {
+ ASSERT_NEAR(u_c[i], u_opt[i], 1) << "mode=" << m << " i=" << i;
+ ASSERT_NEAR(v_c[i], v_opt[i], 1) << "mode=" << m << " i=" << i;
+ }
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i;
+ }
+ }
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ free_aligned_buffer_page_end(yuvc_buf);
+ free_aligned_buffer_page_end(argbc_buf);
+ free_aligned_buffer_page_end(src_argb);
+ free_aligned_buffer_page_end(y_c);
+ free_aligned_buffer_page_end(u_c);
+ free_aligned_buffer_page_end(v_c);
+ free_aligned_buffer_page_end(y_opt);
+ free_aligned_buffer_page_end(u_opt);
+ free_aligned_buffer_page_end(v_opt);
+ free_aligned_buffer_page_end(rgb_c);
+ free_aligned_buffer_page_end(rgb_opt);
+}
+#endif // ENABLE_ROW_TESTS
+
+// Regression test for MergeAR64Plane depth=16 (UBSan shift) and
+// MergeARGB16To8Plane depth=8 saturation on out-of-range >255 samples.
+TEST_F(LibYUVPlanarTest, MergeAR64Depth16AndMergeARGB16To8Depth8Clamp) {
+ const int kWidth = 37;
+ const int kHeight = 2;
+ align_buffer_page_end_16(src_r, kWidth * kHeight);
+ align_buffer_page_end_16(src_g, kWidth * kHeight);
+ align_buffer_page_end_16(src_b, kWidth * kHeight);
+ align_buffer_page_end_16(src_a, kWidth * kHeight);
+ align_buffer_page_end_16(ar64_c, kWidth * 4 * kHeight);
+ align_buffer_page_end_16(ar64_opt, kWidth * 4 * kHeight);
+ align_buffer_page_end(argb_c, kWidth * 4 * kHeight);
+ align_buffer_page_end(argb_opt, kWidth * 4 * kHeight);
+ for (int i = 0; i < kWidth * kHeight; ++i) {
+ src_r[i] = static_cast<uint16_t>(200u + (i * 19u));
+ src_g[i] = static_cast<uint16_t>(100u + (i * 37u));
+ src_b[i] = static_cast<uint16_t>(50u + (i * 53u));
+ src_a[i] = static_cast<uint16_t>(0x8000u + (i * 101u));
+ }
+
+ MaskCpuFlags(disable_cpu_flags_);
+ MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth,
+ ar64_c, kWidth * 4, kWidth, kHeight, 16);
+ MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a,
+ kWidth, argb_c, kWidth * 4, kWidth, kHeight, 8);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth,
+ ar64_opt, kWidth * 4, kWidth, kHeight, 16);
+ MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a,
+ kWidth, argb_opt, kWidth * 4, kWidth, kHeight, 8);
+
+ for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
+ ASSERT_EQ(ar64_c[i], ar64_opt[i]);
+ ASSERT_EQ(argb_c[i], argb_opt[i]);
+ }
+
+ free_aligned_buffer_page_end_16(src_r);
+ free_aligned_buffer_page_end_16(src_g);
+ free_aligned_buffer_page_end_16(src_b);
+ free_aligned_buffer_page_end_16(src_a);
+ free_aligned_buffer_page_end_16(ar64_c);
+ free_aligned_buffer_page_end_16(ar64_opt);
+ free_aligned_buffer_page_end(argb_c);
+ free_aligned_buffer_page_end(argb_opt);
+}
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/rotate_argb_test.cc b/libs/libyuv/unit_test/rotate_argb_test.cc
index d200389596..902aa81294 100644
--- a/libs/libyuv/unit_test/rotate_argb_test.cc
+++ b/libs/libyuv/unit_test/rotate_argb_test.cc
@@ -12,19 +12,20 @@
#include "../unit_test/unit_test.h"
#include "libyuv/cpu_id.h"
+#include "libyuv/planar_functions.h"
#include "libyuv/rotate_argb.h"
namespace libyuv {
-void TestRotateBpp(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- libyuv::RotationMode mode,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- const int kBpp) {
+static void TestRotateBpp(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ const int kBpp) {
if (src_width < 1) {
src_width = 1;
}
@@ -75,7 +76,7 @@ void TestRotateBpp(int src_width,
// Rotation should be exact.
for (int i = 0; i < dst_argb_plane_size; ++i) {
- EXPECT_EQ(dst_argb_c[i], dst_argb_opt[i]);
+ ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]);
}
free_aligned_buffer_page_end(dst_argb_c);
@@ -119,6 +120,20 @@ TEST_F(LibYUVRotateTest, ARGBRotate270_Opt) {
disable_cpu_flags_, benchmark_cpu_info_);
}
+TEST_F(LibYUVRotateTest, ARGBRotate90_Odd) {
+ ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, ARGBRotate270_Odd) {
+ ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
static void TestRotatePlane(int src_width,
int src_height,
int dst_width,
@@ -156,31 +171,271 @@ TEST_F(LibYUVRotateTest, RotatePlane270_Opt) {
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane0_Odd) {
- TestRotatePlane(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate0,
+ TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane90_Odd) {
- TestRotatePlane(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate90,
+ TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane180_Odd) {
- TestRotatePlane(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate180,
+ TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane270_Odd) {
- TestRotatePlane(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate270,
+ TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
+TEST_F(LibYUVRotateTest, RotatePlane90_TestStride) {
+ int argb_plane_size = benchmark_width_ * 4 * abs(benchmark_height_);
+
+ align_buffer_page_end(src_argb, argb_plane_size);
+ align_buffer_page_end(dst_argb, argb_plane_size);
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
+ benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_, kRotate0));
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
+ benchmark_width_ * 4 - 1, benchmark_width_ - 1,
+ benchmark_height_, kRotate0));
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
+ benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_, kRotate180));
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
+ benchmark_width_ * 4 - 1, benchmark_width_ - 1,
+ benchmark_height_, kRotate180));
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
+ abs(benchmark_height_) * 4, benchmark_width_,
+ benchmark_height_, kRotate90));
+
+ ASSERT_EQ(-1, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
+ abs(benchmark_height_) * 4, benchmark_width_ - 1,
+ benchmark_height_, kRotate90));
+
+ ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
+ abs(benchmark_height_) * 4, benchmark_width_,
+ benchmark_height_, kRotate270));
+
+ ASSERT_EQ(-1, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
+ abs(benchmark_height_) * 4, benchmark_width_ - 1,
+ benchmark_height_, kRotate270));
+
+ free_aligned_buffer_page_end(dst_argb);
+ free_aligned_buffer_page_end(src_argb);
+}
+
+static void TestRotatePlane_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height < 1) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ int src_stride = src_width;
+ int src_plane_size = src_stride * abs(src_height);
+ align_buffer_page_end_16(src, src_plane_size);
+ for (int i = 0; i < src_plane_size; ++i) {
+ src[i] = fastrand() & 0xffff;
+ }
+
+ int dst_stride = dst_width;
+ int dst_plane_size = dst_stride * dst_height;
+ align_buffer_page_end_16(dst_c, dst_plane_size);
+ align_buffer_page_end_16(dst_opt, dst_plane_size);
+ memset(dst_c, 2, dst_plane_size * 2);
+ memset(dst_opt, 3, dst_plane_size * 2);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height,
+ mode);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ RotatePlane_16(src, src_stride, dst_opt, dst_stride, src_width, src_height,
+ mode);
+ }
+
+ // Rotation should be exact.
+ for (int i = 0; i < dst_plane_size; ++i) {
+ ASSERT_EQ(dst_c[i], dst_opt[i]);
+ }
+
+ free_aligned_buffer_page_end_16(dst_c);
+ free_aligned_buffer_page_end_16(dst_opt);
+ free_aligned_buffer_page_end_16(src);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane0_16_Opt) {
+ TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate0, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane90_16_Opt) {
+ TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate90, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane180_16_Opt) {
+ TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate180, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane270_16_Opt) {
+ TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate270, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane0_16_Odd) {
+ TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane90_16_Odd) {
+ TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane180_16_Odd) {
+ TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane270_16_Odd) {
+ TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_);
+}
+
+static void TestRotatePlane16_SplitMergeUV(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height < 1) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ const int src_pixels = src_width * src_height;
+ const int dst_pixels = dst_width * dst_height;
+ align_buffer_page_end(src_16, src_pixels * 2);
+ align_buffer_page_end(dst_16, dst_pixels * 2);
+ align_buffer_page_end(ref_16, dst_pixels * 2);
+ align_buffer_page_end(src_lo, src_pixels);
+ align_buffer_page_end(src_hi, src_pixels);
+ align_buffer_page_end(dst_lo, dst_pixels);
+ align_buffer_page_end(dst_hi, dst_pixels);
+ for (int i = 0; i < src_pixels * 2; ++i) {
+ src_16[i] = fastrand() & 0xff;
+ }
+ memset(dst_16, 2, dst_pixels * 2);
+ memset(ref_16, 3, dst_pixels * 2);
+
+ MaskCpuFlags(benchmark_cpu_info);
+ SplitUVPlane(src_16, src_width * 2, src_lo, src_width, src_hi, src_width,
+ src_width, src_height);
+ RotatePlane(src_lo, src_width, dst_lo, dst_width, src_width, src_height,
+ mode);
+ RotatePlane(src_hi, src_width, dst_hi, dst_width, src_width, src_height,
+ mode);
+ MergeUVPlane(dst_lo, dst_width, dst_hi, dst_width, ref_16, dst_width * 2,
+ dst_width, dst_height);
+
+ I410Rotate(reinterpret_cast<const uint16_t*>(src_16), src_width,
+ reinterpret_cast<const uint16_t*>(src_16), src_width,
+ reinterpret_cast<const uint16_t*>(src_16), src_width,
+ reinterpret_cast<uint16_t*>(dst_16), dst_width,
+ reinterpret_cast<uint16_t*>(dst_16), dst_width,
+ reinterpret_cast<uint16_t*>(dst_16), dst_width, src_width,
+ src_height, mode);
+
+ EXPECT_EQ(0, memcmp(ref_16, dst_16, dst_pixels * 2));
+
+ free_aligned_buffer_page_end(dst_hi);
+ free_aligned_buffer_page_end(dst_lo);
+ free_aligned_buffer_page_end(src_hi);
+ free_aligned_buffer_page_end(src_lo);
+ free_aligned_buffer_page_end(ref_16);
+ free_aligned_buffer_page_end(dst_16);
+ free_aligned_buffer_page_end(src_16);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Opt) {
+ TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
+ benchmark_width_, benchmark_height_, kRotate0,
+ benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
+ benchmark_height_, benchmark_width_, kRotate90,
+ benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
+ benchmark_width_, benchmark_height_,
+ kRotate180, benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
+ benchmark_height_, benchmark_width_,
+ kRotate270, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Odd) {
+ TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
+ benchmark_width_ + 3, benchmark_height_ + 5,
+ kRotate0, benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
+ benchmark_height_ + 5, benchmark_width_ + 3,
+ kRotate90, benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
+ benchmark_width_ + 3, benchmark_height_ + 5,
+ kRotate180, benchmark_cpu_info_);
+ TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
+ benchmark_height_ + 5, benchmark_width_ + 3,
+ kRotate270, benchmark_cpu_info_);
+}
+
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/rotate_test.cc b/libs/libyuv/unit_test/rotate_test.cc
index 61941e63e0..5ddf899cbe 100644
--- a/libs/libyuv/unit_test/rotate_test.cc
+++ b/libs/libyuv/unit_test/rotate_test.cc
@@ -14,8 +14,19 @@
#include "libyuv/cpu_id.h"
#include "libyuv/rotate.h"
+#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \
+ !defined(ENABLE_ROW_TESTS)
+#define ENABLE_ROW_TESTS
+#endif
+
+#ifdef ENABLE_ROW_TESTS
+#include "libyuv/rotate_row.h"
+#endif
+
namespace libyuv {
+#define SUBSAMPLE(v, a) ((((v) + (a) - 1)) / (a))
+
static void I420TestRotate(int src_width,
int src_height,
int dst_width,
@@ -72,7 +83,7 @@ static void I420TestRotate(int src_width,
// Rotation should be exact.
for (int i = 0; i < dst_i420_size; ++i) {
- EXPECT_EQ(dst_i420_c[i], dst_i420_opt[i]);
+ ASSERT_EQ(dst_i420_c[i], dst_i420_opt[i]);
}
free_aligned_buffer_page_end(dst_i420_c);
@@ -108,33 +119,121 @@ TEST_F(LibYUVRotateTest, I420Rotate270_Opt) {
// Odd width tests work but disabled because they use C code and can be
// tested by passing an odd width command line or environment variable.
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate0_Odd) {
- I420TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate0,
+ I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate90_Odd) {
- I420TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate90,
+ I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate180_Odd) {
- I420TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate180,
+ I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate270_Odd) {
- I420TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate270,
+ I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
+static void I422TestRotate(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height == 0) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ int src_i422_y_size = src_width * Abs(src_height);
+ int src_i422_uv_size = ((src_width + 1) / 2) * Abs(src_height);
+ int src_i422_size = src_i422_y_size + src_i422_uv_size * 2;
+ align_buffer_page_end(src_i422, src_i422_size);
+ for (int i = 0; i < src_i422_size; ++i) {
+ src_i422[i] = fastrand() & 0xff;
+ }
+
+ int dst_i422_y_size = dst_width * dst_height;
+ int dst_i422_uv_size = ((dst_width + 1) / 2) * dst_height;
+ int dst_i422_size = dst_i422_y_size + dst_i422_uv_size * 2;
+ align_buffer_page_end(dst_i422_c, dst_i422_size);
+ align_buffer_page_end(dst_i422_opt, dst_i422_size);
+ memset(dst_i422_c, 2, dst_i422_size);
+ memset(dst_i422_opt, 3, dst_i422_size);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I422Rotate(src_i422, src_width, src_i422 + src_i422_y_size,
+ (src_width + 1) / 2, src_i422 + src_i422_y_size + src_i422_uv_size,
+ (src_width + 1) / 2, dst_i422_c, dst_width,
+ dst_i422_c + dst_i422_y_size, (dst_width + 1) / 2,
+ dst_i422_c + dst_i422_y_size + dst_i422_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ I422Rotate(
+ src_i422, src_width, src_i422 + src_i422_y_size, (src_width + 1) / 2,
+ src_i422 + src_i422_y_size + src_i422_uv_size, (src_width + 1) / 2,
+ dst_i422_opt, dst_width, dst_i422_opt + dst_i422_y_size,
+ (dst_width + 1) / 2, dst_i422_opt + dst_i422_y_size + dst_i422_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+ }
+
+ // Rotation should be exact.
+ for (int i = 0; i < dst_i422_size; ++i) {
+ ASSERT_EQ(dst_i422_c[i], dst_i422_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(dst_i422_c);
+ free_aligned_buffer_page_end(dst_i422_opt);
+ free_aligned_buffer_page_end(src_i422);
+}
+
+TEST_F(LibYUVRotateTest, I422Rotate0_Opt) {
+ I422TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate0, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I422Rotate90_Opt) {
+ I422TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate90, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I422Rotate180_Opt) {
+ I422TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate180, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I422Rotate270_Opt) {
+ I422TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate270, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
static void I444TestRotate(int src_width,
int src_height,
int dst_width,
@@ -189,7 +288,7 @@ static void I444TestRotate(int src_width,
// Rotation should be exact.
for (int i = 0; i < dst_i444_size; ++i) {
- EXPECT_EQ(dst_i444_c[i], dst_i444_opt[i]);
+ ASSERT_EQ(dst_i444_c[i], dst_i444_opt[i]);
}
free_aligned_buffer_page_end(dst_i444_c);
@@ -225,29 +324,29 @@ TEST_F(LibYUVRotateTest, I444Rotate270_Opt) {
// Odd width tests work but disabled because they use C code and can be
// tested by passing an odd width command line or environment variable.
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate0_Odd) {
- I444TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate0,
+ I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate90_Odd) {
- I444TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate90,
+ I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate180_Odd) {
- I444TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate180,
+ I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate270_Odd) {
- I444TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate270,
+ I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
@@ -307,7 +406,7 @@ static void NV12TestRotate(int src_width,
// Rotation should be exact.
for (int i = 0; i < dst_i420_size; ++i) {
- EXPECT_EQ(dst_i420_c[i], dst_i420_opt[i]);
+ ASSERT_EQ(dst_i420_c[i], dst_i420_opt[i]);
}
free_aligned_buffer_page_end(dst_i420_c);
@@ -340,29 +439,29 @@ TEST_F(LibYUVRotateTest, NV12Rotate270_Opt) {
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate0_Odd) {
- NV12TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate0,
+ NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate90_Odd) {
- NV12TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate90,
+ NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate180_Odd) {
- NV12TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_width_ - 3, benchmark_height_ - 1, kRotate180,
+ NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate270_Odd) {
- NV12TestRotate(benchmark_width_ - 3, benchmark_height_ - 1,
- benchmark_height_ - 1, benchmark_width_ - 3, kRotate270,
+ NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
+ benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
@@ -391,4 +490,566 @@ TEST_F(LibYUVRotateTest, NV12Rotate270_Invert) {
disable_cpu_flags_, benchmark_cpu_info_);
}
+// Test Android 420 to I420 Rotate
+#define TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, \
+ SRC_SUBSAMP_Y, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
+ W1280, N, NEG, OFF, PN, OFF_U, OFF_V, ROT) \
+ TEST_F(LibYUVRotateTest, \
+ SRC_FMT_PLANAR##To##FMT_PLANAR##Rotate##ROT##To##PN##N) { \
+ const int kWidth = W1280; \
+ const int kHeight = benchmark_height_; \
+ const int kSizeUV = \
+ SUBSAMPLE(kWidth, SRC_SUBSAMP_X) * SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); \
+ align_buffer_page_end(src_y, kWidth * kHeight + OFF); \
+ align_buffer_page_end(src_uv, \
+ kSizeUV * ((PIXEL_STRIDE == 3) ? 3 : 2) + OFF); \
+ align_buffer_page_end(dst_y_c, kWidth * kHeight); \
+ align_buffer_page_end(dst_u_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
+ SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ align_buffer_page_end(dst_v_c, SUBSAMPLE(kWidth, SUBSAMP_X) * \
+ SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ align_buffer_page_end(dst_y_opt, kWidth * kHeight); \
+ align_buffer_page_end(dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
+ SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ align_buffer_page_end(dst_v_opt, SUBSAMPLE(kWidth, SUBSAMP_X) * \
+ SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ uint8_t* src_u = src_uv + OFF_U; \
+ uint8_t* src_v = src_uv + (PIXEL_STRIDE == 1 ? kSizeUV : OFF_V); \
+ int src_stride_uv = SUBSAMPLE(kWidth, SUBSAMP_X) * PIXEL_STRIDE; \
+ for (int i = 0; i < kHeight; ++i) \
+ for (int j = 0; j < kWidth; ++j) \
+ src_y[i * kWidth + j + OFF] = (fastrand() & 0xff); \
+ for (int i = 0; i < SUBSAMPLE(kHeight, SRC_SUBSAMP_Y); ++i) { \
+ for (int j = 0; j < SUBSAMPLE(kWidth, SRC_SUBSAMP_X); ++j) { \
+ src_u[(i * src_stride_uv) + j * PIXEL_STRIDE + OFF] = \
+ (fastrand() & 0xff); \
+ src_v[(i * src_stride_uv) + j * PIXEL_STRIDE + OFF] = \
+ (fastrand() & 0xff); \
+ } \
+ } \
+ memset(dst_y_c, 1, kWidth * kHeight); \
+ memset(dst_u_c, 2, \
+ SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_v_c, 3, \
+ SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_y_opt, 101, kWidth * kHeight); \
+ memset(dst_u_opt, 102, \
+ SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ memset(dst_v_opt, 103, \
+ SUBSAMPLE(kWidth, SUBSAMP_X) * SUBSAMPLE(kHeight, SUBSAMP_Y)); \
+ MaskCpuFlags(disable_cpu_flags_); \
+ SRC_FMT_PLANAR##To##FMT_PLANAR##Rotate( \
+ src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), \
+ src_v + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), PIXEL_STRIDE, dst_y_c, \
+ kWidth, dst_u_c, SUBSAMPLE(kWidth, SUBSAMP_X), dst_v_c, \
+ SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight, \
+ (libyuv::RotationMode)ROT); \
+ MaskCpuFlags(benchmark_cpu_info_); \
+ for (int i = 0; i < benchmark_iterations_; ++i) { \
+ SRC_FMT_PLANAR##To##FMT_PLANAR##Rotate( \
+ src_y + OFF, kWidth, src_u + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), \
+ src_v + OFF, SUBSAMPLE(kWidth, SRC_SUBSAMP_X), PIXEL_STRIDE, \
+ dst_y_opt, kWidth, dst_u_opt, SUBSAMPLE(kWidth, SUBSAMP_X), \
+ dst_v_opt, SUBSAMPLE(kWidth, SUBSAMP_X), kWidth, NEG kHeight, \
+ (libyuv::RotationMode)ROT); \
+ } \
+ for (int i = 0; i < kHeight; ++i) { \
+ for (int j = 0; j < kWidth; ++j) { \
+ ASSERT_EQ(dst_y_c[i * kWidth + j], dst_y_opt[i * kWidth + j]); \
+ } \
+ } \
+ for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
+ for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
+ ASSERT_EQ(dst_u_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j], \
+ dst_u_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]); \
+ } \
+ } \
+ for (int i = 0; i < SUBSAMPLE(kHeight, SUBSAMP_Y); ++i) { \
+ for (int j = 0; j < SUBSAMPLE(kWidth, SUBSAMP_X); ++j) { \
+ ASSERT_EQ(dst_v_c[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j], \
+ dst_v_opt[i * SUBSAMPLE(kWidth, SUBSAMP_X) + j]); \
+ } \
+ } \
+ free_aligned_buffer_page_end(dst_y_c); \
+ free_aligned_buffer_page_end(dst_u_c); \
+ free_aligned_buffer_page_end(dst_v_c); \
+ free_aligned_buffer_page_end(dst_y_opt); \
+ free_aligned_buffer_page_end(dst_u_opt); \
+ free_aligned_buffer_page_end(dst_v_opt); \
+ free_aligned_buffer_page_end(src_y); \
+ free_aligned_buffer_page_end(src_uv); \
+ }
+
+#define TESTAPLANARTOP(SRC_FMT_PLANAR, PN, PIXEL_STRIDE, OFF_U, OFF_V, \
+ SRC_SUBSAMP_X, SRC_SUBSAMP_Y, FMT_PLANAR, SUBSAMP_X, \
+ SUBSAMP_Y) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_ + 1, \
+ _Any, +, 0, PN, OFF_U, OFF_V, 0) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \
+ _Unaligned, +, 2, PN, OFF_U, OFF_V, 0) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Invert, \
+ -, 0, PN, OFF_U, OFF_V, 0) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, \
+ 0, PN, OFF_U, OFF_V, 0) \
+ TESTAPLANARTOPI(SRC_FMT_PLANAR, PIXEL_STRIDE, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
+ FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, _Opt, +, \
+ 0, PN, OFF_U, OFF_V, 180)
+
+TESTAPLANARTOP(Android420, I420, 1, 0, 0, 2, 2, I420, 2, 2)
+TESTAPLANARTOP(Android420, NV12, 2, 0, 1, 2, 2, I420, 2, 2)
+TESTAPLANARTOP(Android420, NV21, 2, 1, 0, 2, 2, I420, 2, 2)
+#undef TESTAPLANARTOP
+#undef TESTAPLANARTOPI
+
+static void I010TestRotate(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height == 0) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ int src_i010_y_size = src_width * Abs(src_height);
+ int src_i010_uv_size = ((src_width + 1) / 2) * ((Abs(src_height) + 1) / 2);
+ int src_i010_size = src_i010_y_size + src_i010_uv_size * 2;
+ align_buffer_page_end_16(src_i010, src_i010_size);
+ for (int i = 0; i < src_i010_size; ++i) {
+ src_i010[i] = fastrand() & 0x3ff;
+ }
+
+ int dst_i010_y_size = dst_width * dst_height;
+ int dst_i010_uv_size = ((dst_width + 1) / 2) * ((dst_height + 1) / 2);
+ int dst_i010_size = dst_i010_y_size + dst_i010_uv_size * 2;
+ align_buffer_page_end_16(dst_i010_c, dst_i010_size);
+ align_buffer_page_end_16(dst_i010_opt, dst_i010_size);
+ memset(dst_i010_c, 2, dst_i010_size * 2);
+ memset(dst_i010_opt, 3, dst_i010_size * 2);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I010Rotate(src_i010, src_width, src_i010 + src_i010_y_size,
+ (src_width + 1) / 2, src_i010 + src_i010_y_size + src_i010_uv_size,
+ (src_width + 1) / 2, dst_i010_c, dst_width,
+ dst_i010_c + dst_i010_y_size, (dst_width + 1) / 2,
+ dst_i010_c + dst_i010_y_size + dst_i010_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ I010Rotate(
+ src_i010, src_width, src_i010 + src_i010_y_size, (src_width + 1) / 2,
+ src_i010 + src_i010_y_size + src_i010_uv_size, (src_width + 1) / 2,
+ dst_i010_opt, dst_width, dst_i010_opt + dst_i010_y_size,
+ (dst_width + 1) / 2, dst_i010_opt + dst_i010_y_size + dst_i010_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+ }
+
+ // Rotation should be exact.
+ for (int i = 0; i < dst_i010_size; ++i) {
+ ASSERT_EQ(dst_i010_c[i], dst_i010_opt[i]);
+ }
+
+ free_aligned_buffer_page_end_16(dst_i010_c);
+ free_aligned_buffer_page_end_16(dst_i010_opt);
+ free_aligned_buffer_page_end_16(src_i010);
+}
+
+TEST_F(LibYUVRotateTest, I010Rotate0_Opt) {
+ I010TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate0, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I010Rotate90_Opt) {
+ I010TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate90, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I010Rotate180_Opt) {
+ I010TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate180, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I010Rotate270_Opt) {
+ I010TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate270, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+static void I210TestRotate(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height == 0) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ int src_i210_y_size = src_width * Abs(src_height);
+ int src_i210_uv_size = ((src_width + 1) / 2) * Abs(src_height);
+ int src_i210_size = src_i210_y_size + src_i210_uv_size * 2;
+ align_buffer_page_end_16(src_i210, src_i210_size);
+ for (int i = 0; i < src_i210_size; ++i) {
+ src_i210[i] = fastrand() & 0x3ff;
+ }
+
+ int dst_i210_y_size = dst_width * dst_height;
+ int dst_i210_uv_size = ((dst_width + 1) / 2) * dst_height;
+ int dst_i210_size = dst_i210_y_size + dst_i210_uv_size * 2;
+ align_buffer_page_end_16(dst_i210_c, dst_i210_size);
+ align_buffer_page_end_16(dst_i210_opt, dst_i210_size);
+ memset(dst_i210_c, 2, dst_i210_size * 2);
+ memset(dst_i210_opt, 3, dst_i210_size * 2);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I210Rotate(src_i210, src_width, src_i210 + src_i210_y_size,
+ (src_width + 1) / 2, src_i210 + src_i210_y_size + src_i210_uv_size,
+ (src_width + 1) / 2, dst_i210_c, dst_width,
+ dst_i210_c + dst_i210_y_size, (dst_width + 1) / 2,
+ dst_i210_c + dst_i210_y_size + dst_i210_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ I210Rotate(
+ src_i210, src_width, src_i210 + src_i210_y_size, (src_width + 1) / 2,
+ src_i210 + src_i210_y_size + src_i210_uv_size, (src_width + 1) / 2,
+ dst_i210_opt, dst_width, dst_i210_opt + dst_i210_y_size,
+ (dst_width + 1) / 2, dst_i210_opt + dst_i210_y_size + dst_i210_uv_size,
+ (dst_width + 1) / 2, src_width, src_height, mode);
+ }
+
+ // Rotation should be exact.
+ for (int i = 0; i < dst_i210_size; ++i) {
+ ASSERT_EQ(dst_i210_c[i], dst_i210_opt[i]);
+ }
+
+ free_aligned_buffer_page_end_16(dst_i210_c);
+ free_aligned_buffer_page_end_16(dst_i210_opt);
+ free_aligned_buffer_page_end_16(src_i210);
+}
+
+TEST_F(LibYUVRotateTest, I210Rotate0_Opt) {
+ I210TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate0, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I210Rotate90_Opt) {
+ I210TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate90, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I210Rotate180_Opt) {
+ I210TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate180, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I210Rotate270_Opt) {
+ I210TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate270, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+static void I410TestRotate(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (src_width < 1) {
+ src_width = 1;
+ }
+ if (src_height == 0) {
+ src_height = 1;
+ }
+ if (dst_width < 1) {
+ dst_width = 1;
+ }
+ if (dst_height < 1) {
+ dst_height = 1;
+ }
+ int src_i410_y_size = src_width * Abs(src_height);
+ int src_i410_uv_size = src_width * Abs(src_height);
+ int src_i410_size = src_i410_y_size + src_i410_uv_size * 2;
+ align_buffer_page_end_16(src_i410, src_i410_size);
+ for (int i = 0; i < src_i410_size; ++i) {
+ src_i410[i] = fastrand() & 0x3ff;
+ }
+
+ int dst_i410_y_size = dst_width * dst_height;
+ int dst_i410_uv_size = dst_width * dst_height;
+ int dst_i410_size = dst_i410_y_size + dst_i410_uv_size * 2;
+ align_buffer_page_end_16(dst_i410_c, dst_i410_size);
+ align_buffer_page_end_16(dst_i410_opt, dst_i410_size);
+ memset(dst_i410_c, 2, dst_i410_size * 2);
+ memset(dst_i410_opt, 3, dst_i410_size * 2);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I410Rotate(src_i410, src_width, src_i410 + src_i410_y_size, src_width,
+ src_i410 + src_i410_y_size + src_i410_uv_size, src_width,
+ dst_i410_c, dst_width, dst_i410_c + dst_i410_y_size, dst_width,
+ dst_i410_c + dst_i410_y_size + dst_i410_uv_size, dst_width,
+ src_width, src_height, mode);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (int i = 0; i < benchmark_iterations; ++i) {
+ I410Rotate(src_i410, src_width, src_i410 + src_i410_y_size, src_width,
+ src_i410 + src_i410_y_size + src_i410_uv_size, src_width,
+ dst_i410_opt, dst_width, dst_i410_opt + dst_i410_y_size,
+ dst_width, dst_i410_opt + dst_i410_y_size + dst_i410_uv_size,
+ dst_width, src_width, src_height, mode);
+ }
+
+ // Rotation should be exact.
+ for (int i = 0; i < dst_i410_size; ++i) {
+ ASSERT_EQ(dst_i410_c[i], dst_i410_opt[i]);
+ }
+
+ free_aligned_buffer_page_end_16(dst_i410_c);
+ free_aligned_buffer_page_end_16(dst_i410_opt);
+ free_aligned_buffer_page_end_16(src_i410);
+}
+
+TEST_F(LibYUVRotateTest, I410Rotate0_Opt) {
+ I410TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate0, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I410Rotate90_Opt) {
+ I410TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate90, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I410Rotate180_Opt) {
+ I410TestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
+ benchmark_height_, kRotate180, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+TEST_F(LibYUVRotateTest, I410Rotate270_Opt) {
+ I410TestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
+ benchmark_width_, kRotate270, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_);
+}
+
+#if defined(ENABLE_ROW_TESTS)
+
+TEST_F(LibYUVRotateTest, TransposeWx4_32_Test) {
+ // dst width and height
+ const int width = 4;
+ const int height = 4;
+ int src_pixels[4][4];
+ int dst_pixels_c[4][4];
+ int dst_pixels_opt[4][4];
+
+ for (int i = 0; i < 4; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ src_pixels[i][j] = i * 10 + j;
+ }
+ }
+ memset(dst_pixels_c, 1, width * height * 4);
+ memset(dst_pixels_opt, 2, width * height * 4);
+
+ TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_c, width * 4, width);
+
+ const int benchmark_iterations =
+ (benchmark_iterations_ * benchmark_width_ * benchmark_height_ + 15) /
+ (4 * 4);
+ for (int i = 0; i < benchmark_iterations; ++i) {
+#if defined(HAS_TRANSPOSEWX4_32_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ } else
+#elif defined(HAS_TRANSPOSEWX4_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ } else
+#endif
+ {
+ TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ }
+ }
+
+ for (int i = 0; i < 4; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ ASSERT_EQ(dst_pixels_c[i][j], src_pixels[j][i]);
+ ASSERT_EQ(dst_pixels_c[i][j], dst_pixels_opt[i][j]);
+ }
+ }
+}
+
+TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) {
+ // dst width and height
+ const int width = ((benchmark_width_ * benchmark_height_ + 3) / 4 + 3) & ~3;
+ const int height = 4;
+ align_buffer_page_end(src_pixels, height * width * 4);
+ align_buffer_page_end(dst_pixels_c, width * height * 4);
+ align_buffer_page_end(dst_pixels_opt, width * height * 4);
+
+ MemRandomize(src_pixels, height * width * 4);
+ memset(dst_pixels_c, 1, width * height * 4);
+ memset(dst_pixels_opt, 2, width * height * 4);
+
+ TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_c, width * 4, width);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+#if defined(HAS_TRANSPOSEWX4_32_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ } else
+#elif defined(HAS_TRANSPOSEWX4_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ } else
+#endif
+ {
+ TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
+ (uint8_t*)dst_pixels_opt, width * 4, width);
+ }
+ }
+
+ for (int i = 0; i < width * height; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+TEST_F(LibYUVRotateTest, TransposeWx4_32_Any) {
+ for (int width : {4, 8, 12, 16, 20, 28, 36}) {
+ const int height = 4;
+ align_buffer_page_end(src_pixels, height * width * 4);
+ align_buffer_page_end(dst_pixels_c, width * height * 4);
+ align_buffer_page_end(dst_pixels_opt, width * height * 4);
+
+ MemRandomize(src_pixels, height * width * 4);
+ memset(dst_pixels_c, 1, width * height * 4);
+ memset(dst_pixels_opt, 2, width * height * 4);
+
+ TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_c, width * 4, width);
+#if defined(HAS_TRANSPOSEWX4_32_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ if ((width & 7) == 0) {
+ TransposeWx4_32_AVX2(src_pixels, height * 4, dst_pixels_opt, width * 4,
+ width);
+ } else {
+ TransposeWx4_32_Any_AVX2(src_pixels, height * 4, dst_pixels_opt,
+ width * 4, width);
+ }
+ } else
+#endif
+#if defined(HAS_TRANSPOSEWX4_32_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx4_32_NEON(src_pixels, height * 4, dst_pixels_opt, width * 4,
+ width);
+ } else
+#endif
+ {
+ TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_opt, width * 4,
+ width);
+ }
+
+ ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * height * 4))
+ << "width=" << width;
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+ }
+}
+
+#if defined(HAS_TRANSPOSEWXH_32_AVX2)
+TEST_F(LibYUVRotateTest, TransposeWxH_32_Test) {
+ if (!TestCpuFlag(kCpuHasAVX2)) {
+ return;
+ }
+ const int kMaxW = 35;
+ const int kMaxH = 25;
+ align_buffer_page_end(src_pixels, kMaxW * kMaxH * 4);
+ align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH * 4);
+ align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH * 4);
+ MemRandomize(src_pixels, kMaxW * kMaxH * 4);
+
+ for (int h = 1; h <= kMaxH; ++h) {
+ for (int w = 1; w <= kMaxW; ++w) {
+ const int src_stride = w * 4;
+ const int dst_stride = h * 4;
+ const int src_bytes = src_stride * h;
+ const int dst_bytes = dst_stride * w;
+ const uint8_t* src = src_pixels + kMaxW * kMaxH * 4 - src_bytes;
+ uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH * 4 - dst_bytes;
+ uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH * 4 - dst_bytes;
+
+ memset(dst_c, 1, dst_bytes);
+ memset(dst_opt, 2, dst_bytes);
+ TransposeWxH_32_C(src, src_stride, dst_c, dst_stride, w, h);
+ TransposeWxH_32_AVX2(src, src_stride, dst_opt, dst_stride, w, h);
+ ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h;
+
+ // Also test negative src_stride (90-degree) and negative dst_stride
+ // (270-degree).
+ memset(dst_c, 3, dst_bytes);
+ memset(dst_opt, 4, dst_bytes);
+ TransposeWxH_32_C(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
+ dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w,
+ h);
+ TransposeWxH_32_AVX2(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
+ dst_opt + (ptrdiff_t)(w - 1) * dst_stride,
+ -dst_stride, w, h);
+ ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << "neg " << w << "x" << h;
+ }
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
+
+#endif // ENABLE_ROW_TESTS
+
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/scale_argb_test.cc b/libs/libyuv/unit_test/scale_argb_test.cc
index 93b77f56a9..532bcb5097 100644
--- a/libs/libyuv/unit_test/scale_argb_test.cc
+++ b/libs/libyuv/unit_test/scale_argb_test.cc
@@ -22,6 +22,12 @@ namespace libyuv {
#define STRINGIZE(line) #line
#define FILELINESTR(file, line) file ":" STRINGIZE(line)
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
static int ARGBTestFilter(int src_width,
int src_height,
@@ -114,8 +120,8 @@ static int ARGBTestFilter(int src_width,
return max_diff;
}
-static const int kTileX = 8;
-static const int kTileY = 8;
+static const int kTileX = 64;
+static const int kTileY = 64;
static int TileARGBScale(const uint8_t* src_argb,
int src_stride_argb,
@@ -232,34 +238,49 @@ static int ARGBClipTestFilter(int src_width,
#define DX(x, nom, denom) static_cast<int>((Abs(x) / nom) * nom)
#define SX(x, nom, denom) static_cast<int>((x / nom) * denom)
-#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
+#define TEST_FACTOR1(DISABLED_, name, filter, nom, denom, max_diff) \
TEST_F(LibYUVScaleTest, ARGBScaleDownBy##name##_##filter) { \
int diff = ARGBTestFilter( \
SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
} \
- TEST_F(LibYUVScaleTest, ARGBScaleDownClipBy##name##_##filter) { \
+ TEST_F(LibYUVScaleTest, DISABLED_##ARGBScaleDownClipBy##name##_##filter) { \
int diff = ARGBClipTestFilter( \
SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
kFilter##filter, benchmark_iterations_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
}
// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but
// filtering is different fixed point implementations for SSSE3, Neon and C.
-#define TEST_FACTOR(name, nom, denom) \
- TEST_FACTOR1(name, None, nom, denom, 0) \
- TEST_FACTOR1(name, Linear, nom, denom, 3) \
- TEST_FACTOR1(name, Bilinear, nom, denom, 3) \
- TEST_FACTOR1(name, Box, nom, denom, 3)
+#ifndef DISABLE_SLOW_TESTS
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(, name, None, nom, denom, 0) \
+ TEST_FACTOR1(, name, Linear, nom, denom, 3) \
+ TEST_FACTOR1(, name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(, name, Box, nom, denom, 3)
+#else
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(DISABLED_, name, None, nom, denom, 0) \
+ TEST_FACTOR1(DISABLED_, name, Linear, nom, denom, 3) \
+ TEST_FACTOR1(DISABLED_, name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(DISABLED_, name, Box, nom, denom, 3)
+#else
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(DISABLED_, name, Bilinear, nom, denom, 3)
+#endif
+#endif
TEST_FACTOR(2, 1, 2)
TEST_FACTOR(4, 1, 4)
-// TEST_FACTOR(8, 1, 8) Disable for benchmark performance.
+#ifndef DISABLE_SLOW_TESTS
+TEST_FACTOR(8, 1, 8)
+#endif
TEST_FACTOR(3by4, 3, 4)
TEST_FACTOR(3by8, 3, 8)
TEST_FACTOR(3, 1, 3)
@@ -268,70 +289,108 @@ TEST_FACTOR(3, 1, 3)
#undef SX
#undef DX
-#define TEST_SCALETO1(name, width, height, filter, max_diff) \
+#define TEST_SCALETO1(DISABLED_, name, width, height, filter, max_diff) \
TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \
int diff = ARGBTestFilter(benchmark_width_, benchmark_height_, width, \
height, kFilter##filter, benchmark_iterations_, \
disable_cpu_flags_, benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
} \
TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \
int diff = ARGBTestFilter(width, height, Abs(benchmark_width_), \
Abs(benchmark_height_), kFilter##filter, \
benchmark_iterations_, disable_cpu_flags_, \
benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
} \
- TEST_F(LibYUVScaleTest, name##ClipTo##width##x##height##_##filter) { \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##name##ClipTo##width##x##height##_##filter) { \
int diff = \
ARGBClipTestFilter(benchmark_width_, benchmark_height_, width, height, \
kFilter##filter, benchmark_iterations_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
} \
- TEST_F(LibYUVScaleTest, name##ClipFrom##width##x##height##_##filter) { \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##name##ClipFrom##width##x##height##_##filter) { \
int diff = ARGBClipTestFilter(width, height, Abs(benchmark_width_), \
Abs(benchmark_height_), kFilter##filter, \
benchmark_iterations_); \
- EXPECT_LE(diff, max_diff); \
+ ASSERT_LE(diff, max_diff); \
}
-/// Test scale to a specified size with all 4 filters.
-#define TEST_SCALETO(name, width, height) \
- TEST_SCALETO1(name, width, height, None, 0) \
- TEST_SCALETO1(name, width, height, Linear, 3) \
- TEST_SCALETO1(name, width, height, Bilinear, 3)
+#ifndef DISABLE_SLOW_TESTS
+// Test scale to a specified size with all 4 filters.
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(, name, width, height, None, 0) \
+ TEST_SCALETO1(, name, width, height, Linear, 3) \
+ TEST_SCALETO1(, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(, name, width, height, Box, 3)
+#else
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(DISABLED_, name, width, height, None, 0) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Linear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Box, 3)
+#else
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3)
+#endif
+#endif
TEST_SCALETO(ARGBScale, 1, 1)
-TEST_SCALETO(ARGBScale, 320, 240)
TEST_SCALETO(ARGBScale, 569, 480)
TEST_SCALETO(ARGBScale, 640, 360)
+#ifndef DISABLE_SLOW_TESTS
+TEST_SCALETO(ARGBScale, 50, 1)
+TEST_SCALETO(ARGBScale, 256, 144) /* 128x72 * 2 */
+TEST_SCALETO(ARGBScale, 320, 240)
TEST_SCALETO(ARGBScale, 1280, 720)
TEST_SCALETO(ARGBScale, 1920, 1080)
+#endif // DISABLE_SLOW_TESTS
#undef TEST_SCALETO1
#undef TEST_SCALETO
+#define TEST_SCALESWAPXY1(name, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##SwapXY_##filter) { \
+ int diff = ARGBTestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+// Test scale with swapped width and height with all 3 filters.
+TEST_SCALESWAPXY1(ARGBScale, None, 0)
+TEST_SCALESWAPXY1(ARGBScale, Linear, 0)
+TEST_SCALESWAPXY1(ARGBScale, Bilinear, 0)
+#else
+TEST_SCALESWAPXY1(ARGBScale, Bilinear, 0)
+#endif
+#undef TEST_SCALESWAPXY1
+
// Scale with YUV conversion to ARGB and clipping.
// TODO(fbarchard): Add fourcc support. All 4 ARGB formats is easy to support.
-LIBYUV_API
-int YUVToARGBScaleReference2(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint32_t /* src_fourcc */,
- int src_width,
- int src_height,
- uint8_t* dst_argb,
- int dst_stride_argb,
- uint32_t /* dst_fourcc */,
- int dst_width,
- int dst_height,
- int clip_x,
- int clip_y,
- int clip_width,
- int clip_height,
- enum FilterMode filtering) {
+static int YUVToARGBScaleReference2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint32_t /* src_fourcc */,
+ int src_width,
+ int src_height,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ uint32_t /* dst_fourcc */,
+ int dst_width,
+ int dst_height,
+ int clip_x,
+ int clip_y,
+ int clip_width,
+ int clip_height,
+ enum FilterMode filtering) {
uint8_t* argb_buffer =
static_cast<uint8_t*>(malloc(src_width * src_height * 4));
int r;
@@ -371,12 +430,14 @@ static void FillRamp(uint8_t* buf,
}
// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
-static int YUVToARGBTestFilter(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- FilterMode f,
- int benchmark_iterations) {
+static void YUVToARGBTestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int error_threshold,
+ int* max_diff_out) {
int64_t src_y_plane_size = Abs(src_width) * Abs(src_height);
int64_t src_uv_plane_size =
((Abs(src_width) + 1) / 2) * ((Abs(src_height) + 1) / 2);
@@ -387,13 +448,13 @@ static int YUVToARGBTestFilter(int src_width,
align_buffer_page_end(src_u, src_uv_plane_size);
align_buffer_page_end(src_v, src_uv_plane_size);
- int64_t dst_argb_plane_size = (dst_width) * (dst_height)*4LL;
- int dst_stride_argb = (dst_width)*4;
+ int64_t dst_argb_plane_size = (dst_width) * (dst_height) * 4LL;
+ int dst_stride_argb = (dst_width) * 4;
align_buffer_page_end(dst_argb_c, dst_argb_plane_size);
align_buffer_page_end(dst_argb_opt, dst_argb_plane_size);
if (!dst_argb_c || !dst_argb_opt || !src_y || !src_u || !src_v) {
printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
- return 0;
+ return;
}
// Fill YUV image with continuous ramp, which is less sensitive to
// subsampling and filtering differences for test purposes.
@@ -422,36 +483,199 @@ static int YUVToARGBTestFilter(int src_width,
int abs_diff = Abs(dst_argb_c[(i * dst_stride_argb) + j] -
dst_argb_opt[(i * dst_stride_argb) + j]);
if (abs_diff > max_diff) {
- printf("error %d at %d,%d c %d opt %d", abs_diff, j, i,
+ max_diff = abs_diff;
+ }
+ if (abs_diff > error_threshold) {
+ printf("error %d at %d,%d c %d opt %d\n", abs_diff, j, i,
dst_argb_c[(i * dst_stride_argb) + j],
dst_argb_opt[(i * dst_stride_argb) + j]);
- EXPECT_LE(abs_diff, 40);
- max_diff = abs_diff;
+ goto cleanup;
}
}
}
+cleanup:
+ if (max_diff_out) {
+ *max_diff_out = max_diff;
+ }
free_aligned_buffer_page_end(dst_argb_c);
free_aligned_buffer_page_end(dst_argb_opt);
free_aligned_buffer_page_end(src_y);
free_aligned_buffer_page_end(src_u);
free_aligned_buffer_page_end(src_v);
- return max_diff;
}
TEST_F(LibYUVScaleTest, YUVToRGBScaleUp) {
- int diff =
- YUVToARGBTestFilter(benchmark_width_, benchmark_height_,
- benchmark_width_ * 3 / 2, benchmark_height_ * 3 / 2,
- libyuv::kFilterBilinear, benchmark_iterations_);
- EXPECT_LE(diff, 10);
+ int diff = 0;
+ YUVToARGBTestFilter(benchmark_width_, benchmark_height_,
+ benchmark_width_ * 3 / 2, benchmark_height_ * 3 / 2,
+ libyuv::kFilterBilinear, benchmark_iterations_, 10,
+ &diff);
+ ASSERT_LE(diff, 10);
}
TEST_F(LibYUVScaleTest, YUVToRGBScaleDown) {
- int diff = YUVToARGBTestFilter(
- benchmark_width_ * 3 / 2, benchmark_height_ * 3 / 2, benchmark_width_,
- benchmark_height_, libyuv::kFilterBilinear, benchmark_iterations_);
- EXPECT_LE(diff, 10);
+ int diff = 0;
+ YUVToARGBTestFilter(benchmark_width_ * 3 / 2, benchmark_height_ * 3 / 2,
+ benchmark_width_, benchmark_height_,
+ libyuv::kFilterBilinear, benchmark_iterations_, 10,
+ &diff);
+ ASSERT_LE(diff, 10);
+}
+
+TEST_F(LibYUVScaleTest, ARGBTest3x) {
+ const int kSrcStride = 480 * 4;
+ const int kDstStride = 160 * 4;
+ const int kSize = kSrcStride * 3;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 480 * 3; ++i) {
+ orig_pixels[i * 4 + 0] = i;
+ orig_pixels[i * 4 + 1] = 255 - i;
+ orig_pixels[i * 4 + 2] = i + 1;
+ orig_pixels[i * 4 + 3] = i + 10;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ ARGBScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+ ASSERT_EQ(226, dest_pixels[2]);
+ ASSERT_EQ(235, dest_pixels[3]);
+
+ ARGBScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+ ASSERT_EQ(226, dest_pixels[2]);
+ ASSERT_EQ(235, dest_pixels[3]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, ARGBTest4x) {
+ const int kSrcStride = 640 * 4;
+ const int kDstStride = 160 * 4;
+ const int kSize = kSrcStride * 4;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 640 * 4; ++i) {
+ orig_pixels[i * 4 + 0] = i;
+ orig_pixels[i * 4 + 1] = 255 - i;
+ orig_pixels[i * 4 + 2] = i + 1;
+ orig_pixels[i * 4 + 3] = i + 10;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ ARGBScale(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_NEAR(66, dest_pixels[0], 4);
+ ASSERT_NEAR(255 - 66, dest_pixels[1], 4);
+ ASSERT_NEAR(67, dest_pixels[2], 4);
+ ASSERT_NEAR(76, dest_pixels[3], 4);
+
+ ARGBScale(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(2, dest_pixels[0]);
+ ASSERT_EQ(255 - 2, dest_pixels[1]);
+ ASSERT_EQ(3, dest_pixels[2]);
+ ASSERT_EQ(12, dest_pixels[3]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, ARGBScale_LargeHeightOverflow) {
+ const int src_w = 4;
+ const int src_h = 32767;
+ align_buffer_page_end(orig_pixels, src_w * src_h * 4);
+ align_buffer_page_end(dest_pixels, src_w * 2 * 4);
+ memset(orig_pixels, 128, src_w * src_h * 4);
+
+ // Test both vertical-only scaling (dst_w == src_w) and 2D scaling (dst_w !=
+ // src_w).
+ for (int dst_w : {4, 2}) {
+ int res = ARGBScale(orig_pixels, src_w * 4, src_w, src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterBilinear);
+ EXPECT_EQ(0, res);
+
+ res = ARGBScale(orig_pixels, src_w * 4, src_w, -src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterBilinear);
+ EXPECT_EQ(0, res);
+
+ res = ARGBScale(orig_pixels, src_w * 4, src_w, src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterLinear);
+ EXPECT_EQ(0, res);
+
+ res = ARGBScale(orig_pixels, src_w * 4, src_w, -src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterLinear);
+ EXPECT_EQ(0, res);
+
+ res = ARGBScale(orig_pixels, src_w * 4, src_w, src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterNone);
+ EXPECT_EQ(0, res);
+
+ res = ARGBScale(orig_pixels, src_w * 4, src_w, -src_h, dest_pixels,
+ dst_w * 4, dst_w, 2, kFilterNone);
+ EXPECT_EQ(0, res);
+ }
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, ARGBScaleDimensionOverflow) {
+ uint8_t src[16] = {};
+ uint8_t dst[4] = {};
+ EXPECT_EQ(-1, ARGBScale(src, 4, 32769, 1, dst, 4, 1, 1, kFilterNone));
+ EXPECT_EQ(-1, ARGBScale(src, 4, -32769, 1, dst, 4, 1, 1, kFilterNone));
+ EXPECT_EQ(-1, ARGBScale(src, 4, 1, 32769, dst, 4, 1, 1, kFilterNone));
+ EXPECT_EQ(-1, ARGBScale(src, 4, 1, -32769, dst, 4, 1, 1, kFilterNone));
+ EXPECT_EQ(-1, ARGBScaleClip(src, 4, 32769, 1, dst, 4, 1, 1, 0, 0, 1, 1,
+ kFilterNone));
+ EXPECT_EQ(-1, ARGBScaleClip(src, 4, -32769, 1, dst, 4, 1, 1, 0, 0, 1, 1,
+ kFilterNone));
+ EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, 32769, dst, 4, 1, 1, 0, 0, 1, 1,
+ kFilterNone));
+ EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, -32769, dst, 4, 1, 1, 0, 0, 1, 1,
+ kFilterNone));
+}
+
+#if defined(__LP64__) || defined(_WIN64)
+TEST_F(LibYUVScaleTest, ARGBScaleLargeImageOver2GB) {
+ const int kSrcWidth = 4;
+ const int kSrcHeight = 32768;
+ const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX)
+ const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight;
+ uint8_t* src = static_cast<uint8_t*>(calloc(1, kSrcSize));
+ if (!src) {
+ GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes";
+ }
+ uint8_t dst[8 * 4 * 4] = {};
+
+ for (int dst_w : {4, 2}) {
+ for (FilterMode f :
+ {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
+ EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst,
+ dst_w * 4, dst_w, 2, f));
+ EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst,
+ dst_w * 4, dst_w, 2, f));
+ }
+ }
+ free(src);
}
+#endif
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/scale_plane_test.cc b/libs/libyuv/unit_test/scale_plane_test.cc
new file mode 100644
index 0000000000..012377d5cd
--- /dev/null
+++ b/libs/libyuv/unit_test/scale_plane_test.cc
@@ -0,0 +1,919 @@
+/*
+ * Copyright 2023 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include <new>
+
+#include "../unit_test/unit_test.h"
+#include "libyuv/cpu_id.h"
+#include "libyuv/scale.h"
+#include "libyuv/scale_uv.h"
+
+#ifdef ENABLE_ROW_TESTS
+#include "libyuv/scale_row.h" // For ScaleRowDown2Box_Odd_C
+#endif
+
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer)
+#include <sanitizer/msan_interface.h>
+#define MSAN_POISON(ptr, size) __msan_allocated_memory(ptr, size)
+#endif
+#endif
+#ifndef MSAN_POISON
+#define MSAN_POISON(ptr, size)
+#endif
+
+#define STRINGIZE(line) #line
+#define FILELINESTR(file, line) file ":" STRINGIZE(line)
+
+#if (defined(__riscv) && !defined(__clang__)) || defined(__hexagon__)
+#define DISABLE_SLOW_TESTS
+#undef ENABLE_FULL_TESTS
+#undef ENABLE_ROW_TESTS
+#define LEAN_TESTS
+#endif
+
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
+namespace libyuv {
+
+// POC: int row_stride = src_stride * 2 overflows to a small negative value
+// when src_stride is close to INT_MAX, causing src_ptr to walk backward
+// past the start of the source allocation on the second loop iteration.
+// With src_stride = 0x7FFFFFFE, row_stride = (int)0xFFFFFFFC = -4, so on
+// y=1 ScaleRowDown2Box reads 4 bytes before the heap allocation.
+TEST_F(LibYUVScaleTest, ScalePlaneDown2_RowStrideOverflow) {
+ constexpr int kSrcStride = 0x7FFFFFFE; // INT_MAX - 1
+ constexpr int kSrcW = 64;
+ constexpr int kSrcH = 4;
+ constexpr int kDstW = 32;
+ constexpr int kDstH = 2;
+ // src_size = (kSrcH - 1) * stride + width.
+ size_t src_size = kSrcH - 1;
+ if (src_size > SIZE_MAX / kSrcStride) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size *= kSrcStride;
+ if (src_size > SIZE_MAX - kSrcW) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size += kSrcW;
+
+#if defined(__aarch64__)
+ // Infer malloc can accept a large size for cpu with dot product (a76/a55)
+ int has_large_malloc = TestCpuFlag(kCpuHasNeonDotProd);
+#else
+ int has_large_malloc = 1;
+#endif
+ if (!has_large_malloc) {
+ GTEST_SKIP() << "large allocation may assert for " << src_size << " bytes";
+ }
+
+ uint8_t* src = new (std::nothrow) uint8_t[src_size];
+ if (!src) {
+ GTEST_SKIP() << "could not allocate " << src_size << " bytes";
+ }
+ uint8_t dst[kDstW * kDstH];
+ uint8_t* src_row = src;
+ for (int i = 0; i < kSrcH; i++) {
+ memset(src_row, 0x41, kSrcW);
+ src_row += kSrcStride;
+ }
+ // Force the C row kernel: the SIMD kernels are inline asm that ASAN does not
+ // instrument, so they silently read OOB without a report.
+ MaskCpuFlags(1);
+ // 2*dst == src on both axes -> ScalePlane dispatches to ScalePlaneDown2.
+ // int row_stride = kSrcStride * 2 wraps to -4; on y=1 src_ptr underflows.
+ ScalePlane(src, kSrcStride, kSrcW, kSrcH, dst, kDstW, kDstW, kDstH,
+ kFilterBox);
+ MaskCpuFlags(0);
+ delete[] src;
+}
+
+// POC: same defect in the 1/4 fast path. src_stride = 0x3FFFFFFF gives
+// int row_stride = src_stride * 4 = (int)0xFFFFFFFC = -4.
+TEST_F(LibYUVScaleTest, ScalePlaneDown4_RowStrideOverflow) {
+ constexpr int kSrcStride = 0x3FFFFFFF; // INT_MAX / 4 (rounded down)
+ constexpr int kSrcW = 64;
+ constexpr int kSrcH = 8;
+ constexpr int kDstW = 16;
+ constexpr int kDstH = 2;
+ // src_size = (kSrcH - 1) * stride + width.
+ size_t src_size = kSrcH - 1;
+ if (src_size > SIZE_MAX / kSrcStride) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size *= kSrcStride;
+ if (src_size > SIZE_MAX - kSrcW) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size += kSrcW;
+
+#if defined(__aarch64__)
+ // Infer malloc can accept a large size for cpu with dot product (a76/a55)
+ int has_large_malloc = TestCpuFlag(kCpuHasNeonDotProd);
+#else
+ int has_large_malloc = 1;
+#endif
+ if (!has_large_malloc) {
+ GTEST_SKIP() << "large allocation may assert for " << src_size << " bytes";
+ }
+
+ uint8_t* src = new (std::nothrow) uint8_t[src_size];
+ if (!src) {
+ GTEST_SKIP() << "could not allocate " << src_size << " bytes";
+ }
+ uint8_t dst[kDstW * kDstH];
+ uint8_t* src_row = src;
+ for (int i = 0; i < kSrcH; i++) {
+ memset(src_row, 0x41, kSrcW);
+ src_row += kSrcStride;
+ }
+ // Force the C row kernel: the SIMD kernels are inline asm that ASAN does not
+ // instrument, so they silently read OOB without a report.
+ MaskCpuFlags(1);
+ // 4*dst == src on both axes with kFilterBox -> ScalePlaneDown4.
+ ScalePlane(src, kSrcStride, kSrcW, kSrcH, dst, kDstW, kDstW, kDstH,
+ kFilterBox);
+ MaskCpuFlags(0);
+ delete[] src;
+}
+
+#ifdef ENABLE_ROW_TESTS
+#ifdef HAS_SCALEROWDOWN2_SSSE3
+TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_Odd_SSSE3) {
+ SIMD_ALIGNED(uint8_t orig_pixels[128 * 2]);
+ SIMD_ALIGNED(uint8_t dst_pixels_opt[64]);
+ SIMD_ALIGNED(uint8_t dst_pixels_c[64]);
+ memset(orig_pixels, 0, sizeof(orig_pixels));
+ memset(dst_pixels_opt, 0, sizeof(dst_pixels_opt));
+ memset(dst_pixels_c, 0, sizeof(dst_pixels_c));
+
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ if (!has_ssse3) {
+ printf("Warning SSSE3 not detected; Skipping test.\n");
+ } else {
+ // TL.
+ orig_pixels[0] = 255u;
+ orig_pixels[1] = 0u;
+ orig_pixels[128 + 0] = 0u;
+ orig_pixels[128 + 1] = 0u;
+ // TR.
+ orig_pixels[2] = 0u;
+ orig_pixels[3] = 100u;
+ orig_pixels[128 + 2] = 0u;
+ orig_pixels[128 + 3] = 0u;
+ // BL.
+ orig_pixels[4] = 0u;
+ orig_pixels[5] = 0u;
+ orig_pixels[128 + 4] = 50u;
+ orig_pixels[128 + 5] = 0u;
+ // BR.
+ orig_pixels[6] = 0u;
+ orig_pixels[7] = 0u;
+ orig_pixels[128 + 6] = 0u;
+ orig_pixels[128 + 7] = 20u;
+ // Odd.
+ orig_pixels[126] = 4u;
+ orig_pixels[127] = 255u;
+ orig_pixels[128 + 126] = 16u;
+ orig_pixels[128 + 127] = 255u;
+
+ // Test regular half size.
+ ScaleRowDown2Box_C(orig_pixels, 128, dst_pixels_c, 64);
+
+ ASSERT_EQ(64u, dst_pixels_c[0]);
+ ASSERT_EQ(25u, dst_pixels_c[1]);
+ ASSERT_EQ(13u, dst_pixels_c[2]);
+ ASSERT_EQ(5u, dst_pixels_c[3]);
+ ASSERT_EQ(0u, dst_pixels_c[4]);
+ ASSERT_EQ(133u, dst_pixels_c[63]);
+
+ // Test Odd width version - Last pixel is just 1 horizontal pixel.
+ ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 64);
+
+ ASSERT_EQ(64u, dst_pixels_c[0]);
+ ASSERT_EQ(25u, dst_pixels_c[1]);
+ ASSERT_EQ(13u, dst_pixels_c[2]);
+ ASSERT_EQ(5u, dst_pixels_c[3]);
+ ASSERT_EQ(0u, dst_pixels_c[4]);
+ ASSERT_EQ(10u, dst_pixels_c[63]);
+
+ // Test one pixel less, should skip the last pixel.
+ memset(dst_pixels_c, 0, sizeof(dst_pixels_c));
+ ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 63);
+
+ ASSERT_EQ(64u, dst_pixels_c[0]);
+ ASSERT_EQ(25u, dst_pixels_c[1]);
+ ASSERT_EQ(13u, dst_pixels_c[2]);
+ ASSERT_EQ(5u, dst_pixels_c[3]);
+ ASSERT_EQ(0u, dst_pixels_c[4]);
+ ASSERT_EQ(0u, dst_pixels_c[63]);
+
+ // Test regular half size SSSE3.
+ ScaleRowDown2Box_SSSE3(orig_pixels, 128, dst_pixels_opt, 64);
+
+ ASSERT_EQ(64u, dst_pixels_opt[0]);
+ ASSERT_EQ(25u, dst_pixels_opt[1]);
+ ASSERT_EQ(13u, dst_pixels_opt[2]);
+ ASSERT_EQ(5u, dst_pixels_opt[3]);
+ ASSERT_EQ(0u, dst_pixels_opt[4]);
+ ASSERT_EQ(133u, dst_pixels_opt[63]);
+
+ // Compare C and SSSE3 match.
+ ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 64);
+ ScaleRowDown2Box_Odd_SSSE3(orig_pixels, 128, dst_pixels_opt, 64);
+ for (int i = 0; i < 64; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+ }
+}
+#endif // HAS_SCALEROWDOWN2_SSSE3
+
+TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) {
+ SIMD_ALIGNED(uint16_t orig_pixels[2560 * 2]);
+ SIMD_ALIGNED(uint16_t dst_pixels_c[1280]);
+ SIMD_ALIGNED(uint16_t dst_pixels_opt[1280]);
+
+ memset(orig_pixels, 0, sizeof(orig_pixels));
+ memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
+ memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
+
+ for (int i = 0; i < 2560 * 2; ++i) {
+ orig_pixels[i] = i;
+ }
+ ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_c[0], 1280);
+ for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ if (has_neon) {
+ ScaleRowDown2Box_16_NEON(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+ } else {
+ ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+ }
+#elif defined(HAS_SCALEROWDOWN2_16_AVX2)
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ if (has_avx2) {
+ ScaleRowDown2Box_16_AVX2(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+ } else {
+ ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+ }
+#else
+ ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+#endif
+ }
+
+ for (int i = 0; i < 1280; ++i) {
+ ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ ASSERT_EQ(dst_pixels_c[0], (0 + 1 + 2560 + 2561 + 2) / 4);
+ ASSERT_EQ(dst_pixels_c[1279], 3839);
+}
+#endif // ENABLE_ROW_TESTS
+
+// Test scaling plane with 8 bit C vs 12 bit C and return maximum pixel
+// difference.
+// 0 = exact.
+static int TestPlaneFilter_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+
+ int i;
+ int64_t src_y_plane_size = (Abs(src_width)) * (Abs(src_height));
+ int src_stride_y = Abs(src_width);
+ int dst_y_plane_size = dst_width * dst_height;
+ int dst_stride_y = dst_width;
+
+ align_buffer_page_end(src_y, src_y_plane_size);
+ align_buffer_page_end(src_y_16, src_y_plane_size * 2);
+ align_buffer_page_end(dst_y_8, dst_y_plane_size);
+ align_buffer_page_end(dst_y_16, dst_y_plane_size * 2);
+ uint16_t* p_src_y_16 = reinterpret_cast<uint16_t*>(src_y_16);
+ uint16_t* p_dst_y_16 = reinterpret_cast<uint16_t*>(dst_y_16);
+
+ MemRandomize(src_y, src_y_plane_size);
+ memset(dst_y_8, 0, dst_y_plane_size);
+ memset(dst_y_16, 1, dst_y_plane_size * 2);
+
+ for (i = 0; i < src_y_plane_size; ++i) {
+ p_src_y_16[i] = src_y[i] & 255;
+ }
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y_8, dst_stride_y,
+ dst_width, dst_height, f);
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+
+ for (i = 0; i < benchmark_iterations; ++i) {
+ ScalePlane_16(p_src_y_16, src_stride_y, src_width, src_height, p_dst_y_16,
+ dst_stride_y, dst_width, dst_height, f);
+ }
+
+ // Expect an exact match.
+ int max_diff = 0;
+ for (i = 0; i < dst_y_plane_size; ++i) {
+ int abs_diff = Abs(dst_y_8[i] - p_dst_y_16[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_y_8);
+ free_aligned_buffer_page_end(dst_y_16);
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_y_16);
+
+ return max_diff;
+}
+
+// The following adjustments in dimensions ensure the scale factor will be
+// exactly achieved.
+// 2 is chroma subsample.
+#define DX(x, nom, denom) static_cast<int>(((Abs(x) / nom + 1) / 2) * nom * 2)
+#define SX(x, nom, denom) static_cast<int>(((x / nom + 1) / 2) * denom * 2)
+
+#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
+ TEST_F(LibYUVScaleTest, DISABLED_##ScalePlaneDownBy##name##_##filter##_16) { \
+ int diff = TestPlaneFilter_16( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but
+// filtering is different fixed point implementations for SSSE3, Neon and C.
+#define TEST_FACTOR(name, nom, denom, boxdiff) \
+ TEST_FACTOR1(name, None, nom, denom, 0) \
+ TEST_FACTOR1(name, Linear, nom, denom, boxdiff) \
+ TEST_FACTOR1(name, Bilinear, nom, denom, boxdiff) \
+ TEST_FACTOR1(name, Box, nom, denom, boxdiff)
+
+TEST_FACTOR(2, 1, 2, 0)
+TEST_FACTOR(4, 1, 4, 0)
+// TEST_FACTOR(8, 1, 8, 0) Disable for benchmark performance. Takes 90 seconds.
+TEST_FACTOR(3by4, 3, 4, 1)
+TEST_FACTOR(3by8, 3, 8, 1)
+TEST_FACTOR(3, 1, 3, 0)
+#undef TEST_FACTOR1
+#undef TEST_FACTOR
+#undef SX
+#undef DX
+
+TEST_F(LibYUVScaleTest, PlaneTest3x) {
+ const int kSrcStride = 480;
+ const int kDstStride = 160;
+ const int kSize = kSrcStride * 3;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 480 * 3; ++i) {
+ orig_pixels[i] = i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ ScalePlane(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(225, dest_pixels[0]);
+
+ ScalePlane(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(225, dest_pixels[0]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, PlaneTest4x) {
+ const int kSrcStride = 640;
+ const int kDstStride = 160;
+ const int kSize = kSrcStride * 4;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 640 * 4; ++i) {
+ orig_pixels[i] = i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ ScalePlane(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(66, dest_pixels[0]);
+
+ ScalePlane(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(2, dest_pixels[0]); // expect the 3rd pixel of the 3rd row
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+// Intent is to test 200x50 to 50x200 but width and height can be parameters.
+TEST_F(LibYUVScaleTest, PlaneTestRotate_None) {
+ const int kSize = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < kSize; ++i) {
+ orig_pixels[i] = i;
+ }
+ align_buffer_page_end(dest_opt_pixels, kSize);
+ align_buffer_page_end(dest_c_pixels, kSize);
+
+ MaskCpuFlags(disable_cpu_flags_); // Disable all CPU optimization.
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_, benchmark_height_,
+ dest_c_pixels, benchmark_height_, benchmark_height_,
+ benchmark_width_, kFilterNone);
+ MaskCpuFlags(benchmark_cpu_info_); // Enable all CPU optimization.
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_,
+ benchmark_height_, dest_opt_pixels, benchmark_height_,
+ benchmark_height_, benchmark_width_, kFilterNone);
+ }
+
+ for (int i = 0; i < kSize; ++i) {
+ ASSERT_EQ(dest_c_pixels[i], dest_opt_pixels[i]);
+ }
+
+ free_aligned_buffer_page_end(dest_c_pixels);
+ free_aligned_buffer_page_end(dest_opt_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, PlaneTestRotate_Bilinear) {
+ const int kSize = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < kSize; ++i) {
+ orig_pixels[i] = i;
+ }
+ align_buffer_page_end(dest_opt_pixels, kSize);
+ align_buffer_page_end(dest_c_pixels, kSize);
+
+ MaskCpuFlags(disable_cpu_flags_); // Disable all CPU optimization.
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_, benchmark_height_,
+ dest_c_pixels, benchmark_height_, benchmark_height_,
+ benchmark_width_, kFilterBilinear);
+ MaskCpuFlags(benchmark_cpu_info_); // Enable all CPU optimization.
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_,
+ benchmark_height_, dest_opt_pixels, benchmark_height_,
+ benchmark_height_, benchmark_width_, kFilterBilinear);
+ }
+
+ for (int i = 0; i < kSize; ++i) {
+ ASSERT_EQ(dest_c_pixels[i], dest_opt_pixels[i]);
+ }
+
+ free_aligned_buffer_page_end(dest_c_pixels);
+ free_aligned_buffer_page_end(dest_opt_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+// Intent is to test 200x50 to 50x200 but width and height can be parameters.
+TEST_F(LibYUVScaleTest, PlaneTestRotate_Box) {
+ const int kSize = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < kSize; ++i) {
+ orig_pixels[i] = i;
+ }
+ align_buffer_page_end(dest_opt_pixels, kSize);
+ align_buffer_page_end(dest_c_pixels, kSize);
+
+ MaskCpuFlags(disable_cpu_flags_); // Disable all CPU optimization.
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_, benchmark_height_,
+ dest_c_pixels, benchmark_height_, benchmark_height_,
+ benchmark_width_, kFilterBox);
+ MaskCpuFlags(benchmark_cpu_info_); // Enable all CPU optimization.
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ ScalePlane(orig_pixels, benchmark_width_, benchmark_width_,
+ benchmark_height_, dest_opt_pixels, benchmark_height_,
+ benchmark_height_, benchmark_width_, kFilterBox);
+ }
+
+ for (int i = 0; i < kSize; ++i) {
+ ASSERT_EQ(dest_c_pixels[i], dest_opt_pixels[i]);
+ }
+
+ free_aligned_buffer_page_end(dest_c_pixels);
+ free_aligned_buffer_page_end(dest_opt_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneBox_Any) {
+ const int kDstW = 29;
+ const int kDstH = 3;
+ const int kSrcH = 9;
+ for (int src_w : {87, 174, 73, 155}) {
+ const int kSrcSize = src_w * kSrcH;
+ const int kDstSize = kDstW * kDstH;
+ align_buffer_page_end(src_pixels, kSrcSize);
+ align_buffer_page_end(dst_c_pixels, kDstSize);
+ align_buffer_page_end(dst_opt_pixels, kDstSize);
+ MemRandomize(src_pixels, kSrcSize);
+ MSAN_POISON(dst_opt_pixels, kDstSize);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ ScalePlane(src_pixels, src_w, src_w, kSrcH, dst_c_pixels, kDstW, kDstW,
+ kDstH, kFilterBox);
+ MaskCpuFlags(benchmark_cpu_info_);
+ ScalePlane(src_pixels, src_w, src_w, kSrcH, dst_opt_pixels, kDstW, kDstW,
+ kDstH, kFilterBox);
+
+ for (int i = 0; i < kDstSize; ++i) {
+ ASSERT_EQ(dst_c_pixels[i], dst_opt_pixels[i]);
+ }
+ free_aligned_buffer_page_end(dst_opt_pixels);
+ free_aligned_buffer_page_end(dst_c_pixels);
+ free_aligned_buffer_page_end(src_pixels);
+ }
+}
+
+TEST_F(LibYUVScaleTest, PlaneTest1_Box) {
+ align_buffer_page_end(orig_pixels, 3);
+ align_buffer_page_end(dst_pixels, 3);
+
+ // Pad the 1x1 byte image with invalid values before and after in case libyuv
+ // reads outside the memory boundaries.
+ orig_pixels[0] = 0;
+ orig_pixels[1] = 1; // scale this pixel
+ orig_pixels[2] = 2;
+ dst_pixels[0] = 3;
+ dst_pixels[1] = 3;
+ dst_pixels[2] = 3;
+
+ libyuv::ScalePlane(orig_pixels + 1, /* src_stride= */ 1, /* src_width= */ 1,
+ /* src_height= */ 1, dst_pixels, /* dst_stride= */ 1,
+ /* dst_width= */ 1, /* dst_height= */ 2,
+ libyuv::kFilterBox);
+
+ ASSERT_EQ(dst_pixels[0], 1);
+ ASSERT_EQ(dst_pixels[1], 1);
+ ASSERT_EQ(dst_pixels[2], 3);
+
+ free_aligned_buffer_page_end(dst_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, PlaneTest1_16_Box) {
+ align_buffer_page_end(orig_pixels_alloc, 3 * 2);
+ align_buffer_page_end(dst_pixels_alloc, 3 * 2);
+ uint16_t* orig_pixels = (uint16_t*)orig_pixels_alloc;
+ uint16_t* dst_pixels = (uint16_t*)dst_pixels_alloc;
+
+ // Pad the 1x1 byte image with invalid values before and after in case libyuv
+ // reads outside the memory boundaries.
+ orig_pixels[0] = 0;
+ orig_pixels[1] = 1; // scale this pixel
+ orig_pixels[2] = 2;
+ dst_pixels[0] = 3;
+ dst_pixels[1] = 3;
+ dst_pixels[2] = 3;
+
+ libyuv::ScalePlane_16(
+ orig_pixels + 1, /* src_stride= */ 1, /* src_width= */ 1,
+ /* src_height= */ 1, dst_pixels, /* dst_stride= */ 1,
+ /* dst_width= */ 1, /* dst_height= */ 2, libyuv::kFilterNone);
+
+ ASSERT_EQ(dst_pixels[0], 1);
+ ASSERT_EQ(dst_pixels[1], 1);
+ ASSERT_EQ(dst_pixels[2], 3);
+
+ free_aligned_buffer_page_end(dst_pixels_alloc);
+ free_aligned_buffer_page_end(orig_pixels_alloc);
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneVerticalPointUp) {
+ const int kWidth = 32;
+ const uint8_t kSentinel = 0xFF;
+ const uint8_t expected[] = {10, 10, 20, 30, 30};
+
+ // An extra sentinel row exposes accidental interpolation past the last valid
+ // row. 3->5 is the uneven ratio that used to drop the last source row.
+ align_buffer_page_end(src, kWidth * 4);
+ align_buffer_page_end(dst, kWidth * 5);
+ memset(src, kSentinel, kWidth * 4);
+ memset(dst, kSentinel, kWidth * 5);
+ memset(src, 10, kWidth);
+ memset(src + kWidth, 20, kWidth);
+ memset(src + 2 * kWidth, 30, kWidth);
+
+ ASSERT_EQ(0, ScalePlane(src, kWidth, kWidth, 3, dst, kWidth, kWidth, 5,
+ kFilterNone));
+ for (int y = 0; y < 5; ++y) {
+ for (int x = 0; x < kWidth; ++x) {
+ EXPECT_EQ(expected[y], dst[y * kWidth + x]);
+ }
+ }
+
+ free_aligned_buffer_page_end(dst);
+ free_aligned_buffer_page_end(src);
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneVerticalPointUp_16) {
+ const int kWidth = 32;
+ const uint16_t kSentinel = 0xFFFF;
+ const uint16_t expected[] = {10, 10, 20, 30, 30};
+
+ align_buffer_page_end(src_alloc, kWidth * 4 * 2);
+ align_buffer_page_end(dst_alloc, kWidth * 5 * 2);
+ uint16_t* src = reinterpret_cast<uint16_t*>(src_alloc);
+ uint16_t* dst = reinterpret_cast<uint16_t*>(dst_alloc);
+ for (int i = 0; i < kWidth * 4; ++i) {
+ src[i] = kSentinel;
+ }
+ for (int i = 0; i < kWidth * 5; ++i) {
+ dst[i] = kSentinel;
+ }
+ for (int x = 0; x < kWidth; ++x) {
+ src[x] = 10;
+ src[kWidth + x] = 20;
+ src[2 * kWidth + x] = 30;
+ }
+
+ ASSERT_EQ(0, ScalePlane_16(src, kWidth, kWidth, 3, dst, kWidth, kWidth, 5,
+ kFilterNone));
+ for (int y = 0; y < 5; ++y) {
+ for (int x = 0; x < kWidth; ++x) {
+ EXPECT_EQ(expected[y], dst[y * kWidth + x]);
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_alloc);
+ free_aligned_buffer_page_end(src_alloc);
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneVerticalBilinearUp) {
+ const int kWidth = 32;
+ const uint8_t kSentinel = 0xFF;
+ // Same-width 2x upsampling still uses FixedDiv1. With y = 0 the 8.8
+ // fractions are 0, 85, 170, 255. InterpolateRow_C:
+ // (a * (256 - yf) + b * yf + 128) >> 8.
+ // Nearest 2x would duplicate rows as {10, 10, 200, 200}.
+ const uint8_t expected[] = {10, 73, 136, 199};
+
+ align_buffer_page_end(src, kWidth * 3);
+ align_buffer_page_end(dst, kWidth * 4);
+ memset(src, kSentinel, kWidth * 3);
+ memset(dst, kSentinel, kWidth * 4);
+ memset(src, 10, kWidth);
+ memset(src + kWidth, 200, kWidth);
+
+ ASSERT_EQ(0, ScalePlane(src, kWidth, kWidth, 2, dst, kWidth, kWidth, 4,
+ kFilterBilinear));
+ for (int y = 0; y < 4; ++y) {
+ for (int x = 0; x < kWidth; ++x) {
+ EXPECT_EQ(expected[y], dst[y * kWidth + x]);
+ }
+ }
+
+ free_aligned_buffer_page_end(dst);
+ free_aligned_buffer_page_end(src);
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneVerticalBilinearUp_16) {
+ const int kWidth = 32;
+ const uint16_t kSentinel = 0xFFFF;
+ const uint16_t expected[] = {10, 73, 136, 199};
+
+ align_buffer_page_end(src_alloc, kWidth * 3 * 2);
+ align_buffer_page_end(dst_alloc, kWidth * 4 * 2);
+ uint16_t* src = reinterpret_cast<uint16_t*>(src_alloc);
+ uint16_t* dst = reinterpret_cast<uint16_t*>(dst_alloc);
+ for (int i = 0; i < kWidth * 3; ++i) {
+ src[i] = kSentinel;
+ }
+ for (int i = 0; i < kWidth * 4; ++i) {
+ dst[i] = kSentinel;
+ }
+ for (int x = 0; x < kWidth; ++x) {
+ src[x] = 10;
+ src[kWidth + x] = 200;
+ }
+
+ ASSERT_EQ(0, ScalePlane_16(src, kWidth, kWidth, 2, dst, kWidth, kWidth, 4,
+ kFilterBilinear));
+ for (int y = 0; y < 4; ++y) {
+ for (int x = 0; x < kWidth; ++x) {
+ EXPECT_EQ(expected[y], dst[y * kWidth + x]);
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_alloc);
+ free_aligned_buffer_page_end(src_alloc);
+}
+
+// POC: int * int overflow in ScalePlaneVertical (scale_common.cc).
+//
+// `yi * src_stride` is evaluated as int * int. When the product exceeds
+// INT_MAX it wraps negative and InterpolateRow reads from BEFORE the
+// source allocation.
+//
+// Parameters:
+// - dst_width == src_width
+// -> ScalePlane dispatches to ScalePlaneVertical
+// - src_height == 5, dst_height == 1
+// -> single iteration with yi == 2
+// - src_stride == 0x7FFFFFF8
+// -> 2 * 0x7FFFFFF8 == 0xFFFFFFF0 == -16 (int)
+//
+// The source buffer is sized so that the *correct* 64-bit offset
+// (2 * 0x7FFFFFF8 == 4294967280) plus kWidth bytes is in-bounds. With the
+// bug, the 32-bit product is -16 and ASAN reports a heap-buffer-overflow
+// READ "16 bytes before" the allocation.
+TEST_F(LibYUVScaleTest, ScalePlaneVertical_IntStrideOverflow) {
+ const int kWidth = 16;
+ const int kSrcHeight = 5;
+ const int kDstHeight = 1;
+ const int kStride = 0x7FFFFFF8; // 2147483640
+
+ // src_size is big enough for the only row this call legitimately touches
+ // (yi == 2) when computed in 64-bit: 2 * stride + width = 4 GiB.
+ size_t src_size = kStride;
+ if (src_size > SIZE_MAX / 2) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size *= 2;
+ if (src_size > SIZE_MAX - kWidth) {
+ GTEST_SKIP() << "could not represent allocation size in size_t";
+ }
+ src_size += kWidth;
+
+#if defined(__aarch64__)
+ // Infer malloc can accept a large size for cpu with dot product (a76/a55)
+ int has_large_malloc = TestCpuFlag(kCpuHasNeonDotProd);
+#else
+ int has_large_malloc = 1;
+#endif
+ if (!has_large_malloc) {
+ GTEST_SKIP() << "large allocation may assert for " << src_size << " bytes";
+ }
+
+ uint8_t* src = new (std::nothrow) uint8_t[src_size];
+ if (!src) {
+ GTEST_SKIP() << "could not allocate " << src_size << " bytes";
+ }
+ uint8_t* dst = new uint8_t[kWidth];
+ memset(dst, 0, kWidth);
+
+ // Force the scalar path so the crash site is deterministic
+ // (InterpolateRow_C -> memcpy when yf == 0).
+ MaskCpuFlags(disable_cpu_flags_);
+
+ int r = ScalePlane(src, kStride, kWidth, kSrcHeight, dst, kWidth, kWidth,
+ kDstHeight, kFilterNone);
+
+ // Not reached under ASAN.
+ ASSERT_EQ(0, r);
+ delete[] src;
+ delete[] dst;
+}
+
+TEST_F(LibYUVScaleTest, ScalePlane_InvalidInputs) {
+ uint8_t src[16] = {0};
+ uint8_t dst[16] = {0};
+
+ // NULL src/dst
+ EXPECT_EQ(-1, ScalePlane(nullptr, 4, 4, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 4, nullptr, 4, 4, 4, kFilterNone));
+
+ // Width/height <= 0 (except src_height which can be negative but not 0)
+ EXPECT_EQ(-1, ScalePlane(src, 4, 0, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, -1, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 0, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 4, dst, 4, 0, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 4, dst, 4, -1, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 4, dst, 4, 4, 0, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 4, dst, 4, 4, -1, kFilterNone));
+
+ // Width/height too large (> 32768)
+ EXPECT_EQ(-1, ScalePlane(src, 4, 32769, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, 32769, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane(src, 4, 4, -32769, dst, 4, 4, 4, kFilterNone));
+
+ // Valid edge cases
+ EXPECT_EQ(0, ScalePlane(src, 4, 1, 1, dst, 4, 1, 1, kFilterNone));
+ EXPECT_EQ(0, ScalePlane(src, 4, 1, -1, dst, 4, 1, 1, kFilterNone));
+}
+
+TEST_F(LibYUVScaleTest, ScalePlane_16_InvalidInputs) {
+ uint16_t src[16] = {0};
+ uint16_t dst[16] = {0};
+
+ EXPECT_EQ(-1, ScalePlane_16(nullptr, 4, 4, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_16(src, 4, 4, 4, nullptr, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_16(src, 4, 0, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_16(src, 4, 32769, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_16(src, 4, 4, -32769, dst, 4, 4, 4, kFilterNone));
+}
+
+TEST_F(LibYUVScaleTest, ScalePlane_12_InvalidInputs) {
+ uint16_t src[16] = {0};
+ uint16_t dst[16] = {0};
+
+ EXPECT_EQ(-1, ScalePlane_12(nullptr, 4, 4, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_12(src, 4, 4, 4, nullptr, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_12(src, 4, 0, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_12(src, 4, 32769, 4, dst, 4, 4, 4, kFilterNone));
+ EXPECT_EQ(-1, ScalePlane_12(src, 4, 4, -32769, dst, 4, 4, 4, kFilterNone));
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneOversizeHeight) {
+ const int kSrcWidth = 4;
+ const int kSrcHeight = 32768;
+ uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight];
+ memset(src, 0, kSrcWidth * kSrcHeight);
+ uint8_t dst[4 * 2] = {};
+
+ for (int dst_w : {4, 2}) {
+ for (int dst_h : {1, 2}) {
+ for (FilterMode f :
+ {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
+ EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst,
+ dst_w, dst_w, dst_h, f));
+ EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, -kSrcHeight, dst,
+ dst_w, dst_w, dst_h, f));
+ }
+ }
+ }
+ delete[] src;
+}
+
+TEST_F(LibYUVScaleTest, ScalePlaneBoxMaxSrcHeight) {
+ const int kSrcWidth = 4;
+ const int kSrcHeight = 32768;
+ uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight];
+ memset(src, 0, kSrcWidth * kSrcHeight);
+ uint8_t dst[2 * 4] = {};
+
+ EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst, 2, 2, 4,
+ kFilterBox));
+ delete[] src;
+}
+
+#if defined(__LP64__) || defined(_WIN64)
+TEST_F(LibYUVScaleTest, ScaleLargeImageOver2GB) {
+ const int kSrcWidth = 4;
+ const int kSrcHeight = 32768;
+ const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX)
+ const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight;
+ uint8_t* src = static_cast<uint8_t*>(calloc(1, kSrcSize));
+ if (!src) {
+ GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes";
+ }
+ uint8_t dst[8 * 4 * 4] = {};
+
+ for (FilterMode f :
+ {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
+ EXPECT_EQ(
+ 0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4, 4, 2, f));
+ EXPECT_EQ(0, ScalePlane(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4, 4,
+ 2, f));
+ EXPECT_EQ(
+ 0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8, 8, 2, f));
+ EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4 * 2, 4,
+ 2, f));
+ EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4 * 2, 4,
+ 2, f));
+ EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8 * 2, 8,
+ 2, f));
+ }
+ free(src);
+}
+#endif
+
+} // namespace libyuv
diff --git a/libs/libyuv/unit_test/scale_rgb_test.cc b/libs/libyuv/unit_test/scale_rgb_test.cc
new file mode 100644
index 0000000000..f6fa1e8caa
--- /dev/null
+++ b/libs/libyuv/unit_test/scale_rgb_test.cc
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2022 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stdlib.h>
+#include <time.h>
+
+#include "../unit_test/unit_test.h"
+#include "libyuv/cpu_id.h"
+#include "libyuv/scale_rgb.h"
+
+namespace libyuv {
+
+#define STRINGIZE(line) #line
+#define FILELINESTR(file, line) file ":" STRINGIZE(line)
+
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
+// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
+static int RGBTestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+
+ int i, j;
+ const int b = 0; // 128 to test for padding/stride.
+ int64_t src_rgb_plane_size =
+ (Abs(src_width) + b * 3) * (Abs(src_height) + b * 3) * 3LL;
+ int src_stride_rgb = (b * 3 + Abs(src_width)) * 3;
+
+ align_buffer_page_end(src_rgb, src_rgb_plane_size);
+ if (!src_rgb) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ MemRandomize(src_rgb, src_rgb_plane_size);
+
+ int64_t dst_rgb_plane_size = (dst_width + b * 3) * (dst_height + b * 3) * 3LL;
+ int dst_stride_rgb = (b * 3 + dst_width) * 3;
+
+ align_buffer_page_end(dst_rgb_c, dst_rgb_plane_size);
+ align_buffer_page_end(dst_rgb_opt, dst_rgb_plane_size);
+ if (!dst_rgb_c || !dst_rgb_opt) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ memset(dst_rgb_c, 2, dst_rgb_plane_size);
+ memset(dst_rgb_opt, 3, dst_rgb_plane_size);
+
+ // Warm up both versions for consistent benchmarks.
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ RGBScale(src_rgb + (src_stride_rgb * b) + b * 3, src_stride_rgb, src_width,
+ src_height, dst_rgb_c + (dst_stride_rgb * b) + b * 3, dst_stride_rgb,
+ dst_width, dst_height, f);
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ RGBScale(src_rgb + (src_stride_rgb * b) + b * 3, src_stride_rgb, src_width,
+ src_height, dst_rgb_opt + (dst_stride_rgb * b) + b * 3,
+ dst_stride_rgb, dst_width, dst_height, f);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ double c_time = get_time();
+ RGBScale(src_rgb + (src_stride_rgb * b) + b * 3, src_stride_rgb, src_width,
+ src_height, dst_rgb_c + (dst_stride_rgb * b) + b * 3, dst_stride_rgb,
+ dst_width, dst_height, f);
+
+ c_time = (get_time() - c_time);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ double opt_time = get_time();
+ for (i = 0; i < benchmark_iterations; ++i) {
+ RGBScale(src_rgb + (src_stride_rgb * b) + b * 3, src_stride_rgb, src_width,
+ src_height, dst_rgb_opt + (dst_stride_rgb * b) + b * 3,
+ dst_stride_rgb, dst_width, dst_height, f);
+ }
+ opt_time = (get_time() - opt_time) / benchmark_iterations;
+
+ // Report performance of C vs OPT
+ printf("filter %d - %8d us C - %8d us OPT\n", f,
+ static_cast<int>(c_time * 1e6), static_cast<int>(opt_time * 1e6));
+
+ // C version may be a little off from the optimized. Order of
+ // operations may introduce rounding somewhere. So do a difference
+ // of the buffers and look to see that the max difference isn't
+ // over 2.
+ int max_diff = 0;
+ for (i = b; i < (dst_height + b); ++i) {
+ for (j = b * 3; j < (dst_width + b) * 3; ++j) {
+ int abs_diff = Abs(dst_rgb_c[(i * dst_stride_rgb) + j] -
+ dst_rgb_opt[(i * dst_stride_rgb) + j]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_rgb_c);
+ free_aligned_buffer_page_end(dst_rgb_opt);
+ free_aligned_buffer_page_end(src_rgb);
+ return max_diff;
+}
+
+// The following adjustments in dimensions ensure the scale factor will be
+// exactly achieved.
+#define DX(x, nom, denom) static_cast<int>((Abs(x) / nom) * nom)
+#define SX(x, nom, denom) static_cast<int>((x / nom) * denom)
+
+#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
+ TEST_F(LibYUVScaleTest, RGBScaleDownBy##name##_##filter) { \
+ int diff = RGBTestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but
+// filtering is different fixed point implementations for SSSE3, Neon and C.
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(name, None, nom, denom, 0) \
+ TEST_FACTOR1(name, Linear, nom, denom, 3) \
+ TEST_FACTOR1(name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(name, Box, nom, denom, 3)
+#else
+// Test a scale factor with Bilinear.
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(name, Bilinear, nom, denom, 3)
+#endif
+
+TEST_FACTOR(2, 1, 2)
+#ifndef DISABLE_SLOW_TESTS
+TEST_FACTOR(4, 1, 4)
+// TEST_FACTOR(8, 1, 8) Disable for benchmark performance.
+TEST_FACTOR(3by4, 3, 4)
+TEST_FACTOR(3by8, 3, 8)
+TEST_FACTOR(3, 1, 3)
+#endif
+#undef TEST_FACTOR1
+#undef TEST_FACTOR
+#undef SX
+#undef DX
+
+#define TEST_SCALETO1(name, width, height, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \
+ int diff = RGBTestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \
+ int diff = RGBTestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+/// Test scale to a specified size with all 4 filters.
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(name, width, height, None, 0) \
+ TEST_SCALETO1(name, width, height, Linear, 3) \
+ TEST_SCALETO1(name, width, height, Bilinear, 3)
+#else
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(name, width, height, Bilinear, 3)
+#endif
+
+TEST_SCALETO(RGBScale, 640, 360)
+#ifndef DISABLE_SLOW_TESTS
+TEST_SCALETO(RGBScale, 1, 1)
+TEST_SCALETO(RGBScale, 256, 144) /* 128x72 * 3 */
+TEST_SCALETO(RGBScale, 320, 240)
+TEST_SCALETO(RGBScale, 569, 480)
+TEST_SCALETO(RGBScale, 1280, 720)
+TEST_SCALETO(RGBScale, 1920, 1080)
+#endif // DISABLE_SLOW_TESTS
+#undef TEST_SCALETO1
+#undef TEST_SCALETO
+
+#define TEST_SCALESWAPXY1(name, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##SwapXY_##filter) { \
+ int diff = RGBTestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+// Test scale with swapped width and height with all 3 filters.
+TEST_SCALESWAPXY1(RGBScale, None, 0)
+TEST_SCALESWAPXY1(RGBScale, Linear, 0)
+TEST_SCALESWAPXY1(RGBScale, Bilinear, 0)
+#else
+TEST_SCALESWAPXY1(RGBScale, Bilinear, 0)
+#endif
+#undef TEST_SCALESWAPXY1
+
+TEST_F(LibYUVScaleTest, RGBTest3x) {
+ const int kSrcStride = 480 * 3;
+ const int kDstStride = 160 * 3;
+ const int kSize = kSrcStride * 3;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 480 * 3; ++i) {
+ orig_pixels[i * 3 + 0] = i;
+ orig_pixels[i * 3 + 1] = 255 - i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ RGBScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+
+ RGBScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, RGBTest4x) {
+ const int kSrcStride = 640 * 3;
+ const int kDstStride = 160 * 3;
+ const int kSize = kSrcStride * 4;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 640 * 4; ++i) {
+ orig_pixels[i * 3 + 0] = i;
+ orig_pixels[i * 3 + 1] = 255 - i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ RGBScale(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(66, dest_pixels[0]);
+ ASSERT_EQ(190, dest_pixels[1]);
+
+ RGBScale(orig_pixels, kSrcStride, 64, 4, dest_pixels, kDstStride, 16, 1,
+ kFilterNone);
+
+ ASSERT_EQ(2, dest_pixels[0]); // expect the 3rd pixel of the 3rd row
+ ASSERT_EQ(255 - 2, dest_pixels[1]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+} // namespace libyuv
diff --git a/libs/libyuv/unit_test/scale_test.cc b/libs/libyuv/unit_test/scale_test.cc
index 94a7859006..e634b6c0f8 100644
--- a/libs/libyuv/unit_test/scale_test.cc
+++ b/libs/libyuv/unit_test/scale_test.cc
@@ -22,6 +22,17 @@
#define STRINGIZE(line) #line
#define FILELINESTR(file, line) file ":" STRINGIZE(line)
+#if (defined(__riscv) && !defined(__clang__)) || defined(__hexagon__)
+#define DISABLE_SLOW_TESTS
+#undef ENABLE_FULL_TESTS
+#endif
+
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
namespace libyuv {
// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
@@ -142,6 +153,123 @@ static int I420TestFilter(int src_width,
return max_diff;
}
+// Test scaling with 8 bit C vs 12 bit C and return maximum pixel difference.
+// 0 = exact.
+static int I420TestFilter_12(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+
+ int i;
+ int src_width_uv = (Abs(src_width) + 1) >> 1;
+ int src_height_uv = (Abs(src_height) + 1) >> 1;
+
+ int64_t src_y_plane_size = (Abs(src_width)) * (Abs(src_height));
+ int64_t src_uv_plane_size = (src_width_uv) * (src_height_uv);
+
+ int src_stride_y = Abs(src_width);
+ int src_stride_uv = src_width_uv;
+
+ align_buffer_page_end(src_y, src_y_plane_size);
+ align_buffer_page_end(src_u, src_uv_plane_size);
+ align_buffer_page_end(src_v, src_uv_plane_size);
+ align_buffer_page_end(src_y_12, src_y_plane_size * 2);
+ align_buffer_page_end(src_u_12, src_uv_plane_size * 2);
+ align_buffer_page_end(src_v_12, src_uv_plane_size * 2);
+ if (!src_y || !src_u || !src_v || !src_y_12 || !src_u_12 || !src_v_12) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ uint16_t* p_src_y_12 = reinterpret_cast<uint16_t*>(src_y_12);
+ uint16_t* p_src_u_12 = reinterpret_cast<uint16_t*>(src_u_12);
+ uint16_t* p_src_v_12 = reinterpret_cast<uint16_t*>(src_v_12);
+
+ MemRandomize(src_y, src_y_plane_size);
+ MemRandomize(src_u, src_uv_plane_size);
+ MemRandomize(src_v, src_uv_plane_size);
+
+ for (i = 0; i < src_y_plane_size; ++i) {
+ p_src_y_12[i] = src_y[i];
+ }
+ for (i = 0; i < src_uv_plane_size; ++i) {
+ p_src_u_12[i] = src_u[i];
+ p_src_v_12[i] = src_v[i];
+ }
+
+ int dst_width_uv = (dst_width + 1) >> 1;
+ int dst_height_uv = (dst_height + 1) >> 1;
+
+ int dst_y_plane_size = (dst_width) * (dst_height);
+ int dst_uv_plane_size = (dst_width_uv) * (dst_height_uv);
+
+ int dst_stride_y = dst_width;
+ int dst_stride_uv = dst_width_uv;
+
+ align_buffer_page_end(dst_y_8, dst_y_plane_size);
+ align_buffer_page_end(dst_u_8, dst_uv_plane_size);
+ align_buffer_page_end(dst_v_8, dst_uv_plane_size);
+ align_buffer_page_end(dst_y_12, dst_y_plane_size * 2);
+ align_buffer_page_end(dst_u_12, dst_uv_plane_size * 2);
+ align_buffer_page_end(dst_v_12, dst_uv_plane_size * 2);
+
+ uint16_t* p_dst_y_12 = reinterpret_cast<uint16_t*>(dst_y_12);
+ uint16_t* p_dst_u_12 = reinterpret_cast<uint16_t*>(dst_u_12);
+ uint16_t* p_dst_v_12 = reinterpret_cast<uint16_t*>(dst_v_12);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I420Scale(src_y, src_stride_y, src_u, src_stride_uv, src_v, src_stride_uv,
+ src_width, src_height, dst_y_8, dst_stride_y, dst_u_8,
+ dst_stride_uv, dst_v_8, dst_stride_uv, dst_width, dst_height, f);
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (i = 0; i < benchmark_iterations; ++i) {
+ I420Scale_12(p_src_y_12, src_stride_y, p_src_u_12, src_stride_uv,
+ p_src_v_12, src_stride_uv, src_width, src_height, p_dst_y_12,
+ dst_stride_y, p_dst_u_12, dst_stride_uv, p_dst_v_12,
+ dst_stride_uv, dst_width, dst_height, f);
+ }
+
+ // Expect an exact match.
+ int max_diff = 0;
+ for (i = 0; i < dst_y_plane_size; ++i) {
+ int abs_diff = Abs(dst_y_8[i] - p_dst_y_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ for (i = 0; i < dst_uv_plane_size; ++i) {
+ int abs_diff = Abs(dst_u_8[i] - p_dst_u_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ abs_diff = Abs(dst_v_8[i] - p_dst_v_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_y_8);
+ free_aligned_buffer_page_end(dst_u_8);
+ free_aligned_buffer_page_end(dst_v_8);
+ free_aligned_buffer_page_end(dst_y_12);
+ free_aligned_buffer_page_end(dst_u_12);
+ free_aligned_buffer_page_end(dst_v_12);
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_u);
+ free_aligned_buffer_page_end(src_v);
+ free_aligned_buffer_page_end(src_y_12);
+ free_aligned_buffer_page_end(src_u_12);
+ free_aligned_buffer_page_end(src_v_12);
+
+ return max_diff;
+}
+
// Test scaling with 8 bit C vs 16 bit C and return maximum pixel difference.
// 0 = exact.
static int I420TestFilter_16(int src_width,
@@ -377,6 +505,123 @@ static int I444TestFilter(int src_width,
return max_diff;
}
+// Test scaling with 8 bit C vs 12 bit C and return maximum pixel difference.
+// 0 = exact.
+static int I444TestFilter_12(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+
+ int i;
+ int src_width_uv = Abs(src_width);
+ int src_height_uv = Abs(src_height);
+
+ int64_t src_y_plane_size = (Abs(src_width)) * (Abs(src_height));
+ int64_t src_uv_plane_size = (src_width_uv) * (src_height_uv);
+
+ int src_stride_y = Abs(src_width);
+ int src_stride_uv = src_width_uv;
+
+ align_buffer_page_end(src_y, src_y_plane_size);
+ align_buffer_page_end(src_u, src_uv_plane_size);
+ align_buffer_page_end(src_v, src_uv_plane_size);
+ align_buffer_page_end(src_y_12, src_y_plane_size * 2);
+ align_buffer_page_end(src_u_12, src_uv_plane_size * 2);
+ align_buffer_page_end(src_v_12, src_uv_plane_size * 2);
+ if (!src_y || !src_u || !src_v || !src_y_12 || !src_u_12 || !src_v_12) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ uint16_t* p_src_y_12 = reinterpret_cast<uint16_t*>(src_y_12);
+ uint16_t* p_src_u_12 = reinterpret_cast<uint16_t*>(src_u_12);
+ uint16_t* p_src_v_12 = reinterpret_cast<uint16_t*>(src_v_12);
+
+ MemRandomize(src_y, src_y_plane_size);
+ MemRandomize(src_u, src_uv_plane_size);
+ MemRandomize(src_v, src_uv_plane_size);
+
+ for (i = 0; i < src_y_plane_size; ++i) {
+ p_src_y_12[i] = src_y[i];
+ }
+ for (i = 0; i < src_uv_plane_size; ++i) {
+ p_src_u_12[i] = src_u[i];
+ p_src_v_12[i] = src_v[i];
+ }
+
+ int dst_width_uv = dst_width;
+ int dst_height_uv = dst_height;
+
+ int dst_y_plane_size = (dst_width) * (dst_height);
+ int dst_uv_plane_size = (dst_width_uv) * (dst_height_uv);
+
+ int dst_stride_y = dst_width;
+ int dst_stride_uv = dst_width_uv;
+
+ align_buffer_page_end(dst_y_8, dst_y_plane_size);
+ align_buffer_page_end(dst_u_8, dst_uv_plane_size);
+ align_buffer_page_end(dst_v_8, dst_uv_plane_size);
+ align_buffer_page_end(dst_y_12, dst_y_plane_size * 2);
+ align_buffer_page_end(dst_u_12, dst_uv_plane_size * 2);
+ align_buffer_page_end(dst_v_12, dst_uv_plane_size * 2);
+
+ uint16_t* p_dst_y_12 = reinterpret_cast<uint16_t*>(dst_y_12);
+ uint16_t* p_dst_u_12 = reinterpret_cast<uint16_t*>(dst_u_12);
+ uint16_t* p_dst_v_12 = reinterpret_cast<uint16_t*>(dst_v_12);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ I444Scale(src_y, src_stride_y, src_u, src_stride_uv, src_v, src_stride_uv,
+ src_width, src_height, dst_y_8, dst_stride_y, dst_u_8,
+ dst_stride_uv, dst_v_8, dst_stride_uv, dst_width, dst_height, f);
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ for (i = 0; i < benchmark_iterations; ++i) {
+ I444Scale_12(p_src_y_12, src_stride_y, p_src_u_12, src_stride_uv,
+ p_src_v_12, src_stride_uv, src_width, src_height, p_dst_y_12,
+ dst_stride_y, p_dst_u_12, dst_stride_uv, p_dst_v_12,
+ dst_stride_uv, dst_width, dst_height, f);
+ }
+
+ // Expect an exact match.
+ int max_diff = 0;
+ for (i = 0; i < dst_y_plane_size; ++i) {
+ int abs_diff = Abs(dst_y_8[i] - p_dst_y_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ for (i = 0; i < dst_uv_plane_size; ++i) {
+ int abs_diff = Abs(dst_u_8[i] - p_dst_u_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ abs_diff = Abs(dst_v_8[i] - p_dst_v_12[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_y_8);
+ free_aligned_buffer_page_end(dst_u_8);
+ free_aligned_buffer_page_end(dst_v_8);
+ free_aligned_buffer_page_end(dst_y_12);
+ free_aligned_buffer_page_end(dst_u_12);
+ free_aligned_buffer_page_end(dst_v_12);
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_u);
+ free_aligned_buffer_page_end(src_v);
+ free_aligned_buffer_page_end(src_y_12);
+ free_aligned_buffer_page_end(src_u_12);
+ free_aligned_buffer_page_end(src_v_12);
+
+ return max_diff;
+}
+
// Test scaling with 8 bit C vs 16 bit C and return maximum pixel difference.
// 0 = exact.
static int I444TestFilter_16(int src_width,
@@ -494,411 +739,454 @@ static int I444TestFilter_16(int src_width,
return max_diff;
}
-// The following adjustments in dimensions ensure the scale factor will be
-// exactly achieved.
-// 2 is chroma subsample.
-#define DX(x, nom, denom) static_cast<int>(((Abs(x) / nom + 1) / 2) * nom * 2)
-#define SX(x, nom, denom) static_cast<int>(((x / nom + 1) / 2) * denom * 2)
-
-#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
- TEST_F(LibYUVScaleTest, I420ScaleDownBy##name##_##filter) { \
- int diff = I420TestFilter( \
- SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
- DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
- kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I444ScaleDownBy##name##_##filter) { \
- int diff = I444TestFilter( \
- SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
- DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
- kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I420ScaleDownBy##name##_##filter##_16) { \
- int diff = I420TestFilter_16( \
- SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
- DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
- kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I444ScaleDownBy##name##_##filter##_16) { \
- int diff = I444TestFilter_16( \
- SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
- DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
- kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
+static int NVTestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int subsample_x,
+ int subsample_y,
+ int (*NVScale)(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering),
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
}
-// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but
-// filtering is different fixed point implementations for SSSE3, Neon and C.
-#define TEST_FACTOR(name, nom, denom, boxdiff) \
- TEST_FACTOR1(name, None, nom, denom, 0) \
- TEST_FACTOR1(name, Linear, nom, denom, 3) \
- TEST_FACTOR1(name, Bilinear, nom, denom, 3) \
- TEST_FACTOR1(name, Box, nom, denom, boxdiff)
+ int i, j;
+ int src_width_uv = subsample_x ? ((Abs(src_width) + 1) >> 1) : Abs(src_width);
+ int src_height_uv =
+ subsample_y ? ((Abs(src_height) + 1) >> 1) : Abs(src_height);
-TEST_FACTOR(2, 1, 2, 0)
-TEST_FACTOR(4, 1, 4, 0)
-// TEST_FACTOR(8, 1, 8, 0) Disable for benchmark performance. Takes 90 seconds.
-TEST_FACTOR(3by4, 3, 4, 1)
-TEST_FACTOR(3by8, 3, 8, 1)
-TEST_FACTOR(3, 1, 3, 0)
-#undef TEST_FACTOR1
-#undef TEST_FACTOR
-#undef SX
-#undef DX
+ int64_t src_y_plane_size = (Abs(src_width)) * (Abs(src_height));
+ int64_t src_uv_plane_size = (src_width_uv) * (src_height_uv) * 2;
-#define TEST_SCALETO1(name, width, height, filter, max_diff) \
- TEST_F(LibYUVScaleTest, I420##name##To##width##x##height##_##filter) { \
- int diff = I420TestFilter(benchmark_width_, benchmark_height_, width, \
- height, kFilter##filter, benchmark_iterations_, \
- disable_cpu_flags_, benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I444##name##To##width##x##height##_##filter) { \
- int diff = I444TestFilter(benchmark_width_, benchmark_height_, width, \
- height, kFilter##filter, benchmark_iterations_, \
- disable_cpu_flags_, benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I420##name##To##width##x##height##_##filter##_16) { \
- int diff = I420TestFilter_16( \
- benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I444##name##To##width##x##height##_##filter##_16) { \
- int diff = I444TestFilter_16( \
- benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I420##name##From##width##x##height##_##filter) { \
- int diff = I420TestFilter(width, height, Abs(benchmark_width_), \
- Abs(benchmark_height_), kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, I444##name##From##width##x##height##_##filter) { \
- int diff = I444TestFilter(width, height, Abs(benchmark_width_), \
- Abs(benchmark_height_), kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, \
- I420##name##From##width##x##height##_##filter##_16) { \
- int diff = I420TestFilter_16(width, height, Abs(benchmark_width_), \
- Abs(benchmark_height_), kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
- } \
- TEST_F(LibYUVScaleTest, \
- I444##name##From##width##x##height##_##filter##_16) { \
- int diff = I444TestFilter_16(width, height, Abs(benchmark_width_), \
- Abs(benchmark_height_), kFilter##filter, \
- benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+ int src_stride_y = Abs(src_width);
+ int src_stride_uv = src_width_uv * 2;
+
+ align_buffer_page_end(src_y, src_y_plane_size);
+ align_buffer_page_end(src_uv, src_uv_plane_size);
+ if (!src_y || !src_uv) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
}
+ MemRandomize(src_y, src_y_plane_size);
+ MemRandomize(src_uv, src_uv_plane_size);
-// Test scale to a specified size with all 4 filters.
-#define TEST_SCALETO(name, width, height) \
- TEST_SCALETO1(name, width, height, None, 0) \
- TEST_SCALETO1(name, width, height, Linear, 3) \
- TEST_SCALETO1(name, width, height, Bilinear, 3) \
- TEST_SCALETO1(name, width, height, Box, 3)
+ int dst_width_uv = subsample_x ? ((dst_width + 1) >> 1) : dst_width;
+ int dst_height_uv = subsample_y ? ((dst_height + 1) >> 1) : dst_height;
-TEST_SCALETO(Scale, 1, 1)
-TEST_SCALETO(Scale, 320, 240)
-TEST_SCALETO(Scale, 569, 480)
-TEST_SCALETO(Scale, 640, 360)
-TEST_SCALETO(Scale, 1280, 720)
-TEST_SCALETO(Scale, 1920, 1080)
-#undef TEST_SCALETO1
-#undef TEST_SCALETO
+ int64_t dst_y_plane_size = (dst_width) * (dst_height);
+ int64_t dst_uv_plane_size = (dst_width_uv) * (dst_height_uv) * 2;
-#ifdef ENABLE_ROW_TESTS
-#ifdef HAS_SCALEROWDOWN2_SSSE3
-TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_Odd_SSSE3) {
- SIMD_ALIGNED(uint8_t orig_pixels[128 * 2]);
- SIMD_ALIGNED(uint8_t dst_pixels_opt[64]);
- SIMD_ALIGNED(uint8_t dst_pixels_c[64]);
- memset(orig_pixels, 0, sizeof(orig_pixels));
- memset(dst_pixels_opt, 0, sizeof(dst_pixels_opt));
- memset(dst_pixels_c, 0, sizeof(dst_pixels_c));
-
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- if (!has_ssse3) {
- printf("Warning SSSE3 not detected; Skipping test.\n");
- } else {
- // TL.
- orig_pixels[0] = 255u;
- orig_pixels[1] = 0u;
- orig_pixels[128 + 0] = 0u;
- orig_pixels[128 + 1] = 0u;
- // TR.
- orig_pixels[2] = 0u;
- orig_pixels[3] = 100u;
- orig_pixels[128 + 2] = 0u;
- orig_pixels[128 + 3] = 0u;
- // BL.
- orig_pixels[4] = 0u;
- orig_pixels[5] = 0u;
- orig_pixels[128 + 4] = 50u;
- orig_pixels[128 + 5] = 0u;
- // BR.
- orig_pixels[6] = 0u;
- orig_pixels[7] = 0u;
- orig_pixels[128 + 6] = 0u;
- orig_pixels[128 + 7] = 20u;
- // Odd.
- orig_pixels[126] = 4u;
- orig_pixels[127] = 255u;
- orig_pixels[128 + 126] = 16u;
- orig_pixels[128 + 127] = 255u;
-
- // Test regular half size.
- ScaleRowDown2Box_C(orig_pixels, 128, dst_pixels_c, 64);
-
- EXPECT_EQ(64u, dst_pixels_c[0]);
- EXPECT_EQ(25u, dst_pixels_c[1]);
- EXPECT_EQ(13u, dst_pixels_c[2]);
- EXPECT_EQ(5u, dst_pixels_c[3]);
- EXPECT_EQ(0u, dst_pixels_c[4]);
- EXPECT_EQ(133u, dst_pixels_c[63]);
-
- // Test Odd width version - Last pixel is just 1 horizontal pixel.
- ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 64);
-
- EXPECT_EQ(64u, dst_pixels_c[0]);
- EXPECT_EQ(25u, dst_pixels_c[1]);
- EXPECT_EQ(13u, dst_pixels_c[2]);
- EXPECT_EQ(5u, dst_pixels_c[3]);
- EXPECT_EQ(0u, dst_pixels_c[4]);
- EXPECT_EQ(10u, dst_pixels_c[63]);
-
- // Test one pixel less, should skip the last pixel.
- memset(dst_pixels_c, 0, sizeof(dst_pixels_c));
- ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 63);
-
- EXPECT_EQ(64u, dst_pixels_c[0]);
- EXPECT_EQ(25u, dst_pixels_c[1]);
- EXPECT_EQ(13u, dst_pixels_c[2]);
- EXPECT_EQ(5u, dst_pixels_c[3]);
- EXPECT_EQ(0u, dst_pixels_c[4]);
- EXPECT_EQ(0u, dst_pixels_c[63]);
-
- // Test regular half size SSSE3.
- ScaleRowDown2Box_SSSE3(orig_pixels, 128, dst_pixels_opt, 64);
-
- EXPECT_EQ(64u, dst_pixels_opt[0]);
- EXPECT_EQ(25u, dst_pixels_opt[1]);
- EXPECT_EQ(13u, dst_pixels_opt[2]);
- EXPECT_EQ(5u, dst_pixels_opt[3]);
- EXPECT_EQ(0u, dst_pixels_opt[4]);
- EXPECT_EQ(133u, dst_pixels_opt[63]);
-
- // Compare C and SSSE3 match.
- ScaleRowDown2Box_Odd_C(orig_pixels, 128, dst_pixels_c, 64);
- ScaleRowDown2Box_Odd_SSSE3(orig_pixels, 128, dst_pixels_opt, 64);
- for (int i = 0; i < 64; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
- }
- }
-}
-#endif // HAS_SCALEROWDOWN2_SSSE3
-
-extern "C" void ScaleRowUp2_16_NEON(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-extern "C" void ScaleRowUp2_16_MMI(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-extern "C" void ScaleRowUp2_16_C(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-
-TEST_F(LibYUVScaleTest, TestScaleRowUp2_16) {
- SIMD_ALIGNED(uint16_t orig_pixels[640 * 2 + 1]); // 2 rows + 1 pixel overrun.
- SIMD_ALIGNED(uint16_t dst_pixels_opt[1280]);
- SIMD_ALIGNED(uint16_t dst_pixels_c[1280]);
-
- memset(orig_pixels, 0, sizeof(orig_pixels));
- memset(dst_pixels_opt, 1, sizeof(dst_pixels_opt));
- memset(dst_pixels_c, 2, sizeof(dst_pixels_c));
-
- for (int i = 0; i < 640 * 2 + 1; ++i) {
- orig_pixels[i] = i;
- }
- ScaleRowUp2_16_C(&orig_pixels[0], 640, &dst_pixels_c[0], 1280);
- for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
-#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
- int has_neon = TestCpuFlag(kCpuHasNEON);
- if (has_neon) {
- ScaleRowUp2_16_NEON(&orig_pixels[0], 640, &dst_pixels_opt[0], 1280);
- } else {
- ScaleRowUp2_16_C(&orig_pixels[0], 640, &dst_pixels_opt[0], 1280);
- }
-#elif !defined(LIBYUV_DISABLE_MMI) && defined(_MIPS_ARCH_LOONGSON3A)
- int has_mmi = TestCpuFlag(kCpuHasMMI);
- if (has_mmi) {
- ScaleRowUp2_16_MMI(&orig_pixels[0], 640, &dst_pixels_opt[0], 1280);
- } else {
- ScaleRowUp2_16_C(&orig_pixels[0], 640, &dst_pixels_opt[0], 1280);
- }
-#else
- ScaleRowUp2_16_C(&orig_pixels[0], 640, &dst_pixels_opt[0], 1280);
-#endif
+ int dst_stride_y = dst_width;
+ int dst_stride_uv = dst_width_uv * 2;
+
+ align_buffer_page_end(dst_y_c, dst_y_plane_size);
+ align_buffer_page_end(dst_uv_c, dst_uv_plane_size);
+ align_buffer_page_end(dst_y_opt, dst_y_plane_size);
+ align_buffer_page_end(dst_uv_opt, dst_uv_plane_size);
+ if (!dst_y_c || !dst_uv_c || !dst_y_opt || !dst_uv_opt) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
}
- for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ double c_time = get_time();
+ NVScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width, src_height,
+ dst_y_c, dst_stride_y, dst_uv_c, dst_stride_uv, dst_width, dst_height,
+ f);
+ c_time = (get_time() - c_time);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ double opt_time = get_time();
+ for (i = 0; i < benchmark_iterations; ++i) {
+ NVScale(src_y, src_stride_y, src_uv, src_stride_uv, src_width, src_height,
+ dst_y_opt, dst_stride_y, dst_uv_opt, dst_stride_uv, dst_width,
+ dst_height, f);
}
- EXPECT_EQ(dst_pixels_c[0], (0 * 9 + 1 * 3 + 640 * 3 + 641 * 1 + 8) / 16);
- EXPECT_EQ(dst_pixels_c[1279], 800);
-}
+ opt_time = (get_time() - opt_time) / benchmark_iterations;
+ // Report performance of C vs OPT.
+ printf("filter %d - %8d us C - %8d us OPT\n", f,
+ static_cast<int>(c_time * 1e6), static_cast<int>(opt_time * 1e6));
-extern "C" void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-
-TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) {
- SIMD_ALIGNED(uint16_t orig_pixels[2560 * 2]);
- SIMD_ALIGNED(uint16_t dst_pixels_c[1280]);
- SIMD_ALIGNED(uint16_t dst_pixels_opt[1280]);
-
- memset(orig_pixels, 0, sizeof(orig_pixels));
- memset(dst_pixels_c, 1, sizeof(dst_pixels_c));
- memset(dst_pixels_opt, 2, sizeof(dst_pixels_opt));
-
- for (int i = 0; i < 2560 * 2; ++i) {
- orig_pixels[i] = i;
- }
- ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_c[0], 1280);
- for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
-#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
- int has_neon = TestCpuFlag(kCpuHasNEON);
- if (has_neon) {
- ScaleRowDown2Box_16_NEON(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
- } else {
- ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
+ // C version may be a little off from the optimized. Order of
+ // operations may introduce rounding somewhere. So do a difference
+ // of the buffers and look to see that the max difference is not
+ // over 3.
+ int max_diff = 0;
+ for (i = 0; i < (dst_height); ++i) {
+ for (j = 0; j < (dst_width); ++j) {
+ int abs_diff = Abs(dst_y_c[(i * dst_stride_y) + j] -
+ dst_y_opt[(i * dst_stride_y) + j]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
-#else
- ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
-#endif
}
- for (int i = 0; i < 1280; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ for (i = 0; i < (dst_height_uv); ++i) {
+ for (j = 0; j < (dst_width_uv * 2); ++j) {
+ int abs_diff = Abs(dst_uv_c[(i * dst_stride_uv) + j] -
+ dst_uv_opt[(i * dst_stride_uv) + j]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
}
- EXPECT_EQ(dst_pixels_c[0], (0 + 1 + 2560 + 2561 + 2) / 4);
- EXPECT_EQ(dst_pixels_c[1279], 3839);
+ free_aligned_buffer_page_end(dst_y_c);
+ free_aligned_buffer_page_end(dst_uv_c);
+ free_aligned_buffer_page_end(dst_y_opt);
+ free_aligned_buffer_page_end(dst_uv_opt);
+ free_aligned_buffer_page_end(src_y);
+ free_aligned_buffer_page_end(src_uv);
+
+ return max_diff;
}
-#endif // ENABLE_ROW_TESTS
-// Test scaling plane with 8 bit C vs 16 bit C and return maximum pixel
-// difference.
-// 0 = exact.
-static int TestPlaneFilter_16(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- FilterMode f,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info) {
+static int NV12TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return NVTestFilter(src_width, src_height, dst_width, dst_height, f, 1, 1,
+ NV12Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
+static int NV16TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return NVTestFilter(src_width, src_height, dst_width, dst_height, f, 1, 0,
+ NV16Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
+static int NV24TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return NVTestFilter(src_width, src_height, dst_width, dst_height, f, 0, 0,
+ NV24Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
+static int PxxxTestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int subsample_x,
+ int subsample_y,
+ int (*PxxxScale)(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint16_t* dst_y,
+ int dst_stride_y,
+ uint16_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering),
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
return 0;
}
- int i;
+ int i, j;
+ int src_width_uv = subsample_x ? ((Abs(src_width) + 1) >> 1) : Abs(src_width);
+ int src_height_uv =
+ subsample_y ? ((Abs(src_height) + 1) >> 1) : Abs(src_height);
+
int64_t src_y_plane_size = (Abs(src_width)) * (Abs(src_height));
+ int64_t src_uv_plane_size = (src_width_uv) * (src_height_uv) * 2;
+
int src_stride_y = Abs(src_width);
- int dst_y_plane_size = dst_width * dst_height;
- int dst_stride_y = dst_width;
+ int src_stride_uv = src_width_uv * 2;
- align_buffer_page_end(src_y, src_y_plane_size);
- align_buffer_page_end(src_y_16, src_y_plane_size * 2);
- align_buffer_page_end(dst_y_8, dst_y_plane_size);
- align_buffer_page_end(dst_y_16, dst_y_plane_size * 2);
- uint16_t* p_src_y_16 = reinterpret_cast<uint16_t*>(src_y_16);
- uint16_t* p_dst_y_16 = reinterpret_cast<uint16_t*>(dst_y_16);
+ align_buffer_page_end(src_y, src_y_plane_size * 2);
+ align_buffer_page_end(src_uv, src_uv_plane_size * 2);
+ if (!src_y || !src_uv) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ uint16_t* p_src_y = reinterpret_cast<uint16_t*>(src_y);
+ uint16_t* p_src_uv = reinterpret_cast<uint16_t*>(src_uv);
+ MemRandomize(src_y, src_y_plane_size * 2);
+ MemRandomize(src_uv, src_uv_plane_size * 2);
+ for (i = 0; i < src_y_plane_size; ++i) {
+ p_src_y[i] = static_cast<uint16_t>(p_src_y[i] & 0xffc0);
+ }
+ for (i = 0; i < src_uv_plane_size; ++i) {
+ p_src_uv[i] = static_cast<uint16_t>(p_src_uv[i] & 0xffc0);
+ }
- MemRandomize(src_y, src_y_plane_size);
- memset(dst_y_8, 0, dst_y_plane_size);
- memset(dst_y_16, 1, dst_y_plane_size * 2);
+ int dst_width_uv = subsample_x ? ((dst_width + 1) >> 1) : dst_width;
+ int dst_height_uv = subsample_y ? ((dst_height + 1) >> 1) : dst_height;
- for (i = 0; i < src_y_plane_size; ++i) {
- p_src_y_16[i] = src_y[i] & 255;
+ int64_t dst_y_plane_size = (dst_width) * (dst_height);
+ int64_t dst_uv_plane_size = (dst_width_uv) * (dst_height_uv) * 2;
+
+ int dst_stride_y = dst_width;
+ int dst_stride_uv = dst_width_uv * 2;
+
+ align_buffer_page_end(dst_y_c, dst_y_plane_size * 2);
+ align_buffer_page_end(dst_uv_c, dst_uv_plane_size * 2);
+ align_buffer_page_end(dst_y_opt, dst_y_plane_size * 2);
+ align_buffer_page_end(dst_uv_opt, dst_uv_plane_size * 2);
+ if (!dst_y_c || !dst_uv_c || !dst_y_opt || !dst_uv_opt) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
}
+ uint16_t* p_dst_y_c = reinterpret_cast<uint16_t*>(dst_y_c);
+ uint16_t* p_dst_uv_c = reinterpret_cast<uint16_t*>(dst_uv_c);
+ uint16_t* p_dst_y_opt = reinterpret_cast<uint16_t*>(dst_y_opt);
+ uint16_t* p_dst_uv_opt = reinterpret_cast<uint16_t*>(dst_uv_opt);
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
- ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y_8, dst_stride_y,
- dst_width, dst_height, f);
- MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ double c_time = get_time();
+ PxxxScale(p_src_y, src_stride_y, p_src_uv, src_stride_uv, src_width,
+ src_height, p_dst_y_c, dst_stride_y, p_dst_uv_c, dst_stride_uv,
+ dst_width, dst_height, f);
+ c_time = (get_time() - c_time);
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ double opt_time = get_time();
for (i = 0; i < benchmark_iterations; ++i) {
- ScalePlane_16(p_src_y_16, src_stride_y, src_width, src_height, p_dst_y_16,
- dst_stride_y, dst_width, dst_height, f);
+ PxxxScale(p_src_y, src_stride_y, p_src_uv, src_stride_uv, src_width,
+ src_height, p_dst_y_opt, dst_stride_y, p_dst_uv_opt,
+ dst_stride_uv, dst_width, dst_height, f);
}
+ opt_time = (get_time() - opt_time) / benchmark_iterations;
+ printf("filter %d - %8d us C - %8d us OPT\n", f,
+ static_cast<int>(c_time * 1e6), static_cast<int>(opt_time * 1e6));
- // Expect an exact match.
int max_diff = 0;
- for (i = 0; i < dst_y_plane_size; ++i) {
- int abs_diff = Abs(dst_y_8[i] - p_dst_y_16[i]);
- if (abs_diff > max_diff) {
- max_diff = abs_diff;
+ for (i = 0; i < dst_height; ++i) {
+ for (j = 0; j < dst_width; ++j) {
+ uint16_t c = p_dst_y_c[(i * dst_stride_y) + j];
+ uint16_t opt = p_dst_y_opt[(i * dst_stride_y) + j];
+ // Scaled output is converted back to canonical 10-bit MSB.
+ if ((c & 0x3f) || (opt & 0x3f)) {
+ max_diff = 999;
+ }
+ int abs_diff = Abs((c >> 6) - (opt >> 6));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+ }
+ for (i = 0; i < dst_height_uv; ++i) {
+ for (j = 0; j < (dst_width_uv * 2); ++j) {
+ uint16_t c = p_dst_uv_c[(i * dst_stride_uv) + j];
+ uint16_t opt = p_dst_uv_opt[(i * dst_stride_uv) + j];
+ // MergeUVPlane_16 depth=10 must produce canonical 10-bit MSB samples.
+ if ((c & 0x3f) || (opt & 0x3f)) {
+ max_diff = 999;
+ }
+ int abs_diff = Abs((c >> 6) - (opt >> 6));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
}
- free_aligned_buffer_page_end(dst_y_8);
- free_aligned_buffer_page_end(dst_y_16);
+ free_aligned_buffer_page_end(dst_y_c);
+ free_aligned_buffer_page_end(dst_uv_c);
+ free_aligned_buffer_page_end(dst_y_opt);
+ free_aligned_buffer_page_end(dst_uv_opt);
free_aligned_buffer_page_end(src_y);
- free_aligned_buffer_page_end(src_y_16);
+ free_aligned_buffer_page_end(src_uv);
return max_diff;
}
+static int P010TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return PxxxTestFilter(src_width, src_height, dst_width, dst_height, f, 1, 1,
+ P010Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
+static int P210TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return PxxxTestFilter(src_width, src_height, dst_width, dst_height, f, 1, 0,
+ P210Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
+static int P410TestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ return PxxxTestFilter(src_width, src_height, dst_width, dst_height, f, 0, 0,
+ P410Scale, benchmark_iterations, disable_cpu_flags,
+ benchmark_cpu_info);
+}
+
// The following adjustments in dimensions ensure the scale factor will be
// exactly achieved.
// 2 is chroma subsample.
#define DX(x, nom, denom) static_cast<int>(((Abs(x) / nom + 1) / 2) * nom * 2)
#define SX(x, nom, denom) static_cast<int>(((x / nom + 1) / 2) * denom * 2)
-#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
- TEST_F(LibYUVScaleTest, ScalePlaneDownBy##name##_##filter##_16) { \
- int diff = TestPlaneFilter_16( \
- SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
- DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
- kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
- benchmark_cpu_info_); \
- EXPECT_LE(diff, max_diff); \
+#define TEST_FACTOR1(DISABLED_, name, filter, nom, denom, max_diff) \
+ TEST_F(LibYUVScaleTest, I420ScaleDownBy##name##_##filter) { \
+ int diff = I420TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, I444ScaleDownBy##name##_##filter) { \
+ int diff = I444TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I420ScaleDownBy##name##_##filter##_12) { \
+ int diff = I420TestFilter_12( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I444ScaleDownBy##name##_##filter##_12) { \
+ int diff = I444TestFilter_12( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV12ScaleDownBy##name##_##filter) { \
+ int diff = NV12TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV16ScaleDownBy##name##_##filter) { \
+ int diff = NV16TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV24ScaleDownBy##name##_##filter) { \
+ int diff = NV24TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P010ScaleDownBy##name##_##filter) { \
+ int diff = P010TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P210ScaleDownBy##name##_##filter) { \
+ int diff = P210TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P410ScaleDownBy##name##_##filter) { \
+ int diff = P410TestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
}
// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but
// filtering is different fixed point implementations for SSSE3, Neon and C.
-#define TEST_FACTOR(name, nom, denom, boxdiff) \
- TEST_FACTOR1(name, None, nom, denom, 0) \
- TEST_FACTOR1(name, Linear, nom, denom, boxdiff) \
- TEST_FACTOR1(name, Bilinear, nom, denom, boxdiff) \
- TEST_FACTOR1(name, Box, nom, denom, boxdiff)
+#ifndef DISABLE_SLOW_TESTS
+#define TEST_FACTOR(name, nom, denom, boxdiff) \
+ TEST_FACTOR1(, name, None, nom, denom, 0) \
+ TEST_FACTOR1(, name, Linear, nom, denom, 3) \
+ TEST_FACTOR1(, name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(, name, Box, nom, denom, boxdiff)
+#else
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_FACTOR(name, nom, denom, boxdiff) \
+ TEST_FACTOR1(DISABLED_, name, None, nom, denom, 0) \
+ TEST_FACTOR1(DISABLED_, name, Linear, nom, denom, 3) \
+ TEST_FACTOR1(DISABLED_, name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(DISABLED_, name, Box, nom, denom, boxdiff)
+#else
+#define TEST_FACTOR(name, nom, denom, boxdiff) \
+ TEST_FACTOR1(DISABLED_, name, Bilinear, nom, denom, 3) \
+ TEST_FACTOR1(DISABLED_, name, Box, nom, denom, boxdiff)
+#endif
+#endif
TEST_FACTOR(2, 1, 2, 0)
TEST_FACTOR(4, 1, 4, 0)
-// TEST_FACTOR(8, 1, 8, 0) Disable for benchmark performance. Takes 90 seconds.
+#ifndef DISABLE_SLOW_TESTS
+TEST_FACTOR(8, 1, 8, 0)
+#endif
TEST_FACTOR(3by4, 3, 4, 1)
TEST_FACTOR(3by8, 3, 8, 1)
TEST_FACTOR(3, 1, 3, 0)
@@ -906,4 +1194,311 @@ TEST_FACTOR(3, 1, 3, 0)
#undef TEST_FACTOR
#undef SX
#undef DX
+
+#define TEST_SCALETO1(DISABLED_, name, width, height, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, I420##name##To##width##x##height##_##filter) { \
+ int diff = I420TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, I444##name##To##width##x##height##_##filter) { \
+ int diff = I444TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I420##name##To##width##x##height##_##filter##_12) { \
+ int diff = I420TestFilter_12( \
+ benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I444##name##To##width##x##height##_##filter##_12) { \
+ int diff = I444TestFilter_12( \
+ benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I420##name##To##width##x##height##_##filter##_16) { \
+ int diff = I420TestFilter_16( \
+ benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I444##name##To##width##x##height##_##filter##_16) { \
+ int diff = I444TestFilter_16( \
+ benchmark_width_, benchmark_height_, width, height, kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV12##name##To##width##x##height##_##filter) { \
+ int diff = NV12TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV16##name##To##width##x##height##_##filter) { \
+ int diff = NV16TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV24##name##To##width##x##height##_##filter) { \
+ int diff = NV24TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P010##name##To##width##x##height##_##filter) { \
+ int diff = P010TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P210##name##To##width##x##height##_##filter) { \
+ int diff = P210TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P410##name##To##width##x##height##_##filter) { \
+ int diff = P410TestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, I420##name##From##width##x##height##_##filter) { \
+ int diff = I420TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, I444##name##From##width##x##height##_##filter) { \
+ int diff = I444TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I420##name##From##width##x##height##_##filter##_12) { \
+ int diff = I420TestFilter_12(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I444##name##From##width##x##height##_##filter##_12) { \
+ int diff = I444TestFilter_12(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I420##name##From##width##x##height##_##filter##_16) { \
+ int diff = I420TestFilter_16(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, \
+ DISABLED_##I444##name##From##width##x##height##_##filter##_16) { \
+ int diff = I444TestFilter_16(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV12##name##From##width##x##height##_##filter) { \
+ int diff = NV12TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV16##name##From##width##x##height##_##filter) { \
+ int diff = NV16TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV24##name##From##width##x##height##_##filter) { \
+ int diff = NV24TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P010##name##From##width##x##height##_##filter) { \
+ int diff = P010TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P210##name##From##width##x##height##_##filter) { \
+ int diff = P210TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P410##name##From##width##x##height##_##filter) { \
+ int diff = P410TestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#ifndef DISABLE_SLOW_TESTS
+// Test scale to a specified size with all 4 filters.
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(, name, width, height, None, 0) \
+ TEST_SCALETO1(, name, width, height, Linear, 3) \
+ TEST_SCALETO1(, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(, name, width, height, Box, 3)
+#else
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(DISABLED_, name, width, height, None, 0) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Linear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Box, 3)
+#else
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Box, 3)
+#endif
+#endif
+
+TEST_SCALETO(Scale, 1, 1)
+TEST_SCALETO(Scale, 569, 480)
+TEST_SCALETO(Scale, 640, 360)
+#ifndef DISABLE_SLOW_TESTS
+TEST_SCALETO(Scale, 256, 144) /* 128x72 * 2 */
+TEST_SCALETO(Scale, 264, 216)
+TEST_SCALETO(Scale, 320, 240)
+TEST_SCALETO(Scale, 1280, 720)
+TEST_SCALETO(Scale, 1920, 1080)
+TEST_SCALETO(Scale, 1080, 1920) // for rotated phones
+#endif // DISABLE_SLOW_TESTS
+#undef TEST_SCALETO1
+#undef TEST_SCALETO
+
+#define TEST_SCALESWAPXY1(DISABLED_, name, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, I420##name##SwapXY_##filter) { \
+ int diff = I420TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, I444##name##SwapXY_##filter) { \
+ int diff = I444TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I420##name##SwapXY_##filter##_12) { \
+ int diff = I420TestFilter_12(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I444##name##SwapXY_##filter##_12) { \
+ int diff = I444TestFilter_12(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I420##name##SwapXY_##filter##_16) { \
+ int diff = I420TestFilter_16(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, DISABLED_##I444##name##SwapXY_##filter##_16) { \
+ int diff = I444TestFilter_16(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV12##name##SwapXY_##filter) { \
+ int diff = NV12TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV16##name##SwapXY_##filter) { \
+ int diff = NV16TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, NV24##name##SwapXY_##filter) { \
+ int diff = NV24TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P010##name##SwapXY_##filter) { \
+ int diff = P010TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P210##name##SwapXY_##filter) { \
+ int diff = P210TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, P410##name##SwapXY_##filter) { \
+ int diff = P410TestFilter(benchmark_width_, benchmark_height_, \
+ benchmark_height_, benchmark_width_, \
+ kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+// Test scale to a specified size with all 4 filters.
+#ifndef DISABLE_SLOW_TESTS
+TEST_SCALESWAPXY1(, Scale, None, 0)
+TEST_SCALESWAPXY1(, Scale, Linear, 3)
+TEST_SCALESWAPXY1(, Scale, Bilinear, 3)
+TEST_SCALESWAPXY1(, Scale, Box, 3)
+#else
+#if defined(ENABLE_FULL_TESTS)
+TEST_SCALESWAPXY1(DISABLED_, Scale, None, 0)
+TEST_SCALESWAPXY1(DISABLED_, Scale, Linear, 3)
+TEST_SCALESWAPXY1(DISABLED_, Scale, Bilinear, 3)
+TEST_SCALESWAPXY1(DISABLED_, Scale, Box, 3)
+#else
+TEST_SCALESWAPXY1(DISABLED_, Scale, Bilinear, 3)
+TEST_SCALESWAPXY1(DISABLED_, Scale, Box, 3)
+#endif
+#endif
+#undef TEST_SCALESWAPXY1
+
} // namespace libyuv
diff --git a/libs/libyuv/unit_test/scale_uv_test.cc b/libs/libyuv/unit_test/scale_uv_test.cc
new file mode 100644
index 0000000000..bbb6bbc680
--- /dev/null
+++ b/libs/libyuv/unit_test/scale_uv_test.cc
@@ -0,0 +1,657 @@
+/*
+ * Copyright 2011 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stdlib.h>
+#include <time.h>
+
+#include "../unit_test/unit_test.h"
+#include "libyuv/cpu_id.h"
+#include "libyuv/scale.h"
+#include "libyuv/scale_uv.h"
+
+namespace libyuv {
+
+#define STRINGIZE(line) #line
+#define FILELINESTR(file, line) file ":" STRINGIZE(line)
+
+#if !defined(DISABLE_SLOW_TESTS) || defined(__x86_64__) || defined(__i386__)
+// SLOW TESTS are those that are unoptimized C code.
+// FULL TESTS are optimized but test many variations of the same code.
+#define ENABLE_FULL_TESTS
+#endif
+
+// Test scaling with C vs Opt and return maximum pixel difference. 0 = exact.
+static int UVTestFilter(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+ if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
+ return 0;
+ }
+
+ int i;
+ int64_t src_uv_plane_size = Abs(src_width) * Abs(src_height) * 2LL;
+ int src_stride_uv = Abs(src_width) * 2;
+ int64_t dst_uv_plane_size = dst_width * dst_height * 2LL;
+ int dst_stride_uv = dst_width * 2;
+
+ align_buffer_page_end(src_uv, src_uv_plane_size);
+ align_buffer_page_end(dst_uv_c, dst_uv_plane_size);
+ align_buffer_page_end(dst_uv_opt, dst_uv_plane_size);
+
+ if (!src_uv || !dst_uv_c || !dst_uv_opt) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ MemRandomize(src_uv, src_uv_plane_size);
+ memset(dst_uv_c, 2, dst_uv_plane_size);
+ memset(dst_uv_opt, 123, dst_uv_plane_size);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ double c_time = get_time();
+ UVScale(src_uv, src_stride_uv, src_width, src_height, dst_uv_c, dst_stride_uv,
+ dst_width, dst_height, f);
+ c_time = (get_time() - c_time);
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ double opt_time = get_time();
+ for (i = 0; i < benchmark_iterations; ++i) {
+ UVScale(src_uv, src_stride_uv, src_width, src_height, dst_uv_opt,
+ dst_stride_uv, dst_width, dst_height, f);
+ }
+ opt_time = (get_time() - opt_time) / benchmark_iterations;
+
+ // Report performance of C vs OPT
+ printf("filter %d - %8d us C - %8d us OPT\n", f,
+ static_cast<int>(c_time * 1e6), static_cast<int>(opt_time * 1e6));
+
+ int max_diff = 0;
+ for (i = 0; i < dst_uv_plane_size; ++i) {
+ int abs_diff = Abs(dst_uv_c[i] - dst_uv_opt[i]);
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_uv_c);
+ free_aligned_buffer_page_end(dst_uv_opt);
+ free_aligned_buffer_page_end(src_uv);
+ return max_diff;
+}
+
+// The following adjustments in dimensions ensure the scale factor will be
+// exactly achieved.
+#define DX(x, nom, denom) static_cast<int>((Abs(x) / nom) * nom)
+#define SX(x, nom, denom) static_cast<int>((x / nom) * denom)
+
+#define TEST_FACTOR1(name, filter, nom, denom) \
+ TEST_F(LibYUVScaleTest, UVScaleDownBy##name##_##filter) { \
+ int diff = UVTestFilter( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_EQ(0, diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+// Test a scale factor with all 4 filters. Expect exact for SIMD vs C.
+#define TEST_FACTOR(name, nom, denom) \
+ TEST_FACTOR1(name, None, nom, denom) \
+ TEST_FACTOR1(name, Linear, nom, denom) \
+ TEST_FACTOR1(name, Bilinear, nom, denom) \
+ TEST_FACTOR1(name, Box, nom, denom)
+#else
+// Test a scale factor with Bilinear.
+#define TEST_FACTOR(name, nom, denom) TEST_FACTOR1(name, Bilinear, nom, denom)
+#endif
+
+TEST_FACTOR(2, 1, 2)
+TEST_FACTOR(4, 1, 4)
+// TEST_FACTOR(8, 1, 8) Disable for benchmark performance.
+TEST_FACTOR(3by4, 3, 4)
+TEST_FACTOR(3by8, 3, 8)
+TEST_FACTOR(3, 1, 3)
+#undef TEST_FACTOR1
+#undef TEST_FACTOR
+#undef SX
+#undef DX
+
+#define TEST_SCALETO1(name, width, height, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \
+ int diff = UVTestFilter(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \
+ int diff = UVTestFilter(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+/// Test scale to a specified size with all 4 filters.
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(name, width, height, None, 0) \
+ TEST_SCALETO1(name, width, height, Linear, 3) \
+ TEST_SCALETO1(name, width, height, Bilinear, 3)
+#else
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(name, width, height, Bilinear, 3)
+#endif
+
+TEST_SCALETO(UVScale, 1, 1)
+TEST_SCALETO(UVScale, 569, 480)
+TEST_SCALETO(UVScale, 640, 360)
+#ifndef DISABLE_SLOW_TESTS
+TEST_SCALETO(UVScale, 256, 144) /* 128x72 * 2 */
+TEST_SCALETO(UVScale, 320, 240)
+TEST_SCALETO(UVScale, 1280, 720)
+TEST_SCALETO(UVScale, 1920, 1080)
+#endif // DISABLE_SLOW_TESTS
+#undef TEST_SCALETO1
+#undef TEST_SCALETO
+
+#define TEST_SCALESWAPXY1(name, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##SwapXY_##filter) { \
+ int diff = \
+ UVTestFilter(benchmark_width_, benchmark_height_, benchmark_height_, \
+ benchmark_width_, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+// Test scale with swapped width and height with all 3 filters.
+TEST_SCALESWAPXY1(UVScale, None, 0)
+TEST_SCALESWAPXY1(UVScale, Linear, 0)
+TEST_SCALESWAPXY1(UVScale, Bilinear, 0)
+#else
+TEST_SCALESWAPXY1(UVScale, Bilinear, 0)
+#endif
+#undef TEST_SCALESWAPXY1
+
+TEST_F(LibYUVScaleTest, UVTest3x) {
+ const int kSrcStride = 480 * 2;
+ const int kDstStride = 160 * 2;
+ const int kSize = kSrcStride * 3;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 480 * 3; ++i) {
+ orig_pixels[i * 2 + 0] = i;
+ orig_pixels[i * 2 + 1] = 255 - i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ UVScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+
+ UVScale(orig_pixels, kSrcStride, 480, 3, dest_pixels, kDstStride, 160, 1,
+ kFilterNone);
+
+ ASSERT_EQ(225, dest_pixels[0]);
+ ASSERT_EQ(255 - 225, dest_pixels[1]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+TEST_F(LibYUVScaleTest, UVTest4x) {
+ const int kSrcStride = 640 * 2;
+ const int kDstStride = 160 * 2;
+ const int kSize = kSrcStride * 4;
+ align_buffer_page_end(orig_pixels, kSize);
+ for (int i = 0; i < 640 * 4; ++i) {
+ orig_pixels[i * 2 + 0] = i;
+ orig_pixels[i * 2 + 1] = 255 - i;
+ }
+ align_buffer_page_end(dest_pixels, kDstStride);
+
+ int iterations160 = (benchmark_width_ * benchmark_height_ + (160 - 1)) / 160 *
+ benchmark_iterations_;
+ for (int i = 0; i < iterations160; ++i) {
+ UVScale(orig_pixels, kSrcStride, 640, 4, dest_pixels, kDstStride, 160, 1,
+ kFilterBilinear);
+ }
+
+ ASSERT_EQ(66, dest_pixels[0]);
+ ASSERT_EQ(190, dest_pixels[1]);
+
+ UVScale(orig_pixels, kSrcStride, 64, 4, dest_pixels, kDstStride, 16, 1,
+ kFilterNone);
+
+ ASSERT_EQ(2, dest_pixels[0]); // expect the 3rd pixel of the 3rd row
+ ASSERT_EQ(255 - 2, dest_pixels[1]);
+
+ free_aligned_buffer_page_end(dest_pixels);
+ free_aligned_buffer_page_end(orig_pixels);
+}
+
+// Box filtering an interleaved UV plane has to produce the same bytes as box
+// filtering the U and V planes separately, which is the path I420 takes.
+// C and SIMD agreeing is not enough here: before ScaleUVBox existed both
+// point sampled, so this compares against an independent implementation.
+static int UVTestBoxAgainstPlanes(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ int cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+ if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
+ return 0;
+ }
+ int64_t src_uv_plane_size = src_width * src_height * 2LL;
+ int64_t src_plane_size = src_width * src_height * 1LL;
+ int64_t dst_uv_plane_size = dst_width * dst_height * 2LL;
+ int64_t dst_plane_size = dst_width * dst_height * 1LL;
+
+ align_buffer_page_end(src_uv, src_uv_plane_size);
+ align_buffer_page_end(src_u, src_plane_size);
+ align_buffer_page_end(src_v, src_plane_size);
+ align_buffer_page_end(dst_uv, dst_uv_plane_size);
+ align_buffer_page_end(dst_u, dst_plane_size);
+ align_buffer_page_end(dst_v, dst_plane_size);
+ if (!src_uv || !src_u || !src_v || !dst_uv || !dst_u || !dst_v) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ MemRandomize(src_uv, src_uv_plane_size);
+ memset(dst_uv, 123, dst_uv_plane_size);
+ for (int i = 0; i < src_width * src_height; ++i) {
+ src_u[i] = src_uv[i * 2 + 0];
+ src_v[i] = src_uv[i * 2 + 1];
+ }
+
+ MaskCpuFlags(cpu_info);
+ UVScale(src_uv, src_width * 2, src_width, src_height, dst_uv, dst_width * 2,
+ dst_width, dst_height, kFilterBox);
+ ScalePlane(src_u, src_width, src_width, src_height, dst_u, dst_width,
+ dst_width, dst_height, kFilterBox);
+ ScalePlane(src_v, src_width, src_width, src_height, dst_v, dst_width,
+ dst_width, dst_height, kFilterBox);
+
+ int max_diff = 0;
+ for (int i = 0; i < dst_width * dst_height; ++i) {
+ int du = Abs(dst_uv[i * 2 + 0] - dst_u[i]);
+ int dv = Abs(dst_uv[i * 2 + 1] - dst_v[i]);
+ if (du > max_diff) {
+ max_diff = du;
+ }
+ if (dv > max_diff) {
+ max_diff = dv;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_v);
+ free_aligned_buffer_page_end(dst_u);
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(src_v);
+ free_aligned_buffer_page_end(src_u);
+ free_aligned_buffer_page_end(src_uv);
+ return max_diff;
+}
+
+// Chroma dimensions, ie half the frame sizes named in the comments.
+// max_diff is 0 for everything ScaleUVBox handles. The 1/2 and 1/4 fast paths
+// predate it and keep their own rounding, so 1/4 is allowed to drift by one.
+#define TEST_BOXPLANE(name, swidth, sheight, dwidth, dheight, max_diff) \
+ TEST_F(LibYUVScaleTest, UVScaleBoxMatchesPlane##name) { \
+ int diff_c = UVTestBoxAgainstPlanes(swidth, sheight, dwidth, dheight, \
+ disable_cpu_flags_); \
+ ASSERT_LE(diff_c, max_diff); \
+ int diff_opt = UVTestBoxAgainstPlanes(swidth, sheight, dwidth, dheight, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff_opt, max_diff); \
+ }
+
+TEST_BOXPLANE(By2, 960, 540, 480, 270, 0) /* 1080p to 960x540 */
+TEST_BOXPLANE(By3, 960, 540, 320, 180, 0) /* 1080p to 640x360 */
+TEST_BOXPLANE(By3Rounded, 640, 360, 214, 120, 0) /* 720p to 427x240 */
+TEST_BOXPLANE(By4, 960, 540, 240, 135, 1) /* ScaleUVDown4Box */
+TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */
+TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */
+TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */
+TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */
+TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */
+TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */
+TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */
+TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */
+TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */
+#undef TEST_BOXPLANE
+
+// A box filter has to read the whole box. Every 3x3 block here is 255 except
+// for its top left pixel, so point sampling the block origin yields 0 while
+// the box average is 8 * 255 * (65536 / 9) >> 16 = 226.
+TEST_F(LibYUVScaleTest, UVScaleBoxAverages) {
+ const int kSrcWidth = 48;
+ const int kSrcHeight = 24;
+ const int kDstWidth = kSrcWidth / 3;
+ const int kDstHeight = kSrcHeight / 3;
+ align_buffer_page_end(src_uv, kSrcWidth * kSrcHeight * 2);
+ align_buffer_page_end(dst_uv, kDstWidth * kDstHeight * 2);
+
+ memset(src_uv, 255, kSrcWidth * kSrcHeight * 2);
+ for (int y = 0; y < kSrcHeight; y += 3) {
+ for (int x = 0; x < kSrcWidth; x += 3) {
+ src_uv[(y * kSrcWidth + x) * 2 + 0] = 0;
+ src_uv[(y * kSrcWidth + x) * 2 + 1] = 0;
+ }
+ }
+ memset(dst_uv, 123, kDstWidth * kDstHeight * 2);
+
+ UVScale(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv, kDstWidth * 2,
+ kDstWidth, kDstHeight, kFilterBox);
+
+ for (int i = 0; i < kDstWidth * kDstHeight * 2; ++i) {
+ ASSERT_EQ(226, dst_uv[i]) << "at " << i;
+ }
+
+ free_aligned_buffer_page_end(dst_uv);
+ free_aligned_buffer_page_end(src_uv);
+}
+
+// Test 16-bit UVScale_16 with C vs Opt and return maximum pixel difference.
+static int UVTestFilter_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode f,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+ if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
+ return 0;
+ }
+
+ int i;
+ int64_t src_uv_samples = Abs(src_width) * Abs(src_height) * 2LL;
+ int src_stride_uv = Abs(src_width) * 2;
+ int64_t dst_uv_samples = dst_width * dst_height * 2LL;
+ int dst_stride_uv = dst_width * 2;
+
+ align_buffer_page_end(src_uv_buf, src_uv_samples * 2);
+ align_buffer_page_end(dst_uv_c_buf, dst_uv_samples * 2);
+ align_buffer_page_end(dst_uv_opt_buf, dst_uv_samples * 2);
+
+ if (!src_uv_buf || !dst_uv_c_buf || !dst_uv_opt_buf) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ uint16_t* src_uv = reinterpret_cast<uint16_t*>(src_uv_buf);
+ uint16_t* dst_uv_c = reinterpret_cast<uint16_t*>(dst_uv_c_buf);
+ uint16_t* dst_uv_opt = reinterpret_cast<uint16_t*>(dst_uv_opt_buf);
+
+ MemRandomize(src_uv_buf, src_uv_samples * 2);
+ memset(dst_uv_c_buf, 2, dst_uv_samples * 2);
+ memset(dst_uv_opt_buf, 123, dst_uv_samples * 2);
+
+ MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
+ int rc_c = UVScale_16(src_uv, src_stride_uv, src_width, src_height, dst_uv_c,
+ dst_stride_uv, dst_width, dst_height, f);
+ if (rc_c != 0) {
+ free_aligned_buffer_page_end(dst_uv_c_buf);
+ free_aligned_buffer_page_end(dst_uv_opt_buf);
+ free_aligned_buffer_page_end(src_uv_buf);
+ return 65536;
+ }
+
+ MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
+ int rc_opt = 0;
+ for (i = 0; i < benchmark_iterations; ++i) {
+ rc_opt |= UVScale_16(src_uv, src_stride_uv, src_width, src_height,
+ dst_uv_opt, dst_stride_uv, dst_width, dst_height, f);
+ }
+ if (rc_opt != 0) {
+ free_aligned_buffer_page_end(dst_uv_c_buf);
+ free_aligned_buffer_page_end(dst_uv_opt_buf);
+ free_aligned_buffer_page_end(src_uv_buf);
+ return 65536;
+ }
+
+ int max_diff = 0;
+ for (i = 0; i < dst_uv_samples; ++i) {
+ int abs_diff = Abs(static_cast<int>(dst_uv_c[i]) -
+ static_cast<int>(dst_uv_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_uv_c_buf);
+ free_aligned_buffer_page_end(dst_uv_opt_buf);
+ free_aligned_buffer_page_end(src_uv_buf);
+ return max_diff;
+}
+
+#define DX(x, nom, denom) static_cast<int>((Abs(x) / nom) * nom)
+#define SX(x, nom, denom) static_cast<int>((x / nom) * denom)
+
+#define TEST_FACTOR1_16(name, filter, nom, denom) \
+ TEST_F(LibYUVScaleTest, UVScale_16_DownBy##name##_##filter) { \
+ int diff = UVTestFilter_16( \
+ SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
+ DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
+ kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_EQ(0, diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_FACTOR_16(name, nom, denom) \
+ TEST_FACTOR1_16(name, None, nom, denom) \
+ TEST_FACTOR1_16(name, Linear, nom, denom) \
+ TEST_FACTOR1_16(name, Bilinear, nom, denom) \
+ TEST_FACTOR1_16(name, Box, nom, denom)
+#else
+#define TEST_FACTOR_16(name, nom, denom) \
+ TEST_FACTOR1_16(name, Bilinear, nom, denom)
+#endif
+
+TEST_FACTOR_16(2, 1, 2)
+TEST_FACTOR_16(4, 1, 4)
+TEST_FACTOR_16(3by4, 3, 4)
+TEST_FACTOR_16(3by8, 3, 8)
+TEST_FACTOR_16(3, 1, 3)
+#undef TEST_FACTOR1_16
+#undef TEST_FACTOR_16
+#undef SX
+#undef DX
+
+#define TEST_SCALETO1_16(name, width, height, filter, max_diff) \
+ TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \
+ int diff = UVTestFilter_16(benchmark_width_, benchmark_height_, width, \
+ height, kFilter##filter, benchmark_iterations_, \
+ disable_cpu_flags_, benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ } \
+ TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \
+ int diff = UVTestFilter_16(width, height, Abs(benchmark_width_), \
+ Abs(benchmark_height_), kFilter##filter, \
+ benchmark_iterations_, disable_cpu_flags_, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff, max_diff); \
+ }
+
+#if defined(ENABLE_FULL_TESTS)
+#define TEST_SCALETO_16(name, width, height) \
+ TEST_SCALETO1_16(name, width, height, None, 0) \
+ TEST_SCALETO1_16(name, width, height, Linear, 0) \
+ TEST_SCALETO1_16(name, width, height, Bilinear, 0)
+#else
+#define TEST_SCALETO_16(name, width, height) \
+ TEST_SCALETO1_16(name, width, height, Bilinear, 0)
+#endif
+
+TEST_SCALETO_16(UVScale_16, 1, 1)
+TEST_SCALETO_16(UVScale_16, 569, 480)
+TEST_SCALETO_16(UVScale_16, 640, 360)
+#undef TEST_SCALETO1_16
+#undef TEST_SCALETO_16
+
+// Compare UVScale_16 against scaling U and V planes separately with
+// ScalePlane_16 across Box, Bilinear, Linear, and None filters.
+static int UVTestAgainstPlanes_16(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ FilterMode filtering,
+ int cpu_info) {
+ if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
+ return 0;
+ }
+ if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
+ return 0;
+ }
+ const int abs_src_width = Abs(src_width);
+ const int abs_src_height = Abs(src_height);
+ int64_t src_uv_samples = abs_src_width * abs_src_height * 2LL;
+ int64_t src_plane_samples = abs_src_width * abs_src_height * 1LL;
+ int64_t dst_uv_samples = dst_width * dst_height * 2LL;
+ int64_t dst_plane_samples = dst_width * dst_height * 1LL;
+
+ align_buffer_page_end(src_uv_buf, src_uv_samples * 2);
+ align_buffer_page_end(src_u_buf, src_plane_samples * 2);
+ align_buffer_page_end(src_v_buf, src_plane_samples * 2);
+ align_buffer_page_end(dst_uv_buf, dst_uv_samples * 2);
+ align_buffer_page_end(dst_u_buf, dst_plane_samples * 2);
+ align_buffer_page_end(dst_v_buf, dst_plane_samples * 2);
+ if (!src_uv_buf || !src_u_buf || !src_v_buf || !dst_uv_buf || !dst_u_buf ||
+ !dst_v_buf) {
+ printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
+ return 0;
+ }
+ uint16_t* src_uv = reinterpret_cast<uint16_t*>(src_uv_buf);
+ uint16_t* src_u = reinterpret_cast<uint16_t*>(src_u_buf);
+ uint16_t* src_v = reinterpret_cast<uint16_t*>(src_v_buf);
+ uint16_t* dst_uv = reinterpret_cast<uint16_t*>(dst_uv_buf);
+ uint16_t* dst_u = reinterpret_cast<uint16_t*>(dst_u_buf);
+ uint16_t* dst_v = reinterpret_cast<uint16_t*>(dst_v_buf);
+
+ MemRandomize(src_uv_buf, src_uv_samples * 2);
+ memset(dst_uv_buf, 123, dst_uv_samples * 2);
+ for (int i = 0; i < abs_src_width * abs_src_height; ++i) {
+ src_u[i] = src_uv[i * 2 + 0];
+ src_v[i] = src_uv[i * 2 + 1];
+ }
+
+ MaskCpuFlags(cpu_info);
+ int rc_uv = UVScale_16(src_uv, abs_src_width * 2, src_width, src_height,
+ dst_uv, dst_width * 2, dst_width, dst_height,
+ filtering);
+ int rc_u = ScalePlane_16(src_u, abs_src_width, src_width, src_height, dst_u,
+ dst_width, dst_width, dst_height, filtering);
+ int rc_v = ScalePlane_16(src_v, abs_src_width, src_width, src_height, dst_v,
+ dst_width, dst_width, dst_height, filtering);
+ int max_diff = (rc_uv != 0 || rc_u != 0 || rc_v != 0) ? 65536 : 0;
+ for (int i = 0; i < dst_width * dst_height; ++i) {
+ int du = Abs(static_cast<int>(dst_uv[i * 2 + 0]) -
+ static_cast<int>(dst_u[i]));
+ int dv = Abs(static_cast<int>(dst_uv[i * 2 + 1]) -
+ static_cast<int>(dst_v[i]));
+ if (du > max_diff) {
+ max_diff = du;
+ }
+ if (dv > max_diff) {
+ max_diff = dv;
+ }
+ }
+
+ free_aligned_buffer_page_end(dst_v_buf);
+ free_aligned_buffer_page_end(dst_u_buf);
+ free_aligned_buffer_page_end(dst_uv_buf);
+ free_aligned_buffer_page_end(src_v_buf);
+ free_aligned_buffer_page_end(src_u_buf);
+ free_aligned_buffer_page_end(src_uv_buf);
+ return max_diff;
+}
+
+#define TEST_UVPLANE16(name, swidth, sheight, dwidth, dheight, filter, \
+ max_diff) \
+ TEST_F(LibYUVScaleTest, UVScale_16_MatchesPlane_##name##_##filter) { \
+ int diff_c = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \
+ kFilter##filter, disable_cpu_flags_); \
+ ASSERT_LE(diff_c, max_diff); \
+ int diff_opt = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \
+ kFilter##filter, \
+ benchmark_cpu_info_); \
+ ASSERT_LE(diff_opt, max_diff); \
+ }
+
+TEST_UVPLANE16(By2, 480, 270, 240, 135, Box, 0)
+TEST_UVPLANE16(By2, 480, 270, 240, 135, Bilinear, 0)
+TEST_UVPLANE16(By2, 480, 270, 240, 135, None, 0)
+TEST_UVPLANE16(By3, 480, 270, 160, 90, Box, 0)
+TEST_UVPLANE16(By3Rounded, 320, 180, 107, 60, Box, 0)
+TEST_UVPLANE16(By4, 480, 270, 120, 68, Box, 0)
+TEST_UVPLANE16(By4Exact, 480, 272, 120, 68, Box, 1)
+TEST_UVPLANE16(By6, 360, 180, 60, 30, Box, 0)
+TEST_UVPLANE16(By6, 360, 180, 60, 30, None, 0)
+TEST_UVPLANE16(OddTail, 33, 17, 7, 5, Box, 0)
+TEST_UVPLANE16(By3OddTail, 69, 15, 23, 5, Box, 0)
+TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Bilinear, 0)
+TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Linear, 0)
+TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, None, 0)
+TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Bilinear, 0)
+TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Linear, 0)
+TEST_UVPLANE16(Up2x, 37, 23, 74, 46, None, 0)
+TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Bilinear, 0)
+TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Linear, 0)
+TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, None, 0)
+TEST_UVPLANE16(VerticalOnlyUp, 41, 19, 41, 53, Bilinear, 0)
+TEST_UVPLANE16(VerticalOnlyDown, 41, 53, 41, 19, Bilinear, 0)
+TEST_UVPLANE16(InvertCopy, 31, -17, 31, 17, None, 0)
+TEST_UVPLANE16(InvertUp, 31, -17, 65, 39, Bilinear, 0)
+#undef TEST_UVPLANE16
+
+// Verify 2x kFilterLinear downscale averages pairs horizontally on odd rows
+// without underflowing into the previous row.
+TEST_F(LibYUVScaleTest, UVScale_16_DownBy2_LinearAveragesOddRows) {
+ const int kSrcWidth = 4;
+ const int kSrcHeight = 2;
+ const int kDstWidth = 2;
+ const int kDstHeight = 1;
+ uint16_t src_uv[kSrcWidth * kSrcHeight * 2] = {
+ // Row 0 (even row - must not be read):
+ 60000, 60000, 60000, 60000, 60000, 60000, 60000, 60000,
+ // Row 1 (odd row - horizontally averaged in pairs):
+ 100, 1000, 300, 3000, 500, 5000, 900, 7000,
+ };
+ uint16_t dst_uv[kDstWidth * kDstHeight * 2] = {0};
+
+ ASSERT_EQ(0, UVScale_16(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv,
+ kDstWidth * 2, kDstWidth, kDstHeight, kFilterLinear));
+ EXPECT_EQ(200, dst_uv[0]);
+ EXPECT_EQ(2000, dst_uv[1]);
+ EXPECT_EQ(700, dst_uv[2]);
+ EXPECT_EQ(6000, dst_uv[3]);
+}
+
+} // namespace libyuv
diff --git a/libs/libyuv/unit_test/testdata/juno.txt b/libs/libyuv/unit_test/testdata/juno.txt
deleted file mode 100644
index dd465272b8..0000000000
--- a/libs/libyuv/unit_test/testdata/juno.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-Processor : AArch64 Processor rev 0 (aarch64)
-processor : 0
-processor : 1
-processor : 2
-processor : 3
-processor : 4
-processor : 5
-Features : fp asimd evtstrm aes pmull sha1 sha2 crc32
-CPU implementer : 0x41
-CPU architecture: AArch64
-CPU variant : 0x0
-CPU part : 0xd07
-CPU revision : 0
-
-Hardware : Juno
diff --git a/libs/libyuv/unit_test/testdata/mips.txt b/libs/libyuv/unit_test/testdata/mips.txt
new file mode 100644
index 0000000000..d9f28cbfcb
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/mips.txt
@@ -0,0 +1,7 @@
+system type : generic-loongson-machine
+machine : loongson,generic
+processor : 0
+
+isa : mips1 mips2 mips3 mips4 mips5 mips32r1 mips32r2 mips64r1 mips64r2
+ASEs implemented : vz
+shadow register sets : 1
diff --git a/libs/libyuv/unit_test/testdata/mips_loongson2k.txt b/libs/libyuv/unit_test/testdata/mips_loongson2k.txt
new file mode 100644
index 0000000000..8a88d38f2b
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/mips_loongson2k.txt
@@ -0,0 +1,5 @@
+system type : Loongson2K-SBC
+machine : loongson,LS2k1000-EVP
+processor : 0
+cpu model : Loongson-2K V0.3 FPU V0.1
+BogoMIPS : 1980.41
diff --git a/libs/libyuv/unit_test/testdata/mips_loongson3.txt b/libs/libyuv/unit_test/testdata/mips_loongson3.txt
new file mode 100644
index 0000000000..1f540b1281
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/mips_loongson3.txt
@@ -0,0 +1,10 @@
+system type : generic-loongson-machine
+machine : Unknown
+processor : 0
+cpu model : ICT Loongson-3 V0.9 FPU V0.1
+model name : ICT Loongson-3A R3 (Loongson-3A3000) @ 1500MHz
+BogoMIPS : 2990.15
+
+isa : mips1 mips2 mips3 mips4 mips5 mips32r1 mips32r2 mips64r1 mips64r2
+ASEs implemented : dsp dsp2 vz
+shadow register sets : 1
diff --git a/libs/libyuv/unit_test/testdata/mips_loongson_mmi.txt b/libs/libyuv/unit_test/testdata/mips_loongson_mmi.txt
new file mode 100644
index 0000000000..0f10b8bb2b
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/mips_loongson_mmi.txt
@@ -0,0 +1,7 @@
+system type : generic-loongson-machine
+machine : loongson,generic
+processor : 0
+
+isa : mips1 mips2 mips3 mips4 mips5 mips32r1 mips32r2 mips64r1 mips64r2
+ASEs implemented : vz loongson-mmi loongson-ext
+shadow register sets : 1
diff --git a/libs/libyuv/unit_test/testdata/riscv64.txt b/libs/libyuv/unit_test/testdata/riscv64.txt
new file mode 100644
index 0000000000..fbb4200fc4
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/riscv64.txt
@@ -0,0 +1,4 @@
+processor : 0
+hart : 1
+isa : rv64imac
+mmu : sv48
\ No newline at end of file
diff --git a/libs/libyuv/unit_test/testdata/riscv64_rvv.txt b/libs/libyuv/unit_test/testdata/riscv64_rvv.txt
new file mode 100644
index 0000000000..af1b3f3667
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/riscv64_rvv.txt
@@ -0,0 +1,4 @@
+processor : 0
+hart : 1
+isa : rv64imafdcv
+mmu : sv48
\ No newline at end of file
diff --git a/libs/libyuv/unit_test/testdata/riscv64_rvv_zvfh.txt b/libs/libyuv/unit_test/testdata/riscv64_rvv_zvfh.txt
new file mode 100644
index 0000000000..c416c1aff4
--- /dev/null
+++ b/libs/libyuv/unit_test/testdata/riscv64_rvv_zvfh.txt
@@ -0,0 +1,4 @@
+processor : 0
+hart : 1
+isa : rv64imafdcv_zfh_zvfh
+mmu : sv48
\ No newline at end of file
diff --git a/libs/libyuv/unit_test/unit_test.cc b/libs/libyuv/unit_test/unit_test.cc
index 2aa9cdaad6..c51b184e4e 100644
--- a/libs/libyuv/unit_test/unit_test.cc
+++ b/libs/libyuv/unit_test/unit_test.cc
@@ -11,29 +11,58 @@
#include "../unit_test/unit_test.h"
#include <stdlib.h> // For getenv()
+#if defined(__hexagon__)
+#include <sys/mman.h>
+#endif
#include <cstring>
-#ifdef LIBYUV_USE_GFLAGS
-#include "gflags/gflags.h"
+#if defined(__hexagon__)
+// Hexagon QuRT sys/mman.h declares mmap/munmap without extern "C", and the
+// runtime does not provide them. Emulate anonymous mappings with malloc/free.
+__attribute__((nothrow)) void* mmap(void* addr,
+ size_t length,
+ int prot,
+ int flags,
+ int fd,
+ off_t offset) {
+ (void)addr;
+ (void)prot;
+ (void)flags;
+ (void)fd;
+ (void)offset;
+ return malloc(length);
+}
+
+__attribute__((nothrow)) int munmap(void* addr, size_t length) {
+ (void)length;
+ free(addr);
+ return 0;
+}
#endif
-#ifdef LIBYUV_USE_BASE_FLAGS
-#include "base/commandlineflags.h"
+
+#ifdef LIBYUV_USE_ABSL_FLAGS
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
#endif
#include "libyuv/cpu_id.h"
unsigned int fastrand_seed = 0xfb;
-#ifdef LIBYUV_USE_GFLAGS
-DEFINE_int32(libyuv_width, 0, "width of test image.");
-DEFINE_int32(libyuv_height, 0, "height of test image.");
-DEFINE_int32(libyuv_repeat, 0, "number of times to repeat test.");
-DEFINE_int32(libyuv_flags, 0, "cpu flags for reference code. 1 = C, -1 = SIMD");
-DEFINE_int32(libyuv_cpu_info,
- 0,
- "cpu flags for benchmark code. 1 = C, -1 = SIMD");
+#ifdef LIBYUV_USE_ABSL_FLAGS
+ABSL_FLAG(int32_t, libyuv_width, 0, "width of test image.");
+ABSL_FLAG(int32_t, libyuv_height, 0, "height of test image.");
+ABSL_FLAG(int32_t, libyuv_repeat, 0, "number of times to repeat test.");
+ABSL_FLAG(int32_t,
+ libyuv_flags,
+ 0,
+ "cpu flags for reference code. 1 = C, -1 = SIMD");
+ABSL_FLAG(int32_t,
+ libyuv_cpu_info,
+ 0,
+ "cpu flags for benchmark code. 1 = C, -1 = SIMD");
#else
-// Disable command line parameters if gflags disabled.
+// Disable command line parameters if absl/flags disabled.
static const int32_t FLAGS_libyuv_width = 0;
static const int32_t FLAGS_libyuv_height = 0;
static const int32_t FLAGS_libyuv_repeat = 0;
@@ -41,36 +70,77 @@ static const int32_t FLAGS_libyuv_flags = 0;
static const int32_t FLAGS_libyuv_cpu_info = 0;
#endif
+#ifdef LIBYUV_USE_ABSL_FLAGS
+#define LIBYUV_GET_FLAG(f) absl::GetFlag(f)
+#else
+#define LIBYUV_GET_FLAG(f) f
+#endif
+
// Test environment variable for disabling CPU features. Any non-zero value
// to disable. Zero ignored to make it easy to set the variable on/off.
#if !defined(__native_client__) && !defined(_M_ARM)
-static LIBYUV_BOOL TestEnv(const char* name) {
+static bool TestEnv(const char* name) {
const char* var = getenv(name);
if (var) {
if (var[0] != '0') {
- return LIBYUV_TRUE;
+ return true;
}
}
- return LIBYUV_FALSE;
+ return false;
}
#else // nacl does not support getenv().
-static LIBYUV_BOOL TestEnv(const char*) {
- return LIBYUV_FALSE;
+static bool TestEnv(const char*) {
+ return false;
}
#endif
-int TestCpuEnv(int cpu_info) {
+static int TestCpuEnv(int cpu_info) {
#if defined(__arm__) || defined(__aarch64__)
if (TestEnv("LIBYUV_DISABLE_NEON")) {
cpu_info &= ~libyuv::kCpuHasNEON;
}
#endif
-#if defined(__mips__) && defined(__linux__)
- if (TestEnv("LIBYUV_DISABLE_MSA")) {
- cpu_info &= ~libyuv::kCpuHasMSA;
+#if defined(__aarch64__)
+ if (TestEnv("LIBYUV_DISABLE_NEON_DOTPROD")) {
+ cpu_info &= ~libyuv::kCpuHasNeonDotProd;
+ }
+ if (TestEnv("LIBYUV_DISABLE_NEON_I8MM")) {
+ cpu_info &= ~libyuv::kCpuHasNeonI8MM;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SVE")) {
+ cpu_info &= ~libyuv::kCpuHasSVE;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SVE2")) {
+ cpu_info &= ~libyuv::kCpuHasSVE2;
}
- if (TestEnv("LIBYUV_DISABLE_MMI")) {
- cpu_info &= ~libyuv::kCpuHasMMI;
+ if (TestEnv("LIBYUV_DISABLE_SME")) {
+ cpu_info &= ~libyuv::kCpuHasSME;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SME2")) {
+ cpu_info &= ~libyuv::kCpuHasSME2;
+ }
+#endif
+#if defined(__longarch__) && defined(__linux__)
+ if (TestEnv("LIBYUV_DISABLE_LSX")) {
+ cpu_info &= ~libyuv::kCpuHasLSX;
+ }
+ if (TestEnv("LIBYUV_DISABLE_LASX")) {
+ cpu_info &= ~libyuv::kCpuHasLASX;
+ }
+#endif
+#if defined(__riscv) && defined(__linux__)
+ if (TestEnv("LIBYUV_DISABLE_RVV")) {
+ cpu_info &= ~libyuv::kCpuHasRVV;
+ }
+#endif
+#if defined(__wasm__)
+ if (TestEnv("LIBYUV_DISABLE_WASM")) {
+ cpu_info &= ~libyuv::kCpuHasWASMSIMD;
+ }
+#endif
+#if defined(__hexagon__)
+ if (TestEnv("LIBYUV_DISABLE_HVX")) {
+ cpu_info &= ~libyuv::kCpuHasHVX;
}
#endif
#if !defined(__pnacl__) && !defined(__CLR_VER) && \
@@ -112,6 +182,9 @@ int TestCpuEnv(int cpu_info) {
if (TestEnv("LIBYUV_DISABLE_AVX512VL")) {
cpu_info &= ~libyuv::kCpuHasAVX512VL;
}
+ if (TestEnv("LIBYUV_DISABLE_AVX512VNNI")) {
+ cpu_info &= ~libyuv::kCpuHasAVX512VNNI;
+ }
if (TestEnv("LIBYUV_DISABLE_AVX512VBMI")) {
cpu_info &= ~libyuv::kCpuHasAVX512VBMI;
}
@@ -121,11 +194,26 @@ int TestCpuEnv(int cpu_info) {
if (TestEnv("LIBYUV_DISABLE_AVX512VBITALG")) {
cpu_info &= ~libyuv::kCpuHasAVX512VBITALG;
}
- if (TestEnv("LIBYUV_DISABLE_AVX512VPOPCNTDQ")) {
- cpu_info &= ~libyuv::kCpuHasAVX512VPOPCNTDQ;
+ if (TestEnv("LIBYUV_DISABLE_AVX10")) {
+ cpu_info &= ~libyuv::kCpuHasAVX10;
+ }
+ if (TestEnv("LIBYUV_DISABLE_AVX10_2")) {
+ cpu_info &= ~libyuv::kCpuHasAVX10_2;
+ }
+ if (TestEnv("LIBYUV_DISABLE_AVXVNNI")) {
+ cpu_info &= ~libyuv::kCpuHasAVXVNNI;
+ }
+ if (TestEnv("LIBYUV_DISABLE_AVXVNNIINT8")) {
+ cpu_info &= ~libyuv::kCpuHasAVXVNNIINT8;
+ }
+ if (TestEnv("LIBYUV_DISABLE_AMXINT8")) {
+ cpu_info &= ~libyuv::kCpuHasAMXINT8;
+ }
+ if (TestEnv("LIBYUV_DISABLE_AVX512BMM")) {
+ cpu_info &= ~libyuv::kCpuHasAVX512BMM;
}
- if (TestEnv("LIBYUV_DISABLE_GFNI")) {
- cpu_info &= ~libyuv::kCpuHasGFNI;
+ if (TestEnv("LIBYUV_DISABLE_AMXTF32")) {
+ cpu_info &= ~libyuv::kCpuHasAMXTF32;
}
#endif
if (TestEnv("LIBYUV_DISABLE_ASM")) {
@@ -148,8 +236,8 @@ LibYUVConvertTest::LibYUVConvertTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -159,29 +247,29 @@ LibYUVConvertTest::LibYUVConvertTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -204,8 +292,8 @@ LibYUVColorTest::LibYUVColorTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -215,29 +303,29 @@ LibYUVColorTest::LibYUVColorTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -260,8 +348,8 @@ LibYUVScaleTest::LibYUVScaleTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -271,29 +359,29 @@ LibYUVScaleTest::LibYUVScaleTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -316,8 +404,8 @@ LibYUVRotateTest::LibYUVRotateTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -327,29 +415,29 @@ LibYUVRotateTest::LibYUVRotateTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -372,8 +460,8 @@ LibYUVPlanarTest::LibYUVPlanarTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -383,29 +471,29 @@ LibYUVPlanarTest::LibYUVPlanarTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -428,8 +516,8 @@ LibYUVBaseTest::LibYUVBaseTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -439,29 +527,29 @@ LibYUVBaseTest::LibYUVBaseTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -484,8 +572,8 @@ LibYUVCompareTest::LibYUVCompareTest()
if (repeat) {
benchmark_iterations_ = atoi(repeat); // NOLINT
}
- if (FLAGS_libyuv_repeat) {
- benchmark_iterations_ = FLAGS_libyuv_repeat;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_repeat)) {
+ benchmark_iterations_ = LIBYUV_GET_FLAG(FLAGS_libyuv_repeat);
}
if (benchmark_iterations_ > 1) {
benchmark_width_ = 1280;
@@ -495,29 +583,29 @@ LibYUVCompareTest::LibYUVCompareTest()
if (width) {
benchmark_width_ = atoi(width); // NOLINT
}
- if (FLAGS_libyuv_width) {
- benchmark_width_ = FLAGS_libyuv_width;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_width)) {
+ benchmark_width_ = LIBYUV_GET_FLAG(FLAGS_libyuv_width);
}
const char* height = getenv("LIBYUV_HEIGHT");
if (height) {
benchmark_height_ = atoi(height); // NOLINT
}
- if (FLAGS_libyuv_height) {
- benchmark_height_ = FLAGS_libyuv_height;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_height)) {
+ benchmark_height_ = LIBYUV_GET_FLAG(FLAGS_libyuv_height);
}
const char* cpu_flags = getenv("LIBYUV_FLAGS");
if (cpu_flags) {
disable_cpu_flags_ = atoi(cpu_flags); // NOLINT
}
- if (FLAGS_libyuv_flags) {
- disable_cpu_flags_ = FLAGS_libyuv_flags;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_flags)) {
+ disable_cpu_flags_ = LIBYUV_GET_FLAG(FLAGS_libyuv_flags);
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
- if (FLAGS_libyuv_cpu_info) {
- benchmark_cpu_info_ = FLAGS_libyuv_cpu_info;
+ if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
+ benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
}
disable_cpu_flags_ = TestCpuEnv(disable_cpu_flags_);
benchmark_cpu_info_ = TestCpuEnv(benchmark_cpu_info_);
@@ -532,11 +620,8 @@ LibYUVCompareTest::LibYUVCompareTest()
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
-#ifdef LIBYUV_USE_GFLAGS
- // AllowCommandLineParsing allows us to ignore flags passed on to us by
- // Chromium build bots without having to explicitly disable them.
- google::AllowCommandLineReparsing();
- google::ParseCommandLineFlags(&argc, &argv, true);
+#ifdef LIBYUV_USE_ABSL_FLAGS
+ absl::ParseCommandLine(argc, argv);
#endif
return RUN_ALL_TESTS();
}
diff --git a/libs/libyuv/unit_test/unit_test.h b/libs/libyuv/unit_test/unit_test.h
index 87907fa160..dd4822c268 100644
--- a/libs/libyuv/unit_test/unit_test.h
+++ b/libs/libyuv/unit_test/unit_test.h
@@ -11,16 +11,16 @@
#ifndef UNIT_TEST_UNIT_TEST_H_ // NOLINT
#define UNIT_TEST_UNIT_TEST_H_
-#ifdef WIN32
+#include <stddef.h> // For NULL
+#include <stdint.h>
+#ifdef _WIN32
#include <windows.h>
-#else
-#include <sys/resource.h>
+#elif !defined(__hexagon__)
#include <sys/time.h>
#endif
-#include <gtest/gtest.h>
-
-#include "libyuv/basic_types.h"
+// pragma to disable warning for ASSERT_NE
+#include <gtest/gtest.h> // IWYU pragma: export
#ifndef SIMD_ALIGNED
#if defined(_MSC_VER) && !defined(__CLR_VER)
@@ -69,15 +69,53 @@ static inline bool SizeValid(int src_width,
return true;
}
+#ifdef LIBYUV_MSAN
+#define align_buffer_page_end(var, size) \
+ uint8_t* var = NULL; \
+ uint8_t* var##_mem = \
+ reinterpret_cast<uint8_t*>(calloc(1, ((size) + 4095 + 63) & ~4095)); \
+ if (var##_mem) \
+ var = reinterpret_cast<uint8_t*>( \
+ (intptr_t)(var##_mem + (((size) + 4095 + 63) & ~4095) - (size)) & ~63)
+#else
#define align_buffer_page_end(var, size) \
+ uint8_t* var = NULL; \
uint8_t* var##_mem = \
reinterpret_cast<uint8_t*>(malloc(((size) + 4095 + 63) & ~4095)); \
- uint8_t* var = reinterpret_cast<uint8_t*>( \
+ if (var##_mem) \
+ var = reinterpret_cast<uint8_t*>( \
(intptr_t)(var##_mem + (((size) + 4095 + 63) & ~4095) - (size)) & ~63)
+#endif
#define free_aligned_buffer_page_end(var) \
free(var##_mem); \
- var = 0
+ var = NULL
+
+#ifdef LIBYUV_MSAN
+#define align_buffer_page_end_16(var, size) \
+ uint16_t* var = NULL; \
+ uint8_t* var##_mem = \
+ reinterpret_cast<uint8_t*>(calloc(1, ((size) * 2 + 4095 + 63) & ~4095)); \
+ if (var##_mem) \
+ var = reinterpret_cast<uint16_t*>( \
+ (intptr_t)(var##_mem + (((size) * 2 + 4095 + 63) & ~4095) - \
+ (size) * 2) & \
+ ~63)
+#else
+#define align_buffer_page_end_16(var, size) \
+ uint16_t* var = NULL; \
+ uint8_t* var##_mem = \
+ reinterpret_cast<uint8_t*>(malloc(((size) * 2 + 4095 + 63) & ~4095)); \
+ if (var##_mem) \
+ var = reinterpret_cast<uint16_t*>( \
+ (intptr_t)(var##_mem + (((size) * 2 + 4095 + 63) & ~4095) - \
+ (size) * 2) & \
+ ~63)
+#endif
+
+#define free_aligned_buffer_page_end_16(var) \
+ free(var##_mem); \
+ var = NULL
#ifdef WIN32
static inline double get_time() {
@@ -86,6 +124,10 @@ static inline double get_time() {
QueryPerformanceFrequency(&f);
return static_cast<double>(t.QuadPart) / static_cast<double>(f.QuadPart);
}
+#elif defined(__hexagon__)
+static inline double get_time() {
+ return 0.;
+}
#else
static inline double get_time() {
struct timeval t;
@@ -111,10 +153,13 @@ inline int fastrand() {
return static_cast<int>((fastrand_seed >> 16) & 0xffff);
}
+// ubsan fails if dst is unaligned unless we use uint8
static inline void MemRandomize(uint8_t* dst, int64_t len) {
int64_t i;
for (i = 0; i < len - 1; i += 2) {
- *reinterpret_cast<uint16_t*>(dst) = fastrand();
+ int r = fastrand();
+ dst[0] = static_cast<uint8_t>(r);
+ dst[1] = static_cast<uint8_t>(r >> 8);
dst += 2;
}
for (; i < len; ++i) {
diff --git a/libs/libyuv/unit_test/video_common_test.cc b/libs/libyuv/unit_test/video_common_test.cc
index a84206a2ad..ca91246b64 100644
--- a/libs/libyuv/unit_test/video_common_test.cc
+++ b/libs/libyuv/unit_test/video_common_test.cc
@@ -29,77 +29,87 @@ static bool TestValidFourCC(uint32_t fourcc, int bpp) {
!TestValidChar((fourcc >> 24) & 0xff)) {
return false;
}
- if (bpp < 0 || bpp > 32) {
+ if (bpp < 0 || bpp > 64) {
return false;
}
return true;
}
TEST_F(LibYUVBaseTest, TestCanonicalFourCC) {
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_I420), CanonicalFourCC(FOURCC_IYUV));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_I420), CanonicalFourCC(FOURCC_YU12));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_I422), CanonicalFourCC(FOURCC_YU16));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_I444), CanonicalFourCC(FOURCC_YU24));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_YUY2), CanonicalFourCC(FOURCC_YUYV));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_YUY2), CanonicalFourCC(FOURCC_YUVS));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_UYVY), CanonicalFourCC(FOURCC_HDYC));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_UYVY), CanonicalFourCC(FOURCC_2VUY));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_MJPG), CanonicalFourCC(FOURCC_JPEG));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_MJPG), CanonicalFourCC(FOURCC_DMB1));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_RAW), CanonicalFourCC(FOURCC_RGB3));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_24BG), CanonicalFourCC(FOURCC_BGR3));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_BGRA), CanonicalFourCC(FOURCC_CM32));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_RAW), CanonicalFourCC(FOURCC_CM24));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_RGBO), CanonicalFourCC(FOURCC_L555));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_RGBP), CanonicalFourCC(FOURCC_L565));
- EXPECT_EQ(static_cast<uint32_t>(FOURCC_RGBO), CanonicalFourCC(FOURCC_5551));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_I420), CanonicalFourCC(FOURCC_IYUV));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_I420), CanonicalFourCC(FOURCC_YU12));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_I422), CanonicalFourCC(FOURCC_YU16));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_I444), CanonicalFourCC(FOURCC_YU24));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_YUY2), CanonicalFourCC(FOURCC_YUYV));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_YUY2), CanonicalFourCC(FOURCC_YUVS));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_UYVY), CanonicalFourCC(FOURCC_HDYC));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_UYVY), CanonicalFourCC(FOURCC_2VUY));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_MJPG), CanonicalFourCC(FOURCC_JPEG));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_MJPG), CanonicalFourCC(FOURCC_DMB1));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_RAW), CanonicalFourCC(FOURCC_RGB3));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_24BG), CanonicalFourCC(FOURCC_BGR3));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_BGRA), CanonicalFourCC(FOURCC_CM32));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_RAW), CanonicalFourCC(FOURCC_CM24));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_RGBO), CanonicalFourCC(FOURCC_L555));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_RGBP), CanonicalFourCC(FOURCC_L565));
+ ASSERT_EQ(static_cast<uint32_t>(FOURCC_RGBO), CanonicalFourCC(FOURCC_5551));
}
TEST_F(LibYUVBaseTest, TestFourCC) {
- EXPECT_TRUE(TestValidFourCC(FOURCC_I420, FOURCC_BPP_I420));
- EXPECT_TRUE(TestValidFourCC(FOURCC_I420, FOURCC_BPP_I420));
- EXPECT_TRUE(TestValidFourCC(FOURCC_I422, FOURCC_BPP_I422));
- EXPECT_TRUE(TestValidFourCC(FOURCC_I444, FOURCC_BPP_I444));
- EXPECT_TRUE(TestValidFourCC(FOURCC_I400, FOURCC_BPP_I400));
- EXPECT_TRUE(TestValidFourCC(FOURCC_NV21, FOURCC_BPP_NV21));
- EXPECT_TRUE(TestValidFourCC(FOURCC_NV12, FOURCC_BPP_NV12));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YUY2, FOURCC_BPP_YUY2));
- EXPECT_TRUE(TestValidFourCC(FOURCC_UYVY, FOURCC_BPP_UYVY));
- EXPECT_TRUE(TestValidFourCC(FOURCC_M420, FOURCC_BPP_M420));
- EXPECT_TRUE(TestValidFourCC(FOURCC_Q420, FOURCC_BPP_Q420)); // deprecated.
- EXPECT_TRUE(TestValidFourCC(FOURCC_ARGB, FOURCC_BPP_ARGB));
- EXPECT_TRUE(TestValidFourCC(FOURCC_BGRA, FOURCC_BPP_BGRA));
- EXPECT_TRUE(TestValidFourCC(FOURCC_ABGR, FOURCC_BPP_ABGR));
- EXPECT_TRUE(TestValidFourCC(FOURCC_AR30, FOURCC_BPP_AR30));
- EXPECT_TRUE(TestValidFourCC(FOURCC_AB30, FOURCC_BPP_AB30));
- EXPECT_TRUE(TestValidFourCC(FOURCC_24BG, FOURCC_BPP_24BG));
- EXPECT_TRUE(TestValidFourCC(FOURCC_RAW, FOURCC_BPP_RAW));
- EXPECT_TRUE(TestValidFourCC(FOURCC_RGBA, FOURCC_BPP_RGBA));
- EXPECT_TRUE(TestValidFourCC(FOURCC_RGBP, FOURCC_BPP_RGBP));
- EXPECT_TRUE(TestValidFourCC(FOURCC_RGBO, FOURCC_BPP_RGBO));
- EXPECT_TRUE(TestValidFourCC(FOURCC_R444, FOURCC_BPP_R444));
- EXPECT_TRUE(TestValidFourCC(FOURCC_H420, FOURCC_BPP_H420));
- EXPECT_TRUE(TestValidFourCC(FOURCC_H422, FOURCC_BPP_H422));
- EXPECT_TRUE(TestValidFourCC(FOURCC_H010, FOURCC_BPP_H010));
- EXPECT_TRUE(TestValidFourCC(FOURCC_MJPG, FOURCC_BPP_MJPG));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YV12, FOURCC_BPP_YV12));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YV16, FOURCC_BPP_YV16));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YV24, FOURCC_BPP_YV24));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YU12, FOURCC_BPP_YU12));
- EXPECT_TRUE(TestValidFourCC(FOURCC_IYUV, FOURCC_BPP_IYUV));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YU16, FOURCC_BPP_YU16));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YU24, FOURCC_BPP_YU24));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YUYV, FOURCC_BPP_YUYV));
- EXPECT_TRUE(TestValidFourCC(FOURCC_YUVS, FOURCC_BPP_YUVS));
- EXPECT_TRUE(TestValidFourCC(FOURCC_HDYC, FOURCC_BPP_HDYC));
- EXPECT_TRUE(TestValidFourCC(FOURCC_2VUY, FOURCC_BPP_2VUY));
- EXPECT_TRUE(TestValidFourCC(FOURCC_JPEG, FOURCC_BPP_JPEG));
- EXPECT_TRUE(TestValidFourCC(FOURCC_DMB1, FOURCC_BPP_DMB1));
- EXPECT_TRUE(TestValidFourCC(FOURCC_BA81, FOURCC_BPP_BA81));
- EXPECT_TRUE(TestValidFourCC(FOURCC_RGB3, FOURCC_BPP_RGB3));
- EXPECT_TRUE(TestValidFourCC(FOURCC_BGR3, FOURCC_BPP_BGR3));
- EXPECT_TRUE(TestValidFourCC(FOURCC_H264, FOURCC_BPP_H264));
- EXPECT_TRUE(TestValidFourCC(FOURCC_ANY, FOURCC_BPP_ANY));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I420, FOURCC_BPP_I420));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I420, FOURCC_BPP_I420));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I422, FOURCC_BPP_I422));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I444, FOURCC_BPP_I444));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I400, FOURCC_BPP_I400));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_NV21, FOURCC_BPP_NV21));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_NV12, FOURCC_BPP_NV12));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_NV16, FOURCC_BPP_NV16));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_NV24, FOURCC_BPP_NV24));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YUY2, FOURCC_BPP_YUY2));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_UYVY, FOURCC_BPP_UYVY));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_M420, FOURCC_BPP_M420)); // deprecated.
+ ASSERT_TRUE(TestValidFourCC(FOURCC_Q420, FOURCC_BPP_Q420)); // deprecated.
+ ASSERT_TRUE(TestValidFourCC(FOURCC_ARGB, FOURCC_BPP_ARGB));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_BGRA, FOURCC_BPP_BGRA));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_ABGR, FOURCC_BPP_ABGR));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_AR30, FOURCC_BPP_AR30));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_AB30, FOURCC_BPP_AB30));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_AR64, FOURCC_BPP_AR64));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_AB64, FOURCC_BPP_AB64));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_24BG, FOURCC_BPP_24BG));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_RAW, FOURCC_BPP_RAW));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_RGBA, FOURCC_BPP_RGBA));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_RGBP, FOURCC_BPP_RGBP));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_RGBO, FOURCC_BPP_RGBO));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_R444, FOURCC_BPP_R444));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_H420, FOURCC_BPP_H420));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_H422, FOURCC_BPP_H422));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_H010, FOURCC_BPP_H010));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_H210, FOURCC_BPP_H210));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I010, FOURCC_BPP_I010));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_I210, FOURCC_BPP_I210));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_P010, FOURCC_BPP_P010));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_P210, FOURCC_BPP_P210));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_P410, FOURCC_BPP_P410));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_MJPG, FOURCC_BPP_MJPG));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YV12, FOURCC_BPP_YV12));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YV16, FOURCC_BPP_YV16));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YV24, FOURCC_BPP_YV24));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YU12, FOURCC_BPP_YU12));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_IYUV, FOURCC_BPP_IYUV));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YU16, FOURCC_BPP_YU16));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YU24, FOURCC_BPP_YU24));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YUYV, FOURCC_BPP_YUYV));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_YUVS, FOURCC_BPP_YUVS));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_HDYC, FOURCC_BPP_HDYC));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_2VUY, FOURCC_BPP_2VUY));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_JPEG, FOURCC_BPP_JPEG));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_DMB1, FOURCC_BPP_DMB1));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_BA81, FOURCC_BPP_BA81));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_RGB3, FOURCC_BPP_RGB3));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_BGR3, FOURCC_BPP_BGR3));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_H264, FOURCC_BPP_H264));
+ ASSERT_TRUE(TestValidFourCC(FOURCC_ANY, FOURCC_BPP_ANY));
}
} // namespace libyuv
diff --git a/libs/libyuv/util/color.cc b/libs/libyuv/util/color.cc
new file mode 100644
index 0000000000..8c3bbefd27
--- /dev/null
+++ b/libs/libyuv/util/color.cc
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2021 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// This utility computes values needed to generate yuvconstants based on
+// white point values.
+// The yuv formulas are tuned for 8 bit YUV channels.
+
+// For those MCs that can be represented as kr and kb:
+// Full range
+// float M[3][3]
+// {{1,0,2*(1-kr)},{1,-((2*kb)/((2-kb)*(1-kb-kr))),-((2*kr)/((2-kr)*(1-kb-kr)))},{1,2*(1-kb),0}};
+// float B[3]
+// {1+(256*(1-kr))/255,1-(256*kb)/(255*(2-kb)*(1-kb-kr))-(256*kr)/(255*(2-kr)*(1-kb-kr)),1+(256*(1-kb))/255};
+// Limited range
+// float M[3][3]
+// {{85/73,0,255/112-(255*kr)/112},{85/73,-((255*kb)/(112*(2-kb)*(1-kb-kr))),-((255*kr)/(112*(2-kr)*(1-kb-kr)))},{85/73,255/112-(255*kb)/112,0}};
+// float B[3]
+// {77662/43435-(1537*kr)/1785,203/219-(1537*kb)/(1785*(2-kb)*(1-kb-kr))-(1537*kr)/(1785*(2-kr)*(1-kb-kr)),77662/43435-(1537*kb)/1785};
+
+// mc bt
+// 1 bt.709 KR = 0.2126; KB = 0.0722
+// 4 fcc KR = 0.30; KB = 0.11
+// 6 bt.601 KR = 0.299; KB = 0.114
+// 7 SMPTE 240M KR = 0.212; KB = 0.087
+// 10 bt2020 KR = 0.2627; KB = 0.0593
+
+// BT.709 full range YUV to RGB reference
+// R = Y + V * 1.5748
+// G = Y - U * 0.18732 - V * 0.46812
+// B = Y + U * 1.8556
+// KR = 0.2126
+// KB = 0.0722
+
+// https://mymusing.co/bt601-yuv-to-rgb-conversion-color/
+
+// // Y contribution to R,G,B. Scale and bias.
+// #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+// #define YB 32 /* 64 / 2 */
+//
+// // U and V contributions to R,G,B.
+// #define UB 113 /* round(1.77200 * 64) */
+// #define UG 22 /* round(0.34414 * 64) */
+// #define VG 46 /* round(0.71414 * 64) */
+// #define VR 90 /* round(1.40200 * 64) */
+//
+// // Bias values to round, and subtract 128 from U and V.
+// #define BB (-UB * 128 + YB)
+// #define BG (UG * 128 + VG * 128 + YB)
+// #define BR (-VR * 128 + YB)
+
+int round(float v) {
+ return (int)(v + 0.5);
+}
+
+int main(int argc, const char* argv[]) {
+ if (argc < 2) {
+ printf("color kr kb\n");
+ return -1;
+ }
+ float kr = atof(argv[1]);
+ float kb = atof(argv[2]);
+ float kg = 1 - kr - kb;
+
+ float vr = 2 * (1 - kr);
+ float ug = 2 * ((1 - kb) * kb / kg);
+ float vg = 2 * ((1 - kr) * kr / kg);
+ float ub = 2 * (1 - kb);
+
+ printf("Full range\n");
+ printf("R = Y + V * %5f\n", vr);
+ printf("G = Y - U * %6f - V * %6f\n", ug, vg);
+ printf("B = Y + U * %5f\n", ub);
+
+ printf("KR = %4f; ", kr);
+ printf("KB = %4f\n", kb);
+ // printf("KG = %4f\n", kg);
+ // #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+ // #define YB 32 /* 64 / 2 */
+ //
+ // // U and V contributions to R,G,B.
+
+ printf("UB %-3d /* round(%f * 64) */\n", round(ub * 64), ub);
+ printf("UG %-3d /* round(%f * 64) */\n", round(ug * 64), ug);
+ printf("VG %-3d /* round(%f * 64) */\n", round(vg * 64), vg);
+ printf("VR %-3d /* round(%f * 64) */\n", round(vr * 64), vr);
+
+ vr = 255.f / 224.f * 2 * (1 - kr);
+ ug = 255.f / 224.f * 2 * ((1 - kb) * kb / kg);
+ vg = 255.f / 224.f * 2 * ((1 - kr) * kr / kg);
+ ub = 255.f / 224.f * 2 * (1 - kb);
+
+ printf("Limited range\n");
+ printf("R = (Y - 16) * 1.164 + V * %5f\n", vr);
+ printf("G = (Y - 16) * 1.164 - U * %6f - V * %6f\n", ug, vg);
+ printf("B = (Y - 16) * 1.164 + U * %5f\n", ub);
+
+ // printf("KG = %4f\n", kg);
+ // #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+ // #define YB 32 /* 64 / 2 */
+ //
+ // // U and V contributions to R,G,B.
+
+ printf("UB %-3d /* round(%f * 64) */\n", round(ub * 64), ub);
+ printf("UG %-3d /* round(%f * 64) */\n", round(ug * 64), ug);
+ printf("VG %-3d /* round(%f * 64) */\n", round(vg * 64), vg);
+ printf("VR %-3d /* round(%f * 64) */\n", round(vr * 64), vr);
+
+ return 0;
+}
diff --git a/libs/libyuv/util/compare.cc b/libs/libyuv/util/compare.cc
index a16613ee2f..4ed933a1db 100644
--- a/libs/libyuv/util/compare.cc
+++ b/libs/libyuv/util/compare.cc
@@ -13,55 +13,129 @@
#include <string.h>
#include <time.h>
-#include "libyuv/basic_types.h"
#include "libyuv/compare.h"
#include "libyuv/version.h"
+static bool ExtractResolutionFromFilename(const char* name,
+ int* width_ptr,
+ int* height_ptr) {
+ for (int i = 0; name[i]; ++i) {
+ if ((name[i] == '.' || name[i] == '_') && name[i + 1] >= '0' &&
+ name[i + 1] <= '9') {
+ int n = sscanf(name + i + 1, "%dx%d", width_ptr, height_ptr); // NOLINT
+ if (2 == n) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
int main(int argc, char** argv) {
- if (argc < 1) {
+ const char* name1 = NULL;
+ const char* name2 = NULL;
+ int width = 0;
+ int height = 0;
+
+ for (int i = 1; i < argc; ++i) {
+ if (!strcmp(argv[i], "-s") && i + 2 < argc) {
+ width = atoi(argv[++i]); // NOLINT
+ height = atoi(argv[++i]); // NOLINT
+ } else if (!name1) {
+ name1 = argv[i];
+ } else if (!name2) {
+ name2 = argv[i];
+ } else if (width == 0 && i + 1 < argc) {
+ width = atoi(argv[i]); // NOLINT
+ height = atoi(argv[++i]); // NOLINT
+ }
+ }
+
+ if (!name1) {
printf("libyuv compare v%d\n", LIBYUV_VERSION);
- printf("compare file1.yuv file2.yuv\n");
+ printf("compare file1.yuv [file2.yuv [width height]]\n");
return -1;
}
- char* name1 = argv[1];
- char* name2 = (argc > 2) ? argv[2] : NULL;
+
+ if (width <= 0 || height <= 0) {
+ if (!ExtractResolutionFromFilename(name1, &width, &height) && name2) {
+ ExtractResolutionFromFilename(name2, &width, &height);
+ }
+ }
+
FILE* fin1 = fopen(name1, "rb");
+ if (!fin1) {
+ fprintf(stderr, "Cannot open %s\n", name1);
+ return -1;
+ }
FILE* fin2 = name2 ? fopen(name2, "rb") : NULL;
+ if (name2 && !fin2) {
+ fprintf(stderr, "Cannot open %s\n", name2);
+ fclose(fin1);
+ return -1;
+ }
- const int kBlockSize = 32768;
- uint8_t buf1[kBlockSize];
- uint8_t buf2[kBlockSize];
+ const bool has_dims = (width >= 16 && height >= 16);
+ const int block_width = has_dims ? width : 256;
+ const int y_size = has_dims ? width * height : 32768;
+ const int uv_width = (width + 1) / 2;
+ const int uv_height = (height + 1) / 2;
+ const int uv_size = has_dims ? uv_width * uv_height : 0;
+ const int block_size = has_dims ? (y_size + 2 * uv_size) : 32768;
+
+ uint8_t* buf1 = static_cast<uint8_t*>(malloc(block_size));
+ uint8_t* buf2 = static_cast<uint8_t*>(malloc(block_size));
uint32_t hash1 = 5381;
uint32_t hash2 = 5381;
uint64_t sum_square_err = 0;
uint64_t size_min = 0;
+ double ssim_sum = 0.0;
+ int ssim_count = 0;
int amt1 = 0;
int amt2 = 0;
do {
- amt1 = static_cast<int>(fread(buf1, 1, kBlockSize, fin1));
+ amt1 = static_cast<int>(fread(buf1, 1, block_size, fin1));
if (amt1 > 0) {
hash1 = libyuv::HashDjb2(buf1, amt1, hash1);
}
if (fin2) {
- amt2 = static_cast<int>(fread(buf2, 1, kBlockSize, fin2));
+ amt2 = static_cast<int>(fread(buf2, 1, block_size, fin2));
if (amt2 > 0) {
hash2 = libyuv::HashDjb2(buf2, amt2, hash2);
}
int amt_min = (amt1 < amt2) ? amt1 : amt2;
size_min += amt_min;
sum_square_err += libyuv::ComputeSumSquareError(buf1, buf2, amt_min);
+ if (has_dims && amt_min == block_size) {
+ ssim_sum += libyuv::I420Ssim(
+ buf1, width, buf1 + y_size, uv_width, buf1 + y_size + uv_size,
+ uv_width, buf2, width, buf2 + y_size, uv_width,
+ buf2 + y_size + uv_size, uv_width, width, height);
+ ++ssim_count;
+ } else if (amt_min >= block_width * 8) {
+ ssim_sum += libyuv::CalcFrameSsim(buf1, block_width, buf2, block_width,
+ block_width, amt_min / block_width);
+ ++ssim_count;
+ }
}
} while (amt1 > 0 || amt2 > 0);
printf("hash1 %x", hash1);
if (fin2) {
printf(", hash2 %x", hash2);
- double mse =
- static_cast<double>(sum_square_err) / static_cast<double>(size_min);
+ double mse = (size_min > 0) ? static_cast<double>(sum_square_err) /
+ static_cast<double>(size_min)
+ : 0.0;
printf(", mse %.2f", mse);
double psnr = libyuv::SumSquareErrorToPsnr(sum_square_err, size_min);
- printf(", psnr %.2f\n", psnr);
+ printf(", psnr %.2f", psnr);
+ double ssim = (ssim_count > 0) ? (ssim_sum / ssim_count) : 0.0;
+ printf(", ssim %.6f", ssim);
fclose(fin2);
}
+ printf("\n");
fclose(fin1);
+ free(buf1);
+ free(buf2);
+ return 0;
}
diff --git a/libs/libyuv/util/cpuid.c b/libs/libyuv/util/cpuid.c
index 84c0602287..d855952970 100644
--- a/libs/libyuv/util/cpuid.c
+++ b/libs/libyuv/util/cpuid.c
@@ -12,21 +12,157 @@
#include <stdlib.h>
#include <string.h>
-#define INCLUDE_LIBYUV_COMPARE_H_
-#include "libyuv.h"
-#include "./psnr.h"
-#include "./ssim.h"
+#ifdef __linux__
+#include <ctype.h>
+#include <setjmp.h>
+#include <signal.h>
+#include <sys/utsname.h>
+#endif
+
+#include "libyuv/cpu_id.h"
+
+#ifdef __cplusplus
+using namespace libyuv;
+#endif
+
+#ifdef __linux__
+static void KernelVersion(int* version) {
+ struct utsname buffer;
+ int i = 0;
+
+ version[0] = version[1] = 0;
+ if (uname(&buffer) == 0) {
+ char* v = buffer.release;
+ for (i = 0; *v && i < 2; ++v) {
+ if (isdigit(*v)) {
+ version[i++] = (int)strtol(v, &v, 10);
+ }
+ }
+ }
+}
+#endif
+
+#if defined(__linux__) && (defined(__i386__) || defined(__x86_64__))
+static sigjmp_buf vdpphps_jmpbuf;
+static void vdpphps_sigill_handler(int sig) {
+ (void)sig;
+ siglongjmp(vdpphps_jmpbuf, 1);
+}
+#endif
int main(int argc, const char* argv[]) {
- int cpu_flags = TestCpuFlag(-1);
- int has_arm = TestCpuFlag(kCpuHasARM);
- int has_mips = TestCpuFlag(kCpuHasMIPS);
- int has_x86 = TestCpuFlag(kCpuHasX86);
(void)argc;
(void)argv;
-#if defined(__i386__) || defined(__x86_64__) || \
- defined(_M_IX86) || defined(_M_X64)
+#if defined(__linux__)
+ {
+ int kernelversion[2];
+ KernelVersion(kernelversion);
+ printf("Kernel Version %d.%d\n", kernelversion[0], kernelversion[1]);
+ }
+#endif // defined(__linux__)
+
+#if defined(__arm__) || defined(__aarch64__)
+ int has_arm = TestCpuFlag(kCpuHasARM);
+ if (has_arm) {
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ int has_neon_dotprod = TestCpuFlag(kCpuHasNeonDotProd);
+ int has_neon_i8mm = TestCpuFlag(kCpuHasNeonI8MM);
+ int has_sve = TestCpuFlag(kCpuHasSVE);
+ int has_sve2 = TestCpuFlag(kCpuHasSVE2);
+ int has_sve_f32mm = TestCpuFlag(kCpuHasSVEF32MM);
+ int has_sme = TestCpuFlag(kCpuHasSME);
+ int has_sme2 = TestCpuFlag(kCpuHasSME2);
+ printf("Has Arm 0x%x\n", has_arm);
+ printf("Has Neon 0x%x\n", has_neon);
+ printf("Has Neon DotProd 0x%x\n", has_neon_dotprod);
+ printf("Has Neon I8MM 0x%x\n", has_neon_i8mm);
+ printf("Has SVE 0x%x\n", has_sve);
+ printf("Has SVE2 0x%x\n", has_sve2);
+ printf("Has SVE F32MM 0x%x\n", has_sve_f32mm);
+ printf("Has SME 0x%x\n", has_sme);
+ printf("Has SME2 0x%x\n", has_sme2);
+
+#if __aarch64__
+ // Read and print the SVE and SME vector lengths.
+ if (has_sve) {
+ int sve_vl;
+ __asm__(
+ ".inst 0x04bf5020 \n" // rdvl x0, #1
+ "mov %w[sve_vl], w0 \n"
+ : [sve_vl] "=r"(sve_vl) // %[sve_vl]
+ :
+ : "x0");
+ printf("SVE vector length: %d bytes\n", sve_vl);
+ }
+ if (has_sme) {
+ int sme_vl;
+ __asm__(
+ ".inst 0x04bf5820 \n" // rdsvl x0, #1
+ "mov %w[sme_vl], w0 \n"
+ : [sme_vl] "=r"(sme_vl) // %[sme_vl]
+ :
+ : "x0");
+ printf("SME vector length: %d bytes\n", sme_vl);
+ }
+#endif // defined(__aarch64__)
+ }
+#endif // if defined(__arm__) || defined(__aarch64__)
+
+#if defined(__riscv)
+ int has_riscv = TestCpuFlag(kCpuHasRISCV);
+ if (has_riscv) {
+ int has_rvv = TestCpuFlag(kCpuHasRVV);
+ printf("Has RISCV 0x%x\n", has_riscv);
+ printf("Has RVV 0x%x\n", has_rvv);
+
+ // Read and print the RVV vector length.
+ if (has_rvv) {
+ register uint32_t vlenb __asm__("t0");
+ __asm__(".word 0xC22022F3" /* CSRR t0, vlenb */ : "=r"(vlenb));
+ printf("RVV vector length: %d bytes\n", vlenb);
+ }
+ }
+#endif // defined(__riscv)
+
+#if defined(__loongarch__)
+ int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
+ if (has_loongarch) {
+ int has_lsx = TestCpuFlag(kCpuHasLSX);
+ int has_lasx = TestCpuFlag(kCpuHasLASX);
+ printf("Has LOONGARCH 0x%x\n", has_loongarch);
+ printf("Has LSX 0x%x\n", has_lsx);
+ printf("Has LASX 0x%x\n", has_lasx);
+ }
+#endif // defined(__loongarch__)
+
+#if defined(__wasm__)
+ int has_wasm = TestCpuFlag(kCpuHasWASM);
+ if (has_wasm) {
+ int has_wasm_simd = TestCpuFlag(kCpuHasWASMSIMD);
+ printf("Has WASM 0x%x\n", has_wasm);
+ printf("Has WASM SIMD 0x%x\n", has_wasm_simd);
+ }
+#endif // defined(__wasm__)
+
+#if defined(__hexagon__)
+ int has_hexagon = TestCpuFlag(kCpuHasHexagon);
+ if (has_hexagon) {
+ int has_hvx = TestCpuFlag(kCpuHasHVX);
+ printf("Has Hexagon 0x%x\n", has_hexagon);
+ printf("Has HVX 0x%x\n", has_hvx);
+#if defined(__HVX_ARCH__)
+ printf("HVX arch: v%d\n", __HVX_ARCH__);
+#endif
+#if defined(__HVX_LENGTH__)
+ printf("HVX vector length: %d bytes\n", __HVX_LENGTH__);
+#endif
+ }
+#endif // defined(__hexagon__)
+
+#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
+ defined(_M_X64)
+ int has_x86 = TestCpuFlag(kCpuHasX86);
if (has_x86) {
int family, model, cpu_info[4];
// Vendor ID:
@@ -56,25 +192,9 @@ int main(int argc, const char* argv[]) {
CpuId(1, 0, &cpu_info[0]);
family = ((cpu_info[0] >> 8) & 0x0f) | ((cpu_info[0] >> 16) & 0xff0);
model = ((cpu_info[0] >> 4) & 0x0f) | ((cpu_info[0] >> 12) & 0xf0);
- printf("Cpu Family %d (0x%x), Model %d (0x%x)\n", family, family,
- model, model);
- }
-#endif
- printf("Cpu Flags %x\n", cpu_flags);
- printf("Has ARM %x\n", has_arm);
- printf("Has MIPS %x\n", has_mips);
- printf("Has X86 %x\n", has_x86);
- if (has_arm) {
- int has_neon = TestCpuFlag(kCpuHasNEON);
- printf("Has NEON %x\n", has_neon);
- }
- if (has_mips) {
- int has_msa = TestCpuFlag(kCpuHasMSA);
- printf("Has MSA %x\n", has_msa);
- int has_mmi = TestCpuFlag(kCpuHasMMI);
- printf("Has MMI %x\n", has_mmi);
- }
- if (has_x86) {
+ printf("Cpu Family %d (0x%x), Model %d (0x%x)\n", family, family, model,
+ model);
+
int has_sse2 = TestCpuFlag(kCpuHasSSE2);
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
int has_sse41 = TestCpuFlag(kCpuHasSSE41);
@@ -82,32 +202,74 @@ int main(int argc, const char* argv[]) {
int has_avx = TestCpuFlag(kCpuHasAVX);
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
int has_erms = TestCpuFlag(kCpuHasERMS);
+ int has_fsmr = TestCpuFlag(kCpuHasFSMR);
int has_fma3 = TestCpuFlag(kCpuHasFMA3);
- int has_f16c = TestCpuFlag(kCpuHasF16C);
- int has_gfni = TestCpuFlag(kCpuHasGFNI);
+ int has_f16c = TestCpuFlag(kCpuHasF16C);
int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
int has_avx512vl = TestCpuFlag(kCpuHasAVX512VL);
+ int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI);
int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI);
int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
- int has_avx512vpopcntdq = TestCpuFlag(kCpuHasAVX512VPOPCNTDQ);
- printf("Has SSE2 %x\n", has_sse2);
- printf("Has SSSE3 %x\n", has_ssse3);
- printf("Has SSE4.1 %x\n", has_sse41);
- printf("Has SSE4.2 %x\n", has_sse42);
- printf("Has AVX %x\n", has_avx);
- printf("Has AVX2 %x\n", has_avx2);
- printf("Has ERMS %x\n", has_erms);
- printf("Has FMA3 %x\n", has_fma3);
- printf("Has F16C %x\n", has_f16c);
- printf("Has GFNI %x\n", has_gfni);
- printf("Has AVX512BW %x\n", has_avx512bw);
- printf("Has AVX512VL %x\n", has_avx512vl);
- printf("Has AVX512VBMI %x\n", has_avx512vbmi);
- printf("Has AVX512VBMI2 %x\n", has_avx512vbmi2);
- printf("Has AVX512VBITALG %x\n", has_avx512vbitalg);
- printf("Has AVX512VPOPCNTDQ %x\n", has_avx512vpopcntdq);
+ int has_avx10 = TestCpuFlag(kCpuHasAVX10);
+ int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2);
+ int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI);
+ int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
+ int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
+ int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
+ int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
+ printf("Has X86 0x%x\n", has_x86);
+ printf("Has SSE2 0x%x\n", has_sse2);
+ printf("Has SSSE3 0x%x\n", has_ssse3);
+ printf("Has SSE4.1 0x%x\n", has_sse41);
+ printf("Has SSE4.2 0x%x\n", has_sse42);
+ printf("Has AVX 0x%x\n", has_avx);
+ printf("Has AVX2 0x%x\n", has_avx2);
+ printf("Has ERMS 0x%x\n", has_erms);
+ printf("Has FSMR 0x%x\n", has_fsmr);
+ printf("Has FMA3 0x%x\n", has_fma3);
+ printf("Has F16C 0x%x\n", has_f16c);
+ printf("Has AVX512BW 0x%x\n", has_avx512bw);
+ printf("Has AVX512VL 0x%x\n", has_avx512vl);
+ printf("Has AVX512VNNI 0x%x\n", has_avx512vnni);
+ printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi);
+ printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2);
+ printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg);
+ printf("Has AVX10 0x%x\n", has_avx10);
+ printf("Has AVX10_2 0x%x\n", has_avx10_2);
+ printf("HAS AVXVNNI 0x%x\n", has_avxvnni);
+ printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
+ printf("Has AMXINT8 0x%x\n", has_amxint8);
+ printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
+ printf("Has AMXTF32 0x%x\n", has_amxtf32);
+
+#ifdef __linux__
+ // Test VDPPHPS instruction
+ {
+ struct sigaction act, oldact;
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = vdpphps_sigill_handler;
+ sigaction(SIGILL, &act, &oldact);
+
+ printf("Testing VDPPHPS instruction... ");
+ fflush(stdout);
+
+ if (sigsetjmp(vdpphps_jmpbuf, 1) == 0) {
+ // VDPPHPS xmm0, xmm0, xmm0
+ __asm__ volatile(".byte 0x62, 0xf2, 0x7c, 0x08, 0x52, 0xc0"
+ :
+ :
+ : "xmm0");
+ printf("Works!\n");
+ } else {
+ printf("Crashed (SIGILL)!\n");
+ }
+
+ sigaction(SIGILL, &oldact, NULL);
+ }
+#endif
}
+#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) ||
+ // defined(_M_X64)
return 0;
}
-
diff --git a/libs/libyuv/util/i444tonv12_eg.cc b/libs/libyuv/util/i444tonv12_eg.cc
new file mode 100644
index 0000000000..0fcb4095a8
--- /dev/null
+++ b/libs/libyuv/util/i444tonv12_eg.cc
@@ -0,0 +1,28 @@
+
+#include "libyuv/convert.h"
+
+#include <stdio.h> // for printf
+#include <string.h> // for memset
+
+int main(int, char**) {
+ unsigned char src_i444[640 * 400 * 3];
+ unsigned char dst_nv12[640 * 400 * 3 / 2];
+
+ for (size_t i = 0; i < sizeof(src_i444); ++i) {
+ src_i444[i] = i & 255;
+ }
+ memset(dst_nv12, 0, sizeof(dst_nv12));
+ libyuv::I444ToNV12(&src_i444[0], 640, // source Y
+ &src_i444[640 * 400], 640, // source U
+ &src_i444[640 * 400 * 2], 640, // source V
+ &dst_nv12[0], 640, // dest Y
+ &dst_nv12[640 * 400], 640, // dest UV
+ 640, 400); // width and height
+
+ int checksum = 0;
+ for (size_t i = 0; i < sizeof(dst_nv12); ++i) {
+ checksum += dst_nv12[i];
+ }
+ printf("checksum %x %s\n", checksum, checksum == 0x2ec0c00 ? "PASS" : "FAIL");
+ return 0;
+}
\ No newline at end of file
diff --git a/libs/libyuv/util/psnr_main.cc b/libs/libyuv/util/psnr_main.cc
index a930b202ec..45261805c9 100644
--- a/libs/libyuv/util/psnr_main.cc
+++ b/libs/libyuv/util/psnr_main.cc
@@ -65,9 +65,9 @@ int num_threads = 0;
#endif
// Parse PYUV format. ie name.1920x800_24Hz_P420.yuv
-bool ExtractResolutionFromFilename(const char* name,
- int* width_ptr,
- int* height_ptr) {
+static bool ExtractResolutionFromFilename(const char* name,
+ int* width_ptr,
+ int* height_ptr) {
// Isolate the .width_height. section of the filename by searching for a
// dot or underscore followed by a digit.
for (int i = 0; name[i]; ++i) {
@@ -105,25 +105,12 @@ bool ExtractResolutionFromFilename(const char* name,
return false;
}
-// Scale Y channel from 16..240 to 0..255.
-// This can be useful when comparing codecs that are inconsistant about Y
-uint8_t ScaleY(uint8_t y) {
- int ny = (y - 16) * 256 / 224;
- if (ny < 0) {
- ny = 0;
- }
- if (ny > 255) {
- ny = 255;
- }
- return static_cast<uint8_t>(ny);
-}
-
// MSE = Mean Square Error
-double GetMSE(double sse, double size) {
+static double GetMSE(double sse, double size) {
return sse / size;
}
-void PrintHelp(const char* program) {
+static void PrintHelp(const char* program) {
printf("%s [-options] org_seq rec_seq [rec_seq2.. etc]\n", program);
#ifdef HAVE_JPEG
printf("jpeg or raw YUV 420 supported.\n");
@@ -151,7 +138,7 @@ void PrintHelp(const char* program) {
exit(0);
}
-void ParseOptions(int argc, const char* argv[]) {
+static void ParseOptions(int argc, const char* argv[]) {
if (argc <= 1) {
PrintHelp(argv[0]);
}
@@ -240,21 +227,21 @@ void ParseOptions(int argc, const char* argv[]) {
}
}
-bool UpdateMetrics(uint8_t* ch_org,
- uint8_t* ch_rec,
- const int y_size,
- const int uv_size,
- const size_t total_size,
- int number_of_frames,
- metric* cur_distortion_psnr,
- metric* distorted_frame,
- bool do_psnr) {
+static bool UpdateMetrics(uint8_t* ch_org,
+ uint8_t* ch_rec,
+ const int y_size,
+ const int uv_size,
+ const size_t total_size,
+ int number_of_frames,
+ metric* cur_distortion_psnr,
+ metric* distorted_frame,
+ bool compute_psnr) {
const int uv_offset = (do_swap_uv ? uv_size : 0);
const uint8_t* const u_org = ch_org + y_size + uv_offset;
const uint8_t* const u_rec = ch_rec + y_size;
const uint8_t* const v_org = ch_org + y_size + (uv_size - uv_offset);
const uint8_t* const v_rec = ch_rec + y_size + uv_size;
- if (do_psnr) {
+ if (compute_psnr) {
#ifdef HAVE_JPEG
double y_err = static_cast<double>(
libyuv::ComputeSumSquareError(ch_org, ch_rec, y_size));
@@ -342,9 +329,12 @@ int main(int argc, const char* argv[]) {
exit(1);
}
+ if (num_rec <= 0) {
+ fclose(file_org);
+ exit(1);
+ }
// Open all files to compare to
- FILE** file_rec = new FILE*[num_rec];
- memset(file_rec, 0, num_rec * sizeof(FILE*)); // NOLINT
+ FILE** file_rec = new FILE*[num_rec]();
for (int cur_rec = 0; cur_rec < num_rec; ++cur_rec) {
file_rec[cur_rec] = fopen(argv[fileindex_rec + cur_rec], "rb");
if (file_rec[cur_rec] == NULL) {
diff --git a/libs/libyuv/util/ssim.cc b/libs/libyuv/util/ssim.cc
index 096fbcf061..f8b4509f87 100644
--- a/libs/libyuv/util/ssim.cc
+++ b/libs/libyuv/util/ssim.cc
@@ -244,23 +244,23 @@ double GetSSIMFullKernel(const uint8_t* org,
// Read 8 pixels at line #L, and convert to 16bit, perform weighting
// and acccumulate.
-#define LOAD_LINE_PAIR(L, WEIGHT) \
- do { \
- const __m128i v0 = \
- _mm_loadl_epi64(reinterpret_cast<const __m128i*>(org + (L)*stride)); \
- const __m128i v1 = \
- _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rec + (L)*stride)); \
- const __m128i w0 = _mm_unpacklo_epi8(v0, zero); \
- const __m128i w1 = _mm_unpacklo_epi8(v1, zero); \
- const __m128i ww0 = _mm_mullo_epi16(w0, (WEIGHT).values_.m_); \
- const __m128i ww1 = _mm_mullo_epi16(w1, (WEIGHT).values_.m_); \
- x = _mm_add_epi32(x, _mm_unpacklo_epi16(ww0, zero)); \
- y = _mm_add_epi32(y, _mm_unpacklo_epi16(ww1, zero)); \
- x = _mm_add_epi32(x, _mm_unpackhi_epi16(ww0, zero)); \
- y = _mm_add_epi32(y, _mm_unpackhi_epi16(ww1, zero)); \
- xx = _mm_add_epi32(xx, _mm_madd_epi16(ww0, w0)); \
- xy = _mm_add_epi32(xy, _mm_madd_epi16(ww0, w1)); \
- yy = _mm_add_epi32(yy, _mm_madd_epi16(ww1, w1)); \
+#define LOAD_LINE_PAIR(L, WEIGHT) \
+ do { \
+ const __m128i v0 = \
+ _mm_loadl_epi64(reinterpret_cast<const __m128i*>(org + (L) * stride)); \
+ const __m128i v1 = \
+ _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rec + (L) * stride)); \
+ const __m128i w0 = _mm_unpacklo_epi8(v0, zero); \
+ const __m128i w1 = _mm_unpacklo_epi8(v1, zero); \
+ const __m128i ww0 = _mm_mullo_epi16(w0, (WEIGHT).values_.m_); \
+ const __m128i ww1 = _mm_mullo_epi16(w1, (WEIGHT).values_.m_); \
+ x = _mm_add_epi32(x, _mm_unpacklo_epi16(ww0, zero)); \
+ y = _mm_add_epi32(y, _mm_unpacklo_epi16(ww1, zero)); \
+ x = _mm_add_epi32(x, _mm_unpackhi_epi16(ww0, zero)); \
+ y = _mm_add_epi32(y, _mm_unpackhi_epi16(ww1, zero)); \
+ xx = _mm_add_epi32(xx, _mm_madd_epi16(ww0, w0)); \
+ xy = _mm_add_epi32(xy, _mm_madd_epi16(ww0, w1)); \
+ yy = _mm_add_epi32(yy, _mm_madd_epi16(ww1, w1)); \
} while (0)
#define ADD_AND_STORE_FOUR_EPI32(M, OUT) \
diff --git a/libs/libyuv/util/yuvconstants.c b/libs/libyuv/util/yuvconstants.c
new file mode 100644
index 0000000000..5f52d7553c
--- /dev/null
+++ b/libs/libyuv/util/yuvconstants.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2021 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// This utility computes values needed to generate yuvconstants based on
+// white point values.
+// The yuv formulas are tuned for 8 bit YUV channels.
+
+// See Also
+// https://mymusing.co/bt601-yuv-to-rgb-conversion-color/
+
+// BT.709 full range YUV to RGB reference
+// R = Y + V * 1.5748
+// G = Y - U * 0.18732 - V * 0.46812
+// B = Y + U * 1.8556
+// KR = 0.2126
+// KB = 0.0722
+
+// // Y contribution to R,G,B. Scale and bias.
+// #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+// #define YB 32 /* 64 / 2 */
+//
+// // U and V contributions to R,G,B.
+// #define UB 113 /* round(1.77200 * 64) */
+// #define UG 22 /* round(0.34414 * 64) */
+// #define VG 46 /* round(0.71414 * 64) */
+// #define VR 90 /* round(1.40200 * 64) */
+//
+// // Bias values to round, and subtract 128 from U and V.
+// #define BB (-UB * 128 + YB)
+// #define BG (UG * 128 + VG * 128 + YB)
+// #define BR (-VR * 128 + YB)
+
+int main(int argc, const char* argv[]) {
+ if (argc < 3) {
+ printf("yuvconstants [KR] [KB]\n");
+ printf(" e.g. yuvconstants 0.2126 0.0722\n");
+ printf(" MC BT KR KB\n");
+ printf(" 1 BT.709 KR = 0.2126; KB = 0.0722\n");
+ printf(" 4 FCC KR = 0.30; KB = 0.11\n");
+ printf(" 6 BT.601 KR = 0.299; KB = 0.114\n");
+ printf(" 7 SMPTE 240M KR = 0.212; KB = 0.087\n");
+ printf(" 9 BT.2020 KR = 0.2627; KB = 0.0593\n");
+ return -1;
+ }
+ float kr = (float)atof(argv[1]);
+ float kb = (float)atof(argv[2]);
+ float kg = 1 - kr - kb;
+
+ float vr = 2 * (1 - kr);
+ float ug = 2 * ((1 - kb) * kb / kg);
+ float vg = 2 * ((1 - kr) * kr / kg);
+ float ub = 2 * (1 - kb);
+
+ printf("Full range\n");
+ printf("R = Y + V * %5f\n", vr);
+ printf("G = Y - U * %6f - V * %6f\n", ug, vg);
+ printf("B = Y + U * %5f\n", ub);
+
+ printf("KR = %4f; ", kr);
+ printf("KB = %4f\n", kb);
+ // printf("KG = %4f\n", kg);
+ // #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+ // #define YB 32 /* 64 / 2 */
+ //
+ // // U and V contributions to R,G,B.
+
+ printf("UB %-3.0f /* round(%f * 64 = %8.4f) */\n", round(ub * 64), ub,
+ ub * 64);
+ printf("UG %-3.0f /* round(%f * 64 = %8.4f) */\n", round(ug * 64), ug,
+ ug * 64);
+ printf("VG %-3.0f /* round(%f * 64 = %8.4f) */\n", round(vg * 64), vg,
+ vg * 64);
+ printf("VR %-3.0f /* round(%f * 64 = %8.4f) */\n", round(vr * 64), vr,
+ vr * 64);
+
+ vr = 255.f / 224.f * 2 * (1 - kr);
+ ug = 255.f / 224.f * 2 * ((1 - kb) * kb / kg);
+ vg = 255.f / 224.f * 2 * ((1 - kr) * kr / kg);
+ ub = 255.f / 224.f * 2 * (1 - kb);
+
+ printf("\nLimited range\n");
+ printf("R = (Y - 16) * 1.164 + V * %5f\n", vr);
+ printf("G = (Y - 16) * 1.164 - U * %6f - V * %6f\n", ug, vg);
+ printf("B = (Y - 16) * 1.164 + U * %5f\n", ub);
+
+ // printf("KG = %4f\n", kg);
+ // #define YG 16320 /* round(1.000 * 64 * 256 * 256 / 257) */
+ // #define YB 32 /* 64 / 2 */
+ //
+ // // U and V contributions to R,G,B.
+
+ printf("UB %-3.0f /* round(%f * 64 = %8.4f) */\n", round(ub * 64), ub,
+ ub * 64);
+ printf("UG %-3.0f /* round(%f * 64 = %8.4f) */\n", round(ug * 64), ug,
+ ug * 64);
+ printf("VG %-3.0f /* round(%f * 64 = %8.4f) */\n", round(vg * 64), vg,
+ vg * 64);
+ printf("VR %-3.0f /* round(%f * 64 = %8.4f) */\n", round(vr * 64), vr,
+ vr * 64);
+
+ return 0;
+}
diff --git a/libs/libyuv/util/yuvconvert.cc b/libs/libyuv/util/yuvconvert.cc
index 27cdfe9e37..047e9235dc 100644
--- a/libs/libyuv/util/yuvconvert.cc
+++ b/libs/libyuv/util/yuvconvert.cc
@@ -42,9 +42,9 @@ static __inline uint32_t Abs(int32_t v) {
}
// Parse PYUV format. ie name.1920x800_24Hz_P420.yuv
-bool ExtractResolutionFromFilename(const char* name,
- int* width_ptr,
- int* height_ptr) {
+static bool ExtractResolutionFromFilename(const char* name,
+ int* width_ptr,
+ int* height_ptr) {
// Isolate the .width_height. section of the filename by searching for a
// dot or underscore followed by a digit.
for (int i = 0; name[i]; ++i) {
@@ -59,7 +59,7 @@ bool ExtractResolutionFromFilename(const char* name,
return false;
}
-void PrintHelp(const char* program) {
+static void PrintHelp(const char* program) {
printf("%s [-options] src_argb.raw dst_yuv.raw\n", program);
printf(
" -s <width> <height> .... specify source resolution. "
@@ -78,7 +78,7 @@ void PrintHelp(const char* program) {
exit(0);
}
-void ParseOptions(int argc, const char* argv[]) {
+static void ParseOptions(int argc, const char* argv[]) {
if (argc <= 1) {
PrintHelp(argv[0]);
}
@@ -165,23 +165,23 @@ static int TileARGBScale(const uint8_t* src_argb,
int src_height,
uint8_t* dst_argb,
int dst_stride_argb,
- int dst_width,
- int dst_height,
+ int destination_width,
+ int destination_height,
libyuv::FilterMode filtering) {
- for (int y = 0; y < dst_height; y += kTileY) {
- for (int x = 0; x < dst_width; x += kTileX) {
+ for (int y = 0; y < destination_height; y += kTileY) {
+ for (int x = 0; x < destination_width; x += kTileX) {
int clip_width = kTileX;
- if (x + clip_width > dst_width) {
- clip_width = dst_width - x;
+ if (x + clip_width > destination_width) {
+ clip_width = destination_width - x;
}
int clip_height = kTileY;
- if (y + clip_height > dst_height) {
- clip_height = dst_height - y;
+ if (y + clip_height > destination_height) {
+ clip_height = destination_height - y;
}
int r = libyuv::ARGBScaleClip(src_argb, src_stride_argb, src_width,
src_height, dst_argb, dst_stride_argb,
- dst_width, dst_height, x, y, clip_width,
- clip_height, filtering);
+ destination_width, destination_height, x, y,
+ clip_width, clip_height, filtering);
if (r) {
return r;
}
@@ -201,8 +201,12 @@ int main(int argc, const char* argv[]) {
}
// Open all files to convert to
- FILE** file_rec = new FILE*[num_rec];
- memset(file_rec, 0, num_rec * sizeof(FILE*)); // NOLINT
+ FILE** file_rec = static_cast<FILE**>(calloc(num_rec, sizeof(FILE*)));
+ if (file_rec == NULL) {
+ fprintf(stderr, "No memory available\n");
+ fclose(file_org);
+ exit(1);
+ }
for (int cur_rec = 0; cur_rec < num_rec; ++cur_rec) {
file_rec[cur_rec] = fopen(argv[fileindex_rec + cur_rec], "wb");
if (file_rec[cur_rec] == NULL) {
@@ -211,7 +215,7 @@ int main(int argc, const char* argv[]) {
for (int i = 0; i < cur_rec; ++i) {
fclose(file_rec[i]);
}
- delete[] file_rec;
+ free(file_rec);
exit(1);
}
}
@@ -243,19 +247,19 @@ int main(int argc, const char* argv[]) {
fseek(file_org, num_skip_org * total_size, SEEK_SET);
#endif
- uint8_t* const ch_org = new uint8_t[org_size];
- uint8_t* const ch_dst = new uint8_t[dst_size];
- uint8_t* const ch_rec = new uint8_t[total_size];
- if (ch_org == NULL || ch_rec == NULL) {
+ uint8_t* const ch_org = static_cast<uint8_t*>(malloc(org_size));
+ uint8_t* const ch_dst = static_cast<uint8_t*>(malloc(dst_size));
+ uint8_t* const ch_rec = static_cast<uint8_t*>(malloc(total_size));
+ if (ch_org == NULL || ch_dst == NULL || ch_rec == NULL) {
fprintf(stderr, "No memory available\n");
fclose(file_org);
for (int i = 0; i < num_rec; ++i) {
fclose(file_rec[i]);
}
- delete[] ch_org;
- delete[] ch_dst;
- delete[] ch_rec;
- delete[] file_rec;
+ free(ch_org);
+ free(ch_dst);
+ free(ch_rec);
+ free(file_rec);
exit(1);
}
@@ -359,9 +363,9 @@ int main(int argc, const char* argv[]) {
for (int cur_rec = 0; cur_rec < num_rec; ++cur_rec) {
fclose(file_rec[cur_rec]);
}
- delete[] ch_org;
- delete[] ch_dst;
- delete[] ch_rec;
- delete[] file_rec;
+ free(ch_org);
+ free(ch_dst);
+ free(ch_rec);
+ free(file_rec);
return 0;
}
diff --git a/libs/libyuv/vpython.toml b/libs/libyuv/vpython.toml
new file mode 100644
index 0000000000..52d8b3adc5
--- /dev/null
+++ b/libs/libyuv/vpython.toml
@@ -0,0 +1,76 @@
+# Copyright 2026 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+requires-python = '>=3.11,<3.12'
+dependencies = [
+ 'protobuf==4.25.1',
+ 'six==1.15.0',
+ 'numpy==1.23.5+chromium.4',
+ 'psutil==5.9.8',
+ 'requests==2.31.0',
+ 'mock==2.0.0',
+ 'parameterized==0.7.1',
+ 'pbr==3.0.0',
+ 'setuptools==68.0.0 ; python_version < "3.14"',
+ 'setuptools>=75.0.0 ; python_version >= "3.14"',
+ 'pyfakefs==3.7.2',
+ 'jsonlines==1.2.0',
+ 'python-dateutil==2.9.0',
+ 'charset-normalizer==2.0.4',
+ 'pyasn1==0.4.5',
+ 'pyasn1-modules==0.2.4',
+ 'rsa==3.4.2',
+ 'cachetools==2.0.1',
+ 'uritemplate==3.0.0',
+ 'google-auth==1.25.0',
+ 'googleapis-common-protos==1.52.0',
+ 'google-api-core==1.25.1',
+ 'google-auth-httplib2==0.1.0',
+ 'google-api-python-client==2.2.0',
+ 'oauth2client==3.0.0',
+ 'html5lib==1.0.1',
+ 'mozdebug==0.2',
+ 'mozinfo==1.2.2',
+ 'mozlog==7.1.0',
+ 'mozprocess==1.2.1',
+ 'urllib3==1.26.6',
+ 'brotli==1.0.9',
+ 'blessings==1.7',
+ 'mozfile==2.0.0',
+ 'mozterm==1.0.0',
+ 'webencodings==0.5.1',
+ 'certifi==2020.11.8',
+ 'chardet==3.0.4',
+ 'idna==2.8',
+ 'distro==1.4.0',
+ 'pillow==10.4.0',
+ 'aioquic==1.2.0+chromium.1',
+ 'pyopenssl==24.2.1',
+ 'service-identity==24.1.0',
+ 'tzdata==2023.4',
+ 'pylsqpack==0.3.12',
+ 'cryptography==43.0.0',
+ 'cffi==1.15.1+chromium.2',
+ 'pycparser==2.19',
+ 'iniconfig==1.1.1',
+ 'packaging==16.8',
+ 'pyparsing==2.4.7',
+ 'toml==0.10.1',
+ 'pytest==6.2.2',
+ 'pytest-asyncio==0.14.0',
+ 'attrs==20.3.0',
+ 'more-itertools==4.1.0',
+ 'pluggy==0.13.1',
+ 'py==1.10.0',
+ 'funcsigs==1.0.2',
+ 'atomicwrites==1.3.0',
+ 'colorama==0.4.1',
+ 'coverage==7.3.1',
+ 'pathos==0.3.0+chromium.2',
+ 'pytz==2024.1',
+ "pywin32==308; sys_platform == 'win32'",
+ "pyobjc==10.0; sys_platform == 'darwin'",
+ 'httplib2==0.19.1',
+ "pandas==2.2.3+chromium.1; sys_platform != 'linux' or platform_machine != 'aarch64'"
+]
diff --git a/libs/libyuv/vpython.toml.uv.lock b/libs/libyuv/vpython.toml.uv.lock
new file mode 100644
index 0000000000..c4570ede52
--- /dev/null
+++ b/libs/libyuv/vpython.toml.uv.lock
@@ -0,0 +1,1722 @@
+aioquic==1.2.0+chromium.1 \
+ --hash=sha256:091c63339525b44f869b36c80c72715f1521e10e8d8dc66f0f1d76ed9bb186c5 \
+ --hash=sha256:4c33abef5832db9b6a8473c5073b7b6d9765915025158d0d1b4f668904ae9eb2 \
+ --hash=sha256:5c0f27ec0e5386743db8e3a0e17c1f118e7bb59915c4a1168c44aae8af4735e7 \
+ --hash=sha256:6c118aac0b8844f133cda053d542adf4dd689cbd587d23e69314bd69cff8cec9 \
+ --hash=sha256:7022d962b525fe3ae4a33278d11a9f757b88bfaab6cc74887c95c5d6226d694d \
+ --hash=sha256:823e7c5addce32335f67108ea0ef0cf754cedaf2345f58ec349bb1604e7b55b2 \
+ --hash=sha256:841ffbbf28eebf558b6f356215ccb34d8963351f1f5e18ee6f4265da974cc703 \
+ --hash=sha256:8e2a7b8dec2cd23a19103894f24207d85094f5db18cfafe42d31a25a8e7ec4b2 \
+ --hash=sha256:9e8c5c3a715e48491e83e5323a1518f079d65de90a829599cdff37c14998748e \
+ --hash=sha256:aa043efcfff94803237104db48ca62a9f23c192291a10339554223af74ee6021 \
+ --hash=sha256:b7a3164babb8173cba2e35d43f4bf3f6159e19b4a28601b21ee9a5163f1a2bf9 \
+ --hash=sha256:e2c23c2c0b376208fe138c0bdb3ba572257d51d788837807250b0ce6444d3fa1 \
+ --hash=sha256:e53155d0542f8968ea03005785cf384af1f29ae1a04864af711d6c02690b6958 \
+ --hash=sha256:f99f43c4961c885581513d603af5172660314007eace52523e05464d1c03b41d
+atomicwrites==1.3.0 \
+ --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \
+ --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \
+ --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \
+ --hash=sha256:6bb9f1a7743fc65fe70e76ab1225522a3d1efc511bca66b5b89e4c41ef94ccca \
+ --hash=sha256:6bb9f1a7743fc65fe70e76ab1225522a3d1efc511bca66b5b89e4c41ef94ccca \
+ --hash=sha256:6bb9f1a7743fc65fe70e76ab1225522a3d1efc511bca66b5b89e4c41ef94ccca
+ # via pytest
+attrs==20.3.0 \
+ --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \
+ --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \
+ --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \
+ --hash=sha256:be113fc296f1e704a1ae1a26a1f7af5082217cf0af1cdef3fa93afc543bab3f7 \
+ --hash=sha256:f37869632f36155a5bea58fd2c2a4b970e51b49fc9037989640280ba6a23bc14 \
+ --hash=sha256:f37869632f36155a5bea58fd2c2a4b970e51b49fc9037989640280ba6a23bc14
+ # via
+ # pytest
+ # service-identity
+blessings==1.7 \
+ --hash=sha256:74e662a1b980c6982356b113be1160074e32f1dba038da4829f2742736a41318 \
+ --hash=sha256:9d110e490ecb455308c2e4b9fabed4a2aa27fc61ae9c929e226be189a9e2d3f9 \
+ --hash=sha256:9d110e490ecb455308c2e4b9fabed4a2aa27fc61ae9c929e226be189a9e2d3f9 \
+ --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \
+ --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e \
+ --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e
+ # via mozlog
+brotli==1.0.9 \
+ --hash=sha256:1590964412a2f2b59540b995cd89f382faf4657f5bc8d034240e9e0a10713ae5 \
+ --hash=sha256:1e43abc36411833f42e12cc7ac95f9b66c8d896e636f30aefbee8997c486bca7 \
+ --hash=sha256:2037368f12f750aba61f74329d9c2111a57c36e586c671686aef93f524008e97 \
+ --hash=sha256:2e7c834df41cb201c35e622e08b2b1e0d9ad51a2d6f0ff8c0f42c4b110564cc7 \
+ --hash=sha256:4258cc1b975b088abd0d8a6dcace6488f2b2cab658c67a6bc1ade04c76697fa6 \
+ --hash=sha256:633eacc9633199b897ddbb31242aa08ed0da04b0e3a52af166988a66f9168c11 \
+ --hash=sha256:672875325d005af939bb1f417dd74e4c67fe3d9eba04ece818cc0c741640f20e \
+ --hash=sha256:717773b68db3ac0f26729867d5a6d9bc4b8611e75da3995fcc25e4efde859e75 \
+ --hash=sha256:87a7bb50c2cce9cf9d279442ee3ee79731fc9c39421fa6c60896c7932ed5df32 \
+ --hash=sha256:90fd84cd7fd28a8dccbd1253916bfd1b58ffc79a4c8f9725fcd7abd5e0922ce5 \
+ --hash=sha256:99e081f61a21f13996e61b5fcb311c0ffaa47dbdf509d94576578ae784f752ca \
+ --hash=sha256:a16e5e7ab858d1ecf4521b91f527a3150212d3038f41d9f878706fd9c9756727 \
+ --hash=sha256:a172a8fcda96953ffb3ba0c46ac87226e2fa787a5e3314d808355acdfd9556cb \
+ --hash=sha256:b47f558b2274209c59c1a24e3d350491480c0b5d871fa052cc20224e28da5fa5 \
+ --hash=sha256:c02ce929bad0afa590a32d41a610c417b0c0860f7c37ec0249553f9842846ce1 \
+ --hash=sha256:ed1d451150616224ce20cfc2664ed6f8716739c7b52e6814aba73d6753e62bca \
+ --hash=sha256:f6430b9d07dfe90fd16d0877a3d4e2d464316b35d4b755e57e79be4bdb7869d0
+cachetools==2.0.1 \
+ --hash=sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe \
+ --hash=sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe \
+ --hash=sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe \
+ --hash=sha256:452888d3eac7decf889530a271923254abb7380a0255cd65693cd29c2c66b4f8 \
+ --hash=sha256:452888d3eac7decf889530a271923254abb7380a0255cd65693cd29c2c66b4f8 \
+ --hash=sha256:452888d3eac7decf889530a271923254abb7380a0255cd65693cd29c2c66b4f8
+ # via google-auth
+certifi==2020.11.8 \
+ --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \
+ --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \
+ --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \
+ --hash=sha256:83899d4fd2a1071acca79abb10435484cdfd9f223522fd26ac2449b2c4c52d9b \
+ --hash=sha256:e538978d4526342d7b37cfcdd503eaf48f32c7f362d7ca8cb32ee5491b872841 \
+ --hash=sha256:e538978d4526342d7b37cfcdd503eaf48f32c7f362d7ca8cb32ee5491b872841
+ # via
+ # aioquic
+ # requests
+cffi==1.15.1+chromium.2 \
+ --hash=sha256:10bdaddfd9bc761f9cf6ea80a0cb3dbcf85a6034c3038879634948d75dc903e1 \
+ --hash=sha256:220efee8db3410f448fc08a1337345b168853387cc0fd11c288cd4024d005308 \
+ --hash=sha256:2252d280c0ab8be58f52469f4106743073ae37f956e4e483ed8d4e46efaf8566 \
+ --hash=sha256:4171ff5206e90549e6f42693a69849c8fcc448936d301d3b3a4e244b221f3f83 \
+ --hash=sha256:50b3834673f3ebfcdf4fdbf42a3c34c7ad05362c42a16cea26e5d193cdb9fff5 \
+ --hash=sha256:51013c858d2bb7540ac5860644efe82020dc492396105887349c28908913c3b0 \
+ --hash=sha256:5b54af79e781dc7fcbc17038ed339fb70bbf8e1f0dff989ef12cdebc53c820f4 \
+ --hash=sha256:6291010bcc1e76ae6431cfee3b32fa526b402c3450e2cb8830e409b3e4d93028 \
+ --hash=sha256:80916c530265a4935f44f13b1e3ff711b82f5e6312279be2164e958990920d58 \
+ --hash=sha256:860f9af71776499a4a9684255be1af58f1ed713ddbf9eee6603023af8133d447 \
+ --hash=sha256:a7f78589e2e8d1af2381a3abe778b0e10d2d5691fb14d83a3a352847747fe379 \
+ --hash=sha256:aa5d44a118e77bc6e7aea22a6b7e9c9a207eee23695516865ba0b95e270ffa41 \
+ --hash=sha256:b3613b9aa31c33af8c8956786697fec59296543fb4dbf3afeec80daf3b7381a2 \
+ --hash=sha256:f1c983b1b1b32e87f63bc8180de89b780bb8b6a35eec36ba2ae0490ccce3e9cf \
+ --hash=sha256:f9a328b8d39fa3cf6659d80e61b52560d5896a53ea6abcebcbdbce2c05496270
+ # via cryptography
+chardet==3.0.4 \
+ --hash=sha256:a3d948e8a23a4a1072c0f972b2a7d9a04369bde7863d7ecd7b1cee31937a8f6f \
+ --hash=sha256:a3d948e8a23a4a1072c0f972b2a7d9a04369bde7863d7ecd7b1cee31937a8f6f \
+ --hash=sha256:a3d948e8a23a4a1072c0f972b2a7d9a04369bde7863d7ecd7b1cee31937a8f6f \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
+charset-normalizer==2.0.4 \
+ --hash=sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b \
+ --hash=sha256:8290b03e2f31a4b3399e1727f0323607253bb2bd38f0cb23d4e0cc6d76a5ec72
+ # via requests
+colorama==0.4.1 \
+ --hash=sha256:758fb3cafa52391ea645703f283afd57f002011140d470793379790398876733 \
+ --hash=sha256:758fb3cafa52391ea645703f283afd57f002011140d470793379790398876733 \
+ --hash=sha256:758fb3cafa52391ea645703f283afd57f002011140d470793379790398876733 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48 \
+ --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48
+ # via pytest
+coverage==7.3.1 \
+ --hash=sha256:2b7f038e155de2156bd1073068b0ba39e97ed8ca340889a680ae2d7685dd60ea \
+ --hash=sha256:37df007c1cb77d75ddf9e8753d2f69ba904d4beaec7d696fff0081a61ce81cd8 \
+ --hash=sha256:40c0f331b8bde5128d97fa3c284078958b1aab42a86697c40f65b650417ed904 \
+ --hash=sha256:412d5ef5437e9bad534e7464ea80094d195a998803146a9a856850e79083a93c \
+ --hash=sha256:4c16428216025c980a8d160c885c5225d143353f609a82537e8b957a2109330e \
+ --hash=sha256:4cf8ed834637fe317552ab068048c7202068b64706593159d67fa1936753a19b \
+ --hash=sha256:6bc0892076728c5c5d3c92a7e968ddc1c52b49e79479e9419a50431df200566d \
+ --hash=sha256:79a16c5b61a81ac581d07d74f2b257a77853d31f3e48d9f674226f4fd42a77ef \
+ --hash=sha256:7f3c398881de3a558f9fb94f232757c73d3498a420462bb232590c4ecfb9a908 \
+ --hash=sha256:85d78e23c50f3820666670e4689f1b30c0915e7db0020a22a41be4158d78e7f7 \
+ --hash=sha256:afd828ea3ffebf4851053258a3e9d52079bfd1c60d233b0d0467bba73ebd044f \
+ --hash=sha256:b81ff0d20e6da0552923e36b9b9c5c6e7dcd6192653fc9192b059137f50938c9 \
+ --hash=sha256:bdaedd684fdf5c2104be67e2c1b92f73e06b83a1cb0cab19d598a4e4894ccb56 \
+ --hash=sha256:ea1637565ebc68eca9ed5f6414c8231614c79b5f2f9d3409b8e8f1894902b7ee \
+ --hash=sha256:ec748a35bfb05af56249a15db789773644902de436f6867129eb183479f17010 \
+ --hash=sha256:fbd870567a278eddd1903e206969bb29f76288b21a5c1a61dab6e0d8260ec75f
+cryptography==43.0.0 \
+ --hash=sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709 \
+ --hash=sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e \
+ --hash=sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778 \
+ --hash=sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22 \
+ --hash=sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895 \
+ --hash=sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895 \
+ --hash=sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74 \
+ --hash=sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74 \
+ --hash=sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc \
+ --hash=sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66 \
+ --hash=sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66 \
+ --hash=sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66 \
+ --hash=sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5 \
+ --hash=sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5
+ # via
+ # aioquic
+ # pyopenssl
+ # service-identity
+dill==0.4.1 \
+ --hash=sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d \
+ --hash=sha256:6de43e59480656b19447122e9d653748231ecbd1e1e084e1bc249d4880bdfb57
+ # via
+ # multiprocess
+ # pathos
+distro==1.4.0 \
+ --hash=sha256:b9fb5628b296e58599f257d50ad9744d154893b28a19eba10e5c17835891398e \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 \
+ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4
+ # via mozinfo
+funcsigs==1.0.2 \
+ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+ --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+ --hash=sha256:c75adf3fb65c5e85c480444a9182c2045f43996a805ff042207d28418e7289b8 \
+ --hash=sha256:c75adf3fb65c5e85c480444a9182c2045f43996a805ff042207d28418e7289b8 \
+ --hash=sha256:c75adf3fb65c5e85c480444a9182c2045f43996a805ff042207d28418e7289b8
+google-api-core==1.25.1 \
+ --hash=sha256:292dd636ed381098d24b7093ccb826b2278a12d886a3fc982084069aa24a8fbb \
+ --hash=sha256:292dd636ed381098d24b7093ccb826b2278a12d886a3fc982084069aa24a8fbb \
+ --hash=sha256:292dd636ed381098d24b7093ccb826b2278a12d886a3fc982084069aa24a8fbb \
+ --hash=sha256:340e5db8a4ae911ce74c886841bc9479572a2dbca30b922044ed24fc69f500ba \
+ --hash=sha256:db582cdba02f4c7d66e9c30421e0c06cf1ae6886e66611a51c235c7210d4f2cc \
+ --hash=sha256:db582cdba02f4c7d66e9c30421e0c06cf1ae6886e66611a51c235c7210d4f2cc
+ # via google-api-python-client
+google-api-python-client==2.2.0 \
+ --hash=sha256:330787e6e8ab10a1629b783a2dd98ae31196ad8f218f9fddc105dc8ac788808f \
+ --hash=sha256:9edac3b071588e08e4b3d14455f2a3c2a57637cab6535f70cff0c01200e22909
+google-auth==1.25.0 \
+ --hash=sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d \
+ --hash=sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d \
+ --hash=sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d \
+ --hash=sha256:c4db95010a903477b81f06e5c37de92ffffd19299e04c02d0edc3c6939310844 \
+ --hash=sha256:c4db95010a903477b81f06e5c37de92ffffd19299e04c02d0edc3c6939310844 \
+ --hash=sha256:c4db95010a903477b81f06e5c37de92ffffd19299e04c02d0edc3c6939310844
+ # via
+ # google-api-core
+ # google-api-python-client
+ # google-auth-httplib2
+google-auth-httplib2==0.1.0 \
+ --hash=sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10 \
+ --hash=sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10 \
+ --hash=sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10 \
+ --hash=sha256:3a16b4a034b717c9fdb53723e7824bc33f56d20ff3b3673496798c99f0b7de69 \
+ --hash=sha256:3a16b4a034b717c9fdb53723e7824bc33f56d20ff3b3673496798c99f0b7de69 \
+ --hash=sha256:3a16b4a034b717c9fdb53723e7824bc33f56d20ff3b3673496798c99f0b7de69
+ # via google-api-python-client
+googleapis-common-protos==1.52.0 \
+ --hash=sha256:536edff5a2d8a18613e2795577f307c656c905ee53a5e7fd64bd39e620de0557 \
+ --hash=sha256:536edff5a2d8a18613e2795577f307c656c905ee53a5e7fd64bd39e620de0557 \
+ --hash=sha256:536edff5a2d8a18613e2795577f307c656c905ee53a5e7fd64bd39e620de0557 \
+ --hash=sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24 \
+ --hash=sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24 \
+ --hash=sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24
+ # via google-api-core
+html5lib==1.0.1 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3 \
+ --hash=sha256:e88d4002933d893f8b23ebb05db9f49996533d4ac6ddcbfd51b62b9b9793b397 \
+ --hash=sha256:e88d4002933d893f8b23ebb05db9f49996533d4ac6ddcbfd51b62b9b9793b397 \
+ --hash=sha256:e88d4002933d893f8b23ebb05db9f49996533d4ac6ddcbfd51b62b9b9793b397
+httplib2==0.19.1 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:2ad195faf9faf079723f6714926e9a9061f694d07724b846658ce08d40f522b4 \
+ --hash=sha256:b7cd9ee0a01ce4e658366e13a2047b54c84fc71f1f0ff268325a1ed5b5aa119d
+ # via
+ # google-api-python-client
+ # google-auth-httplib2
+ # oauth2client
+idna==2.8 \
+ --hash=sha256:6c68f10e99f5f99caff1bf6b897f7032748b7fa206d16f507c86867bc078312d \
+ --hash=sha256:6c68f10e99f5f99caff1bf6b897f7032748b7fa206d16f507c86867bc078312d \
+ --hash=sha256:6c68f10e99f5f99caff1bf6b897f7032748b7fa206d16f507c86867bc078312d \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+ # via requests
+iniconfig==1.1.1 \
+ --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
+ --hash=sha256:9d81514370059186276e9a1799e5a9752e4c060233c254d6954b53fad2ff8a5e
+ # via pytest
+jsonlines==1.2.0 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:0ebd5b0c3efe0d4b5018b320fb0ee1a7b680ab39f6eb853715859f818d386cc8 \
+ --hash=sha256:73011f8c3ffffa7e2f1222168c0f6b792d609bdf3a19a4742a7ea8f046228398 \
+ --hash=sha256:73011f8c3ffffa7e2f1222168c0f6b792d609bdf3a19a4742a7ea8f046228398 \
+ --hash=sha256:73011f8c3ffffa7e2f1222168c0f6b792d609bdf3a19a4742a7ea8f046228398
+mock==2.0.0 \
+ --hash=sha256:1005a7e0d4cc802865427a9a1928d717023653e10cc7e54451981fe08ea267a1 \
+ --hash=sha256:2f61f7f5133a2fe0ba86872efbd85ff86340aa3f4b46cadf7cca961dc651b9db \
+ --hash=sha256:2f61f7f5133a2fe0ba86872efbd85ff86340aa3f4b46cadf7cca961dc651b9db \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1
+more-itertools==4.1.0 \
+ --hash=sha256:0dd8f72eeab0d2c3bd489025bb2f6a1b8342f9b198f6fc37b52d15cfa4531fea \
+ --hash=sha256:11a625025954c20145b37ff6309cd54e39ca94f72f6bb9576d1195db6fa2442e \
+ --hash=sha256:11a625025954c20145b37ff6309cd54e39ca94f72f6bb9576d1195db6fa2442e \
+ --hash=sha256:7586f4373cc401db9ebaf156b52fe5cb3e136be27b1bb6dd6ee1e08b35b3de79 \
+ --hash=sha256:f6199f71f4c81c6a6323be83693b04b6417d6c1810a2693e74863d43c30be9b7 \
+ --hash=sha256:f6199f71f4c81c6a6323be83693b04b6417d6c1810a2693e74863d43c30be9b7
+mozdebug==0.2 \
+ --hash=sha256:3e6fea693a7380639d7c29c2381d5bab9a7827f4ecfd819d6f1971778766f17f \
+ --hash=sha256:3e6fea693a7380639d7c29c2381d5bab9a7827f4ecfd819d6f1971778766f17f \
+ --hash=sha256:3e6fea693a7380639d7c29c2381d5bab9a7827f4ecfd819d6f1971778766f17f \
+ --hash=sha256:bed08684bf2d415595bed781b1498084dc55d72cd51fd1d94386e136572844fb \
+ --hash=sha256:bed08684bf2d415595bed781b1498084dc55d72cd51fd1d94386e136572844fb \
+ --hash=sha256:bed08684bf2d415595bed781b1498084dc55d72cd51fd1d94386e136572844fb
+mozfile==2.0.0 \
+ --hash=sha256:7355643fca705f43d9412d444fb59588fbbd25f0d1e7592b9482f84e3d23bc8e \
+ --hash=sha256:7355643fca705f43d9412d444fb59588fbbd25f0d1e7592b9482f84e3d23bc8e \
+ --hash=sha256:7355643fca705f43d9412d444fb59588fbbd25f0d1e7592b9482f84e3d23bc8e \
+ --hash=sha256:bb744a07c2f6b8dd2a87d3532da6d54fd454328d08a2a1a1689dc68fc420791f \
+ --hash=sha256:bb744a07c2f6b8dd2a87d3532da6d54fd454328d08a2a1a1689dc68fc420791f \
+ --hash=sha256:bb744a07c2f6b8dd2a87d3532da6d54fd454328d08a2a1a1689dc68fc420791f
+ # via
+ # mozinfo
+ # mozlog
+mozinfo==1.2.2 \
+ --hash=sha256:139044387a3488b531fa4520cdab251c0dec2aa2bd0e8c975c3ed35491b7228f \
+ --hash=sha256:3834efcbe15fc459e7f505d7388794540ea952d4d266978be5ca024355032265 \
+ --hash=sha256:7c9ce6f1e04c96cba5d2687e0115b61e922e935dfde8a3823b06fdc17ac42053 \
+ --hash=sha256:9c106ffa5bdb3c1b548f4d7c8a99f492cdddf2b5964727229248273d7e1b4c65 \
+ --hash=sha256:9c106ffa5bdb3c1b548f4d7c8a99f492cdddf2b5964727229248273d7e1b4c65 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5 \
+ --hash=sha256:e8a2e67b44be1db042e38fa881de10918aea4588514fbabf6423de1c6728c3f5
+ # via
+ # mozdebug
+ # mozprocess
+mozlog==7.1.0 \
+ --hash=sha256:19952b32b781667db2b0f6b2d0aec96d4f0f8c2bfa625899758dc151730b7fb8 \
+ --hash=sha256:38ce1ddaf279aa5c4284c73fdcacd428114bda5ea05a22191c60f4d6c3309b86 \
+ --hash=sha256:54b9a1e781ce31fc10079dc8aec509fff7feca83714edeae6c981e279ceb796f \
+ --hash=sha256:54b9a1e781ce31fc10079dc8aec509fff7feca83714edeae6c981e279ceb796f \
+ --hash=sha256:54b9a1e781ce31fc10079dc8aec509fff7feca83714edeae6c981e279ceb796f \
+ --hash=sha256:9155ce75cd57dc2d54aa129f28207fd7d9d7a8c777b867894be81c45bf232734
+mozprocess==1.2.1 \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:08e1036b53819fd144331f6dfbbb17fc8ca782bbed2e28b4aa771b8b91f7dffb \
+ --hash=sha256:4eb345bb6990a2137ce37107dff1d5bab6395848254295a1e9c6089a79bfec54 \
+ --hash=sha256:4eb345bb6990a2137ce37107dff1d5bab6395848254295a1e9c6089a79bfec54 \
+ --hash=sha256:4eb345bb6990a2137ce37107dff1d5bab6395848254295a1e9c6089a79bfec54
+mozterm==1.0.0 \
+ --hash=sha256:602d9d93a7b4fed803d034cf73c061c7b62496ac9698b77bce9fe1e37571ffb2 \
+ --hash=sha256:602d9d93a7b4fed803d034cf73c061c7b62496ac9698b77bce9fe1e37571ffb2 \
+ --hash=sha256:602d9d93a7b4fed803d034cf73c061c7b62496ac9698b77bce9fe1e37571ffb2 \
+ --hash=sha256:f5eafa25c23d391e2a2bb1dd45ee928fc9e3c811977a3856b5a5a0778011053c \
+ --hash=sha256:f5eafa25c23d391e2a2bb1dd45ee928fc9e3c811977a3856b5a5a0778011053c \
+ --hash=sha256:f5eafa25c23d391e2a2bb1dd45ee928fc9e3c811977a3856b5a5a0778011053c
+ # via mozlog
+multiprocess==0.70.14 \
+ --hash=sha256:150f1d52633d4fc9baab1300005cfe83db6219397be4971a77d7fa77ab1ca25f \
+ --hash=sha256:1f526675380b915b1e38b3c34e8194285ee854cc3eeab3c87de18adbf0761864 \
+ --hash=sha256:40983e76911725cea7031791ceb707ae23039a5804649599c2fc0bd807ced40e \
+ --hash=sha256:79e431dfb5f8eeee31457884ac157b8f985b4d1aad267c44f75cdffa82040870 \
+ --hash=sha256:7ac72d24fc7486ed60397e9ff60dbc033def6756fecf48232addc426ad20dca4 \
+ --hash=sha256:caaf3420bc77adb23217238856e6e218db5d6ea3372af9f3c9600754a1fb09eb \
+ --hash=sha256:e0117c25d394a3b3a947ecb4f3ba62597848c5f82b56d6c19793ff28b6ceb4c5
+ # via pathos
+numpy==1.23.5+chromium.4 \
+ --hash=sha256:1f09f09cd02263534f3f3376dc290bb33b2d16c10eb85fd1b0491b02028f559c \
+ --hash=sha256:298e858d71fd8e2d963414be9d85288f4e04df92c1025128d7917b42a82768cd \
+ --hash=sha256:3b6a0b96bcd6ce7393a92adbc00892d4af02f6c04936f671979dc285d9ed7c38 \
+ --hash=sha256:3f58475348b7b887ebc9cf63ec63173b52acf988faa9d8e5b797c7befbf9a1c4 \
+ --hash=sha256:44e321bb428153a6e396e7328d808def77c3cef8b3d93c5f9e4108d04c558997 \
+ --hash=sha256:48f8428d17325de892ac883561d6ba485978f7b924e0b610d9f31ce968dbabe9 \
+ --hash=sha256:5af4510cebaf0bb81ea9af5368fbf5a6ce21b4963dbd3e31fe0f796cffefc9ab \
+ --hash=sha256:69098ac2618e6a95064a9096b8eb1e8a737b2105aeec1a21b4eac62ba8d48fa7 \
+ --hash=sha256:785d2cec75a1dfb812be1a3a24da0c82ba60729c7b53e6bd132fda1479f6250f \
+ --hash=sha256:9241565b3fb920fed1268068efbc0e46fbc4f2d6056495574507adaa32741f5d \
+ --hash=sha256:929b83e970651c3971f1cb83ed2c3345266507487702155cb7007d2507d18771 \
+ --hash=sha256:9b59a1ae3c94fd499099e304a23dacb486c030f13e826f79eb77470fa618d979 \
+ --hash=sha256:a5706db7526e7edfafafe4fe620625c346a30f4b97acc8484f55ec5e83d25a74 \
+ --hash=sha256:b80a749316f06b7d6a48cb1e7c5b0cf3de9a2b4ac6d001b3a6a0e5e800ea0f20 \
+ --hash=sha256:eaf5507daa4d82341ad08bb2e5fd780c10e021ad48f8555ebf36027e2809787f
+ # via
+ # --override /tmp/overrides.txt
+ # pandas
+oauth2client==3.0.0 \
+ --hash=sha256:7c47d7f77c785398c13452821ea11142ebe764229333ec0cf505502f6756291c \
+ --hash=sha256:800663b3d5c94dcb5fd5fdfcedf72cbc281e0e309818487d20f93385eccfb8be \
+ --hash=sha256:8bccbd064280d1b2c00aa6c45f471234e75c2af75aa504a9362ddc9a88b0a937 \
+ --hash=sha256:9eedc9d8e25a38492172805e6aa690f9e213bde26eadde3a26076920dee78384 \
+ --hash=sha256:a6034847934e8dc6b676dc5648486e7a8bcd76a0669c3ca7140a5b86162df243 \
+ --hash=sha256:d91e655716b132d06ea9bf4bf02d0a652ada98fde38c9ee0ab6a844b9aa54833 \
+ --hash=sha256:f1a083936e34f1d72eb307ac9fc0ae1655200707600f8e4c600400dd08ca9575
+packaging==16.8 \
+ --hash=sha256:12ab96d61fb0cf16023ec1d79d87e7372868c5a883d08378eb29741b1015d243 \
+ --hash=sha256:55eaf1bd3efbcc9bbfd7562abaf4b152b9b67f06da364280d04731c18eefde6e \
+ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
+ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
+ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
+ --hash=sha256:cfd34b5b8bab8dd8ae8563cb2871894dd24cae16c8b3cace8b0ab5768d40f1bd
+ # via pytest
+pandas==2.2.3+chromium.1 ; platform_machine != 'aarch64' or sys_platform != 'linux' \
+ --hash=sha256:21c49f76db22c6f13370b357112ac28443d0b57c8c8476613385caa5bf8d7d23 \
+ --hash=sha256:3322b4dba5f4d132abb393e202f22ec7777db4b9b5e7eddba5060ead0d975169 \
+ --hash=sha256:c6f279f603a83f59fbc1b61306e644ca82c48b61ada7b335aa6dbb63c1e38f12 \
+ --hash=sha256:d30cace0d7fc615a216e863bbda793f9eb8ed83b561af458bcbc6d7c421ddc63
+ # via --override /tmp/overrides.txt
+parameterized==0.7.1 \
+ --hash=sha256:aa0891c15d6899b49751bffb204a2a799f9fa71b075c312071fa59f60701e651 \
+ --hash=sha256:aa0891c15d6899b49751bffb204a2a799f9fa71b075c312071fa59f60701e651 \
+ --hash=sha256:aa0891c15d6899b49751bffb204a2a799f9fa71b075c312071fa59f60701e651 \
+ --hash=sha256:ea0326ba5bbbe7c427329a27b75003410df07d1173ca254976f8f5a64922c322 \
+ --hash=sha256:ea0326ba5bbbe7c427329a27b75003410df07d1173ca254976f8f5a64922c322 \
+ --hash=sha256:ea0326ba5bbbe7c427329a27b75003410df07d1173ca254976f8f5a64922c322
+pathos==0.3.0+chromium.2 \
+ --hash=sha256:063f0c266aeca5b10f74a9878531d9e34d6933403bdcb6d1ad7a4a454c8ced60 \
+ --hash=sha256:063f0c266aeca5b10f74a9878531d9e34d6933403bdcb6d1ad7a4a454c8ced60 \
+ --hash=sha256:12bba7d6829ea348a42065c2fd0d1398e421d1ebd37f19701e60bff22723c2b5 \
+ --hash=sha256:2ec15ab1dda26b4f46a8cffbf08477412c0dd23225f0ff4de8395f440499ab15 \
+ --hash=sha256:2ec15ab1dda26b4f46a8cffbf08477412c0dd23225f0ff4de8395f440499ab15 \
+ --hash=sha256:4459c545a931689c34f75a45a677abb7fb08140bbfe9de5ace08d4c6a468d655 \
+ --hash=sha256:4459c545a931689c34f75a45a677abb7fb08140bbfe9de5ace08d4c6a468d655 \
+ --hash=sha256:5ab009f9825e372091d273b6815b791c85475db824b8dc3e62219f184672d626 \
+ --hash=sha256:6507053f4e89d5ce9f5ec11f331b8d423a777dbe199c84337b9e2d172e1cc120 \
+ --hash=sha256:b70f874130da27fe245b9b0fa2adbe6ac585b0bf2bdeefe2e117a646c7bd9008 \
+ --hash=sha256:c389796934467ea9209d741665bd4f3a83c331810abdb33f3d483bee6988d12e \
+ --hash=sha256:cb1f42b2e6ebed6a035a1a78f4eda9cf5dbfb4ec2332e52e19a28ac32c5644aa \
+ --hash=sha256:fc63d03deb79a9296f6fcb875a24ac95d335ebcfa0c6e085675f46cc4eebf846
+pbr==3.0.0 \
+ --hash=sha256:86f0e548dfbdc1ad12559725a66b5e1ca0b2e0744940da1ec89991fd11a2b32b \
+ --hash=sha256:86f0e548dfbdc1ad12559725a66b5e1ca0b2e0744940da1ec89991fd11a2b32b \
+ --hash=sha256:86f0e548dfbdc1ad12559725a66b5e1ca0b2e0744940da1ec89991fd11a2b32b \
+ --hash=sha256:de776c05e4a75e4d266e03a3d82ba13894e2c51a2fa52501187361ac180d4468 \
+ --hash=sha256:de776c05e4a75e4d266e03a3d82ba13894e2c51a2fa52501187361ac180d4468 \
+ --hash=sha256:de776c05e4a75e4d266e03a3d82ba13894e2c51a2fa52501187361ac180d4468
+ # via mock
+pillow==10.4.0 \
+ --hash=sha256:026dbe011f1f3c78487614add2b5987c5040139c32113e1027a8f2882d2529d3 \
+ --hash=sha256:37251922614046bc350a732d801320fcab619b8235d506976c5d0601cedc7d4b \
+ --hash=sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3 \
+ --hash=sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6 \
+ --hash=sha256:635d61f9fbd8176a8674cf78deb7ae2120efded91b7f4400e1cf688b7bfd20ff \
+ --hash=sha256:6e9d3cec497f3af115e22e4cfcfb873ec34d8824948a1bf84585f282cad4ae65 \
+ --hash=sha256:721f288a22b8c11dd8c1842d9dffb36dc60fc57526e50080006a4ec45e1b6b9e \
+ --hash=sha256:8205c5353fbc00ececdf4fbc1eef6d93123481849ee9404fa6659a639e1520b8 \
+ --hash=sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736 \
+ --hash=sha256:96e334e6aefd6351a50390480d766cc4b016f54f9b1e2957dbf7b60b38f67b21 \
+ --hash=sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2 \
+ --hash=sha256:b650012afc6a3aabf574564e929aa38e1638a7b0ad66be08582d06e00861b258 \
+ --hash=sha256:c711e74ff7c6ccecf38fcfa86ccd908977c25fbe670e09e14940654c8ccaa2de \
+ --hash=sha256:d26a252c34d97a4c8e595182e83bd5e5561621f583a48edbe607fa90abf63042 \
+ --hash=sha256:d3a08281dcc4517dc91933befea03770b1327b8aea655592ee7c6b9a84063f2d \
+ --hash=sha256:e8731c73977a1bf6f3be6e499c60ea901afe914814ae9dc5dbe03b92f0e06521
+pluggy==0.13.1 \
+ --hash=sha256:8421107c23be143ff540859538d9ab530b99911d3f65bf77b625dc93b03c92c3 \
+ --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d
+ # via pytest
+pox==0.3.2 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61 \
+ --hash=sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61
+ # via pathos
+ppft==1.7.6.6 \
+ --hash=sha256:f355d2caeed8bd7c9e4a860c471f31f7e66d1ada2791ab5458ea7dca15a51e41
+ # via pathos
+protobuf==4.25.1 \
+ --hash=sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6 \
+ --hash=sha256:ff433d2aa814b498df6c54a8e14b716522b3ea61cb1a16950a25770374383eb1
+ # via
+ # google-api-core
+ # googleapis-common-protos
+psutil==5.9.8 \
+ --hash=sha256:048f45fadd4abbff7b2777537149ff17954180c84b0ae8f83e1223dedfe9a9f1 \
+ --hash=sha256:14b2aa38686bf66b915482c2985b69eb2d38f6027c6b4dabfbe9f2b6e57b0c09 \
+ --hash=sha256:338de09c5db9257deb16d9895558863c832d4af7a809616bbc4400efc99a7bbc \
+ --hash=sha256:39f8270b61fdd9647919257ebe588989b2a5e9bf31c2f1598e9aceac897f52c4 \
+ --hash=sha256:47998e2bbb7c54add76df84a337bfdc1fe9c325852047da746c7b5bffef06c75 \
+ --hash=sha256:52deb67064b88cdc33219b7378ba3959f0c3811b2a1bcfdc10fba05d5a8155ce \
+ --hash=sha256:5ae2b403ea7c5c040070b718d1f2276822b88f5562941b51ba7fb6603409cb75 \
+ --hash=sha256:84b14ade9c0df00e95bf52a6723a3876ac48ec77b58f77b5c0df11e934bc1cea \
+ --hash=sha256:8b523fc1f8baba813c1f4bdb9af601616e6e061fe22229803bcef6d70c248142 \
+ --hash=sha256:9230458145f4ef87ad6528bb4747d2bfe254dda325849c26577192823755c09a \
+ --hash=sha256:aa4b0dffd73bf962630a7128c0a6b9831f1b750455a262f1fc37cf9930068e84 \
+ --hash=sha256:b38e45716e7bb107c65875e47af505023972b3762c68bf36b221786607e301b8 \
+ --hash=sha256:c2244067b9e05678afaff2812f86e6742d1467037068dc4795ed0a333f1b0a8f \
+ --hash=sha256:d2ef673e3ee83d69238f55dbd6a15f20a840ea10d4e65d7facf73392666dcd66 \
+ --hash=sha256:e36c1e1000e2774fafb908f69c564a0e491a448d5f5fcb28bc917586596d8f7b \
+ --hash=sha256:f7eb76683ac9558cbf4cef8cc7f859a65ed6412862073862fb3e855e4796d48e
+py==1.10.0 \
+ --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a \
+ --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
+ # via pytest
+pyasn1==0.4.5 \
+ --hash=sha256:58bbb1feb769c42205eaf2cfc25fc2075c38f34528bddf7c14306f2f4d40145f \
+ --hash=sha256:58bbb1feb769c42205eaf2cfc25fc2075c38f34528bddf7c14306f2f4d40145f \
+ --hash=sha256:90aed86e2b1bb2ddf5b208f1c640578f36e8b093b6d93840f8c2fb6c2a50f68c \
+ --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e \
+ --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e \
+ --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e
+ # via
+ # oauth2client
+ # pyasn1-modules
+ # rsa
+ # service-identity
+pyasn1-modules==0.2.4 \
+ --hash=sha256:0d0c7fe74b8a90ca500a4575dd144a901e7243c97529adeeec9f318079aae181 \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:79580acf813e3b7d6e69783884e6e83ac94bf4617b36a135b85c599d8a818a7b \
+ --hash=sha256:f85c2e8bfac150869a431b58bb77596fd22c02f4206fc7bf71aacc215a2391d5 \
+ --hash=sha256:f85c2e8bfac150869a431b58bb77596fd22c02f4206fc7bf71aacc215a2391d5
+ # via
+ # google-auth
+ # oauth2client
+ # service-identity
+pycparser==2.19 \
+ --hash=sha256:0a95ac18b01740fdfdb6e4e9a910d68da6d4e16188908fc9f90a90c365ceebfe \
+ --hash=sha256:0b124f28a2dd63b172100564eae285503e3d05e248b340a08ece41dffef30ee8 \
+ --hash=sha256:631c2972be26c3c77f523e0b8681e496dbc1813b34f7aa88230a22d98d61447a \
+ --hash=sha256:67015b4cf53a668f2d70c9a09a1e062afe677355ceee54cdaeac94897b37c70a \
+ --hash=sha256:7353ea66ac83bd9b1d0df4551c8ec1f8fea420e9fc57fb489aff235321266a35 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218 \
+ --hash=sha256:e315b9d9958839880f12409b385d4fdc41207a0b392f56ef5ead452489fe7218
+ # via cffi
+pyfakefs==3.7.2 \
+ --hash=sha256:8259042941a0690b89d92d55ed8fb1720ca4a83db87713d2fb9717cfd5e5f18e \
+ --hash=sha256:9b3718c246ec98ca8269c6d19e9c33c1cc3af32e22bf189bddb921bc7fc16e39 \
+ --hash=sha256:9e98f7179507c650ce17824e6493a4effef03e726ffe9f84917f5800266edcc9 \
+ --hash=sha256:f415f38fe488b46974700af95f22b37e951a51159403eb9011ca37db0fde8b97 \
+ --hash=sha256:f415f38fe488b46974700af95f22b37e951a51159403eb9011ca37db0fde8b97 \
+ --hash=sha256:f415f38fe488b46974700af95f22b37e951a51159403eb9011ca37db0fde8b97
+pylsqpack==0.3.12 \
+ --hash=sha256:0bbb56fd2d4d3a76fca985ba82db4738a96e01f4b9af8abe6979ce5265265cd7 \
+ --hash=sha256:458b00d38d6cb79c9c60c04945392e1a80ff8504ea8e26a95d9d74cf5e509873 \
+ --hash=sha256:5d2451d7855c41cd394a3ebc35b349e7d4086556339e8c25950f4dc8639cf9cb \
+ --hash=sha256:65528edbbb2754b018a9b0c4a28fd2199c9c18cf5732b3844807db13fd8c52e5 \
+ --hash=sha256:695da4bc1d597ae4c77f9468c22d56d17e6d5c6b758df20915bc083515d3de0a \
+ --hash=sha256:7b14af83f1adbc2ef1da48514f309e99ba03ac9a67eada9409738ec3abfdf9bb \
+ --hash=sha256:8410c3cd5ead8df2eee83df07b01ebab8be62e3c2f3d643587a0cb0a727bea4b \
+ --hash=sha256:90cbbb01550fd7ddc41d89debcead03624f95b40148a195f11347e916f3a0259 \
+ --hash=sha256:9423ca122fb276052490978ea60a389cad97db8be6dcd7382523d580b329edff \
+ --hash=sha256:b3c8e1d15b944fc74b02ac1529bf8cf4702f50367d4015907a5575dd2710d1ba \
+ --hash=sha256:b3c8e1d15b944fc74b02ac1529bf8cf4702f50367d4015907a5575dd2710d1ba \
+ --hash=sha256:b9696910c4fecccffbb9d576b6a9dfb42709f69056b93c37c3d9552ea521155b \
+ --hash=sha256:c1afb58dcd895c06ab4714acadaff1f239f2b6e3b6d0fe3d40d75af11bb1fc86 \
+ --hash=sha256:e6f3932e4b72215e87739863b08db7da226e89d8652495f4c5d16dbccb747577
+ # via aioquic
+pyobjc==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1cc046cb45aaccb34d9b1c04f631102f0e012aeae36e032c4c5b26b0a2a0890b \
+ --hash=sha256:3b878065a1dcea5a5bc7821a815f68fb4899684f3ff90a5c9e34d5654355d090 \
+ --hash=sha256:49dd78e35d11e2abea30bf331641bb44a68bb4546c65467f23c707ee349ee6a0 \
+ --hash=sha256:ea39b44bb15701bb86747f4aac6dd8c0ab0cf4bf13a9550848d9837bff8f88ac
+pyobjc-core==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:02d0a674a86fc0ae19ec19592bb58bd3dc995fa45885e266b9d779d8f1fabb36 \
+ --hash=sha256:0d5b711d1cc49f471e7b3ce298faec5b84f9a14955941d92af055fb21ab5bfed \
+ --hash=sha256:45c6ca02801171697e7d7b1916a8a0b8c651e1ef9c3c6ac7548c89dc0c706e5e \
+ --hash=sha256:f1f584a85c7d77dd815aad2e65da1d718f5211c511e59ca1cfe5c287ecb27782
+ # via
+ # pyobjc
+ # pyobjc-framework-accounts
+ # pyobjc-framework-addressbook
+ # pyobjc-framework-adsupport
+ # pyobjc-framework-applescriptkit
+ # pyobjc-framework-applescriptobjc
+ # pyobjc-framework-applicationservices
+ # pyobjc-framework-audiovideobridging
+ # pyobjc-framework-authenticationservices
+ # pyobjc-framework-automaticassessmentconfiguration
+ # pyobjc-framework-automator
+ # pyobjc-framework-avfoundation
+ # pyobjc-framework-avkit
+ # pyobjc-framework-businesschat
+ # pyobjc-framework-calendarstore
+ # pyobjc-framework-cfnetwork
+ # pyobjc-framework-cloudkit
+ # pyobjc-framework-cocoa
+ # pyobjc-framework-collaboration
+ # pyobjc-framework-colorsync
+ # pyobjc-framework-contacts
+ # pyobjc-framework-contactsui
+ # pyobjc-framework-coreaudio
+ # pyobjc-framework-coreaudiokit
+ # pyobjc-framework-corebluetooth
+ # pyobjc-framework-coredata
+ # pyobjc-framework-corehaptics
+ # pyobjc-framework-corelocation
+ # pyobjc-framework-coremedia
+ # pyobjc-framework-coremediaio
+ # pyobjc-framework-coremidi
+ # pyobjc-framework-coreml
+ # pyobjc-framework-coremotion
+ # pyobjc-framework-coreservices
+ # pyobjc-framework-corespotlight
+ # pyobjc-framework-coretext
+ # pyobjc-framework-corewlan
+ # pyobjc-framework-cryptotokenkit
+ # pyobjc-framework-devicecheck
+ # pyobjc-framework-dictionaryservices
+ # pyobjc-framework-discrecording
+ # pyobjc-framework-discrecordingui
+ # pyobjc-framework-diskarbitration
+ # pyobjc-framework-dvdplayback
+ # pyobjc-framework-eventkit
+ # pyobjc-framework-exceptionhandling
+ # pyobjc-framework-executionpolicy
+ # pyobjc-framework-externalaccessory
+ # pyobjc-framework-fileprovider
+ # pyobjc-framework-fileproviderui
+ # pyobjc-framework-findersync
+ # pyobjc-framework-fsevents
+ # pyobjc-framework-gamecenter
+ # pyobjc-framework-gamecontroller
+ # pyobjc-framework-gamekit
+ # pyobjc-framework-gameplaykit
+ # pyobjc-framework-imagecapturecore
+ # pyobjc-framework-inputmethodkit
+ # pyobjc-framework-installerplugins
+ # pyobjc-framework-instantmessage
+ # pyobjc-framework-intents
+ # pyobjc-framework-iobluetooth
+ # pyobjc-framework-iobluetoothui
+ # pyobjc-framework-iosurface
+ # pyobjc-framework-ituneslibrary
+ # pyobjc-framework-latentsemanticmapping
+ # pyobjc-framework-launchservices
+ # pyobjc-framework-libdispatch
+ # pyobjc-framework-libxpc
+ # pyobjc-framework-linkpresentation
+ # pyobjc-framework-localauthentication
+ # pyobjc-framework-mapkit
+ # pyobjc-framework-mediaaccessibility
+ # pyobjc-framework-medialibrary
+ # pyobjc-framework-mediaplayer
+ # pyobjc-framework-mediatoolbox
+ # pyobjc-framework-metal
+ # pyobjc-framework-metalkit
+ # pyobjc-framework-metalperformanceshaders
+ # pyobjc-framework-modelio
+ # pyobjc-framework-multipeerconnectivity
+ # pyobjc-framework-naturallanguage
+ # pyobjc-framework-netfs
+ # pyobjc-framework-network
+ # pyobjc-framework-networkextension
+ # pyobjc-framework-notificationcenter
+ # pyobjc-framework-opendirectory
+ # pyobjc-framework-osakit
+ # pyobjc-framework-oslog
+ # pyobjc-framework-pencilkit
+ # pyobjc-framework-photos
+ # pyobjc-framework-photosui
+ # pyobjc-framework-preferencepanes
+ # pyobjc-framework-pubsub
+ # pyobjc-framework-pushkit
+ # pyobjc-framework-quartz
+ # pyobjc-framework-quicklookthumbnailing
+ # pyobjc-framework-safariservices
+ # pyobjc-framework-scenekit
+ # pyobjc-framework-screensaver
+ # pyobjc-framework-scriptingbridge
+ # pyobjc-framework-searchkit
+ # pyobjc-framework-security
+ # pyobjc-framework-securityfoundation
+ # pyobjc-framework-securityinterface
+ # pyobjc-framework-servicemanagement
+ # pyobjc-framework-social
+ # pyobjc-framework-soundanalysis
+ # pyobjc-framework-speech
+ # pyobjc-framework-spritekit
+ # pyobjc-framework-storekit
+ # pyobjc-framework-syncservices
+ # pyobjc-framework-systemconfiguration
+ # pyobjc-framework-systemextensions
+ # pyobjc-framework-usernotifications
+ # pyobjc-framework-videosubscriberaccount
+ # pyobjc-framework-videotoolbox
+ # pyobjc-framework-vision
+ # pyobjc-framework-webkit
+pyobjc-framework-accounts==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:519e28359a3144cfec7adbbc4cf8136706b9bfc2d9f7ba34b84802297fe39499 \
+ --hash=sha256:519e28359a3144cfec7adbbc4cf8136706b9bfc2d9f7ba34b84802297fe39499 \
+ --hash=sha256:519e28359a3144cfec7adbbc4cf8136706b9bfc2d9f7ba34b84802297fe39499 \
+ --hash=sha256:ac870dd990d5e184dfb47d314627af342f5d71935881e98cdff2bba083582fd5
+ # via
+ # pyobjc
+ # pyobjc-framework-cloudkit
+pyobjc-framework-addressbook==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:534412e5adab44bc799d87a7e7247d19259739cdf1cb935090e438ddc2d00882 \
+ --hash=sha256:55347434ea271a89c807a89344c8f420a810a1479076294b080ed1914c8184da \
+ --hash=sha256:5fe07df44850074a02f37e84c97ccd07659ec1c8c753f87d410ce1cc0bb6aa7b \
+ --hash=sha256:95783846dcef30ad056746907b4b156fae6e3cf3827e222395bb2002bc1b73dd
+ # via pyobjc
+pyobjc-framework-adsupport==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:31060beb83841f6ffd4fb6c6bfab6d537a51192720ef7b554661d5432570a052 \
+ --hash=sha256:c3f0258c288313baec5db08f650d9fba6e0455a2f09b86e916e90df21efb0e78 \
+ --hash=sha256:eff4562cc80c25876f1659adc4133c1b35a2799050c7fc99796a551bb2649627 \
+ --hash=sha256:eff4562cc80c25876f1659adc4133c1b35a2799050c7fc99796a551bb2649627
+ # via pyobjc
+pyobjc-framework-applescriptkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3a4212f6015678a37414757cc8f9bffba9080af8a7c59a996bdb5aaaa481dbe4 \
+ --hash=sha256:dfa3df63532214d7dfd345776c0dd0d6ecc95ffa52950e507f56495218234b96 \
+ --hash=sha256:dfa3df63532214d7dfd345776c0dd0d6ecc95ffa52950e507f56495218234b96 \
+ --hash=sha256:f7020b37cbcb01ba88638f3af35b62d541b3266360cd8be1a15d5e899a701c08
+ # via pyobjc
+pyobjc-framework-applescriptobjc==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:24601d5773bf46be4654650cc119860e07f9e99778856b581d7db9237f32c633 \
+ --hash=sha256:24601d5773bf46be4654650cc119860e07f9e99778856b581d7db9237f32c633 \
+ --hash=sha256:c1109ad7fe08abc53e4d9971d77f3878e0c2422b008737a404d716e28301d7f6 \
+ --hash=sha256:e305081556cf4b75adaa807b7e3e424ee949e685f47682dc6c2092e9587fe012
+ # via pyobjc
+pyobjc-framework-applicationservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0482704c28e52d702df654e879d2a5eff073305ffe523a69eab7c94f68dff670 \
+ --hash=sha256:56435724dca1779a19421a253a84a34cf398af0ac1e644b9906e8b035a8d4e9b \
+ --hash=sha256:a55b35f9990488287aa4bd9f6ce6d146b192c5e665c0b22e2e6517bc470a1c1e \
+ --hash=sha256:ce10afeb4a601f48f758d41b9887610947d78ff20163e5e9b383a0a5b97fabd8
+ # via pyobjc
+pyobjc-framework-audiovideobridging==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4f97c1378712ce96e368cb027bffcf84dc380985da0a2df9763f95a4d00ffe24 \
+ --hash=sha256:512e1d92897ce6956719661349a43905abe69fe842357d40c9af79ddf996338d \
+ --hash=sha256:97954ded73eb0d465405a67176bbf12a14deec8688bf37a8979e42088d14837e \
+ --hash=sha256:97954ded73eb0d465405a67176bbf12a14deec8688bf37a8979e42088d14837e
+ # via pyobjc
+pyobjc-framework-authenticationservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:13680c98d5582cd02293ec18b2c03359129edf3ae54d961b80196f01a0639525 \
+ --hash=sha256:52592293003563ffbf10ac226b49e8d066ff7ecb74d93bb4cfa26f40922a4980 \
+ --hash=sha256:8c8dc4b20143e9501567d02e3d80f913f306035dda1d5ba641a6b74464da42f7 \
+ --hash=sha256:fee53c77bb665ddfaabdfd898c3d7000efc8e94181d8f946cbca01e55720aaac
+ # via pyobjc
+pyobjc-framework-automaticassessmentconfiguration==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1695816971e09b684206bfe5983083067b04cd8a00a1668752a0882c15a2aa9e \
+ --hash=sha256:6908e94c74b6d1b0a0e8edbef57b6a4c90e2a44dcb72e299a76e6fee15524847 \
+ --hash=sha256:97bea072d37ed8b5ce541bd821a78cd48de5b63d6f0d55cffcee7e5a6c756428 \
+ --hash=sha256:ee0b15954ff9583cf4d5d89fcce95d2ef2e48c6bbf2f2ff23b452caa02610d49
+ # via pyobjc
+pyobjc-framework-automator==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0ab20df31e526f0b049487724fb7ea27c811065bfbf5b69486685688e0f3a499 \
+ --hash=sha256:160cefc779519f5b3afc2daad5c9af9f93b5827dd704377a910e98ed378ed065 \
+ --hash=sha256:160cefc779519f5b3afc2daad5c9af9f93b5827dd704377a910e98ed378ed065 \
+ --hash=sha256:d9f82a8d6aebfa3f46f764a67e66142c68b6214cc62579db9fd61ad95274ac5d
+ # via pyobjc
+pyobjc-framework-avfoundation==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:027db0bb63c40e3f6b116ab20bbf51749e7d42ec17b626e3bfb2c32d6249d97e \
+ --hash=sha256:304815544be66c56c367cdd815bab054f82e138284b027dc812c747a973397a6 \
+ --hash=sha256:81ba59ed0de8b8c9e3230f94aae7744ecc7ad4facb66381d44e857399ce67128 \
+ --hash=sha256:ab9d1b21ea2ce7b01030b549029ee2a3146cdb1734d2e4ca07fca394c8d00362
+ # via
+ # pyobjc
+ # pyobjc-framework-mediaplayer
+pyobjc-framework-avkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1823907932e185c6de552fa23bc0307db05fe40737e06afa4bd21f61f616867d \
+ --hash=sha256:5ffda7e7595070720277f43d7c55bbc5ce1bcca757a4d6400a0b99bc0289f643 \
+ --hash=sha256:6d5db4368a402c4be4c02d15b1ef6ccd921bd557d380356c940eec30df5eb8e4 \
+ --hash=sha256:909b16edc8aff3763b10872afa910dc24e0aa5cd526580cef1b022b4f91775da
+ # via pyobjc
+pyobjc-framework-businesschat==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:badcd9dfe1a97fa460e39aec5d9e1fcfbb9c5a271f8b58301dd488a4c0bc6a02 \
+ --hash=sha256:badcd9dfe1a97fa460e39aec5d9e1fcfbb9c5a271f8b58301dd488a4c0bc6a02 \
+ --hash=sha256:f52fdaad5af7759ee1df4a3c3db45a025b2e3989cfe6e5879d5927b60bdf9259 \
+ --hash=sha256:f851eb32f3722a345267cd8accaacd3535d01a4e1750a1e5c4b989c9b4ca4948
+ # via pyobjc
+pyobjc-framework-calendarstore==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:26d576b80a054a502e569b977b07df5afd195b9e63c036cab018805dfbde7e0d \
+ --hash=sha256:4b392d396f48b2990c4fdef4bd253a839723d02f8faa0007c63b0a11186a4ace \
+ --hash=sha256:8d5a52bf2c9f42ddb1d2dd9cbb73bd10765e101e26200f9518ce6af55fdf4c12 \
+ --hash=sha256:8d5a52bf2c9f42ddb1d2dd9cbb73bd10765e101e26200f9518ce6af55fdf4c12
+ # via pyobjc
+pyobjc-framework-cfnetwork==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:540caeafea6560335eae7cc2e2c9639c9dd6253bcf65517fb2b5c8c5e58ce820 \
+ --hash=sha256:65d5bb0a146b1f4531631c1a1e8ca90fdc5fc4448331b77848c87512e1c27833 \
+ --hash=sha256:729e053a58c90871d65e1b65ab5d1eeb4794aca97b03668154677390565786ed \
+ --hash=sha256:ba7801708c0b38980552966daf9d26e68da6c29d8e0d0e4f2a3f259c3b54455d
+ # via pyobjc
+pyobjc-framework-cloudkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:8b4a0cfc5614c88e0d328357a1a7bb7b2e8a2de1f68f31621c0f3ab4d226b32e \
+ --hash=sha256:a76aca73ed9c961fb10cada82970d2ef6d01604bc97b4ee4e9a59bae8ce4f22c \
+ --hash=sha256:a76aca73ed9c961fb10cada82970d2ef6d01604bc97b4ee4e9a59bae8ce4f22c \
+ --hash=sha256:a81b5bc266fdfb26afede14d075bebf0700340f6bfdf299dcdeb0fc2f2b6ef41
+ # via pyobjc
+pyobjc-framework-cocoa==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:08a22ede8cba3ab93dac216a3be9a79e7546f13d578536975f94dd05b876fb77 \
+ --hash=sha256:93e2b103677bb0b814e75b03e959f2cb07f2914f1ff6b4b6acd07ee1e209ea86 \
+ --hash=sha256:bf21e38e3aa6e326da6668f94b24f2dbba6bdb124b5b8e6bdc60d9d238f98a14 \
+ --hash=sha256:d29c9586eb647c14531904dc302189e37c95241d8b8509cbce150cadc5b8b16d
+ # via
+ # pyobjc
+ # pyobjc-framework-accounts
+ # pyobjc-framework-addressbook
+ # pyobjc-framework-adsupport
+ # pyobjc-framework-applescriptkit
+ # pyobjc-framework-applescriptobjc
+ # pyobjc-framework-applicationservices
+ # pyobjc-framework-audiovideobridging
+ # pyobjc-framework-authenticationservices
+ # pyobjc-framework-automaticassessmentconfiguration
+ # pyobjc-framework-automator
+ # pyobjc-framework-avfoundation
+ # pyobjc-framework-avkit
+ # pyobjc-framework-businesschat
+ # pyobjc-framework-calendarstore
+ # pyobjc-framework-cfnetwork
+ # pyobjc-framework-cloudkit
+ # pyobjc-framework-collaboration
+ # pyobjc-framework-colorsync
+ # pyobjc-framework-contacts
+ # pyobjc-framework-contactsui
+ # pyobjc-framework-coreaudio
+ # pyobjc-framework-coreaudiokit
+ # pyobjc-framework-corebluetooth
+ # pyobjc-framework-coredata
+ # pyobjc-framework-corehaptics
+ # pyobjc-framework-corelocation
+ # pyobjc-framework-coremedia
+ # pyobjc-framework-coremediaio
+ # pyobjc-framework-coremidi
+ # pyobjc-framework-coreml
+ # pyobjc-framework-coremotion
+ # pyobjc-framework-corespotlight
+ # pyobjc-framework-coretext
+ # pyobjc-framework-corewlan
+ # pyobjc-framework-cryptotokenkit
+ # pyobjc-framework-devicecheck
+ # pyobjc-framework-discrecording
+ # pyobjc-framework-discrecordingui
+ # pyobjc-framework-diskarbitration
+ # pyobjc-framework-dvdplayback
+ # pyobjc-framework-eventkit
+ # pyobjc-framework-exceptionhandling
+ # pyobjc-framework-executionpolicy
+ # pyobjc-framework-externalaccessory
+ # pyobjc-framework-fileprovider
+ # pyobjc-framework-findersync
+ # pyobjc-framework-fsevents
+ # pyobjc-framework-gamecenter
+ # pyobjc-framework-gamecontroller
+ # pyobjc-framework-gamekit
+ # pyobjc-framework-gameplaykit
+ # pyobjc-framework-imagecapturecore
+ # pyobjc-framework-inputmethodkit
+ # pyobjc-framework-installerplugins
+ # pyobjc-framework-instantmessage
+ # pyobjc-framework-intents
+ # pyobjc-framework-iobluetooth
+ # pyobjc-framework-iosurface
+ # pyobjc-framework-ituneslibrary
+ # pyobjc-framework-latentsemanticmapping
+ # pyobjc-framework-linkpresentation
+ # pyobjc-framework-localauthentication
+ # pyobjc-framework-mapkit
+ # pyobjc-framework-mediaaccessibility
+ # pyobjc-framework-medialibrary
+ # pyobjc-framework-mediatoolbox
+ # pyobjc-framework-metal
+ # pyobjc-framework-metalkit
+ # pyobjc-framework-modelio
+ # pyobjc-framework-multipeerconnectivity
+ # pyobjc-framework-naturallanguage
+ # pyobjc-framework-netfs
+ # pyobjc-framework-network
+ # pyobjc-framework-networkextension
+ # pyobjc-framework-notificationcenter
+ # pyobjc-framework-opendirectory
+ # pyobjc-framework-osakit
+ # pyobjc-framework-oslog
+ # pyobjc-framework-pencilkit
+ # pyobjc-framework-photos
+ # pyobjc-framework-photosui
+ # pyobjc-framework-preferencepanes
+ # pyobjc-framework-pubsub
+ # pyobjc-framework-pushkit
+ # pyobjc-framework-quartz
+ # pyobjc-framework-quicklookthumbnailing
+ # pyobjc-framework-safariservices
+ # pyobjc-framework-scenekit
+ # pyobjc-framework-screensaver
+ # pyobjc-framework-scriptingbridge
+ # pyobjc-framework-security
+ # pyobjc-framework-securityfoundation
+ # pyobjc-framework-securityinterface
+ # pyobjc-framework-servicemanagement
+ # pyobjc-framework-social
+ # pyobjc-framework-soundanalysis
+ # pyobjc-framework-speech
+ # pyobjc-framework-spritekit
+ # pyobjc-framework-storekit
+ # pyobjc-framework-syncservices
+ # pyobjc-framework-systemconfiguration
+ # pyobjc-framework-systemextensions
+ # pyobjc-framework-usernotifications
+ # pyobjc-framework-videosubscriberaccount
+ # pyobjc-framework-videotoolbox
+ # pyobjc-framework-vision
+ # pyobjc-framework-webkit
+pyobjc-framework-collaboration==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:c8c7f65b271f37d06dfed4af7b228008b24117edbe9c2b6cb4fa739ce42a184e \
+ --hash=sha256:f3cbea0595f9ec4ea8072573f75eaf8ce0740009bf81e53bebed409a774406ca \
+ --hash=sha256:f3cbea0595f9ec4ea8072573f75eaf8ce0740009bf81e53bebed409a774406ca \
+ --hash=sha256:f3cbea0595f9ec4ea8072573f75eaf8ce0740009bf81e53bebed409a774406ca
+ # via pyobjc
+pyobjc-framework-colorsync==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3f9d3c0f6c9ad96b256f4e0eccef73f7dea500f3ecc36a10a8fe16cba3d78d68 \
+ --hash=sha256:3f9d3c0f6c9ad96b256f4e0eccef73f7dea500f3ecc36a10a8fe16cba3d78d68 \
+ --hash=sha256:3f9d3c0f6c9ad96b256f4e0eccef73f7dea500f3ecc36a10a8fe16cba3d78d68 \
+ --hash=sha256:934700933994ccf58695d49cbfa28b2138a6885462914dd93847910ac48ec912
+ # via pyobjc
+pyobjc-framework-contacts==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2ebe8c9d120f46ceb923f37cc58245512b2ebdf986cb51f2bdc94092554d4f06 \
+ --hash=sha256:59826b286e4be9456c0405dc872a4b8b746578cf7d03cd2695f331baf2fb55af \
+ --hash=sha256:919001923baaa2903bb687c6a2365d8a7646580e573da4d6a3a83b05489c1971 \
+ --hash=sha256:ab9196d0bfe38a24c4e645c466301507f322f7574348100491bc0a5106d3f4bc
+ # via
+ # pyobjc
+ # pyobjc-framework-contactsui
+pyobjc-framework-contactsui==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:7740d9d68cc10271443b69c1f8a1cd52dc618e3b7330088582a992816671916a \
+ --hash=sha256:a911ee29b0d701c9b77a7c8f655c84f565b560da623f7f654264f1c4e3f3a13c \
+ --hash=sha256:df5031428fecf3055aa7986859b523a37cf80cba0aa802e4f403a283c111961d \
+ --hash=sha256:f2e0a05b1b370fa55d05718cde916b3e0460eba5b848881f1e880f79cfe1ac6a
+ # via pyobjc
+pyobjc-framework-coreaudio==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2c86a13859740665aaa8edd6029aead3bc08adaedf86e74f49c4416d89fb77f6 \
+ --hash=sha256:7e4622ff6bbbabcf596da1c5e9dff923b4aa1fac0db93dc72dfac318a72a6aa4 \
+ --hash=sha256:a49426dd5eaa5991e971f67bb1a328e9a6681959be46004cbc715f7013ac45a9 \
+ --hash=sha256:e2b7f55db3edf599a5bf13df0dbebb2c4281b14161ba68a7889e54f6aaa10cf4
+ # via
+ # pyobjc
+ # pyobjc-framework-avfoundation
+ # pyobjc-framework-coreaudiokit
+pyobjc-framework-coreaudiokit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:5d7979ab8f8af1b9154b682eff3f2ac58720b5782686f86683f8cecbb811db8d \
+ --hash=sha256:7d188105da7acd29fe5b0e1836dbecb93350105cb51bdf7bf6fcb4a8e6f8797a \
+ --hash=sha256:8155057538c15720b97afadfc7fe69846b06647e767d05d04ea1ae08ea8d9c0f \
+ --hash=sha256:e1d9fe8517300472053057b6d0604f0342b052942848613eb7ffbf9be9d45937
+ # via pyobjc
+pyobjc-framework-corebluetooth==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:13a6338ee65dff08466851131dd7e6eb5f08acc4821ca48633f0872e0fe8b282 \
+ --hash=sha256:961ac9be9f46e058c876097a9f9334111aec92ed917522ab0b9313536a3a5ff8 \
+ --hash=sha256:9cbb9376f0307dfbb711328826e76b50da75bd01547a9f1f56286f1a0174ff38 \
+ --hash=sha256:af540b321314227ea3b92bd3729a00280f50f7379571810419e6e542b776e1e9 \
+ --hash=sha256:df4e40d17d755a887b701d7255bef89a64139ef5431f3b2781825854ed17f280
+ # via pyobjc
+pyobjc-framework-coredata==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:de944867061c32afc6ae82af0dbac4c756f56929a3fbca815dc1949625f17fb4 \
+ --hash=sha256:df38856b6295626ffdd30944c407e0aa4d9597c6669244dd810043ab9b5e4aaf \
+ --hash=sha256:e1426496c099f2e8183ce6970df800a1144cf0f6b72ff7dfca0939e205cf360b \
+ --hash=sha256:f8b4c66909469b90504130b54df953a39e7e279b17161e9c91e495a02d4e6344
+ # via
+ # pyobjc
+ # pyobjc-framework-cloudkit
+ # pyobjc-framework-syncservices
+pyobjc-framework-corehaptics==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:457774c6007c78136a6c2cc7e1d8a0070ac65b9fef6d059392a8931b3e52381a \
+ --hash=sha256:adb746f0aa3136b7d91c7531f7d67453203684fe7b93eb496ff9961c4377e046 \
+ --hash=sha256:adb746f0aa3136b7d91c7531f7d67453203684fe7b93eb496ff9961c4377e046 \
+ --hash=sha256:f28fb3cb4fe37d38e9c7b9960ddbfc872664fdbcef4e4afe8b6769fb480463ac
+ # via pyobjc
+pyobjc-framework-corelocation==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4a953b93788d5a31c6bdc87c41201ae55e1318407bf0cec99ac0c9217178f4dd \
+ --hash=sha256:62888393ee3958eaeb18c5e96c2464726a13f50ec8000b8ab86c6dae89003566 \
+ --hash=sha256:8de1ba44b98bdd12d48605694cf648671962e27a585a6ff0e162916cdc71634e \
+ --hash=sha256:96317eebb54a64f3ef243502065c3e515becab5436d7e11d2ab9ff2c610f2e06
+ # via
+ # pyobjc
+ # pyobjc-framework-cloudkit
+ # pyobjc-framework-mapkit
+pyobjc-framework-coremedia==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:9bcabfbe1d700de44653b6ef56b20ee8a2666d561407d372c80ae02d97eee4a2 \
+ --hash=sha256:c365aa3cb00694e62c235dbfaeee6377f87d0a0dda17bf866ac5b47969e503ca \
+ --hash=sha256:e87394fdb45707e027b3b11b2a70c59fb5bdacd9341bba3490b05d9e74641925 \
+ --hash=sha256:fe7f6728308a51dc45bd9d898d9b69d4a1d2c06f4a5793d6e8e74f3c5c184c35
+ # via
+ # pyobjc
+ # pyobjc-framework-avfoundation
+ # pyobjc-framework-oslog
+ # pyobjc-framework-videotoolbox
+pyobjc-framework-coremediaio==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:09ed675a443ce55704a3734b5aa9a34f50e855b6720d8ab35a49794243907e43 \
+ --hash=sha256:0e9b3d4a3e4461114462332854b00e1af6942e25b798c81977acdc542324ccda \
+ --hash=sha256:1de6e69649251772587fce29ce2f3e4f9e47bb7f2eb22e47b9cf616fde5bd3c7 \
+ --hash=sha256:325223c061f9fb37886c9ff9a00e590971ded5e1844ca2da4c4f76a4d1c20d13
+ # via pyobjc
+pyobjc-framework-coremidi==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:47c2d23e592ae69b554a80ab99d3a668322cbf42a90e5131d4c16d8c516b48fd \
+ --hash=sha256:5c128011244f333f13915f3f27a258244902bfbc676746471fd6e03108410271 \
+ --hash=sha256:bfd372170ce3cb95d514afdd31c1df27e316414b37066ce8eacb999c662a48ab \
+ --hash=sha256:defb9f064158c81ebe5f7882337474bd9bc91b2cb643c0db2cf3675d2bef5aae
+ # via pyobjc
+pyobjc-framework-coreml==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0d3b4b7aca34fe83b0b0e3c77c6b884a5172bcf6d9e87d13520568dc35df50ec \
+ --hash=sha256:7fafbcd6da6a36494e974c09cf5b6b322953ed78547da43a39b2597695a95e21 \
+ --hash=sha256:d29bad388f42eb0561d66d2eabdb34ba2bbca3a0b71df9de7e8de99991125dd2 \
+ --hash=sha256:e78b6788f21d8883e2ef6b8b297cf45717ca3af6c1d8baaae4be79e2bd6172e6
+ # via
+ # pyobjc
+ # pyobjc-framework-vision
+pyobjc-framework-coremotion==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0fbf4fe506b5dd195aaedbbd61381fbc19bd088f4115ea5d54159c86cef6f259 \
+ --hash=sha256:362b2c0a5b07ed3399fc59e23cc95d54e6b53bee71395ed17f6b467c3ec3bbce \
+ --hash=sha256:8dfb44b07c5ec50ffd17865f1faeab2eb36e09281aa3f59dfc199c5f431a5541 \
+ --hash=sha256:a93fa4aed245aa1af19d30f9e70dbec4c75dd38609d4d9a536150a4f448db74a
+ # via pyobjc
+pyobjc-framework-coreservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:715ee843718f9e7d7e4b0de2dbc49b9205032fa163b1eb015d61206d1cc07512 \
+ --hash=sha256:b1f4fac38b2a9dd0a2b1146d634212a23decda3506292a1b493cbf57bfd47bcb \
+ --hash=sha256:b6d55920868020ffd5f67f54b7f7b101a3d99b36d8ef634ba8797a4bbf6f9b45 \
+ --hash=sha256:d60dc5f0078826176c4ecdce31ddd6c799050bef05c4261a121485b798a9ced9
+ # via
+ # pyobjc
+ # pyobjc-framework-dictionaryservices
+ # pyobjc-framework-launchservices
+ # pyobjc-framework-searchkit
+pyobjc-framework-corespotlight==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2aed583a34dd835dfdd667b92f2eb6a06b048414688053fe5db7be3854a5d992 \
+ --hash=sha256:39f2223e094769c9121a2b304b16a7a1d5c95c1efdb3fda2bb5987bc2e87604f \
+ --hash=sha256:406532292cc64bef5ab06b74d15f3ed92754dfb14971696c5564806e47ea7b0f \
+ --hash=sha256:ff0fccaac19702b5a251cbabb0785dda78fc6827a5ec99c33e3249045f6fcb6e
+ # via pyobjc
+pyobjc-framework-coretext==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:076a963b0438c5b4fa380dd7a521028891e2f4dbe7e87daef0462cb285a66896 \
+ --hash=sha256:7591ff17c6aa2641727341aacb0675105b3ff0410fae7c20e11a9ab26dc154c8 \
+ --hash=sha256:de6f1d040a09c94a4c156628ae8593c2cb0ddff814603941a214b71876de4a4a \
+ --hash=sha256:e6d242e7e4ed62431dff952b90c18c2deb1cb332d472f51db1e5ca66a06dedcf
+ # via pyobjc
+pyobjc-framework-corewlan==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0dabd26dda1c68d90cdaf996175888830ec06177c7f4513622ace6fc91ac0b6c \
+ --hash=sha256:3007e20dc45cdd3d395f166063446938bb597e88e6a005e594cc76b83f991ef6 \
+ --hash=sha256:38bd451577c07f7d1162dd187400773ae120c9c05fec1ff910d4a0474f84046e \
+ --hash=sha256:a09f1986b92b84de09ac2d3e1136c93da4e051a89d59a639f47f4901e0317a33
+ # via pyobjc
+pyobjc-framework-cryptotokenkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:39e592aa144a8cfe7dadc04d1e91775f6f1a4e598630c34510fa6e8fae42d14e \
+ --hash=sha256:930843a526f95eacf23b05bbc1e56bd116303ff2f0bdf4d0e210c1de7c14952d \
+ --hash=sha256:abaf92e04fa73c030ea43f72d1ab13331c6ff5800b41532980ff3cbf1a733212 \
+ --hash=sha256:cb1896d93aefb7acc6eec22e8a2d095e2abf43cf4ff2dcc605b72e42f6a7fedb
+ # via pyobjc
+pyobjc-framework-devicecheck==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:581b92945ae2ab4e0e95224113f25b63774689ac8d392a234d09d5b5ff09ec41 \
+ --hash=sha256:581b92945ae2ab4e0e95224113f25b63774689ac8d392a234d09d5b5ff09ec41 \
+ --hash=sha256:581b92945ae2ab4e0e95224113f25b63774689ac8d392a234d09d5b5ff09ec41 \
+ --hash=sha256:911f5eb7b2f4080577d4331483cb24ab36b3310e5a6ac9f5681ed9066249c535
+ # via pyobjc
+pyobjc-framework-dictionaryservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:5a0a526356826e3f855d6e551a0521a9fb1719b76ce9c33d00c6fa2f58fa49bb \
+ --hash=sha256:5a0a526356826e3f855d6e551a0521a9fb1719b76ce9c33d00c6fa2f58fa49bb \
+ --hash=sha256:5a0a526356826e3f855d6e551a0521a9fb1719b76ce9c33d00c6fa2f58fa49bb \
+ --hash=sha256:6a398871acd1ef85855877292987384aa4ff2c9624e260853616eeb03120416c
+ # via pyobjc
+pyobjc-framework-discrecording==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:934ac640f189797534824f3ca5ddb2c21cb8cf9b6f5864457eb399169ad485ad \
+ --hash=sha256:9504d5d55194dfb7ad09c3c13609d9a7d2df0c7d3221402097cf75485f158615 \
+ --hash=sha256:b1c7f85978049e5787de6881cca58dd16790e6c6856a09f2567eda52c94e99fa \
+ --hash=sha256:dd728b91987a2bb5c7e80d09d62abbd2ff26a079d4b319c35fef437db3554035
+ # via
+ # pyobjc
+ # pyobjc-framework-discrecordingui
+pyobjc-framework-discrecordingui==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:473a634eb76069aa9a105e914dec7dbb886036b0d9e04bcee9f920c51cb0d3e7 \
+ --hash=sha256:473a634eb76069aa9a105e914dec7dbb886036b0d9e04bcee9f920c51cb0d3e7 \
+ --hash=sha256:473a634eb76069aa9a105e914dec7dbb886036b0d9e04bcee9f920c51cb0d3e7 \
+ --hash=sha256:a274930dd19c324dc94e6e319f1623accb70f62f6d3a66b745e63420741a822c
+ # via pyobjc
+pyobjc-framework-diskarbitration==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:17441138d7c142117efb94c856ce5a3b481ea5a7908c7593b5e2a5c135752323 \
+ --hash=sha256:17441138d7c142117efb94c856ce5a3b481ea5a7908c7593b5e2a5c135752323 \
+ --hash=sha256:17441138d7c142117efb94c856ce5a3b481ea5a7908c7593b5e2a5c135752323 \
+ --hash=sha256:9fcc49a36daabcaf2c308c1818a550a5d34c5c83721bad3d2497cd3047b22cd5
+ # via pyobjc
+pyobjc-framework-dvdplayback==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:191f1b2abe33f065fbb776462f43b4cb5a8a5bfefa488b5f9332d2785e8b1062 \
+ --hash=sha256:5c7eb8935a3cb70d46bb4317dadfc32b68b702d6eb9acd22dd8970c790d6b4cc \
+ --hash=sha256:5c7eb8935a3cb70d46bb4317dadfc32b68b702d6eb9acd22dd8970c790d6b4cc \
+ --hash=sha256:5c7eb8935a3cb70d46bb4317dadfc32b68b702d6eb9acd22dd8970c790d6b4cc
+ # via pyobjc
+pyobjc-framework-eventkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:064f1d1bb3caee2c914018ecc9d52a4872a77946e86e6ede9bd3a577df001eb1 \
+ --hash=sha256:5205fa87d02ec6e37f83a611788d57dfcb74c94ddbf484209eebc5f619852dd4 \
+ --hash=sha256:c7ce0148405370ecf38878d45d95adddf74e3d7b4c58254fae0e9228c14d8c4d \
+ --hash=sha256:c7ce0148405370ecf38878d45d95adddf74e3d7b4c58254fae0e9228c14d8c4d
+ # via pyobjc
+pyobjc-framework-exceptionhandling==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:192d9869af5593cf432e19334fd12ae0e61cb709ba4ff9a3e2f8d4be3ea8a363 \
+ --hash=sha256:36669ca43a4962a590cbd2f3f96b2b9fae48a4e920c0e525a07c63f11c974ffb \
+ --hash=sha256:36669ca43a4962a590cbd2f3f96b2b9fae48a4e920c0e525a07c63f11c974ffb \
+ --hash=sha256:46bc333ef718fca942741441932e3541d9d4c819a07a758207eb2c8e01e13d59
+ # via pyobjc
+pyobjc-framework-executionpolicy==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1dd12021d06c6c6dfa79f24c919e3b8e9ef970ecc5a5aafd3521214a841e381a \
+ --hash=sha256:1dd12021d06c6c6dfa79f24c919e3b8e9ef970ecc5a5aafd3521214a841e381a \
+ --hash=sha256:2baff57684bb5ed4fb24fb1cbdb85b63113916b8ec5a7080e6916663b1d1658a \
+ --hash=sha256:f7a8fcc03758e2f0b8de759320db30ca2b486299dcac6b2e5d1012b06abeb957
+ # via pyobjc
+pyobjc-framework-externalaccessory==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:00c69a9b09af663618b22c4a1601b958733e266ffc38b85864a87b2c82cf403f \
+ --hash=sha256:1114babdf062e9a5111a9534a9fae37135c9c4cdf09ddff35058c3314b6cc66a \
+ --hash=sha256:c9582d6a1b7224ead3a5a0944df080adbdaaab0dc7b428eae82fcf7e9ef1b9f3 \
+ --hash=sha256:f3b648aa4963a9db0c7057eb384245803d689492dd59cea7de822f663ec36e7d
+ # via pyobjc
+pyobjc-framework-fileprovider==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4b2d58f0bc91dc820e37dcf198a073d0e5f1a8bb8a51a8b293faf26532c99f64 \
+ --hash=sha256:9cf8f59536b5e2db223ccb09f54889cc4589cbae46a033b659e3a532f4f18ed3 \
+ --hash=sha256:bfa37d4819d355dd23b6ba8dc1741ac4a5872546946bf5a96767ea617fd37e20 \
+ --hash=sha256:f498af5cdde27fe133386d43d8733ecb4cfe4190cbcf30b785adda8ee3e17ade
+ # via
+ # pyobjc
+ # pyobjc-framework-fileproviderui
+pyobjc-framework-fileproviderui==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:53ad098926d66269254921494f7aa6425df66a8cc8d9bf6eba8a086cb77038af \
+ --hash=sha256:f67a44d6ca5646f282f0fbd1cca5ad70c19e8654272fa7903da0d30c106ba617 \
+ --hash=sha256:f67a44d6ca5646f282f0fbd1cca5ad70c19e8654272fa7903da0d30c106ba617 \
+ --hash=sha256:f67a44d6ca5646f282f0fbd1cca5ad70c19e8654272fa7903da0d30c106ba617
+ # via pyobjc
+pyobjc-framework-findersync==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2fe04f59bf5a44361da2f0968c705481da69af88c13ec92c5ba62fb70764635b \
+ --hash=sha256:2fe04f59bf5a44361da2f0968c705481da69af88c13ec92c5ba62fb70764635b \
+ --hash=sha256:6050c0ee480f8db1b50e9f52809792cc2322e7d30ed2915dc5db57544a34295f \
+ --hash=sha256:aa78025cf3c1f7df6a68e8b2edf679314f59709cf9dd0b54b1bba691008faffe
+ # via pyobjc
+pyobjc-framework-fsevents==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2b293eda3731cf875a0735f1573a7ddb0b294af25e881253341e4dfe77a4f101 \
+ --hash=sha256:55eed25064fb595f26439a684ea5a7e088294d0d61c11c2250825458ddca3ff7 \
+ --hash=sha256:66f0ce01af35f57d28c005686684f05c896068186da12c76e3dbb77b2a7dd73e \
+ --hash=sha256:ee2a03df22b33ad93d1613f707bd574a069c0dda3355a95e773d4758c7389bcf
+ # via
+ # pyobjc
+ # pyobjc-framework-coreservices
+pyobjc-framework-gamecenter==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:28c90420ae47507271b416ac916e9dcbd7d48c648fd94578ecbd92b447463aa8 \
+ --hash=sha256:52c40b01cd3b3cc0b0b95334797daa031aaa8a8216ef5be26a40ef79bc8e89d9 \
+ --hash=sha256:6d25b4f14826d76340b8ff07b6e8f6e56b2c920a4d76b1d434656d0ba6e375cb \
+ --hash=sha256:df43bc50275e25db78b64ef82d8e76f38486b5b4ad9ba6ccbbdc43f73519f26a
+ # via pyobjc
+pyobjc-framework-gamecontroller==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:09a153310bc58adf005f372c5b9c1cb210374cdcd6eb45c8826d3907f9b46f8a \
+ --hash=sha256:2c8f1e30b7012d5aa799949760323bfe2df569134b2a4256aa45ccbe0bcfee40 \
+ --hash=sha256:8725563213e696b7e6a29587d8b9562baaff9341606fc7453b2cd928c97d877d \
+ --hash=sha256:9a2fcce3c8552994b979e5dbf816c6246ba27a1b7cd6672881d19acc163144b7
+ # via pyobjc
+pyobjc-framework-gamekit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1223c243b3a10bccb773ac471b20baf04e007a16513f3d6a78b8921ed358e40d \
+ --hash=sha256:3e3910b22638afca7c864885a914a210fdbf4f444ded9f3c8cffaa9ba708f9da \
+ --hash=sha256:6634c9ac96bfe4d6e0a2c5fc1dc655397d154f75cdee226d6c02d39fa5781819 \
+ --hash=sha256:93a8d53a98483ec7297a66a4769062855c21cabb8f16468356b7c72e1997f39d
+ # via pyobjc
+pyobjc-framework-gameplaykit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:387481e798dadceffebd27d0488fc7dcb7631dc2e2c06c5e845204f9cea266c5 \
+ --hash=sha256:512215ef9eb5147434d65e3378c4ae61602c875ebf3db957600c2d09df3d2a2b \
+ --hash=sha256:728cf8c8c2f374398ab96ee40480bc433b18bb910117c6e28821d4c1ee206e39 \
+ --hash=sha256:ed6235a84feddd8019b91d81f5bf7db1c8a17198e4242e2931cbe68ff1519794
+ # via pyobjc
+pyobjc-framework-imagecapturecore==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1a7b26665c6e557d8bc270a14e8ae4a2a18a8ecc02a04d321cc152739257f823 \
+ --hash=sha256:2c25d6d2e3bf275de845973a43ac75d2479cd41cb6be6d73cc6fa99d4542271f \
+ --hash=sha256:4b3af1bb2aeb0c3d5434f3b29d21e8a967fa640e5bd2b76395d9e5d999b5cd2c \
+ --hash=sha256:a8f9dda226c954c96bf67c651ba751cf961f186901b52fb171b50eee4c199c66
+ # via pyobjc
+pyobjc-framework-inputmethodkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:10b4b649e51e5199a40852466d5236b62f104c61dbd74b5e0ca869e1cdc00378 \
+ --hash=sha256:4f2ed5b0a85c1805e5978aee163dbd1dcdbf815b1ae32faa953eb7161fd65d65 \
+ --hash=sha256:7cbb1628ae650661a00e86f4f5f660a83848b14295a62f4101ebf0248ea2c3a0 \
+ --hash=sha256:8d5e6a3813920c94821caef5db1a955870d6b398c6ee999eac61f0549a1d33e3
+ # via pyobjc
+pyobjc-framework-installerplugins==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:7d96c00f9b1c3049be333433cdabb0dba851c525a3698a3e295c3ed1e6162a67 \
+ --hash=sha256:7d96c00f9b1c3049be333433cdabb0dba851c525a3698a3e295c3ed1e6162a67 \
+ --hash=sha256:7d96c00f9b1c3049be333433cdabb0dba851c525a3698a3e295c3ed1e6162a67 \
+ --hash=sha256:bae9b4d81b965328b2a94bb084bba18ad6ff8d08a7f92bda2d433b07ec67cb2d
+ # via pyobjc
+pyobjc-framework-instantmessage==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:48c5219851038cb815ee1bfa20662120285abecd793be144f9fb3cd604d4695b \
+ --hash=sha256:f3ea71c542c67a3752bc185b1d4031e224054efed2398158dc0da691233ceeb0 \
+ --hash=sha256:f3ea71c542c67a3752bc185b1d4031e224054efed2398158dc0da691233ceeb0 \
+ --hash=sha256:f3ea71c542c67a3752bc185b1d4031e224054efed2398158dc0da691233ceeb0
+ # via pyobjc
+pyobjc-framework-intents==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0c3bd2bfddc7d38be87e3d4d16bc29441450962ea32ecf68d119591ce8f5b19f \
+ --hash=sha256:26200c14bbfe4c25a2c8d72b4d57d34af2a4921661a7f97802017217aa4a6b7b \
+ --hash=sha256:a30279bb879c8288f655ac08a98c6bdcd295b7ab0a74ee3442a014e3e4b06b43 \
+ --hash=sha256:abf36f100c38abbbd0015c8120e5f018d394896075116028df3f15c2a7a06af6
+ # via pyobjc
+pyobjc-framework-iobluetooth==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1f9fe213d00bbfc61367164172f8a898dc2c5a0cb7f66882ce1d74f58d098f24 \
+ --hash=sha256:49cec2fe3a8d6afc5e1167c439ad840a76ea58b78e0e2a81fddc42f65f1c1e2d \
+ --hash=sha256:85695b26332916700d5a0d161fda903b9dfec18137949df5f10224486489b4df \
+ --hash=sha256:e6ed2991f4043134c740a9d0621617817ac57b1222e85952d6c53859436bed3d
+ # via
+ # pyobjc
+ # pyobjc-framework-iobluetoothui
+pyobjc-framework-iobluetoothui==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:043c9307069c3f125127043155e02b135a8ad604d1b4b623175aee0183751d37 \
+ --hash=sha256:043c9307069c3f125127043155e02b135a8ad604d1b4b623175aee0183751d37 \
+ --hash=sha256:043c9307069c3f125127043155e02b135a8ad604d1b4b623175aee0183751d37 \
+ --hash=sha256:f985031886bcf94777495774ceebd76d7f96e5bfe24456d92f2e13e2f98675e3
+ # via pyobjc
+pyobjc-framework-iosurface==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:002d32147d4daedeb9322321d79343bd7c0ff5dfac19d2edebeaff218d57ba64 \
+ --hash=sha256:6b5dca0e5600c78cf698aef586b2ebeaa0787cf2e7d9c6a2bec44a2340f4d83d \
+ --hash=sha256:bb59b5790c0a77e2dae7d20daab13dd9897451f97a7836c26de005d66a75453f \
+ --hash=sha256:bb59b5790c0a77e2dae7d20daab13dd9897451f97a7836c26de005d66a75453f
+ # via pyobjc
+pyobjc-framework-ituneslibrary==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3212bc7acc5e19a12f609c952a389e8a52f86ba950bd4e39f458ab154913f0cc \
+ --hash=sha256:51f163ac282ce38fc29d6d477515eba61233569e7629709f747319495a2035cc \
+ --hash=sha256:51f163ac282ce38fc29d6d477515eba61233569e7629709f747319495a2035cc \
+ --hash=sha256:cf1ebd986d27798fb687dc6f55589eb1b9229c4dc1f27922e8eaff9d7ee0acad
+ # via pyobjc
+pyobjc-framework-latentsemanticmapping==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2530af11a23f5bb2230920ed28d0c3766f09d2e41ebfcc21254d84077e32943c \
+ --hash=sha256:a04ef00377f0ab795c9e05cb9e0f4e43db3f769dceb2f4534b692caef4de3df4 \
+ --hash=sha256:a04ef00377f0ab795c9e05cb9e0f4e43db3f769dceb2f4534b692caef4de3df4 \
+ --hash=sha256:a04ef00377f0ab795c9e05cb9e0f4e43db3f769dceb2f4534b692caef4de3df4
+ # via pyobjc
+pyobjc-framework-launchservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:291131cf0fcbb19e84052c4aa8a24a987f1a94fa80b973c0026dd9ad1ef82a65 \
+ --hash=sha256:6c7ec57587248f193fdb317e55dd5e6903157c262d469faefb73417cc2c02a29 \
+ --hash=sha256:6c7ec57587248f193fdb317e55dd5e6903157c262d469faefb73417cc2c02a29 \
+ --hash=sha256:94abbdc5a3dfe364a5d97c05cb20667947cc230060fb9a348d9a79f8e7f672b9
+ # via pyobjc
+pyobjc-framework-libdispatch==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:31ece9451b25f70a27dfc409c3bfc855bc95def9106ff4f58cafeb2842f0bbd2 \
+ --hash=sha256:3de4744a4b55bb58ad33603095722f6c3d167d22caf0ed1af5ddf43a1b081cb4 \
+ --hash=sha256:6c16500e529a68330bc607019929ff7f7f99dceda569f2f2885636c140a1ff89 \
+ --hash=sha256:f726d4df928a9db4c0582ef02a838284e21a7bb5b5d362b5fff0d136d27e6eaf
+ # via pyobjc
+pyobjc-framework-libxpc==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0406acb0206816c6b063fe88c14922ec4f47d9f9bc19af9507918ae129d54601 \
+ --hash=sha256:6ea9f88ba998731a27a4946bd920767fa3087e922397f9f84a8fc7c6fed975ae \
+ --hash=sha256:96926a5f7e319523297f666e62b32854e133aec65fe65f510da166bc68488093 \
+ --hash=sha256:a7faa551bb2c83d173750c4bae9a26544371c79dd8ed199636637518cb391b79
+ # via pyobjc
+pyobjc-framework-linkpresentation==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3333c2f863bb75388d753da2298ec0e1de51d0ec363c1a4c4d25dace1052a893 \
+ --hash=sha256:e69a5a8fbe647e3c201fb89db3b94573f566c20e67b5f8e677b40152ae1bb9ea \
+ --hash=sha256:e69a5a8fbe647e3c201fb89db3b94573f566c20e67b5f8e677b40152ae1bb9ea \
+ --hash=sha256:e69a5a8fbe647e3c201fb89db3b94573f566c20e67b5f8e677b40152ae1bb9ea
+ # via pyobjc
+pyobjc-framework-localauthentication==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:24aeb0edf1af6eaa17cd1eb9dbe16988e88ce62fbe31943936152a1d3f07060e \
+ --hash=sha256:7a7b09a58025d71e1cc0f7df83d4f71eaf76f988917a25145f06adbc805338e5 \
+ --hash=sha256:e63e242e82ac1ed17917599ac5c29577451c8881dd4aef17ae4096bd2e361b4e \
+ --hash=sha256:e63e242e82ac1ed17917599ac5c29577451c8881dd4aef17ae4096bd2e361b4e
+ # via pyobjc
+pyobjc-framework-mapkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2265de3dfdd987f98258c100d18a37516b9da297be17ca34ef3c8595e1c48b89 \
+ --hash=sha256:8999be166009807fcf2adf3c2807216acb020b4e82acf4c62d67e4d4a9b1c09e \
+ --hash=sha256:bec114d216061075308baf55f19e737cc2cd5f792518e6a813d3859c335ef1da \
+ --hash=sha256:c94503e72b588641afaaa9b1aaa85fcca114537ae07718f00f66b66092250bf5
+ # via pyobjc
+pyobjc-framework-mediaaccessibility==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4c241ea7402c5ee52ec6af8c5b71327be69e1ab54c70e7824cf17f7d61e8372e \
+ --hash=sha256:52e7322fec9f6a8c535abfd2af6ddabf017466584a63e7454f2374b79a39fd81 \
+ --hash=sha256:52e7322fec9f6a8c535abfd2af6ddabf017466584a63e7454f2374b79a39fd81 \
+ --hash=sha256:b1a81602f37a999b36180de08afcb5bab0e6bc12740cc5ee9710b5527596fa9c
+ # via pyobjc
+pyobjc-framework-medialibrary==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:5556f813551be8dbd2c17d08637f6a00092b74efefe369f580df8c1469f81eb5 \
+ --hash=sha256:95e6072f07796c68e566c12efaf374fd837350e776418b17c1b6215a8c176250 \
+ --hash=sha256:97ee76bbb59e7185fda910d6a543108fe5e4099a2bf934365d480015816f91bd \
+ --hash=sha256:97ee76bbb59e7185fda910d6a543108fe5e4099a2bf934365d480015816f91bd
+ # via pyobjc
+pyobjc-framework-mediaplayer==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4ec50a9839806dfc893063cc42ee738a9a9827fb73f2e5897b1e8e7d34e68f9d \
+ --hash=sha256:4ec50a9839806dfc893063cc42ee738a9a9827fb73f2e5897b1e8e7d34e68f9d \
+ --hash=sha256:4ec50a9839806dfc893063cc42ee738a9a9827fb73f2e5897b1e8e7d34e68f9d \
+ --hash=sha256:d446bac0c05fd8700a16ebe30b7caf596f6a5b4ab86e7fe517b73673cd49ce84
+ # via pyobjc
+pyobjc-framework-mediatoolbox==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:550d49dea607c96e0539c33ade1ae806c58d6898f873ca6b9415d966a570432e \
+ --hash=sha256:6bb53a630f1f6956a17b8300fe78d32e5c40aa369c486696d2553b1da453fca6 \
+ --hash=sha256:8a648a57c486555710eac3ee01f2b9d89892cead96aa420943946c65ad0544a9 \
+ --hash=sha256:aef53cff8f6e0aa5ffd930c82af68e97a401cc0ef00400ed8cd889e0744ecc3b
+ # via pyobjc
+pyobjc-framework-metal==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3072b03a28067e1142fa566b71654bbbbe860d9f11941b11f6ee02795a303176 \
+ --hash=sha256:42810b82d7797011a29b9f10b0bdc14bfa278bcb3856a8b74adcaae88ab9cce2 \
+ --hash=sha256:43f60957623e25dd5592d740edab895308d08c3a0f29c63ca8c7c594304aecdb \
+ --hash=sha256:57b2df28aba9fe58a1f4d91838dba4462ef2c19ba62bb0be36d72217a1c8b6f6
+ # via
+ # pyobjc
+ # pyobjc-framework-metalkit
+ # pyobjc-framework-metalperformanceshaders
+pyobjc-framework-metalkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3f5252f932aa10b600b9837ed21b927ad1a5b80c31ed88fb5e0e590f1ac7baad \
+ --hash=sha256:587b29b2eaabc7b5689fa9239248299d6be629fb12ca4894e0539fffc1d9833a \
+ --hash=sha256:e1ad86dc89b162e8d05b7f95a723d9c02b93eda2f1f226d1d04e96a615c4eff7 \
+ --hash=sha256:fc760d01b78f328f12d69aadca1a3d5c66fc160be75fa32b18d7f04def5411ea
+ # via pyobjc
+pyobjc-framework-metalperformanceshaders==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1e4734aecceb2cc948f00ce3065f48153437668f5a84e43301c319563c9f7fd0 \
+ --hash=sha256:7461e6fe30b4b31dee698db1558c1a6fccf96f0cdf78fcacbc14a09c2b36121d \
+ --hash=sha256:cd4d156dfbcf8f00e8dbc7048c954a6559a32d709cb36ff0d0cf5bd3dc311d76 \
+ --hash=sha256:e13d9b866cf8c67ec1a0f350e5a33975b962582f549333a12ac50c81c8a4ff38
+ # via pyobjc
+pyobjc-framework-modelio==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:34d5a7020a7f0e2483360046d227a0709105ab0b70c72808bc564b8740b3a70c \
+ --hash=sha256:4d12021dc80bec3eabd3b66cdd88836f12d304cfb26753ebf6d8a6201f9bb598 \
+ --hash=sha256:8ebeccf7e637dd76900e933c040f7dd43172833915e91165da970532dc677325 \
+ --hash=sha256:decf52dcf325708b1a9ff3fa584cbafb4584d5aed899732f2631aa9d7e0db783
+ # via pyobjc
+pyobjc-framework-multipeerconnectivity==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:2b4afa1ff6d9d203e667786e8c100f0299bcc33d7a9f8996a446e0ac0c8efb7b \
+ --hash=sha256:4dfa1c88c6ca91355b06f96843f3c737c5dfb1cb74cc9cdefffaf2239b09cb1f \
+ --hash=sha256:789e2e190d1c6d244487fb8e8fcd9214d5c98560d31c3714fe5e06f6d9e19960 \
+ --hash=sha256:990f38b75fa3cf9ae009228626efed1ab917ff63985ac994837d48db2f95a87b
+ # via pyobjc
+pyobjc-framework-naturallanguage==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:53c66e0cf3b5b8d22e331f8916c2a1a05f009d4c65f1f393e422dc47f027b54c \
+ --hash=sha256:53c66e0cf3b5b8d22e331f8916c2a1a05f009d4c65f1f393e422dc47f027b54c \
+ --hash=sha256:53c66e0cf3b5b8d22e331f8916c2a1a05f009d4c65f1f393e422dc47f027b54c \
+ --hash=sha256:dbcf1469d0d6b4d0cc8172454a3bcbdeae20b04692019472199a42eb1cdcbd35
+ # via pyobjc
+pyobjc-framework-netfs==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:25899eeb3bbeceeae3e83afa7e8e112a7aeb2acd2141857fc4a2dbba479f7b39 \
+ --hash=sha256:bc65b11917e5c4722206e829137ae8d2bf2627c061fab2886614860379e24487 \
+ --hash=sha256:bc65b11917e5c4722206e829137ae8d2bf2627c061fab2886614860379e24487 \
+ --hash=sha256:bc65b11917e5c4722206e829137ae8d2bf2627c061fab2886614860379e24487
+ # via pyobjc
+pyobjc-framework-network==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1e96f09c85a819acaaad307c2f8ced8b7758bb396c528a07b0dcc94a5c447aee \
+ --hash=sha256:30966dcdbf96209113776b42edd2be4479513a83e15a88e9f88e17d9684ce811 \
+ --hash=sha256:964d1a10d3de1aa282bf33bba5360e94ae6b09f1e10efdaf4357c74f3fc58ba7 \
+ --hash=sha256:af717ff1ab8124246e94a0b6c4da3541c486b96fe898301e20b0b10ac2b57ab5
+ # via pyobjc
+pyobjc-framework-networkextension==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:386909eeacd3afb861201f9951c9c33627957d27884c31c289e134c34ec0ca07 \
+ --hash=sha256:5c9d0d09d286524639cf5070bed85252b6699353b064eb6d602c332ba4610215 \
+ --hash=sha256:bbf3f34265c324935c2379e2e9dc985295d966e9fbee3cd985654160f4640b49 \
+ --hash=sha256:d895e04a12f933f477c0b1f77b13c07479b4ac38df2d23391ab313e54e9665e5
+ # via pyobjc
+pyobjc-framework-notificationcenter==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1e23dfb29af137b10f0c389505e0e77bde84ca7290e99fb39b393bcab24254eb \
+ --hash=sha256:4c4644ff6a7128cd822289590697dc9ee9d28da6d640ce86de421d7722af5b35 \
+ --hash=sha256:5f744fb9b2a693cd49c9931733791b5467ac65badb68c888e8fbe43ce60de8d9 \
+ --hash=sha256:fd8abb44282970353875d3c012c79e7e79b82e143784b409a4265c0ec9babba8
+ # via pyobjc
+pyobjc-framework-opendirectory==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:bac760f838fa89557f8e8d3d5e50c948053da2fc95652158f657d5dd1c4d1629 \
+ --hash=sha256:ece772476d16d1076b0b54fa7e6d26e7e5112884795c2490fe82f74cd8c03b76 \
+ --hash=sha256:ece772476d16d1076b0b54fa7e6d26e7e5112884795c2490fe82f74cd8c03b76 \
+ --hash=sha256:ece772476d16d1076b0b54fa7e6d26e7e5112884795c2490fe82f74cd8c03b76
+ # via pyobjc
+pyobjc-framework-osakit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1ce1bd389f8c55dd8cd3275db63a52143decc0cad00a5601266c65e3761cb37f \
+ --hash=sha256:df5d0cd733215fb717ceffc4a8e4d6a2cabd779d4c37a9a543e83e080d889356 \
+ --hash=sha256:f711267b7a01233f3eeb638ebb3aec2714151f3a5bbcf622bb3eb566385d051d \
+ --hash=sha256:f711267b7a01233f3eeb638ebb3aec2714151f3a5bbcf622bb3eb566385d051d
+ # via pyobjc
+pyobjc-framework-oslog==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:262dbabf344c94e221d5adf6f17396e68ca78e57036faaa3500842c69172ef29 \
+ --hash=sha256:7d5bcfbcaff0ac7a686b6ab091f1f587af447e834ddd576e7a0333a0e994d679 \
+ --hash=sha256:9097148f396bf61f74286805aa74483a553c28e8f2977169a7e8e98332463614 \
+ --hash=sha256:c20cdd6576cc1465d291afec97a31257fde78ee9566aa2caf4a175b8469e5e77
+ # via pyobjc
+pyobjc-framework-pencilkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:5a987280e6f240ff83ef9ccceddc0527883e60df45080505481a112c7e059933 \
+ --hash=sha256:d2c8aec7e6f5f8ecf544f3d7636549d45bc9a7284d7c6ddc341f672329549048 \
+ --hash=sha256:fa4d6a887196f01299e60e709eb698a6ec17d45429ead1316bd3055bbd13762a \
+ --hash=sha256:fa4d6a887196f01299e60e709eb698a6ec17d45429ead1316bd3055bbd13762a
+ # via pyobjc
+pyobjc-framework-photos==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1bc42e76541196ec0b57bf335b454c8b3f00e975d0ea889e2436e5e02eecafca \
+ --hash=sha256:546f864293a9c6a5c0a168e05144f6cb6786f479329fcdc4e7f32c025526cef9 \
+ --hash=sha256:9aa102cb75c53972603b9a00621161ee236e28ee6d6e1e8d6547433d0055b061 \
+ --hash=sha256:a6824fd47dccecbce50baa4238ba6c5b8c9aa9fcd6138a5d27b64d441dafaf12
+ # via pyobjc
+pyobjc-framework-photosui==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:073865bcb080ed2c8b18c09f9261b871588bc7224ac7dd11ebb8bd9982b15669 \
+ --hash=sha256:1538c459d0971a8c6788efdb8cc530ad965d0e39e5a612849fca91573051a47b \
+ --hash=sha256:64a4696598635e7759eea3afc218f37b66e4faac68b430e4904550679d9bb0a1 \
+ --hash=sha256:c03360b0b1df2369c0a57863766cd891a0939133a09b9be80a2902e7e5ae7e9a
+ # via pyobjc
+pyobjc-framework-preferencepanes==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:02b95ca7dfe66b0487422008bb67cb6202d27f57dbb68c3c160a2812247eaf77 \
+ --hash=sha256:bf4875293f66b6176512d85c5febe2084b6dd5a500d7564c7fb47bed38468012 \
+ --hash=sha256:d51cf15b50d08c9213c34ed10067532c387a9e00d035ac8e6f139eeaac4de890 \
+ --hash=sha256:d51cf15b50d08c9213c34ed10067532c387a9e00d035ac8e6f139eeaac4de890
+ # via pyobjc
+pyobjc-framework-pubsub==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1bde2f9d7dd2baf82d8d5094d8adf7c0d4b507ec1c32106f8c89ba953bc9b287 \
+ --hash=sha256:1bde2f9d7dd2baf82d8d5094d8adf7c0d4b507ec1c32106f8c89ba953bc9b287 \
+ --hash=sha256:966534d65daa59d79d9662b4218e6fed2c13e4375840434e559805c933b09da3 \
+ --hash=sha256:9cac04d2901108fb22bd891245dd908b0ba63bb4be697fb68717ab3407adc00a
+ # via pyobjc
+pyobjc-framework-pushkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:93eb706b1030e059befd657bbde55179b103b705f67ce5fda7d59ef00bd52597 \
+ --hash=sha256:cfb9fb736f0c42809e27ee7c1b3da43c59d67303ce752209f7a037ee66f68499 \
+ --hash=sha256:d62f405feb0427036630bce41e3a6582b9ed9a636c6445572f300270e2422951 \
+ --hash=sha256:f93657477660862bac633620d3d3c9515296749117abf1418d31e997e7efcea3
+ # via pyobjc
+pyobjc-framework-quartz==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:03f97df672c4c4d702e5fa3b16d32a4c2e9b1cbf38592917f2529bc8697e6131 \
+ --hash=sha256:0b208b6e7fe22caf101ef3cbe756d156e91697a8a9b70acb0dfee238a7aaf37e \
+ --hash=sha256:a543d4ab5c03719801b23cbeaca580179e23c1579fb7e1d09f2e7fb0dccd2f95 \
+ --hash=sha256:eae3261ca8afc159eb14977d5d02bea6cd0906e84ba7741ab2668532320dcb5c
+ # via
+ # pyobjc
+ # pyobjc-framework-applicationservices
+ # pyobjc-framework-avfoundation
+ # pyobjc-framework-avkit
+ # pyobjc-framework-coretext
+ # pyobjc-framework-gamekit
+ # pyobjc-framework-instantmessage
+ # pyobjc-framework-linkpresentation
+ # pyobjc-framework-mapkit
+ # pyobjc-framework-medialibrary
+ # pyobjc-framework-modelio
+ # pyobjc-framework-oslog
+ # pyobjc-framework-quicklookthumbnailing
+ # pyobjc-framework-scenekit
+ # pyobjc-framework-spritekit
+ # pyobjc-framework-videotoolbox
+ # pyobjc-framework-vision
+pyobjc-framework-quicklookthumbnailing==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:447a8ab915bccd99d68698f072c87051fbbb857d319d5f260d5b3f92237f0288 \
+ --hash=sha256:447a8ab915bccd99d68698f072c87051fbbb857d319d5f260d5b3f92237f0288 \
+ --hash=sha256:447a8ab915bccd99d68698f072c87051fbbb857d319d5f260d5b3f92237f0288 \
+ --hash=sha256:cb94da8097724fc1e8dde4b3d825cf6fc78569f146ed7b195538ca028d61a9cf
+ # via pyobjc
+pyobjc-framework-safariservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:65701896ee1c958d27646e39457a57a3092322651289c26baa20731370ca6f6a \
+ --hash=sha256:7e6083ff340bbbbe2896d7560576e6721574d0fcdc3c4613a1ea00cb1d33df86 \
+ --hash=sha256:81c91b816349a8043b187f7b67ffe46ef3c85373241eaca90ad558d3c56642ce \
+ --hash=sha256:89060e2ba2a8b4ae9cda6637f0dd1d7dd2e6ba1bee3e2be35a8b3481c20de06b
+ # via pyobjc
+pyobjc-framework-scenekit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:47ee19ac48c067a1af7186a8addae3cd1fa42fcefdeccee519bb2f5f3f7456ad \
+ --hash=sha256:4cfa0dfccd821fcb3641d2e45a5ad8ade493ae691bac17728e40c8eba7ff488a \
+ --hash=sha256:62c9c60a514119e295ed2467a0a0a7bc702db19efb209582cdd682bcc35ecf8c \
+ --hash=sha256:a1c59d3e08e9abe06a1c0a1a93db740a4ae58a77cd87683d84354f59ccb88041
+ # via pyobjc
+pyobjc-framework-screensaver==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1d1fe276e06221d100bf3feccdf50466e755c6eb8d5c1efa17ef71508f4f9b24 \
+ --hash=sha256:871ebe82221e4663448ddec589f660ac7538fd0ff2a24f92ab385ebc604845c2 \
+ --hash=sha256:abba8626d1efb51ad9d1474937cf2328c9752671676fad3e1db7d8ce697ee6d3 \
+ --hash=sha256:ed6bac6a081a811c7823e79af82fd8db25451e79bf5a270093456b83e7170207
+ # via pyobjc
+pyobjc-framework-scriptingbridge==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0e9934c9d247ed3dc7ba6c2db9c9aed9ec0113b32f7b06b606c1822393322c9e \
+ --hash=sha256:1e8c8ac310e1ae825830240c22a0877673a668402218eeff24bfa947142c7b83 \
+ --hash=sha256:44f65133d089f5571a96ddf5c62b00f81557a2978d7da722c8288b042b15ca58 \
+ --hash=sha256:f9f87616491d7317f4fd967e64b65c692744a8b215ba23debdb9b4850817cc96
+ # via pyobjc
+pyobjc-framework-searchkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1f6728f2b08d39b161d77bdaafca2db6f64efae33592d4b9c1c458e7800c07cc \
+ --hash=sha256:1f6728f2b08d39b161d77bdaafca2db6f64efae33592d4b9c1c458e7800c07cc \
+ --hash=sha256:1f6728f2b08d39b161d77bdaafca2db6f64efae33592d4b9c1c458e7800c07cc \
+ --hash=sha256:5657fae70317fa96c145c6c6baa3d94e3dcd5176bd98ae5952eaa208c4cd4026
+ # via pyobjc
+pyobjc-framework-security==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:4743dc3f12a679ce13b75116de4cd96ec3e10fff899d7d340c41a95360f06e9d \
+ --hash=sha256:5d2cf2b1e9dd06c0f41a3ecd399b5e70aae15fc013ecbc1c69c67d1d11c2126f \
+ --hash=sha256:661ccabede6c4e86719a75e0f1c889ed594c059fc5ce211ea3855bf6513ac69c \
+ --hash=sha256:db084ba1cedc7ff14f4d94358b3f59defc50883cffbc12810121e5252895ea15
+ # via
+ # pyobjc
+ # pyobjc-framework-localauthentication
+ # pyobjc-framework-securityfoundation
+ # pyobjc-framework-securityinterface
+pyobjc-framework-securityfoundation==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:6390bafd47b858e9695067884869c766d974fc7495b222ad21a2d31dc34a9180 \
+ --hash=sha256:6390bafd47b858e9695067884869c766d974fc7495b222ad21a2d31dc34a9180 \
+ --hash=sha256:6390bafd47b858e9695067884869c766d974fc7495b222ad21a2d31dc34a9180 \
+ --hash=sha256:a3546ed431f31e74c383e4f33a671454f198ca6e2b1062aa6d7cbb61626cf0f5
+ # via pyobjc
+pyobjc-framework-securityinterface==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:328049de977b8e66bae412e066c04ba1b40c3a551ee429404ce32326d8a36629 \
+ --hash=sha256:6d0588d92308a6bebd4cff6e232f326f255450313f5d6d6241c5bb158b20fcaf \
+ --hash=sha256:c51a9020d2e2cdfc7c4e616efeb7c0c377b356d70987f4c2d4d458c1984c44a9 \
+ --hash=sha256:ee56f02e9efaf5de9a82b5dc580567232dd928110cf8dbac130917c627f387f2
+ # via pyobjc
+pyobjc-framework-servicemanagement==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:50b96930e9ee129698384c3053d2b2f9d11f72351e8979505d4796a6675a4d42 \
+ --hash=sha256:c4d5723893507cdcc764dce02a67f788ef2cbc26e2c8ac4362b1779de0ea6337 \
+ --hash=sha256:c4d5723893507cdcc764dce02a67f788ef2cbc26e2c8ac4362b1779de0ea6337 \
+ --hash=sha256:c4d5723893507cdcc764dce02a67f788ef2cbc26e2c8ac4362b1779de0ea6337
+ # via pyobjc
+pyobjc-framework-social==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:062ffea31d6543aab0f4c51332baaa4388a4ba5c826caf4092d7b068cc9ff4a2 \
+ --hash=sha256:062ffea31d6543aab0f4c51332baaa4388a4ba5c826caf4092d7b068cc9ff4a2 \
+ --hash=sha256:062ffea31d6543aab0f4c51332baaa4388a4ba5c826caf4092d7b068cc9ff4a2 \
+ --hash=sha256:990e0c758c12708493297c391826a8c6d40a9ec6f548162a5cded571d7a5eb33
+ # via pyobjc
+pyobjc-framework-soundanalysis==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:00a8d9bf2fa38d0c1960edae3b5470123496e3b8599244db3f5f9946445dba28 \
+ --hash=sha256:00a8d9bf2fa38d0c1960edae3b5470123496e3b8599244db3f5f9946445dba28 \
+ --hash=sha256:d47f224f7c2e7935a718b1958af71af2828921cad86504a9ebf970d15247d59a \
+ --hash=sha256:e448cd9b73fcae89191c98903c6754ca029af2fc8eece4a95d11d167e0dfbe86
+ # via pyobjc
+pyobjc-framework-speech==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:346c4df2e64b080c2ab957ca4d6578a74abaf2ed43e575cfe7cc6a36dd19d042 \
+ --hash=sha256:4e327139b5d1307d00788533a7d2450f65aabff5e65d405805125c72c093fa1a \
+ --hash=sha256:7e551fe41d6f459996c3d7544ce9929d0ca2ae059fedbf51cbc3a42011faf11d \
+ --hash=sha256:bea5346c5f33f70f1add7430cca18d3fca484c3642f7046b63a3703fcbc2b50f
+ # via pyobjc
+pyobjc-framework-spritekit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:54ce361a0b95cfc7e2fe08d2fb514d61ec386406b13a0c205076ff8995dcfdee \
+ --hash=sha256:bf2524d2159a5378b514806a7735a4ee5f358fca5a7036c983503a9e9deb59ea \
+ --hash=sha256:da874aba3d540e3808c294e877250e4cb04ee8dd68de0194d862a2d5b717ca8f \
+ --hash=sha256:e23e56743abb21d1113baab879558124a10e6a57490e80a825b3c2760fcb70ba
+ # via
+ # pyobjc
+ # pyobjc-framework-gameplaykit
+pyobjc-framework-storekit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:15466cc1b40621e0f823abd428c48d1cba0c3fa22810782d5be3c2136f533498 \
+ --hash=sha256:42c1966626cd412f8d87d016088962a67dd5357386bd7a525fac117bca4de12e \
+ --hash=sha256:739531d040d090c7318b2d65b600364db99147656cafb513e655742e9b2433cf \
+ --hash=sha256:fa3cde89dac6cfd30b09755ea5e3dcaa8c494b2c928996aac23658434fb230e1
+ # via pyobjc
+pyobjc-framework-syncservices==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:1d25d30ef22e2ce6ceac01d275c9f39627c5db48a8bd232e5553b30d9a02daad \
+ --hash=sha256:2432e04784bf1e68ef1951b40d8febcbddbd572a53931302fd126f7c7639b93c \
+ --hash=sha256:6a97d452b933d61267d229529909df26d5b7c40ae9029c95a891585d96208cc5 \
+ --hash=sha256:73d7404936938c18208ab48e50285b304bff6cc7b00649263925412231cfed18
+ # via pyobjc
+pyobjc-framework-systemconfiguration==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:452bd8c52aa6cea18c619786f31e6ad064dd8b0a3693e2200f425400173b9733 \
+ --hash=sha256:775aae2e4f48cba3b831c080f853fe76812124b4476ac6b5e329009f0a6fa8c2 \
+ --hash=sha256:a669274a35113402c461f9f788084972db1d7eb57847667268ad31b4c1323c63 \
+ --hash=sha256:f697e2169b089905b5e494856b32aa8a011e9c51b94338e443833adb9a6f4512
+ # via pyobjc
+pyobjc-framework-systemextensions==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:39912c06675109a0d9780cb5d0accdabae5d1968e9140e26378b8f12442186cd \
+ --hash=sha256:428ca84d425f25e8bcdea47df06dc9342761cc2f8550a2cb137b2a6a57fd6824 \
+ --hash=sha256:6f423d1e31831bbf15f446004c1b4c51428006ff886e1461e31dcb24bb72c8e3 \
+ --hash=sha256:bc37f18c98d3f52b0cd7434b81b57d22c9a6e3da568ee36af2770706419494f5
+ # via pyobjc
+pyobjc-framework-usernotifications==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:055379dae7caf4d9fa0a4ebba07115a8c972ca8c458266d59d610262b08f0a1f \
+ --hash=sha256:08841d80265b29f7c578d634344941138cc582d42fdfb745e06d30f6cabb5789 \
+ --hash=sha256:3eff0b8b5391a89e085dce964f48ef7460ace6ddcb743b01b5d567ce67bd6ea8 \
+ --hash=sha256:7bd4df365174ca92011bc1926873319f9bb71d924b2e7721f23059c815f71c44
+ # via pyobjc
+pyobjc-framework-videosubscriberaccount==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:0b609c0b4ffac32ea6a520f373a0c980a2d6a1a2280b375dfeea15ed0c84456b \
+ --hash=sha256:0b609c0b4ffac32ea6a520f373a0c980a2d6a1a2280b375dfeea15ed0c84456b \
+ --hash=sha256:0b609c0b4ffac32ea6a520f373a0c980a2d6a1a2280b375dfeea15ed0c84456b \
+ --hash=sha256:6d7c621fa87465d84296f8fc6f40a2f7a2a0916f49e59ccf15912cef99d7c318
+ # via pyobjc
+pyobjc-framework-videotoolbox==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:5bb2466e50cce92376814726d31afe7fd7cdc78d7763c9e47b29c44aa2920aa5 \
+ --hash=sha256:6b1f1b3ce72a0979b1532fb53b084114a85377262e8e012fa8dbdf5c1a992015 \
+ --hash=sha256:8d5e78465ab2296c929834dfbf82cf7f8e30e50d6b1506de217937ce00edf7b2 \
+ --hash=sha256:933f616585620b08a0d7c1c1831e94e8cd2410ab57e8efe2e8d1c16720db8ef3
+ # via pyobjc
+pyobjc-framework-vision==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:133d276bf1a828739e910bd585f13417a39421ece400418057498cc303f55196 \
+ --hash=sha256:38cba672cac6edf4250a956ba9ed3d2aa4fda5cd79e2d1d805586f55e1f5ea91 \
+ --hash=sha256:6f5009063cf5a05a39df6a067f1165cf1e9e7415b391608fea3ec957887405e3 \
+ --hash=sha256:d37d43c9593cdb6daa7b551dd889aa89b3dcef441dacf21079faed93d498dd87
+ # via pyobjc
+pyobjc-framework-webkit==10.0 ; sys_platform == 'darwin' \
+ --hash=sha256:3df3faba2ddabfbb3a8e05a3777b75c4a48f28ee75bffbfe19227a95ce7ad743 \
+ --hash=sha256:688c9af93374196d47806cc5c322f590a26d590be23ecedecdec8c8441a0cfd9 \
+ --hash=sha256:9a77c7db46dc6775ba6dc12909275aa548619e88d8dc4a6a2c3438902b6d5b5b \
+ --hash=sha256:df6508526bfcdfdeb24db17ff3e6bbddb65b955e529efe828e0afba86a77c49b
+ # via pyobjc
+pyopenssl==24.2.1 \
+ --hash=sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d \
+ --hash=sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d \
+ --hash=sha256:d80942d4141a8569a529d6bf3a80923ca56aabd1a565248bf2108cb3cd4ebe2f
+ # via aioquic
+pyparsing==2.4.7 \
+ --hash=sha256:e353556f0615c2246ea24b7afd6089cb096c13c2f5c26b0ad0e0a542b5db6d19 \
+ --hash=sha256:e353556f0615c2246ea24b7afd6089cb096c13c2f5c26b0ad0e0a542b5db6d19 \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
+ --hash=sha256:f10f6d3fc1598d55e192fe92b58c020281a7ad211a6f8f0ca9c24b3e97ecd699
+ # via
+ # httplib2
+ # packaging
+pytest==6.2.2 \
+ --hash=sha256:1054ad59b89e53209f5e917515fbe5d6972ad4d63c5beda90bc97dd72701635e \
+ --hash=sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839
+ # via pytest-asyncio
+pytest-asyncio==0.14.0 \
+ --hash=sha256:2eae1e34f6c68fc0a9dc12d4bea190483843ff4708d24277c41568d6b6044f1d \
+ --hash=sha256:b01c3c3fd2f19380aff0c436e339fe3b8ae4aeed6f5c1619e671d744223a0698
+python-dateutil==2.9.0 \
+ --hash=sha256:cbf2f1da5e6083ac2fbfd4da39a25f34312230110440f424a14c7558bb85d82e \
+ --hash=sha256:d6e8ba261767bc6626b87eafbaff60e52b68417ae5d245ba5b703e59a384a169
+ # via pandas
+pytz==2024.1 \
+ --hash=sha256:23d0cb73c431924d7c4e8884520c798647165ab49f34b5f3aa6d44ae702d7111 \
+ --hash=sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319
+ # via
+ # google-api-core
+ # pandas
+pywin32==308 ; sys_platform == 'win32' \
+ --hash=sha256:48e9e39c028aae1f5dde236379070c8374393b695be23631c447b93ee9b62a5e \
+ --hash=sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b \
+ --hash=sha256:c7022dd1acce30dc2c668599a2228f6b5859fc6c5323a37f2f72262bfc2e7b52
+requests==2.31.0 \
+ --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
+ --hash=sha256:7c024df3d83d462cc8127c6c9a2dae2e5d2bddb54c1039624ab3baf485d752ca
+ # via google-api-core
+rsa==3.4.2 \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd \
+ --hash=sha256:b77867f9c1b9d489e4c3afbe77ce5ec47b402affb5ab4b801483b2d247d2f7d7 \
+ --hash=sha256:b77867f9c1b9d489e4c3afbe77ce5ec47b402affb5ab4b801483b2d247d2f7d7 \
+ --hash=sha256:b77867f9c1b9d489e4c3afbe77ce5ec47b402affb5ab4b801483b2d247d2f7d7
+ # via
+ # google-auth
+ # oauth2client
+service-identity==24.1.0 \
+ --hash=sha256:16bd4928f1f2ea771c7c0f52eb25e42b4d4e504ea821a782bef966325bd35367 \
+ --hash=sha256:a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a
+ # via aioquic
+setuptools==68.0.0 ; python_full_version < '3.14' \
+ --hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
+ --hash=sha256:25c8300cbd8c41a350e4890f8340ca0bbadff88967f7b26de4c8ccfe5d851a51
+ # via
+ # google-api-core
+ # google-auth
+setuptools==75.6.0 ; python_full_version >= '3.14' \
+ --hash=sha256:95e7b6c4ee1eb64afe479ba9f0c89ef4fb8e0da33319d829ecb2ad1dcd131dfd \
+ --hash=sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d
+ # via
+ # google-api-core
+ # google-auth
+six==1.15.0 \
+ --hash=sha256:7bb413e52c2f9981580e74743d53187be17e108df9b4f9fab1964aab2e94b9d6 \
+ --hash=sha256:7bb413e52c2f9981580e74743d53187be17e108df9b4f9fab1964aab2e94b9d6 \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+ --hash=sha256:c7dfbdb6d68189c1942e9405fae9eef6bdf2ddff305866df52b806c11da74323
+ # via
+ # blessings
+ # google-api-core
+ # google-api-python-client
+ # google-auth
+ # google-auth-httplib2
+ # html5lib
+ # jsonlines
+ # mock
+ # more-itertools
+ # mozfile
+ # mozlog
+ # mozterm
+ # oauth2client
+ # packaging
+ # python-dateutil
+toml==0.10.1 \
+ --hash=sha256:4099063d9b50219bf09bfa693ab8dcd96ce585cee9fdadce5a48a08ee23b0646 \
+ --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88
+ # via pytest
+tzdata==2023.4 \
+ --hash=sha256:3bfb77cffc5b0b14bdb11a65b1074fe50cccbae0f3107024e344b48912c89184 \
+ --hash=sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3
+ # via pandas
+uritemplate==3.0.0 \
+ --hash=sha256:015cd3c3e66e9d9ccd8ada54fd7167ba51c50832044d27f1439a07d4fbd76430 \
+ --hash=sha256:01c69f4fe8ed503b2951bef85d996a9d22434d2431584b5b107b2981ff416fbd \
+ --hash=sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd \
+ --hash=sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd \
+ --hash=sha256:7ad3e6d74caeb1d2c5f2496885e652dad1b3894abf28fdf2da9e98f4ffe8919a \
+ --hash=sha256:98801f9862158b078d7252d835c7dbee5b66bf776c3226131669164ce93ae54f
+ # via google-api-python-client
+urllib3==1.26.6 \
+ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \
+ --hash=sha256:fa608904f610002ec11bb0347f46ccf36c1fe0fb399c8044fce17e8fc02936db
+ # via requests
+webencodings==0.5.1 \
+ --hash=sha256:5d65b7d725951f58509b737028fb8ba2a576ee87c4d454ea0a121cda690875ed \
+ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+ --hash=sha256:ea5ad2d6c5efc2f7de6b721fb3712f27428a9a9ad6d2a96b9abd414d84089a73 \
+ --hash=sha256:ea5ad2d6c5efc2f7de6b721fb3712f27428a9a9ad6d2a96b9abd414d84089a73
+ # via html5lib
diff --git a/libs/libyuv/winarm.mk b/libs/libyuv/winarm.mk
index c4307a431f..b0a344ae06 100644
--- a/libs/libyuv/winarm.mk
+++ b/libs/libyuv/winarm.mk
@@ -31,6 +31,7 @@ LOCAL_OBJ_FILES = \
source/scale_any.o\
source/scale_argb.o\
source/scale_common.o\
+ source/scale_uv.o\
source/video_common.o
.cc.o:
diff --git a/libs/win32/libyuv/libyuv.2017.vcxproj b/libs/win32/libyuv/libyuv.2017.vcxproj
index f74f7e1203..8d1dc74fba 100644
--- a/libs/win32/libyuv/libyuv.2017.vcxproj
+++ b/libs/win32/libyuv/libyuv.2017.vcxproj
@@ -133,10 +133,10 @@
<ClCompile Include="..\..\libyuv\source\compare.cc" />
<ClCompile Include="..\..\libyuv\source\compare_common.cc" />
<ClCompile Include="..\..\libyuv\source\compare_gcc.cc" />
- <ClCompile Include="..\..\libyuv\source\compare_mmi.cc" />
- <ClCompile Include="..\..\libyuv\source\compare_msa.cc" />
+ <ClCompile Include="..\..\libyuv\source\compare_hvx.cc" />
<ClCompile Include="..\..\libyuv\source\compare_neon.cc" />
<ClCompile Include="..\..\libyuv\source\compare_neon64.cc" />
+ <ClCompile Include="..\..\libyuv\source\compare_wasm.cc" />
<ClCompile Include="..\..\libyuv\source\compare_win.cc" />
<ClCompile Include="..\..\libyuv\source\convert.cc" />
<ClCompile Include="..\..\libyuv\source\convert_argb.cc" />
@@ -154,27 +154,43 @@
<ClCompile Include="..\..\libyuv\source\rotate_argb.cc" />
<ClCompile Include="..\..\libyuv\source\rotate_common.cc" />
<ClCompile Include="..\..\libyuv\source\rotate_gcc.cc" />
- <ClCompile Include="..\..\libyuv\source\rotate_mmi.cc" />
- <ClCompile Include="..\..\libyuv\source\rotate_msa.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_hvx.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_lasx.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_lsx.cc" />
<ClCompile Include="..\..\libyuv\source\rotate_neon.cc" />
<ClCompile Include="..\..\libyuv\source\rotate_neon64.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_rvv.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_sme.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_sve.cc" />
+ <ClCompile Include="..\..\libyuv\source\rotate_wasm.cc" />
<ClCompile Include="..\..\libyuv\source\rotate_win.cc" />
<ClCompile Include="..\..\libyuv\source\row_any.cc" />
<ClCompile Include="..\..\libyuv\source\row_common.cc" />
<ClCompile Include="..\..\libyuv\source\row_gcc.cc" />
- <ClCompile Include="..\..\libyuv\source\row_mmi.cc" />
- <ClCompile Include="..\..\libyuv\source\row_msa.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_hvx.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_lasx.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_lsx.cc" />
<ClCompile Include="..\..\libyuv\source\row_neon.cc" />
<ClCompile Include="..\..\libyuv\source\row_neon64.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_rvv.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_sme.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_sve.cc" />
+ <ClCompile Include="..\..\libyuv\source\row_wasm.cc" />
<ClCompile Include="..\..\libyuv\source\row_win.cc" />
<ClCompile Include="..\..\libyuv\source\scale.cc" />
<ClCompile Include="..\..\libyuv\source\scale_any.cc" />
<ClCompile Include="..\..\libyuv\source\scale_argb.cc" />
<ClCompile Include="..\..\libyuv\source\scale_common.cc" />
- <ClCompile Include="..\..\libyuv\source\scale_mmi.cc" />
- <ClCompile Include="..\..\libyuv\source\scale_msa.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_gcc.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_hvx.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_lsx.cc" />
<ClCompile Include="..\..\libyuv\source\scale_neon.cc" />
<ClCompile Include="..\..\libyuv\source\scale_neon64.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_rgb.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_rvv.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_sme.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_uv.cc" />
+ <ClCompile Include="..\..\libyuv\source\scale_wasm.cc" />
<ClCompile Include="..\..\libyuv\source\scale_win.cc" />
<ClCompile Include="..\..\libyuv\source\video_common.cc" />
</ItemGroup>
@@ -187,16 +203,20 @@
<ClInclude Include="..\..\libyuv\include\libyuv\convert_from.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\convert_from_argb.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\cpu_id.h" />
- <ClInclude Include="..\..\libyuv\include\libyuv\macros_msa.h" />
+ <ClInclude Include="..\..\libyuv\include\libyuv\cpu_support.h" />
+ <ClInclude Include="..\..\libyuv\include\libyuv\loongson_intrinsics.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\mjpeg_decoder.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\planar_functions.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\rotate.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\rotate_argb.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\rotate_row.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\row.h" />
+ <ClInclude Include="..\..\libyuv\include\libyuv\row_sve.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\scale.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\scale_argb.h" />
+ <ClInclude Include="..\..\libyuv\include\libyuv\scale_rgb.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\scale_row.h" />
+ <ClInclude Include="..\..\libyuv\include\libyuv\scale_uv.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\version.h" />
<ClInclude Include="..\..\libyuv\include\libyuv\video_common.h" />
</ItemGroup>