Merge d0f3ad23cf4f ("Merge tag 'spi-v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi") into android-mainline
Steps on the way to 6.2-rc1
Change-Id: Ief1fd74715d43fed43513c728d4817c9cd270cad
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
diff --git a/BUILD.bazel b/BUILD.bazel
new file mode 100644
index 0000000..c89e71f
--- /dev/null
+++ b/BUILD.bazel
@@ -0,0 +1,534 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2021 The Android Open Source Project
+
+load("//build/bazel_common_rules/dist:dist.bzl", "copy_to_dist_dir")
+load("//build/kernel/kleaf:common_kernels.bzl", "define_common_kernels", "define_db845c")
+load("//build/kernel/kleaf:kernel.bzl", "ddk_headers", "kernel_build", "kernel_images", "kernel_modules_install")
+load(":modules.bzl", "COMMON_GKI_MODULES_LIST")
+
+package(
+ default_visibility = [
+ "//visibility:public",
+ ],
+)
+
+define_common_kernels(target_configs = {
+ "kernel_aarch64": {
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+ "kernel_aarch64_16k": {
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+ "kernel_aarch64_debug": {
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+ "kernel_riscv64": {
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+ "kernel_x86_64": {
+ "kmi_symbol_list_strict_mode": False,
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+ "kernel_x86_64_debug": {
+ "kmi_symbol_list_strict_mode": False,
+ "module_implicit_outs": COMMON_GKI_MODULES_LIST,
+ },
+})
+
+define_db845c(
+ name = "db845c",
+ outs = [
+ "arch/arm64/boot/dts/qcom/qrb5165-rb5.dtb",
+ "arch/arm64/boot/dts/qcom/sdm845-db845c.dtb",
+ "arch/arm64/boot/dts/qcom/sm8450-qrd.dtb",
+ ],
+ module_outs = [
+ # keep sorted
+ "crypto/michael_mic.ko",
+ "drivers/base/regmap/regmap-sdw.ko",
+ "drivers/base/regmap/regmap-slimbus.ko",
+ "drivers/bus/mhi/host/mhi.ko",
+ "drivers/clk/qcom/clk-qcom.ko",
+ "drivers/clk/qcom/clk-rpmh.ko",
+ "drivers/clk/qcom/clk-spmi-pmic-div.ko",
+ "drivers/clk/qcom/dispcc-sdm845.ko",
+ "drivers/clk/qcom/dispcc-sm8250.ko",
+ "drivers/clk/qcom/gcc-sdm845.ko",
+ "drivers/clk/qcom/gcc-sm8250.ko",
+ "drivers/clk/qcom/gcc-sm8450.ko",
+ "drivers/clk/qcom/gpucc-sdm845.ko",
+ "drivers/clk/qcom/gpucc-sm8250.ko",
+ "drivers/clk/qcom/lpass-gfm-sm8250.ko",
+ "drivers/clk/qcom/videocc-sdm845.ko",
+ "drivers/clk/qcom/videocc-sm8250.ko",
+ "drivers/cpufreq/qcom-cpufreq-hw.ko",
+ "drivers/dma-buf/heaps/system_heap.ko",
+ "drivers/dma/qcom/bam_dma.ko",
+ "drivers/dma/qcom/gpi.ko",
+ "drivers/extcon/extcon-usb-gpio.ko",
+ "drivers/firmware/qcom-scm.ko",
+ "drivers/gpio/gpio-wcd934x.ko",
+ "drivers/gpu/drm/bridge/display-connector.ko",
+ "drivers/gpu/drm/bridge/lontium-lt9611.ko",
+ "drivers/gpu/drm/bridge/lontium-lt9611uxc.ko",
+ "drivers/gpu/drm/display/drm_display_helper.ko",
+ "drivers/gpu/drm/display/drm_dp_aux_bus.ko",
+ "drivers/gpu/drm/msm/msm.ko",
+ "drivers/gpu/drm/scheduler/gpu-sched.ko",
+ "drivers/hwspinlock/qcom_hwspinlock.ko",
+ "drivers/i2c/busses/i2c-designware-core.ko",
+ "drivers/i2c/busses/i2c-designware-platform.ko",
+ "drivers/i2c/busses/i2c-qcom-geni.ko",
+ "drivers/i2c/busses/i2c-qup.ko",
+ "drivers/i2c/busses/i2c-rk3x.ko",
+ "drivers/i2c/i2c-dev.ko",
+ "drivers/i2c/i2c-mux.ko",
+ "drivers/i2c/muxes/i2c-mux-pca954x.ko",
+ "drivers/iio/adc/qcom-spmi-adc5.ko",
+ "drivers/iio/adc/qcom-vadc-common.ko",
+ "drivers/input/misc/pm8941-pwrkey.ko",
+ "drivers/interconnect/qcom/icc-bcm-voter.ko",
+ "drivers/interconnect/qcom/icc-osm-l3.ko",
+ "drivers/interconnect/qcom/icc-rpmh.ko",
+ "drivers/interconnect/qcom/qnoc-sdm845.ko",
+ "drivers/interconnect/qcom/qnoc-sm8250.ko",
+ "drivers/interconnect/qcom/qnoc-sm8450.ko",
+ "drivers/iommu/arm/arm-smmu/arm_smmu.ko",
+ "drivers/irqchip/qcom-pdc.ko",
+ "drivers/leds/rgb/leds-qcom-lpg.ko",
+ "drivers/mailbox/qcom-apcs-ipc-mailbox.ko",
+ "drivers/mailbox/qcom-ipcc.ko",
+ "drivers/media/platform/qcom/venus/venus-core.ko",
+ "drivers/media/platform/qcom/venus/venus-dec.ko",
+ "drivers/media/platform/qcom/venus/venus-enc.ko",
+ "drivers/mfd/qcom-spmi-pmic.ko",
+ "drivers/mfd/wcd934x.ko",
+ "drivers/misc/fastrpc.ko",
+ "drivers/mmc/host/cqhci.ko",
+ "drivers/mmc/host/sdhci-msm.ko",
+ "drivers/net/can/spi/mcp251xfd/mcp251xfd.ko",
+ "drivers/net/wireless/ath/ath.ko",
+ "drivers/net/wireless/ath/ath10k/ath10k_core.ko",
+ "drivers/net/wireless/ath/ath10k/ath10k_pci.ko",
+ "drivers/net/wireless/ath/ath10k/ath10k_snoc.ko",
+ "drivers/net/wireless/ath/ath11k/ath11k.ko",
+ "drivers/net/wireless/ath/ath11k/ath11k_ahb.ko",
+ "drivers/net/wireless/ath/ath11k/ath11k_pci.ko",
+ "drivers/nvmem/nvmem_qfprom.ko",
+ "drivers/phy/qualcomm/phy-qcom-qmp-combo.ko",
+ "drivers/phy/qualcomm/phy-qcom-qmp-pcie.ko",
+ "drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.ko",
+ "drivers/phy/qualcomm/phy-qcom-qmp-ufs.ko",
+ "drivers/phy/qualcomm/phy-qcom-qmp-usb.ko",
+ "drivers/phy/qualcomm/phy-qcom-qusb2.ko",
+ "drivers/phy/qualcomm/phy-qcom-snps-femto-v2.ko",
+ "drivers/phy/qualcomm/phy-qcom-usb-hs.ko",
+ "drivers/pinctrl/qcom/pinctrl-lpass-lpi.ko",
+ "drivers/pinctrl/qcom/pinctrl-msm.ko",
+ "drivers/pinctrl/qcom/pinctrl-sdm845.ko",
+ "drivers/pinctrl/qcom/pinctrl-sm8250.ko",
+ "drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.ko",
+ "drivers/pinctrl/qcom/pinctrl-sm8450.ko",
+ "drivers/pinctrl/qcom/pinctrl-spmi-gpio.ko",
+ "drivers/pinctrl/qcom/pinctrl-spmi-mpp.ko",
+ "drivers/power/reset/qcom-pon.ko",
+ "drivers/power/reset/reboot-mode.ko",
+ "drivers/power/reset/syscon-reboot-mode.ko",
+ "drivers/regulator/gpio-regulator.ko",
+ "drivers/regulator/qcom-rpmh-regulator.ko",
+ "drivers/regulator/qcom_spmi-regulator.ko",
+ "drivers/regulator/qcom_usb_vbus-regulator.ko",
+ "drivers/remoteproc/qcom_common.ko",
+ "drivers/remoteproc/qcom_pil_info.ko",
+ "drivers/remoteproc/qcom_q6v5.ko",
+ "drivers/remoteproc/qcom_q6v5_adsp.ko",
+ "drivers/remoteproc/qcom_q6v5_mss.ko",
+ "drivers/remoteproc/qcom_q6v5_pas.ko",
+ "drivers/remoteproc/qcom_q6v5_wcss.ko",
+ "drivers/remoteproc/qcom_sysmon.ko",
+ "drivers/reset/reset-qcom-aoss.ko",
+ "drivers/reset/reset-qcom-pdc.ko",
+ "drivers/rpmsg/qcom_glink.ko",
+ "drivers/rpmsg/qcom_glink_rpm.ko",
+ "drivers/rpmsg/qcom_glink_smem.ko",
+ "drivers/rpmsg/qcom_smd.ko",
+ "drivers/rpmsg/rpmsg_ns.ko",
+ "drivers/rtc/rtc-pm8xxx.ko",
+ "drivers/slimbus/slim-qcom-ngd-ctrl.ko",
+ "drivers/slimbus/slimbus.ko",
+ "drivers/soc/qcom/apr.ko",
+ "drivers/soc/qcom/cmd-db.ko",
+ "drivers/soc/qcom/cpr.ko",
+ "drivers/soc/qcom/llcc-qcom.ko",
+ "drivers/soc/qcom/mdt_loader.ko",
+ "drivers/soc/qcom/pdr_interface.ko",
+ "drivers/soc/qcom/qcom_aoss.ko",
+ "drivers/soc/qcom/qcom_rpmh.ko",
+ "drivers/soc/qcom/qmi_helpers.ko",
+ "drivers/soc/qcom/rmtfs_mem.ko",
+ "drivers/soc/qcom/rpmhpd.ko",
+ "drivers/soc/qcom/smem.ko",
+ "drivers/soc/qcom/smp2p.ko",
+ "drivers/soc/qcom/smsm.ko",
+ "drivers/soc/qcom/socinfo.ko",
+ "drivers/soc/qcom/spm.ko",
+ "drivers/soundwire/soundwire-bus.ko",
+ "drivers/soundwire/soundwire-qcom.ko",
+ "drivers/spi/spi-geni-qcom.ko",
+ "drivers/spi/spi-pl022.ko",
+ "drivers/spi/spi-qcom-qspi.ko",
+ "drivers/spi/spi-qup.ko",
+ "drivers/spmi/spmi-pmic-arb.ko",
+ "drivers/thermal/qcom/lmh.ko",
+ "drivers/thermal/qcom/qcom-spmi-adc-tm5.ko",
+ "drivers/thermal/qcom/qcom-spmi-temp-alarm.ko",
+ "drivers/thermal/qcom/qcom_tsens.ko",
+ "drivers/tty/serial/msm_serial.ko",
+ "drivers/ufs/host/ufs_qcom.ko",
+ "drivers/usb/common/ulpi.ko",
+ "drivers/usb/host/ohci-hcd.ko",
+ "drivers/usb/host/ohci-pci.ko",
+ "drivers/usb/host/ohci-platform.ko",
+ "drivers/usb/typec/qcom-pmic-typec.ko",
+ "drivers/watchdog/pm8916_wdt.ko",
+ "drivers/watchdog/qcom-wdt.ko",
+ "net/qrtr/qrtr.ko",
+ "net/qrtr/qrtr-mhi.ko",
+ "net/qrtr/qrtr-smd.ko",
+ "net/qrtr/qrtr-tun.ko",
+ "sound/soc/codecs/snd-soc-dmic.ko",
+ "sound/soc/codecs/snd-soc-hdmi-codec.ko",
+ "sound/soc/codecs/snd-soc-lpass-macro-common.ko",
+ "sound/soc/codecs/snd-soc-lpass-va-macro.ko",
+ "sound/soc/codecs/snd-soc-lpass-wsa-macro.ko",
+ "sound/soc/codecs/snd-soc-max98927.ko",
+ "sound/soc/codecs/snd-soc-rl6231.ko",
+ "sound/soc/codecs/snd-soc-rt5663.ko",
+ "sound/soc/codecs/snd-soc-wcd-mbhc.ko",
+ "sound/soc/codecs/snd-soc-wcd9335.ko",
+ "sound/soc/codecs/snd-soc-wcd934x.ko",
+ "sound/soc/codecs/snd-soc-wsa881x.ko",
+ "sound/soc/qcom/qdsp6/q6adm.ko",
+ "sound/soc/qcom/qdsp6/q6afe.ko",
+ "sound/soc/qcom/qdsp6/q6afe-clocks.ko",
+ "sound/soc/qcom/qdsp6/q6afe-dai.ko",
+ "sound/soc/qcom/qdsp6/q6apm-dai.ko",
+ "sound/soc/qcom/qdsp6/q6apm-lpass-dais.ko",
+ "sound/soc/qcom/qdsp6/q6asm.ko",
+ "sound/soc/qcom/qdsp6/q6asm-dai.ko",
+ "sound/soc/qcom/qdsp6/q6core.ko",
+ "sound/soc/qcom/qdsp6/q6prm.ko",
+ "sound/soc/qcom/qdsp6/q6prm-clocks.ko",
+ "sound/soc/qcom/qdsp6/q6routing.ko",
+ "sound/soc/qcom/qdsp6/snd-q6apm.ko",
+ "sound/soc/qcom/qdsp6/snd-q6dsp-common.ko",
+ "sound/soc/qcom/snd-soc-qcom-common.ko",
+ "sound/soc/qcom/snd-soc-sdm845.ko",
+ "sound/soc/qcom/snd-soc-sm8250.ko",
+ ],
+)
+
+# TODO(b/258259749): Convert rockpi4 to mixed build
+kernel_build(
+ name = "rockpi4",
+ outs = [
+ "Image",
+ "System.map",
+ "modules.builtin",
+ "modules.builtin.modinfo",
+ "rk3399-rock-pi-4b.dtb",
+ "vmlinux",
+ "vmlinux.symvers",
+ ],
+ build_config = "build.config.rockpi4",
+ dtstree = "//common-modules/virtual-device:rockpi4_dts",
+ module_outs = COMMON_GKI_MODULES_LIST + [
+ # keep sorted
+ "drivers/block/virtio_blk.ko",
+ "drivers/char/hw_random/virtio-rng.ko",
+ "drivers/clk/clk-rk808.ko",
+ "drivers/cpufreq/cpufreq-dt.ko",
+ "drivers/dma/pl330.ko",
+ "drivers/gpu/drm/bridge/analogix/analogix_dp.ko",
+ "drivers/gpu/drm/bridge/synopsys/dw-hdmi.ko",
+ "drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.ko",
+ "drivers/gpu/drm/display/drm_display_helper.ko",
+ "drivers/gpu/drm/drm_dma_helper.ko",
+ "drivers/gpu/drm/rockchip/rockchipdrm.ko",
+ "drivers/i2c/busses/i2c-rk3x.ko",
+ "drivers/iio/adc/rockchip_saradc.ko",
+ "drivers/iio/buffer/industrialio-triggered-buffer.ko",
+ "drivers/iio/buffer/kfifo_buf.ko",
+ "drivers/mfd/rk808.ko",
+ "drivers/mmc/core/pwrseq_simple.ko",
+ "drivers/mmc/host/cqhci.ko",
+ "drivers/mmc/host/dw_mmc.ko",
+ "drivers/mmc/host/dw_mmc-pltfm.ko",
+ "drivers/mmc/host/dw_mmc-rockchip.ko",
+ "drivers/mmc/host/sdhci-of-arasan.ko",
+ "drivers/net/ethernet/stmicro/stmmac/dwmac-rk.ko",
+ "drivers/net/ethernet/stmicro/stmmac/stmmac.ko",
+ "drivers/net/ethernet/stmicro/stmmac/stmmac-platform.ko",
+ "drivers/net/net_failover.ko",
+ "drivers/net/pcs/pcs_xpcs.ko",
+ "drivers/net/virtio_net.ko",
+ "drivers/pci/controller/pcie-rockchip-host.ko",
+ "drivers/phy/rockchip/phy-rockchip-emmc.ko",
+ "drivers/phy/rockchip/phy-rockchip-inno-usb2.ko",
+ "drivers/phy/rockchip/phy-rockchip-pcie.ko",
+ "drivers/phy/rockchip/phy-rockchip-typec.ko",
+ "drivers/pwm/pwm-rockchip.ko",
+ "drivers/regulator/fan53555.ko",
+ "drivers/regulator/pwm-regulator.ko",
+ "drivers/regulator/rk808-regulator.ko",
+ "drivers/rtc/rtc-rk808.ko",
+ "drivers/soc/rockchip/io-domain.ko",
+ "drivers/thermal/rockchip_thermal.ko",
+ "drivers/usb/host/ohci-hcd.ko",
+ "drivers/usb/host/ohci-platform.ko",
+ "drivers/virtio/virtio_pci.ko",
+ "drivers/virtio/virtio_pci_legacy_dev.ko",
+ "drivers/virtio/virtio_pci_modern_dev.ko",
+ "drivers/watchdog/dw_wdt.ko",
+ "net/core/failover.ko",
+ ],
+)
+
+kernel_modules_install(
+ name = "rockpi4_modules_install",
+ kernel_build = "//common:rockpi4",
+)
+
+kernel_images(
+ name = "rockpi4_images",
+ build_initramfs = True,
+ kernel_build = "//common:rockpi4",
+ kernel_modules_install = "//common:rockpi4_modules_install",
+)
+
+copy_to_dist_dir(
+ name = "rockpi4_dist",
+ data = [
+ ":rockpi4",
+ ":rockpi4_images",
+ ":rockpi4_modules_install",
+ ],
+ dist_dir = "out/rockpi4/dist",
+ flat = True,
+)
+
+# allmodconfig build tests.
+# These are build tests only, so:
+# - outs are intentionally set to empty to not copy anything to DIST_DIR
+# - --allow-undeclared-modules must be used so modules are not declared or copied.
+# - No dist target because these are build tests. We don't care about the artifacts.
+
+# tools/bazel build --allow_undeclared_modules //common:kernel_aarch64_allmodconfig
+kernel_build(
+ name = "kernel_aarch64_allmodconfig",
+ # Hack to actually check the build.
+ # Otherwise, Bazel thinks that there are no output files, and skip building.
+ outs = [".config"],
+ build_config = "build.config.allmodconfig.aarch64",
+ visibility = ["//visibility:private"],
+)
+
+# tools/bazel build --allow_undeclared_modules //common:kernel_x86_64_allmodconfig
+kernel_build(
+ name = "kernel_x86_64_allmodconfig",
+ # Hack to actually check the build.
+ # Otherwise, Bazel thinks that there are no output files, and skip building.
+ outs = [".config"],
+ build_config = "build.config.allmodconfig.x86_64",
+ visibility = ["//visibility:private"],
+)
+
+# tools/bazel build --allow_undeclared_modules //common:kernel_arm_allmodconfig
+kernel_build(
+ name = "kernel_arm_allmodconfig",
+ # Hack to actually check the build.
+ # Otherwise, Bazel thinks that there are no output files, and skip building.
+ outs = [".config"],
+ build_config = "build.config.allmodconfig.arm",
+ visibility = ["//visibility:private"],
+)
+
+# DDK Headers
+# All headers. These are the public targets for DDK modules to use.
+alias(
+ name = "all_headers",
+ actual = "all_headers_aarch64",
+ visibility = ["//visibility:public"],
+)
+
+ddk_headers(
+ name = "all_headers_aarch64",
+ hdrs = [":all_headers_allowlist_aarch64"] + select({
+ "//build/kernel/kleaf:allow_ddk_unsafe_headers_set": [":all_headers_unsafe"],
+ "//conditions:default": [],
+ }),
+ visibility = ["//visibility:public"],
+)
+
+ddk_headers(
+ name = "all_headers_arm",
+ hdrs = [":all_headers_allowlist_arm"] + select({
+ "//build/kernel/kleaf:allow_ddk_unsafe_headers_set": [":all_headers_unsafe"],
+ "//conditions:default": [],
+ }),
+ visibility = ["//visibility:public"],
+)
+
+ddk_headers(
+ name = "all_headers_riscv64",
+ hdrs = [":all_headers_allowlist_riscv64"] + select({
+ "//build/kernel/kleaf:allow_ddk_unsafe_headers_set": [":all_headers_unsafe"],
+ "//conditions:default": [],
+ }),
+ visibility = ["//visibility:public"],
+)
+
+ddk_headers(
+ name = "all_headers_x86_64",
+ hdrs = [":all_headers_allowlist_x86_64"] + select({
+ "//build/kernel/kleaf:allow_ddk_unsafe_headers_set": [":all_headers_unsafe"],
+ "//conditions:default": [],
+ }),
+ visibility = ["//visibility:public"],
+)
+
+# Implementation details for DDK headers. The targets below cannot be directly
+# depended on by DDK modules.
+
+# DDK headers allowlist. This is the list of all headers and include
+# directories that are safe to use in DDK modules.
+ddk_headers(
+ name = "all_headers_allowlist_aarch64",
+ hdrs = [
+ ":all_headers_allowlist_aarch64_globs",
+ ":all_headers_allowlist_common_globs",
+ ],
+ # The list of include directories where source files can #include headers
+ # from. In other words, these are the `-I` option to the C compiler.
+ # These are prepended to LINUXINCLUDE.
+ linux_includes = [
+ "arch/arm64/include",
+ "arch/arm64/include/uapi",
+ "include",
+ "include/uapi",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+ddk_headers(
+ name = "all_headers_allowlist_arm",
+ hdrs = [
+ ":all_headers_allowlist_arm_globs",
+ ":all_headers_allowlist_common_globs",
+ ],
+ # The list of include directories where source files can #include headers
+ # from. In other words, these are the `-I` option to the C compiler.
+ # These are prepended to LINUXINCLUDE.
+ linux_includes = [
+ "arch/arm/include",
+ "arch/arm/include/uapi",
+ "include",
+ "include/uapi",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+ddk_headers(
+ name = "all_headers_allowlist_riscv64",
+ hdrs = [
+ ":all_headers_allowlist_common_globs",
+ ":all_headers_allowlist_riscv64_globs",
+ ],
+ # The list of include directories where source files can #include headers
+ # from. In other words, these are the `-I` option to the C compiler.
+ # These are prepended to LINUXINCLUDE.
+ linux_includes = [
+ "arch/riscv/include",
+ "arch/riscv/include/uapi",
+ "include",
+ "include/uapi",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+ddk_headers(
+ name = "all_headers_allowlist_x86_64",
+ hdrs = [
+ ":all_headers_allowlist_common_globs",
+ ":all_headers_allowlist_x86_64_globs",
+ ],
+ # The list of include directories where source files can #include headers
+ # from. In other words, these are the `-I` option to the C compiler.
+ # These are prepended to LINUXINCLUDE.
+ linux_includes = [
+ "arch/x86/include",
+ "arch/x86/include/uapi",
+ "include",
+ "include/uapi",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+# List of DDK headers allowlist that are glob()-ed to avoid changes of BUILD
+# file when the list of files changes. All headers in these directories
+# are safe to use.
+# These are separate filegroup targets so the all_headers_allowlist_* are
+# more friendly to batch BUILD file update tools like buildozer.
+
+# globs() for arm64 only
+filegroup(
+ name = "all_headers_allowlist_aarch64_globs",
+ srcs = glob(["arch/arm64/include/**/*.h"]),
+ visibility = ["//visibility:private"],
+)
+
+# globs() for arm only
+filegroup(
+ name = "all_headers_allowlist_arm_globs",
+ srcs = glob(["arch/arm/include/**/*.h"]),
+ visibility = ["//visibility:private"],
+)
+
+# globs() for riscv64 only
+filegroup(
+ name = "all_headers_allowlist_riscv64_globs",
+ srcs = glob(["arch/riscv/include/**/*.h"]),
+ visibility = ["//visibility:private"],
+)
+
+# globs() for x86 only
+filegroup(
+ name = "all_headers_allowlist_x86_64_globs",
+ srcs = glob(["arch/x86/include/**/*.h"]),
+ visibility = ["//visibility:private"],
+)
+
+# globs() for all architectures
+filegroup(
+ name = "all_headers_allowlist_common_globs",
+ srcs = glob(["include/**/*.h"]),
+ visibility = ["//visibility:private"],
+)
+
+# DDK headers unsafe list. This is the list of all headers and include
+# directories that may be used during migration from kernel_module's, but
+# should be avoided in general.
+# Use with caution; items may:
+# - be removed without notice
+# - be moved into all_headers
+ddk_headers(
+ name = "all_headers_unsafe",
+ hdrs = [
+ "drivers/gpu/drm/virtio/virtgpu_trace.h",
+ ],
+ # The list of include directories where source files can #include headers
+ # from. In other words, these are the `-I` option to the C compiler.
+ # Unsafe include directories are appended to ccflags-y.
+ includes = [],
+ visibility = ["//visibility:private"],
+)
diff --git a/Documentation/ABI/testing/OWNERS b/Documentation/ABI/testing/OWNERS
new file mode 100644
index 0000000..3ab5dca
--- /dev/null
+++ b/Documentation/ABI/testing/OWNERS
@@ -0,0 +1 @@
+per-file sysfs-fs-f2fs=file:/fs/f2fs/OWNERS
diff --git a/Documentation/ABI/testing/sysfs-fs-incfs b/Documentation/ABI/testing/sysfs-fs-incfs
new file mode 100644
index 0000000..690c687
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-fs-incfs
@@ -0,0 +1,64 @@
+What: /sys/fs/incremental-fs/features/corefs
+Date: 2019
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Reads 'supported'. Always present.
+
+What: /sys/fs/incremental-fs/features/v2
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Reads 'supported'. Present if all v2 features of incfs are
+ supported.
+
+What: /sys/fs/incremental-fs/features/zstd
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Reads 'supported'. Present if zstd compression is supported
+ for data blocks.
+
+What: /sys/fs/incremental-fs/instances/[name]
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Folder created when incfs is mounted with the sysfs_name=[name]
+ option. If this option is used, the following values are created
+ in this folder.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_delayed_min
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns a count of the number of reads that were delayed as a
+ result of the per UID read timeouts min time setting.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_delayed_min_us
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns total delay time for all files since first mount as a
+ result of the per UID read timeouts min time setting.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_delayed_pending
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns a count of the number of reads that were delayed as a
+ result of waiting for a pending read.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_delayed_pending_us
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns total delay time for all files since first mount as a
+ result of waiting for a pending read.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_failed_hash_verification
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns number of reads that failed because of hash verification
+ failures.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_failed_other
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns number of reads that failed for reasons other than
+ timing out or hash failures.
+
+What: /sys/fs/incremental-fs/instances/[name]/reads_failed_timed_out
+Date: April 2021
+Contact: Paul Lawrence <paullawrence@google.com>
+Description: Returns number of reads that timed out.
diff --git a/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons
new file mode 100644
index 0000000..acb19b9
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons
@@ -0,0 +1,16 @@
+What: /sys/kernel/wakeup_reasons/last_resume_reason
+Date: February 2014
+Contact: Ruchi Kandoi <kandoiruchi@google.com>
+Description:
+ The /sys/kernel/wakeup_reasons/last_resume_reason is
+ used to report wakeup reasons after system exited suspend.
+
+What: /sys/kernel/wakeup_reasons/last_suspend_time
+Date: March 2015
+Contact: jinqian <jinqian@google.com>
+Description:
+ The /sys/kernel/wakeup_reasons/last_suspend_time is
+ used to report time spent in last suspend cycle. It contains
+ two numbers (in seconds) separated by space. First number is
+ the time spent in suspend and resume processes. Second number
+ is the time spent in sleep state.
\ No newline at end of file
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index fb388c6..7680d38 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1782,6 +1782,10 @@
If specified, z/VM IUCV HVC accepts connections
from listed z/VM user IDs only.
+ hvc_dcc.enable= [ARM,ARM64] Enable DCC driver at runtime. For GKI,
+ disabled at runtime by default to prevent
+ crashes in devices which do not support DCC.
+
hv_nopvspin [X86,HYPER_V] Disables the paravirt spinlock optimizations
which allow the hypervisor to 'idle' the
guest on lock contention.
@@ -1792,6 +1796,10 @@
between unregistering the boot console and initializing
the real console.
+ kunit.enable= [KUNIT] Enable executing KUnit tests in modules on load.
+ Requires CONFIG_KUNIT to be enabled.
+ Default is 0 (disabled)
+
i2c_bus= [HW] Override the default board specific I2C bus speed
or register an additional I2C bus that is not
registered from board initialization code.
diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst
index f9bf18e..4fd2b11 100644
--- a/Documentation/block/inline-encryption.rst
+++ b/Documentation/block/inline-encryption.rst
@@ -77,10 +77,10 @@
============
We introduce ``struct blk_crypto_key`` to represent an inline encryption key and
-how it will be used. This includes the actual bytes of the key; the size of the
-key; the algorithm and data unit size the key will be used with; and the number
-of bytes needed to represent the maximum data unit number the key will be used
-with.
+how it will be used. This includes the type of the key (standard or
+hardware-wrapped); the actual bytes of the key; the size of the key; the
+algorithm and data unit size the key will be used with; and the number of bytes
+needed to represent the maximum data unit number the key will be used with.
We introduce ``struct bio_crypt_ctx`` to represent an encryption context. It
contains a data unit number and a pointer to a blk_crypto_key. We add pointers
@@ -302,3 +302,216 @@
When the crypto API fallback is enabled, this means that all bios with and
encryption context will use the fallback, and IO will complete as usual. When
the fallback is disabled, a bio with an encryption context will be failed.
+
+.. _hardware_wrapped_keys:
+
+Hardware-wrapped keys
+=====================
+
+Motivation and threat model
+---------------------------
+
+Linux storage encryption (dm-crypt, fscrypt, eCryptfs, etc.) traditionally
+relies on the raw encryption key(s) being present in kernel memory so that the
+encryption can be performed. This traditionally isn't seen as a problem because
+the key(s) won't be present during an offline attack, which is the main type of
+attack that storage encryption is intended to protect from.
+
+However, there is an increasing desire to also protect users' data from other
+types of attacks (to the extent possible), including:
+
+- Cold boot attacks, where an attacker with physical access to a system suddenly
+ powers it off, then immediately dumps the system memory to extract recently
+ in-use encryption keys, then uses these keys to decrypt user data on-disk.
+
+- Online attacks where the attacker is able to read kernel memory without fully
+ compromising the system, followed by an offline attack where any extracted
+ keys can be used to decrypt user data on-disk. An example of such an online
+ attack would be if the attacker is able to run some code on the system that
+ exploits a Meltdown-like vulnerability but is unable to escalate privileges.
+
+- Online attacks where the attacker fully compromises the system, but their data
+ exfiltration is significantly time-limited and/or bandwidth-limited, so in
+ order to completely exfiltrate the data they need to extract the encryption
+ keys to use in a later offline attack.
+
+Hardware-wrapped keys are a feature of inline encryption hardware that is
+designed to protect users' data from the above attacks (to the extent possible),
+without introducing limitations such as a maximum number of keys.
+
+Note that it is impossible to **fully** protect users' data from these attacks.
+Even in the attacks where the attacker "just" gets read access to kernel memory,
+they can still extract any user data that is present in memory, including
+plaintext pagecache pages of encrypted files. The focus here is just on
+protecting the encryption keys, as those instantly give access to **all** user
+data in any following offline attack, rather than just some of it (where which
+data is included in that "some" might not be controlled by the attacker).
+
+Solution overview
+-----------------
+
+Inline encryption hardware typically has "keyslots" into which software can
+program keys for the hardware to use; the contents of keyslots typically can't
+be read back by software. As such, the above security goals could be achieved
+if the kernel simply erased its copy of the key(s) after programming them into
+keyslot(s) and thereafter only referred to them via keyslot number.
+
+However, that naive approach runs into the problem that it limits the number of
+unlocked keys to the number of keyslots, which typically is a small number. In
+cases where there is only one encryption key system-wide (e.g., a full-disk
+encryption key), that can be tolerable. However, in general there can be many
+logged-in users with many different keys, and/or many running applications with
+application-specific encrypted storage areas. This is especially true if
+file-based encryption (e.g. fscrypt) is being used.
+
+Thus, it is important for the kernel to still have a way to "remind" the
+hardware about a key, without actually having the raw key itself. This would
+ensure that the number of hardware keyslots only limits the number of active I/O
+requests, not other things such as the number of logged-in users, the number of
+running apps, or the number of encrypted storage areas that apps can create.
+
+Somewhat less importantly, it is also desirable that the raw keys are never
+visible to software at all, even while being initially unlocked. This would
+ensure that a read-only compromise of system memory will never allow a key to be
+extracted to be used off-system, even if it occurs when a key is being unlocked.
+
+To solve all these problems, some vendors of inline encryption hardware have
+made their hardware support *hardware-wrapped keys*. Hardware-wrapped keys
+are encrypted keys that can only be unwrapped (decrypted) and used by hardware
+-- either by the inline encryption hardware itself, or by a dedicated hardware
+block that can directly provision keys to the inline encryption hardware.
+
+(We refer to them as "hardware-wrapped keys" rather than simply "wrapped keys"
+to add some clarity in cases where there could be other types of wrapped keys,
+such as in file-based encryption. Key wrapping is a commonly used technique.)
+
+The key which wraps (encrypts) hardware-wrapped keys is a hardware-internal key
+that is never exposed to software; it is either a persistent key (a "long-term
+wrapping key") or a per-boot key (an "ephemeral wrapping key"). The long-term
+wrapped form of the key is what is initially unlocked, but it is erased from
+memory as soon as it is converted into an ephemerally-wrapped key. In-use
+hardware-wrapped keys are always ephemerally-wrapped, not long-term wrapped.
+
+As inline encryption hardware can only be used to encrypt/decrypt data on-disk,
+the hardware also includes a level of indirection; it doesn't use the unwrapped
+key directly for inline encryption, but rather derives both an inline encryption
+key and a "software secret" from it. Software can use the "software secret" for
+tasks that can't use the inline encryption hardware, such as filenames
+encryption. The software secret is not protected from memory compromise.
+
+Key hierarchy
+-------------
+
+Here is the key hierarchy for a hardware-wrapped key::
+
+ Hardware-wrapped key
+ |
+ |
+ <Hardware KDF>
+ |
+ -----------------------------
+ | |
+ Inline encryption key Software secret
+
+The components are:
+
+- *Hardware-wrapped key*: a key for the hardware's KDF (Key Derivation
+ Function), in ephemerally-wrapped form. The key wrapping algorithm is a
+ hardware implementation detail that doesn't impact kernel operation, but a
+ strong authenticated encryption algorithm such as AES-256-GCM is recommended.
+
+- *Hardware KDF*: a KDF (Key Derivation Function) which the hardware uses to
+ derive subkeys after unwrapping the wrapped key. The hardware's choice of KDF
+ doesn't impact kernel operation, but it does need to be known for testing
+ purposes, and it's also assumed to have at least a 256-bit security strength.
+ All known hardware uses the SP800-108 KDF in Counter Mode with AES-256-CMAC,
+ with a particular choice of labels and contexts; new hardware should use this
+ already-vetted KDF.
+
+- *Inline encryption key*: a derived key which the hardware directly provisions
+ to a keyslot of the inline encryption hardware, without exposing it to
+ software. In all known hardware, this will always be an AES-256-XTS key.
+ However, in principle other encryption algorithms could be supported too.
+ Hardware must derive distinct subkeys for each supported encryption algorithm.
+
+- *Software secret*: a derived key which the hardware returns to software so
+ that software can use it for cryptographic tasks that can't use inline
+ encryption. This value is cryptographically isolated from the inline
+ encryption key, i.e. knowing one doesn't reveal the other. (The KDF ensures
+ this.) Currently, the software secret is always 32 bytes and thus is suitable
+ for cryptographic applications that require up to a 256-bit security strength.
+ Some use cases (e.g. full-disk encryption) won't require the software secret.
+
+Example: in the case of fscrypt, the fscrypt master key (the key that protects a
+particular set of encrypted directories) is made hardware-wrapped. The inline
+encryption key is used as the file contents encryption key, while the software
+secret (rather than the master key directly) is used to key fscrypt's KDF
+(HKDF-SHA512) to derive other subkeys such as filenames encryption keys.
+
+Note that currently this design assumes a single inline encryption key per
+hardware-wrapped key, without any further key derivation. Thus, in the case of
+fscrypt, currently hardware-wrapped keys are only compatible with the "inline
+encryption optimized" settings, which use one file contents encryption key per
+encryption policy rather than one per file. This design could be extended to
+make the hardware derive per-file keys using per-file nonces passed down the
+storage stack, and in fact some hardware already supports this; future work is
+planned to remove this limitation by adding the corresponding kernel support.
+
+Kernel support
+--------------
+
+The inline encryption support of the kernel's block layer ("blk-crypto") has
+been extended to support hardware-wrapped keys as an alternative to standard
+keys, when hardware support is available. This works in the following way:
+
+- A ``key_types_supported`` field is added to the crypto capabilities in
+ ``struct blk_crypto_profile``. This allows device drivers to declare that
+ they support standard keys, hardware-wrapped keys, or both.
+
+- ``struct blk_crypto_key`` can now contain a hardware-wrapped key as an
+ alternative to a standard key; a ``key_type`` field is added to
+ ``struct blk_crypto_config`` to distinguish between the different key types.
+ This allows users of blk-crypto to en/decrypt data using a hardware-wrapped
+ key in a way very similar to using a standard key.
+
+- A new method ``blk_crypto_ll_ops::derive_sw_secret`` is added. Device drivers
+ that support hardware-wrapped keys must implement this method. Users of
+ blk-crypto can call ``blk_crypto_derive_sw_secret()`` to access this method.
+
+- The programming and eviction of hardware-wrapped keys happens via
+ ``blk_crypto_ll_ops::keyslot_program`` and
+ ``blk_crypto_ll_ops::keyslot_evict``, just like it does for standard keys. If
+ a driver supports hardware-wrapped keys, then it must handle hardware-wrapped
+ keys being passed to these methods.
+
+blk-crypto-fallback doesn't support hardware-wrapped keys. Therefore,
+hardware-wrapped keys can only be used with actual inline encryption hardware.
+
+Currently, the kernel only works with hardware-wrapped keys in
+ephemerally-wrapped form. No generic kernel interfaces are provided for
+generating or importing hardware-wrapped keys in the first place, or converting
+them to ephemerally-wrapped form. In Android, SoC vendors are required to
+support these operations in their KeyMint implementation (a hardware abstraction
+layer in userspace); for details, see the `Android documentation
+<https://source.android.com/security/encryption/hw-wrapped-keys>`_.
+
+Testability
+-----------
+
+Both the hardware KDF and the inline encryption itself are well-defined
+algorithms that don't depend on any secrets other than the unwrapped key.
+Therefore, if the unwrapped key is known to software, these algorithms can be
+reproduced in software in order to verify the ciphertext that is written to disk
+by the inline encryption hardware.
+
+However, the unwrapped key will only be known to software for testing if the
+"import" functionality is used. Proper testing is not possible in the
+"generate" case where the hardware generates the key itself. The correct
+operation of the "generate" mode thus relies on the security and correctness of
+the hardware RNG and its use to generate the key, as well as the testing of the
+"import" mode as that should cover all parts other than the key generation.
+
+For an example of a test that verifies the ciphertext written to disk in the
+"import" mode, see the fscrypt hardware-wrapped key tests in xfstests, or
+`Android's vts_kernel_encryption_test
+<https://android.googlesource.com/platform/test/vts-testcase/kernel/+/refs/heads/master/encryption/>`_.
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index b3593ae..bb38fa8 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -18,6 +18,15 @@
faq
running_tips
+.. warning::
+ AOSP only supports running tests loaded with modules. Built-in
+ test execution support has been disabled. In addition, in order
+ to fully enable running module loaded tests both CONFIG_KUNIT
+ needs to be enabled and kernel command line argument
+ `kunit.enable` needs to be set to 1.
+
+ The remaining KUnit documentation has been left as-is.
+
This section details the kernel unit testing framework.
Introduction
diff --git a/Documentation/filesystems/OWNERS b/Documentation/filesystems/OWNERS
new file mode 100644
index 0000000..a63dbf4
--- /dev/null
+++ b/Documentation/filesystems/OWNERS
@@ -0,0 +1 @@
+per-file f2fs**=file:/fs/f2fs/OWNERS
diff --git a/Documentation/filesystems/incfs.rst b/Documentation/filesystems/incfs.rst
new file mode 100644
index 0000000..19db303
--- /dev/null
+++ b/Documentation/filesystems/incfs.rst
@@ -0,0 +1,82 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================================================
+incfs: A stacked incremental filesystem for Linux
+=================================================
+
+/sys/fs interface
+=================
+
+Please update Documentation/ABI/testing/sysfs-fs-incfs if you update this
+section.
+
+incfs creates the following files in /sys/fs.
+
+Features
+--------
+
+/sys/fs/incremental-fs/features/corefs
+ Reads 'supported'. Always present.
+
+/sys/fs/incremental-fs/features/v2
+ Reads 'supported'. Present if all v2 features of incfs are supported. These
+ are:
+ fs-verity support
+ inotify support
+ ioclts:
+ INCFS_IOC_SET_READ_TIMEOUTS
+ INCFS_IOC_GET_READ_TIMEOUTS
+ INCFS_IOC_GET_BLOCK_COUNT
+ INCFS_IOC_CREATE_MAPPED_FILE
+ .incomplete folder
+ .blocks_written pseudo file
+ report_uid mount option
+
+/sys/fs/incremental-fs/features/zstd
+ Reads 'supported'. Present if zstd compression is supported for data blocks.
+
+Optional per mount
+------------------
+
+For each incfs mount, the mount option sysfs_name=[name] creates a /sys/fs
+node called:
+
+/sys/fs/incremental-fs/instances/[name]
+
+This will contain the following files:
+
+/sys/fs/incremental-fs/instances/[name]/reads_delayed_min
+ Returns a count of the number of reads that were delayed as a result of the
+ per UID read timeouts min time setting.
+
+/sys/fs/incremental-fs/instances/[name]/reads_delayed_min_us
+ Returns total delay time for all files since first mount as a result of the
+ per UID read timeouts min time setting.
+
+/sys/fs/incremental-fs/instances/[name]/reads_delayed_pending
+ Returns a count of the number of reads that were delayed as a result of
+ waiting for a pending read.
+
+/sys/fs/incremental-fs/instances/[name]/reads_delayed_pending_us
+ Returns total delay time for all files since first mount as a result of
+ waiting for a pending read.
+
+/sys/fs/incremental-fs/instances/[name]/reads_failed_hash_verification
+ Returns number of reads that failed because of hash verification failures.
+
+/sys/fs/incremental-fs/instances/[name]/reads_failed_other
+ Returns number of reads that failed for reasons other than timing out or
+ hash failures.
+
+/sys/fs/incremental-fs/instances/[name]/reads_failed_timed_out
+ Returns number of reads that timed out.
+
+For reads_delayed_*** settings, note that a file can count for both
+reads_delayed_min and reads_delayed_pending if incfs first waits for a pending
+read then has to wait further for the min time. In that case, the time spent
+waiting is split between reads_delayed_pending_us, which is increased by the
+time spent waiting for the pending read, and reads_delayed_min_us, which is
+increased by the remainder of the time spent waiting.
+
+Reads that timed out are not added to the reads_delayed_pending or the
+reads_delayed_pending_us counters.
diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst
index 4c76fda..6e316f8 100644
--- a/Documentation/filesystems/overlayfs.rst
+++ b/Documentation/filesystems/overlayfs.rst
@@ -195,7 +195,7 @@
1. return EXDEV error: this error is returned by rename(2) when trying to
move a file or directory across filesystem boundaries. Hence
- applications are usually prepared to hande this error (mv(1) for example
+ applications are usually prepared to handle this error (mv(1) for example
recursively copies the directory tree). This is the default behavior.
2. If the "redirect_dir" feature is enabled, then the directory will be
@@ -324,6 +324,30 @@
The resulting access permissions should be the same. The difference is in
the time of copy (on-demand vs. up-front).
+### Non overlapping credentials
+
+As noted above, all access to the upper, lower and work directories is the
+recorded mounter's MAC and DAC credentials. The incoming accesses are
+checked against the caller's credentials.
+
+In the case where caller MAC or DAC credentials do not overlap the mounter, a
+use case available in older versions of the driver, the override_creds mount
+flag can be turned off. For when the use pattern has caller with legitimate
+credentials where the mounter does not. For example init may have been the
+mounter, but the caller would have execute or read MAC permissions where
+init would not. override_creds off means all access, incoming, upper, lower
+or working, will be tested against the caller.
+
+Several unintended side effects will occur though. The caller without certain
+key capabilities or lower privilege will not always be able to delete files or
+directories, create nodes, or search some restricted directories. The ability
+to search and read a directory entry is spotty as a result of the cache
+mechanism not re-testing the credentials because of the assumption, a
+privileged caller can fill cache, then a lower privilege can read the directory
+cache. The uneven security model where cache, upperdir and workdir are opened
+at privilege, but accessed without creating a form of privilege escalation,
+should only be used with strict understanding of the side effects and of the
+security policies.
Multiple lower layers
---------------------
diff --git a/Documentation/kbuild/modules.rst b/Documentation/kbuild/modules.rst
index a1f3eb7..e9b6fe2 100644
--- a/Documentation/kbuild/modules.rst
+++ b/Documentation/kbuild/modules.rst
@@ -21,6 +21,7 @@
--- 4.1 Kernel Includes
--- 4.2 Single Subdirectory
--- 4.3 Several Subdirectories
+ --- 4.4 UAPI Headers Installation
=== 5. Module Installation
--- 5.1 INSTALL_MOD_PATH
--- 5.2 INSTALL_MOD_DIR
@@ -131,6 +132,10 @@
/lib/modules/<kernel_release>/extra/, but a prefix may
be added with INSTALL_MOD_PATH (discussed in section 5).
+ headers_install
+ Export headers in a format suitable for userspace. The default
+ location is $PWD/usr. INSTALL_HDR_PATH can change this path.
+
clean
Remove all generated files in the module directory only.
@@ -406,6 +411,17 @@
pointing to the directory where the currently executing kbuild
file is located.
+4.4 UAPI Headers Installation
+-----------------------------
+
+ External modules may export headers to userspace in a similar
+ fashion to the in-tree counterpart drivers. kbuild supports
+ running headers_install target in an out-of-tree. The location
+ where kbuild searches for headers is $(M)/include/uapi and
+ $(M)/arch/$(SRCARCH)/include/uapi.
+
+ See also Documentation/kbuild/headers_install.rst.
+
5. Module Installation
======================
diff --git a/Documentation/scheduler/sched-energy.rst b/Documentation/scheduler/sched-energy.rst
index 8fbce5e..83f1179 100644
--- a/Documentation/scheduler/sched-energy.rst
+++ b/Documentation/scheduler/sched-energy.rst
@@ -402,7 +402,7 @@
because it is the only one providing some degree of consistency between
frequency requests and energy predictions.
-Using EAS with any other governor than schedutil is not supported.
+Using EAS with any other governor than schedutil is not recommended.
6.5 Scale-invariant utilization signals
diff --git a/Kconfig b/Kconfig
index 745bc77..57a142d 100644
--- a/Kconfig
+++ b/Kconfig
@@ -30,3 +30,6 @@
source "lib/Kconfig.debug"
source "Documentation/Kconfig"
+
+# ANDROID: Set KCONFIG_EXT_PREFIX to decend into an external project.
+source "$(KCONFIG_EXT_PREFIX)Kconfig.ext"
diff --git a/Kconfig.ext b/Kconfig.ext
new file mode 100644
index 0000000..48d805f
--- /dev/null
+++ b/Kconfig.ext
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+# This file is intentionally empty. It's used as a placeholder for when
+# KCONFIG_EXT_PREFIX isn't defined.
diff --git a/MAINTAINERS b/MAINTAINERS
index fb2686a..4069815 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -10076,6 +10076,13 @@
F: drivers/hwmon/ina2xx.c
F: include/linux/platform_data/ina2xx.h
+INCREMENTAL FILE SYSTEM
+M: Paul Lawrence <paullawrence@google.com>
+L: linux-unionfs@vger.kernel.org
+S: Supported
+F: fs/incfs/
+F: tools/testing/selftests/filesystems/incfs/
+
INDEX OF FURTHER KERNEL DOCUMENTATION
M: Carlos Bilbao <carlos.bilbao@amd.com>
S: Maintained
diff --git a/Makefile b/Makefile
index bb60045..3327d5b 100644
--- a/Makefile
+++ b/Makefile
@@ -148,6 +148,24 @@
export KBUILD_EXTMOD
+# ANDROID: set up mixed-build support. mixed-build allows device kernel modules
+# to be compiled against a GKI kernel. This approach still uses the headers and
+# Kbuild from device kernel, so care must be taken to ensure that those headers match.
+ifdef KBUILD_MIXED_TREE
+# Need vmlinux.symvers for modpost and System.map for depmod, check whether they exist in KBUILD_MIXED_TREE
+required_mixed_files=vmlinux.symvers System.map
+$(if $(filter-out $(words $(required_mixed_files)), \
+ $(words $(wildcard $(add-prefix $(KBUILD_MIXED_TREE)/,$(required_mixed_files))))),,\
+ $(error KBUILD_MIXED_TREE=$(KBUILD_MIXED_TREE) doesn't contain $(required_mixed_files)))
+endif
+
+mixed-build-prefix = $(if $(KBUILD_MIXED_TREE),$(KBUILD_MIXED_TREE)/)
+export KBUILD_MIXED_TREE
+# This is a hack for kleaf to set mixed-build-prefix within the execution of a make rule, e.g.
+# within __modinst_pre.
+# TODO(b/205893923): Revert this hack once it is properly handled.
+export mixed-build-prefix
+
# Kbuild will save output files in the current working directory.
# This does not need to match to the root of the kernel source tree.
#
@@ -522,7 +540,7 @@
KBZIP2 = bzip2
KLZOP = lzop
LZMA = lzma
-LZ4 = lz4c
+LZ4 = lz4
XZ = xz
ZSTD = zstd
@@ -748,11 +766,13 @@
libs-y := lib/
endif # KBUILD_EXTMOD
+ifndef KBUILD_MIXED_TREE
# The all: target is the default when no target is given on the
# command line.
# This allow a user to issue only 'make' to build a kernel including modules
# Defaults to vmlinux, but the arch makefile usually adds further targets
all: vmlinux
+endif
CFLAGS_GCOV := -fprofile-arcs -ftest-coverage
ifdef CONFIG_CC_IS_GCC
@@ -980,7 +1000,13 @@
else
CC_FLAGS_LTO := -flto
endif
+
+ifeq ($(SRCARCH),x86)
+# Workaround for compiler / linker bug
CC_FLAGS_LTO += -fvisibility=hidden
+else
+CC_FLAGS_LTO += -fvisibility=default
+endif
# Limit inlining across translation units to reduce binary size
KBUILD_LDFLAGS += -mllvm -import-instr-limit=5
@@ -1170,6 +1196,40 @@
export MODORDER := $(extmod_prefix)modules.order
export MODULES_NSDEPS := $(extmod_prefix)modules.nsdeps
+# ---------------------------------------------------------------------------
+# Kernel headers
+
+PHONY += headers
+
+#Default location for installed headers
+ifeq ($(KBUILD_EXTMOD),)
+PHONY += archheaders archscripts
+hdr-inst := -f $(srctree)/scripts/Makefile.headersinst obj
+headers: $(version_h) scripts_unifdef uapi-asm-generic archheaders archscripts
+else
+hdr-prefix = $(KBUILD_EXTMOD)/
+hdr-inst := -f $(srctree)/scripts/Makefile.headersinst dst=$(KBUILD_EXTMOD)/usr/include objtree=$(objtree)/$(KBUILD_EXTMOD) obj
+endif
+
+export INSTALL_HDR_PATH = $(objtree)/$(hdr-prefix)usr
+
+quiet_cmd_headers_install = INSTALL $(INSTALL_HDR_PATH)/include
+ cmd_headers_install = \
+ mkdir -p $(INSTALL_HDR_PATH); \
+ rsync -mrl --include='*/' --include='*\.h' --exclude='*' \
+ $(hdr-prefix)usr/include $(INSTALL_HDR_PATH);
+
+PHONY += headers_install
+headers_install: headers
+ $(call cmd,headers_install)
+
+headers:
+ifeq ($(KBUILD_EXTMOD),)
+ $(if $(filter um, $(SRCARCH)), $(error Headers not exportable for UML))
+endif
+ $(Q)$(MAKE) $(hdr-inst)=$(hdr-prefix)include/uapi
+ $(Q)$(MAKE) $(hdr-inst)=$(hdr-prefix)arch/$(SRCARCH)/include/uapi
+
ifeq ($(KBUILD_EXTMOD),)
build-dir := .
@@ -1226,6 +1286,7 @@
vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt autoksyms_recursive FORCE
$(call if_changed,ar_vmlinux.a)
+ifndef KBUILD_MIXED_TREE
PHONY += vmlinux_o
vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS)
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o
@@ -1236,13 +1297,15 @@
PHONY += vmlinux
vmlinux: vmlinux.o $(KBUILD_LDS) modpost
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux
+endif
# The actual objects are generated when descending,
# make sure no implicit rule kicks in
$(sort $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)): . ;
filechk_kernel.release = \
- echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))"
+ echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion \
+ $(srctree) $(BRANCH) $(KMI_GENERATION))"
# Store (new) KERNELRELEASE string in include/config/kernel.release
include/config/kernel.release: FORCE
@@ -1342,32 +1405,6 @@
$(Q)find $(srctree)/include/ -name '*.h' | xargs --max-args 1 \
$(srctree)/scripts/headerdep.pl -I$(srctree)/include
-# ---------------------------------------------------------------------------
-# Kernel headers
-
-#Default location for installed headers
-export INSTALL_HDR_PATH = $(objtree)/usr
-
-quiet_cmd_headers_install = INSTALL $(INSTALL_HDR_PATH)/include
- cmd_headers_install = \
- mkdir -p $(INSTALL_HDR_PATH); \
- rsync -mrl --include='*/' --include='*\.h' --exclude='*' \
- usr/include $(INSTALL_HDR_PATH)
-
-PHONY += headers_install
-headers_install: headers
- $(call cmd,headers_install)
-
-PHONY += archheaders archscripts
-
-hdr-inst := -f $(srctree)/scripts/Makefile.headersinst obj
-
-PHONY += headers
-headers: $(version_h) scripts_unifdef uapi-asm-generic archheaders archscripts
- $(if $(filter um, $(SRCARCH)), $(error Headers not exportable for UML))
- $(Q)$(MAKE) $(hdr-inst)=include/uapi
- $(Q)$(MAKE) $(hdr-inst)=arch/$(SRCARCH)/include/uapi
-
ifdef CONFIG_HEADERS_INSTALL
prepare: headers
endif
@@ -1445,7 +1482,9 @@
# Devicetree files
ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/boot/dts/),)
-dtstree := arch/$(SRCARCH)/boot/dts
+# ANDROID: allow this to be overridden by the build environment. This allows
+# one to compile a device tree that is located out-of-tree.
+dtstree ?= arch/$(SRCARCH)/boot/dts
endif
ifneq ($(dtstree),)
@@ -1559,8 +1598,8 @@
ln -s $(CURDIR) $(MODLIB)/build ; \
fi
@sed 's:^:kernel/:' modules.order > $(MODLIB)/modules.order
- @cp -f modules.builtin $(MODLIB)/
- @cp -f $(objtree)/modules.builtin.modinfo $(MODLIB)/
+ @cp -f $(mixed-build-prefix)modules.builtin $(MODLIB)/
+ @cp -f $(or $(mixed-build-prefix),$(objtree)/)modules.builtin.modinfo $(MODLIB)/
endif # CONFIG_MODULES
@@ -1889,6 +1928,8 @@
@echo ''
@echo ' modules - default target, build the module(s)'
@echo ' modules_install - install the module'
+ @echo ' headers_install - Install sanitised kernel headers to INSTALL_HDR_PATH'
+ @echo ' (default: $(abspath $(INSTALL_HDR_PATH)))'
@echo ' clean - remove generated files in module directory only'
@echo ''
@@ -1917,7 +1958,7 @@
quiet_cmd_depmod = DEPMOD $(MODLIB)
cmd_depmod = $(CONFIG_SHELL) $(srctree)/scripts/depmod.sh $(DEPMOD) \
- $(KERNELRELEASE)
+ $(KERNELRELEASE) $(mixed-build-prefix)
modules_install:
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modinst
@@ -1941,7 +1982,7 @@
endif # CONFIG_MODULES
PHONY += modpost
-modpost: $(if $(single-build),, $(if $(KBUILD_BUILTIN), vmlinux.o)) \
+modpost: $(if $(single-build),, $(if $(KBUILD_MIXED_TREE),,$(if $(KBUILD_BUILTIN), vmlinux.o))) \
$(if $(KBUILD_MODULES), modules_check)
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost
@@ -1991,7 +2032,7 @@
# Error messages still appears in the original language
PHONY += $(build-dir)
$(build-dir): prepare
- $(Q)$(MAKE) $(build)=$@ need-builtin=1 need-modorder=1 $(single-goals)
+ $(Q)$(MAKE) $(build)=$@ $(if $(KBUILD_MIXED_TREE),,need-builtin=1) need-modorder=1 $(single-goals)
clean-dirs := $(addprefix _clean_, $(clean-dirs))
PHONY += $(clean-dirs) clean
@@ -2000,7 +2041,9 @@
clean: $(clean-dirs)
$(call cmd,rmfiles)
- @find $(or $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \
+ @find $(or $(KBUILD_EXTMOD), .) \
+ $(if $(filter-out arch/$(SRCARCH)/boot/dts, $(dtstree)), $(dtstree)) \
+ $(RCS_FIND_IGNORE) \
\( -name '*.[aios]' -o -name '*.rsi' -o -name '*.ko' -o -name '.*.cmd' \
-o -name '*.ko.*' \
-o -name '*.dtb' -o -name '*.dtbo' \
@@ -2041,7 +2084,7 @@
cmd_gen_compile_commands = $(PYTHON3) $< -a $(AR) -o $@ $(filter-out $<, $(real-prereqs))
$(extmod_prefix)compile_commands.json: scripts/clang-tools/gen_compile_commands.py \
- $(if $(KBUILD_EXTMOD),, vmlinux.a $(KBUILD_VMLINUX_LIBS)) \
+ $(if $(KBUILD_EXTMOD)$(KBUILD_MIXED_TREE),, vmlinux.a $(KBUILD_VMLINUX_LIBS)) \
$(if $(CONFIG_MODULES), $(MODORDER)) FORCE
$(call if_changed,gen_compile_commands)
@@ -2098,7 +2141,8 @@
$(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH)
kernelrelease:
- @echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))"
+ @echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion \
+ $(srctree) $(BRANCH) $(KMI_GENERATION))"
kernelversion:
@echo $(KERNELVERSION)
diff --git a/OWNERS b/OWNERS
new file mode 100644
index 0000000..a8b2615
--- /dev/null
+++ b/OWNERS
@@ -0,0 +1,13 @@
+# The full list of approvers is defined in
+# https://android.googlesource.com/kernel/common/+/refs/meta/config/OWNERS
+
+# The following OWNERS are defined at the top level to improve the OWNERS
+# suggestions through any user interface. Consider those people the ones that
+# can help with finding the best person to review.
+adelva@google.com
+gregkh@google.com
+maennich@google.com
+saravanak@google.com
+smuckle@google.com
+surenb@google.com
+tkjos@google.com
diff --git a/OWNERS_DrNo b/OWNERS_DrNo
new file mode 100644
index 0000000..c4b8c0d5
--- /dev/null
+++ b/OWNERS_DrNo
@@ -0,0 +1,23 @@
+# Authoritative list of Dr. No reviewers to approve changes on GKI release
+# branches, such as android12-5.10.
+#
+# This file has no effect in this branch, but is referred to from release
+# branches. So, please do not move or rename.
+#
+# See the GKI release documentation (go/gki-dr-no) for further details.
+
+# Main reviewers
+adelva@google.com
+maennich@google.com
+saravanak@google.com
+vmartensson@google.com
+tkjos@google.com
+willdeacon@google.com
+
+# GKI Release Team
+howardsoc@google.com #{LAST_RESORT_SUGGESTION}
+szuweilin@google.com #{LAST_RESORT_SUGGESTION}
+
+# Backup
+sspatil@google.com #{LAST_RESORT_SUGGESTION}
+malchev@google.com #{LAST_RESORT_SUGGESTION}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..4a1deb3
--- /dev/null
+++ b/README.md
@@ -0,0 +1,150 @@
+# How do I submit patches to Android Common Kernels
+
+1. BEST: Make all of your changes to upstream Linux. If appropriate, backport to the stable releases.
+ These patches will be merged automatically in the corresponding common kernels. If the patch is already
+ in upstream Linux, post a backport of the patch that conforms to the patch requirements below.
+ - Do not send patches upstream that contain only symbol exports. To be considered for upstream Linux,
+additions of `EXPORT_SYMBOL_GPL()` require an in-tree modular driver that uses the symbol -- so include
+the new driver or changes to an existing driver in the same patchset as the export.
+ - When sending patches upstream, the commit message must contain a clear case for why the patch
+is needed and beneficial to the community. Enabling out-of-tree drivers or functionality is not
+not a persuasive case.
+
+2. LESS GOOD: Develop your patches out-of-tree (from an upstream Linux point-of-view). Unless these are
+ fixing an Android-specific bug, these are very unlikely to be accepted unless they have been
+ coordinated with kernel-team@android.com. If you want to proceed, post a patch that conforms to the
+ patch requirements below.
+
+# Common Kernel patch requirements
+
+- All patches must conform to the Linux kernel coding standards and pass `scripts/checkpatch.pl`
+- Patches shall not break gki_defconfig or allmodconfig builds for arm, arm64, x86, x86_64 architectures
+(see https://source.android.com/setup/build/building-kernels)
+- If the patch is not merged from an upstream branch, the subject must be tagged with the type of patch:
+`UPSTREAM:`, `BACKPORT:`, `FROMGIT:`, `FROMLIST:`, or `ANDROID:`.
+- All patches must have a `Change-Id:` tag (see https://gerrit-review.googlesource.com/Documentation/user-changeid.html)
+- If an Android bug has been assigned, there must be a `Bug:` tag.
+- All patches must have a `Signed-off-by:` tag by the author and the submitter
+
+Additional requirements are listed below based on patch type
+
+## Requirements for backports from mainline Linux: `UPSTREAM:`, `BACKPORT:`
+
+- If the patch is a cherry-pick from Linux mainline with no changes at all
+ - tag the patch subject with `UPSTREAM:`.
+ - add upstream commit information with a `(cherry picked from commit ...)` line
+ - Example:
+ - if the upstream commit message is
+```
+ important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+```
+>- then Joe Smith would upload the patch for the common kernel as
+```
+ UPSTREAM: important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+
+ Bug: 135791357
+ Change-Id: I4caaaa566ea080fa148c5e768bb1a0b6f7201c01
+ (cherry picked from commit c31e73121f4c1ec41143423ac6ce3ce6dafdcec1)
+ Signed-off-by: Joe Smith <joe.smith@foo.org>
+```
+
+- If the patch requires any changes from the upstream version, tag the patch with `BACKPORT:`
+instead of `UPSTREAM:`.
+ - use the same tags as `UPSTREAM:`
+ - add comments about the changes under the `(cherry picked from commit ...)` line
+ - Example:
+```
+ BACKPORT: important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+
+ Bug: 135791357
+ Change-Id: I4caaaa566ea080fa148c5e768bb1a0b6f7201c01
+ (cherry picked from commit c31e73121f4c1ec41143423ac6ce3ce6dafdcec1)
+ [joe: Resolved minor conflict in drivers/foo/bar.c ]
+ Signed-off-by: Joe Smith <joe.smith@foo.org>
+```
+
+## Requirements for other backports: `FROMGIT:`, `FROMLIST:`,
+
+- If the patch has been merged into an upstream maintainer tree, but has not yet
+been merged into Linux mainline
+ - tag the patch subject with `FROMGIT:`
+ - add info on where the patch came from as `(cherry picked from commit <sha1> <repo> <branch>)`. This
+must be a stable maintainer branch (not rebased, so don't use `linux-next` for example).
+ - if changes were required, use `BACKPORT: FROMGIT:`
+ - Example:
+ - if the commit message in the maintainer tree is
+```
+ important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+```
+>- then Joe Smith would upload the patch for the common kernel as
+```
+ FROMGIT: important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+
+ Bug: 135791357
+ (cherry picked from commit 878a2fd9de10b03d11d2f622250285c7e63deace
+ https://git.kernel.org/pub/scm/linux/kernel/git/foo/bar.git test-branch)
+ Change-Id: I4caaaa566ea080fa148c5e768bb1a0b6f7201c01
+ Signed-off-by: Joe Smith <joe.smith@foo.org>
+```
+
+
+- If the patch has been submitted to LKML, but not accepted into any maintainer tree
+ - tag the patch subject with `FROMLIST:`
+ - add a `Link:` tag with a link to the submittal on lore.kernel.org
+ - add a `Bug:` tag with the Android bug (required for patches not accepted into
+a maintainer tree)
+ - if changes were required, use `BACKPORT: FROMLIST:`
+ - Example:
+```
+ FROMLIST: important patch from upstream
+
+ This is the detailed description of the important patch
+
+ Signed-off-by: Fred Jones <fred.jones@foo.org>
+
+ Bug: 135791357
+ Link: https://lore.kernel.org/lkml/20190619171517.GA17557@someone.com/
+ Change-Id: I4caaaa566ea080fa148c5e768bb1a0b6f7201c01
+ Signed-off-by: Joe Smith <joe.smith@foo.org>
+```
+
+## Requirements for Android-specific patches: `ANDROID:`
+
+- If the patch is fixing a bug to Android-specific code
+ - tag the patch subject with `ANDROID:`
+ - add a `Fixes:` tag that cites the patch with the bug
+ - Example:
+```
+ ANDROID: fix android-specific bug in foobar.c
+
+ This is the detailed description of the important fix
+
+ Fixes: 1234abcd2468 ("foobar: add cool feature")
+ Change-Id: I4caaaa566ea080fa148c5e768bb1a0b6f7201c01
+ Signed-off-by: Joe Smith <joe.smith@foo.org>
+```
+
+- If the patch is a new feature
+ - tag the patch subject with `ANDROID:`
+ - add a `Bug:` tag with the Android bug (required for android-specific features)
+
diff --git a/android/abi_gki_protected_exports b/android/abi_gki_protected_exports
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/android/abi_gki_protected_exports
diff --git a/android/gki_protected_modules b/android/gki_protected_modules
new file mode 100644
index 0000000..fa169fc
--- /dev/null
+++ b/android/gki_protected_modules
@@ -0,0 +1,47 @@
+drivers/bluetooth/btbcm.ko
+drivers/bluetooth/btqca.ko
+drivers/bluetooth/btsdio.ko
+drivers/bluetooth/hci_uart.ko
+drivers/net/can/dev/can-dev.ko
+drivers/net/can/slcan/slcan.ko
+drivers/net/can/vcan.ko
+drivers/net/ppp/bsd_comp.ko
+drivers/net/ppp/ppp_deflate.ko
+drivers/net/ppp/ppp_generic.ko
+drivers/net/ppp/ppp_mppe.ko
+drivers/net/ppp/pppox.ko
+drivers/net/ppp/pptp.ko
+drivers/net/slip/slhc.ko
+drivers/usb/class/cdc-acm.ko
+drivers/usb/serial/ftdi_sio.ko
+drivers/usb/serial/usbserial.ko
+lib/crypto/libarc4.ko
+net/6lowpan/6lowpan.ko
+net/6lowpan/nhc_dest.ko
+net/6lowpan/nhc_fragment.ko
+net/6lowpan/nhc_hop.ko
+net/6lowpan/nhc_ipv6.ko
+net/6lowpan/nhc_mobility.ko
+net/6lowpan/nhc_routing.ko
+net/6lowpan/nhc_udp.ko
+net/8021q/8021q.ko
+net/bluetooth/bluetooth.ko
+net/bluetooth/hidp/hidp.ko
+net/bluetooth/rfcomm/rfcomm.ko
+net/can/can.ko
+net/can/can-bcm.ko
+net/can/can-gw.ko
+net/can/can-raw.ko
+net/ieee802154/6lowpan/ieee802154_6lowpan.ko
+net/ieee802154/ieee802154.ko
+net/ieee802154/ieee802154_socket.ko
+net/l2tp/l2tp_core.ko
+net/l2tp/l2tp_ppp.ko
+net/mac80211/mac80211.ko
+net/mac802154/mac802154.ko
+net/nfc/nfc.ko
+net/rfkill/rfkill.ko
+net/tipc/diag.ko
+net/tipc/tipc.ko
+net/wireless/cfg80211.ko
+
diff --git a/android/gki_system_dlkm_modules b/android/gki_system_dlkm_modules
new file mode 100644
index 0000000..c790142
--- /dev/null
+++ b/android/gki_system_dlkm_modules
@@ -0,0 +1,49 @@
+drivers/block/zram/zram.ko
+drivers/bluetooth/btbcm.ko
+drivers/bluetooth/btqca.ko
+drivers/bluetooth/btsdio.ko
+drivers/bluetooth/hci_uart.ko
+drivers/net/can/dev/can-dev.ko
+drivers/net/can/slcan/slcan.ko
+drivers/net/can/vcan.ko
+drivers/net/ppp/bsd_comp.ko
+drivers/net/ppp/ppp_deflate.ko
+drivers/net/ppp/ppp_generic.ko
+drivers/net/ppp/ppp_mppe.ko
+drivers/net/ppp/pppox.ko
+drivers/net/ppp/pptp.ko
+drivers/net/slip/slhc.ko
+drivers/usb/class/cdc-acm.ko
+drivers/usb/serial/ftdi_sio.ko
+drivers/usb/serial/usbserial.ko
+lib/crypto/libarc4.ko
+mm/zsmalloc.ko
+net/6lowpan/6lowpan.ko
+net/6lowpan/nhc_dest.ko
+net/6lowpan/nhc_fragment.ko
+net/6lowpan/nhc_hop.ko
+net/6lowpan/nhc_ipv6.ko
+net/6lowpan/nhc_mobility.ko
+net/6lowpan/nhc_routing.ko
+net/6lowpan/nhc_udp.ko
+net/8021q/8021q.ko
+net/bluetooth/bluetooth.ko
+net/bluetooth/hidp/hidp.ko
+net/bluetooth/rfcomm/rfcomm.ko
+net/can/can.ko
+net/can/can-bcm.ko
+net/can/can-gw.ko
+net/can/can-raw.ko
+net/ieee802154/6lowpan/ieee802154_6lowpan.ko
+net/ieee802154/ieee802154.ko
+net/ieee802154/ieee802154_socket.ko
+net/l2tp/l2tp_core.ko
+net/l2tp/l2tp_ppp.ko
+net/mac80211/mac80211.ko
+net/mac802154/mac802154.ko
+net/nfc/nfc.ko
+net/rfkill/rfkill.ko
+net/tipc/diag.ko
+net/tipc/tipc.ko
+net/wireless/cfg80211.ko
+
diff --git a/arch/arm/OWNERS b/arch/arm/OWNERS
new file mode 100644
index 0000000..54f66d6
--- /dev/null
+++ b/arch/arm/OWNERS
@@ -0,0 +1 @@
+include ../arm64/OWNERS
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 978db2d..3751c25a 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -51,6 +51,10 @@
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_raise);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_entry);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_exit);
+
/*
* as from 2.5, kernels no longer have an init_tasks structure
* so we need some other way of telling a new secondary core
diff --git a/arch/arm/nwfpe/Makefile b/arch/arm/nwfpe/Makefile
index 303400f..2aec85a 100644
--- a/arch/arm/nwfpe/Makefile
+++ b/arch/arm/nwfpe/Makefile
@@ -11,3 +11,9 @@
entry.o
nwfpe-$(CONFIG_FPE_NWFPE_XP) += extended_cpdo.o
+
+# Try really hard to avoid generating calls to __aeabi_uldivmod() from
+# float64_rem() due to loop elision.
+ifdef CONFIG_CC_IS_CLANG
+CFLAGS_softfloat.o += -mllvm -replexitval=never
+endif
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index d62bd22..3adbbf3 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -166,7 +166,10 @@
KBUILD_IMAGE := $(boot)/vmlinuz.efi
endif
+# Don't compile Image in mixed build with "all" target
+ifndef KBUILD_MIXED_TREE
all: $(notdir $(KBUILD_IMAGE))
+endif
Image vmlinuz.efi: vmlinux
diff --git a/arch/arm64/OWNERS b/arch/arm64/OWNERS
new file mode 100644
index 0000000..f362e24
--- /dev/null
+++ b/arch/arm64/OWNERS
@@ -0,0 +1,4 @@
+per-file crypto/**=file:/crypto/OWNERS
+per-file {include,kernel,kvm,lib}/**=mzyngier@google.com,willdeacon@google.com
+per-file mm/**=file:/mm/OWNERS
+per-file net/**=file:/net/OWNERS
diff --git a/arch/arm64/configs/16k_gki.fragment b/arch/arm64/configs/16k_gki.fragment
new file mode 100644
index 0000000..b923493
--- /dev/null
+++ b/arch/arm64/configs/16k_gki.fragment
@@ -0,0 +1,3 @@
+CONFIG_ARM64_16K_PAGES=y
+# b/241785095
+# CONFIG_INCREMENTAL_FS is not set
diff --git a/arch/arm64/configs/amlogic_gki.fragment b/arch/arm64/configs/amlogic_gki.fragment
new file mode 100644
index 0000000..e577f55
--- /dev/null
+++ b/arch/arm64/configs/amlogic_gki.fragment
@@ -0,0 +1,142 @@
+#
+# Generic drivers/frameworks
+#
+CONFIG_COMMON_CLK_PWM=m
+CONFIG_REGULATOR_PWM=m
+CONFIG_PWRSEQ_EMMC=m
+CONFIG_PWRSEQ_SIMPLE=m
+CONFIG_USB_DWC2=m
+CONFIG_LEDS_GPIO=m
+
+#
+# Networking
+#
+CONFIG_REALTEK_PHY=m
+CONFIG_STMMAC_ETH=m
+CONFIG_STMMAC_PLATFORM=m
+
+#
+# WLAN
+#
+CONFIG_WLAN_VENDOR_BROADCOM=y
+CONFIG_BRCMUTIL=m
+CONFIG_BRCMFMAC=m
+CONFIG_BRCMFMAC_PROTO_BCDC=y
+CONFIG_BRCMFMAC_SDIO=y
+
+#
+# Amlogic
+#
+CONFIG_ARCH_MESON=y
+CONFIG_SERIAL_MESON=m
+CONFIG_SERIAL_MESON_CONSOLE=y
+
+#
+# Amlogic drivers as modules
+#
+
+# core
+CONFIG_MESON_SM=m
+CONFIG_RESET_MESON=m
+CONFIG_MESON_IRQ_GPIO=m
+
+# clocks
+CONFIG_COMMON_CLK_MESON_REGMAP=m
+CONFIG_COMMON_CLK_MESON_DUALDIV=m
+CONFIG_COMMON_CLK_MESON_MPLL=m
+CONFIG_COMMON_CLK_MESON_PHASE=m
+CONFIG_COMMON_CLK_MESON_PLL=m
+CONFIG_COMMON_CLK_MESON_SCLK_DIV=m
+CONFIG_COMMON_CLK_MESON_VID_PLL_DIV=m
+CONFIG_COMMON_CLK_MESON_AO_CLKC=m
+CONFIG_COMMON_CLK_MESON_EE_CLKC=m
+CONFIG_COMMON_CLK_MESON_CPU_DYNDIV=m
+CONFIG_COMMON_CLK_GXBB=m
+CONFIG_COMMON_CLK_AXG=m
+CONFIG_COMMON_CLK_G12A=m
+
+# PHY
+CONFIG_PHY_MESON8B_USB2=m
+CONFIG_PHY_MESON_GXL_USB2=m
+CONFIG_PHY_MESON_G12A_USB2=m
+CONFIG_PHY_MESON_G12A_USB3_PCIE=m
+CONFIG_PHY_MESON_AXG_PCIE=m
+CONFIG_PHY_MESON_AXG_MIPI_PCIE_ANALOG=m
+
+# peripherals
+CONFIG_I2C_MESON=m
+CONFIG_MMC_MESON_GX=m
+CONFIG_HW_RANDOM_MESON=m
+CONFIG_USB_DWC3_MESON_G12A=m
+CONFIG_MESON_SARADC=m
+CONFIG_SPI_MESON_SPICC=m
+CONFIG_SPI_MESON_SPIFC=m
+CONFIG_PCI_MESON=m
+CONFIG_DWMAC_MESON=m
+CONFIG_MDIO_BUS_MUX_MESON_G12A=m
+CONFIG_MESON_GXL_PHY=m
+CONFIG_PINCTRL_MESON=m
+CONFIG_PINCTRL_MESON_GXBB=m
+CONFIG_PINCTRL_MESON_GXL=m
+CONFIG_PINCTRL_MESON_AXG=m
+CONFIG_PINCTRL_MESON_AXG_PMX=m
+CONFIG_PINCTRL_MESON_G12A=m
+CONFIG_MESON_GXBB_WATCHDOG=m
+CONFIG_MESON_WATCHDOG=m
+CONFIG_MTD_NAND_MESON=m
+CONFIG_PWM_MESON=m
+CONFIG_IR_MESON=m
+CONFIG_MESON_EFUSE=m
+CONFIG_MFD_KHADAS_MCU=m
+CONFIG_KHADAS_MCU_FAN_THERMAL=m
+CONFIG_AMLOGIC_THERMAL=m
+
+# sound
+CONFIG_SND_MESON_AXG_SOUND_CARD=m
+CONFIG_SND_MESON_GX_SOUND_CARD=m
+CONFIG_SND_MESON_G12A_TOHDMITX=m
+
+# display / video
+CONFIG_DRM_MESON=m
+CONFIG_DRM_MESON_DW_HDMI=m
+CONFIG_DRM_DW_HDMI=m
+CONFIG_DRM_DW_HDMI_AHB_AUDIO=m
+CONFIG_DRM_DW_HDMI_I2S_AUDIO=m
+CONFIG_DRM_DW_HDMI_CEC=m
+CONFIG_CEC_MESON_AO=m
+CONFIG_CEC_MESON_G12A_AO=m
+CONFIG_VIDEO_MESON_GE2D=m
+
+# SoC drivers
+CONFIG_MESON_CANVAS=m
+CONFIG_MESON_CLK_MEASURE=m
+CONFIG_MESON_GX_PM_DOMAINS=m
+CONFIG_MESON_EE_PM_DOMAINS=m
+CONFIG_MESON_SECURE_PM_DOMAINS=m
+
+#
+# Amlogic drivers disable
+#
+
+# 32-bit SoC drivers
+CONFIG_MESON6_TIMER=n
+CONFIG_MESON_MX_SOCINFO=n
+
+# only needed by DRM on S805X
+CONFIG_MESON_GX_SOCINFO=n
+
+#
+# Debug / Testing
+#
+
+# devtmpfs needed for buildroot/udev module loading, serial console
+#CONFIG_DEVTMPFS=y
+#CONFIG_DEVTMPFS_MOUNT=y
+
+# debug/testing with FB console
+#CONFIG_DRM_KMS_FB_HELPER=y
+#CONFIG_DRM_FBDEV_EMULATION=y
+#CONFIG_FB=y
+#CONFIG_VT=y
+#CONFIG_FRAMEBUFFER_CONSOLE=y
+#CONFIG_LOGO=y
diff --git a/arch/arm64/configs/db845c_gki.fragment b/arch/arm64/configs/db845c_gki.fragment
new file mode 100644
index 0000000..7cb0dc9e
--- /dev/null
+++ b/arch/arm64/configs/db845c_gki.fragment
@@ -0,0 +1,310 @@
+# CONFIG_MODULE_SIG_ALL is not set
+CONFIG_QRTR=m
+CONFIG_QRTR_TUN=m
+CONFIG_SCSI_UFS_QCOM=m
+CONFIG_INPUT_PM8941_PWRKEY=m
+CONFIG_SERIAL_MSM=m
+CONFIG_I2C_QCOM_GENI=m
+CONFIG_I2C_QUP=m
+CONFIG_PINCTRL_MSM=m
+CONFIG_PINCTRL_QCOM_SPMI_PMIC=m
+CONFIG_PINCTRL_SDM845=m
+CONFIG_POWER_RESET_QCOM_PON=m
+CONFIG_SYSCON_REBOOT_MODE=m
+CONFIG_QCOM_TSENS=m
+CONFIG_QCOM_WDT=m
+CONFIG_PM8916_WATCHDOG=m
+CONFIG_MFD_SPMI_PMIC=m
+CONFIG_SPMI_MSM_PMIC_ARB=m
+CONFIG_REGULATOR_QCOM_RPMH=m
+CONFIG_REGULATOR_QCOM_SPMI=m
+CONFIG_DRM_MSM=m
+# CONFIG_DRM_MSM_DSI_28NM_PHY is not set
+# CONFIG_DRM_MSM_DSI_20NM_PHY is not set
+# CONFIG_DRM_MSM_DSI_28NM_8960_PHY is not set
+CONFIG_DRM_LONTIUM_LT9611=m
+CONFIG_USB_OHCI_HCD=m
+CONFIG_USB_OHCI_HCD_PLATFORM=m
+# CONFIG_USB_DWC3_HAPS is not set
+# CONFIG_USB_DWC3_OF_SIMPLE is not set
+CONFIG_USB_GADGET_VBUS_DRAW=500
+# CONFIG_USB_DUMMY_HCD is not set
+CONFIG_USB_ROLE_SWITCH=m
+CONFIG_USB_ULPI_BUS=m
+CONFIG_MMC_SDHCI_MSM=m
+CONFIG_RTC_DRV_PM8XXX=m
+CONFIG_COMMON_CLK_QCOM=m
+CONFIG_SDM_GPUCC_845=m
+CONFIG_QCOM_CLK_RPMH=m
+CONFIG_SDM_DISPCC_845=m
+CONFIG_HWSPINLOCK_QCOM=m
+CONFIG_QCOM_LLCC=m
+CONFIG_QCOM_RMTFS_MEM=m
+CONFIG_QCOM_SMEM=m
+CONFIG_QCOM_SMSM=m
+CONFIG_EXTCON_USB_GPIO=m
+CONFIG_RESET_QCOM_AOSS=m
+CONFIG_RESET_QCOM_PDC=m
+CONFIG_PHY_QCOM_QMP=m
+CONFIG_PHY_QCOM_QUSB2=m
+CONFIG_PHY_QCOM_USB_HS=m
+CONFIG_NVMEM_QCOM_QFPROM=m
+CONFIG_INTERCONNECT_QCOM=y
+CONFIG_INTERCONNECT_QCOM_OSM_L3=m
+CONFIG_INTERCONNECT_QCOM_SDM845=m
+CONFIG_QCOM_RPMH=m
+CONFIG_QCOM_RPMHPD=m
+CONFIG_WLAN_VENDOR_ATH=y
+CONFIG_ATH10K_AHB=y
+CONFIG_ATH10K=m
+CONFIG_ATH10K_PCI=m
+CONFIG_ATH10K_SNOC=m
+CONFIG_QRTR_SMD=m
+CONFIG_QCOM_FASTRPC=m
+CONFIG_QCOM_APCS_IPC=m
+CONFIG_QCOM_Q6V5_COMMON=m
+CONFIG_QCOM_RPROC_COMMON=m
+CONFIG_QCOM_Q6V5_ADSP=m
+CONFIG_QCOM_Q6V5_MSS=m
+CONFIG_QCOM_Q6V5_PAS=m
+CONFIG_QCOM_Q6V5_WCSS=m
+CONFIG_QCOM_SYSMON=m
+CONFIG_RPMSG_QCOM_GLINK_SMEM=m
+CONFIG_RPMSG_QCOM_SMD=m
+CONFIG_QCOM_AOSS_QMP=m
+CONFIG_QCOM_SMP2P=m
+CONFIG_QCOM_SOCINFO=m
+CONFIG_QCOM_APR=m
+CONFIG_QCOM_GLINK_SSR=m
+CONFIG_RPMSG_QCOM_GLINK_RPM=m
+CONFIG_QCOM_PDC=m
+CONFIG_QCOM_SCM=m
+CONFIG_ARM_SMMU=m
+CONFIG_ARM_QCOM_CPUFREQ_HW=m
+# XXX Audio bits start here
+CONFIG_I2C_CHARDEV=m
+CONFIG_I2C_MUX=m
+CONFIG_I2C_MUX_PCA954x=m
+CONFIG_I2C_DESIGNWARE_PLATFORM=m
+CONFIG_I2C_RK3X=m
+CONFIG_SPI_PL022=m
+CONFIG_SPI_QCOM_QSPI=m
+CONFIG_SPI_QUP=m
+CONFIG_SPI_QCOM_GENI=m
+CONFIG_GPIO_WCD934X=m
+CONFIG_MFD_WCD934X=m
+CONFIG_REGULATOR_GPIO=m
+CONFIG_SND_SOC_QCOM=m
+CONFIG_SND_SOC_QCOM_COMMON=m
+CONFIG_SND_SOC_QDSP6_COMMON=m
+CONFIG_SND_SOC_QDSP6_CORE=m
+CONFIG_SND_SOC_QDSP6_AFE=m
+CONFIG_SND_SOC_QDSP6_AFE_DAI=m
+CONFIG_SND_SOC_QDSP6_ADM=m
+CONFIG_SND_SOC_QDSP6_ROUTING=m
+CONFIG_SND_SOC_QDSP6_ASM=m
+CONFIG_SND_SOC_QDSP6_ASM_DAI=m
+CONFIG_SND_SOC_QDSP6=m
+CONFIG_SND_SOC_SDM845=m
+CONFIG_SND_SOC_DMIC=m
+CONFIG_SND_SOC_WCD9335=m
+CONFIG_SND_SOC_WCD934X=m
+CONFIG_SND_SOC_WSA881X=m
+CONFIG_QCOM_BAM_DMA=m
+CONFIG_QCOM_GPI_DMA=m
+CONFIG_SPMI_PMIC_CLKDIV=m
+CONFIG_SOUNDWIRE=m
+CONFIG_SOUNDWIRE_QCOM=m
+CONFIG_SLIMBUS=m
+CONFIG_SLIM_QCOM_NGD_CTRL=m
+CONFIG_DMABUF_HEAPS_SYSTEM=m
+CONFIG_VIDEO_QCOM_VENUS=m
+CONFIG_SDM_VIDEOCC_845=m
+# CONFIG_CXD2880_SPI_DRV is not set
+# CONFIG_MEDIA_TUNER_SIMPLE is not set
+# CONFIG_MEDIA_TUNER_TDA18250 is not set
+# CONFIG_MEDIA_TUNER_TDA8290 is not set
+# CONFIG_MEDIA_TUNER_TDA827X is not set
+# CONFIG_MEDIA_TUNER_TDA18271 is not set
+# CONFIG_MEDIA_TUNER_TDA9887 is not set
+# CONFIG_MEDIA_TUNER_TEA5761 is not set
+# CONFIG_MEDIA_TUNER_TEA5767 is not set
+# CONFIG_MEDIA_TUNER_MSI001 is not set
+# CONFIG_MEDIA_TUNER_MT20XX is not set
+# CONFIG_MEDIA_TUNER_MT2060 is not set
+# CONFIG_MEDIA_TUNER_MT2063 is not set
+# CONFIG_MEDIA_TUNER_MT2266 is not set
+# CONFIG_MEDIA_TUNER_MT2131 is not set
+# CONFIG_MEDIA_TUNER_QT1010 is not set
+# CONFIG_MEDIA_TUNER_XC2028 is not set
+# CONFIG_MEDIA_TUNER_XC5000 is not set
+# CONFIG_MEDIA_TUNER_XC4000 is not set
+# CONFIG_MEDIA_TUNER_MXL5005S is not set
+# CONFIG_MEDIA_TUNER_MXL5007T is not set
+# CONFIG_MEDIA_TUNER_MC44S803 is not set
+# CONFIG_MEDIA_TUNER_MAX2165 is not set
+# CONFIG_MEDIA_TUNER_TDA18218 is not set
+# CONFIG_MEDIA_TUNER_FC0011 is not set
+# CONFIG_MEDIA_TUNER_FC0012 is not set
+# CONFIG_MEDIA_TUNER_FC0013 is not set
+# CONFIG_MEDIA_TUNER_TDA18212 is not set
+# CONFIG_MEDIA_TUNER_E4000 is not set
+# CONFIG_MEDIA_TUNER_FC2580 is not set
+# CONFIG_MEDIA_TUNER_M88RS6000T is not set
+# CONFIG_MEDIA_TUNER_TUA9001 is not set
+# CONFIG_MEDIA_TUNER_SI2157 is not set
+# CONFIG_MEDIA_TUNER_IT913X is not set
+# CONFIG_MEDIA_TUNER_R820T is not set
+# CONFIG_MEDIA_TUNER_MXL301RF is not set
+# CONFIG_MEDIA_TUNER_QM1D1C0042 is not set
+# CONFIG_MEDIA_TUNER_QM1D1B0004 is not set
+# CONFIG_DVB_STB0899 is not set
+# CONFIG_DVB_STB6100 is not set
+# CONFIG_DVB_STV090x is not set
+# CONFIG_DVB_STV0910 is not set
+# CONFIG_DVB_STV6110x is not set
+# CONFIG_DVB_STV6111 is not set
+# CONFIG_DVB_MXL5XX is not set
+# CONFIG_DVB_M88DS3103 is not set
+# CONFIG_DVB_DRXK is not set
+# CONFIG_DVB_TDA18271C2DD is not set
+# CONFIG_DVB_SI2165 is not set
+# CONFIG_DVB_MN88472 is not set
+# CONFIG_DVB_MN88473 is not set
+# CONFIG_DVB_CX24110 is not set
+# CONFIG_DVB_CX24123 is not set
+# CONFIG_DVB_MT312 is not set
+# CONFIG_DVB_ZL10036 is not set
+# CONFIG_DVB_ZL10039 is not set
+# CONFIG_DVB_S5H1420 is not set
+# CONFIG_DVB_STV0288 is not set
+# CONFIG_DVB_STB6000 is not set
+# CONFIG_DVB_STV0299 is not set
+# CONFIG_DVB_STV6110 is not set
+# CONFIG_DVB_STV0900 is not set
+# CONFIG_DVB_TDA8083 is not set
+# CONFIG_DVB_TDA10086 is not set
+# CONFIG_DVB_TDA8261 is not set
+# CONFIG_DVB_VES1X93 is not set
+# CONFIG_DVB_TUNER_ITD1000 is not set
+# CONFIG_DVB_TUNER_CX24113 is not set
+# CONFIG_DVB_TDA826X is not set
+# CONFIG_DVB_TUA6100 is not set
+# CONFIG_DVB_CX24116 is not set
+# CONFIG_DVB_CX24117 is not set
+# CONFIG_DVB_CX24120 is not set
+# CONFIG_DVB_SI21XX is not set
+# CONFIG_DVB_TS2020 is not set
+# CONFIG_DVB_DS3000 is not set
+# CONFIG_DVB_MB86A16 is not set
+# CONFIG_DVB_TDA10071 is not set
+# CONFIG_DVB_SP8870 is not set
+# CONFIG_DVB_SP887X is not set
+# CONFIG_DVB_CX22700 is not set
+# CONFIG_DVB_CX22702 is not set
+# CONFIG_DVB_S5H1432 is not set
+# CONFIG_DVB_DRXD is not set
+# CONFIG_DVB_L64781 is not set
+# CONFIG_DVB_TDA1004X is not set
+# CONFIG_DVB_NXT6000 is not set
+# CONFIG_DVB_MT352 is not set
+# CONFIG_DVB_ZL10353 is not set
+# CONFIG_DVB_DIB3000MB is not set
+# CONFIG_DVB_DIB3000MC is not set
+# CONFIG_DVB_DIB7000M is not set
+# CONFIG_DVB_DIB7000P is not set
+# CONFIG_DVB_DIB9000 is not set
+# CONFIG_DVB_TDA10048 is not set
+# CONFIG_DVB_AF9013 is not set
+# CONFIG_DVB_EC100 is not set
+# CONFIG_DVB_STV0367 is not set
+# CONFIG_DVB_CXD2820R is not set
+# CONFIG_DVB_CXD2841ER is not set
+# CONFIG_DVB_RTL2830 is not set
+# CONFIG_DVB_RTL2832 is not set
+# CONFIG_DVB_RTL2832_SDR is not set
+# CONFIG_DVB_SI2168 is not set
+# CONFIG_DVB_ZD1301_DEMOD is not set
+# CONFIG_DVB_CXD2880 is not set
+# CONFIG_DVB_VES1820 is not set
+# CONFIG_DVB_TDA10021 is not set
+# CONFIG_DVB_TDA10023 is not set
+# CONFIG_DVB_STV0297 is not set
+# CONFIG_DVB_NXT200X is not set
+# CONFIG_DVB_OR51211 is not set
+# CONFIG_DVB_OR51132 is not set
+# CONFIG_DVB_BCM3510 is not set
+# CONFIG_DVB_LGDT330X is not set
+# CONFIG_DVB_LGDT3305 is not set
+# CONFIG_DVB_LGDT3306A is not set
+# CONFIG_DVB_LG2160 is not set
+# CONFIG_DVB_S5H1409 is not set
+# CONFIG_DVB_AU8522_DTV is not set
+# CONFIG_DVB_AU8522_V4L is not set
+# CONFIG_DVB_S5H1411 is not set
+# CONFIG_DVB_S921 is not set
+# CONFIG_DVB_DIB8000 is not set
+# CONFIG_DVB_MB86A20S is not set
+# CONFIG_DVB_TC90522 is not set
+# CONFIG_DVB_MN88443X is not set
+# CONFIG_DVB_PLL is not set
+# CONFIG_DVB_TUNER_DIB0070 is not set
+# CONFIG_DVB_TUNER_DIB0090 is not set
+# CONFIG_DVB_DRX39XYJ is not set
+# CONFIG_DVB_LNBH25 is not set
+# CONFIG_DVB_LNBH29 is not set
+# CONFIG_DVB_LNBP21 is not set
+# CONFIG_DVB_LNBP22 is not set
+# CONFIG_DVB_ISL6405 is not set
+# CONFIG_DVB_ISL6421 is not set
+# CONFIG_DVB_ISL6423 is not set
+# CONFIG_DVB_A8293 is not set
+# CONFIG_DVB_LGS8GL5 is not set
+# CONFIG_DVB_LGS8GXX is not set
+# CONFIG_DVB_ATBM8830 is not set
+# CONFIG_DVB_TDA665x is not set
+# CONFIG_DVB_IX2505V is not set
+# CONFIG_DVB_M88RS2000 is not set
+# CONFIG_DVB_AF9033 is not set
+# CONFIG_DVB_HORUS3A is not set
+# CONFIG_DVB_ASCOT2E is not set
+# CONFIG_DVB_HELENE is not set
+# CONFIG_DVB_CXD2099 is not set
+# CONFIG_DVB_SP2 is not set
+CONFIG_QCOM_COMMAND_DB=m
+CONFIG_QCOM_LMH=m
+# XXX RB5 bits start here
+CONFIG_QCOM_IPCC=m
+CONFIG_QCOM_SPMI_ADC5=m
+CONFIG_QCOM_SPMI_TEMP_ALARM=m
+CONFIG_RPMSG_NS=m
+CONFIG_CAN_MCP251XFD=m
+CONFIG_ATH11K=m
+CONFIG_ATH11K_AHB=m
+CONFIG_ATH11K_PCI=m
+CONFIG_PINCTRL_SM8250=m
+CONFIG_PINCTRL_SM8250_LPASS_LPI=m
+CONFIG_PINCTRL_LPASS_LPI=m
+CONFIG_QCOM_SPMI_ADC_TM5=m
+CONFIG_MFD_QCOM_QCA639X=m
+CONFIG_REGULATOR_QCOM_USB_VBUS=m
+CONFIG_DRM_DISPLAY_CONNECTOR=m
+CONFIG_DRM_LONTIUM_LT9611UXC=m
+CONFIG_SND_SOC_SM8250=m
+CONFIG_SND_SOC_LPASS_WSA_MACRO=m
+CONFIG_SND_SOC_LPASS_VA_MACRO=m
+CONFIG_TYPEC_QCOM_PMIC=m
+CONFIG_LEDS_QCOM_LPG=m
+CONFIG_SM_GPUCC_8250=m
+CONFIG_SM_DISPCC_8250=m
+CONFIG_SM_VIDEOCC_8250=m
+CONFIG_CLK_GFM_LPASS_SM8250=m
+CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=m
+CONFIG_INTERCONNECT_QCOM_SM8250=m
+CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_QCOM_CPR=m
+CONFIG_QCOM_SPM=m
+# XXX SM8450 bits start here
+CONFIG_PINCTRL_SM8450=m
+CONFIG_SM_GCC_8450=m
+CONFIG_INTERCONNECT_QCOM_SM8450=m
diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig
new file mode 100644
index 0000000..fd37c2d
--- /dev/null
+++ b/arch/arm64/configs/gki_defconfig
@@ -0,0 +1,697 @@
+CONFIG_UAPI_HEADER_TEST=y
+CONFIG_LOCALVERSION="-mainline"
+CONFIG_AUDIT=y
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
+CONFIG_PREEMPT=y
+CONFIG_IRQ_TIME_ACCOUNTING=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+CONFIG_PSI=y
+CONFIG_RCU_EXPERT=y
+CONFIG_RCU_NOCB_CPU=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=y
+CONFIG_UCLAMP_TASK=y
+CONFIG_UCLAMP_BUCKETS_COUNT=20
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_UCLAMP_TASK_GROUP=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_BPF=y
+CONFIG_NAMESPACES=y
+# CONFIG_PID_NS is not set
+# CONFIG_RD_BZIP2 is not set
+# CONFIG_RD_LZMA is not set
+# CONFIG_RD_XZ is not set
+# CONFIG_RD_LZO is not set
+CONFIG_BOOT_CONFIG=y
+# CONFIG_SYSFS_SYSCALL is not set
+# CONFIG_FHANDLE is not set
+CONFIG_KALLSYMS_ALL=y
+# CONFIG_RSEQ is not set
+CONFIG_EMBEDDED=y
+CONFIG_PROFILING=y
+CONFIG_ARCH_SUNXI=y
+CONFIG_ARCH_HISI=y
+CONFIG_ARCH_QCOM=y
+CONFIG_SCHED_MC=y
+CONFIG_NR_CPUS=32
+CONFIG_PARAVIRT_TIME_ACCOUNTING=y
+CONFIG_ARM64_SW_TTBR0_PAN=y
+CONFIG_COMPAT=y
+CONFIG_ARMV8_DEPRECATED=y
+CONFIG_SWP_EMULATION=y
+CONFIG_CP15_BARRIER_EMULATION=y
+CONFIG_SETEND_EMULATION=y
+CONFIG_RANDOMIZE_BASE=y
+# CONFIG_RANDOMIZE_MODULE_REGION_FULL is not set
+CONFIG_CMDLINE="console=ttynull stack_depot_disable=on cgroup_disable=pressure kasan.stacktrace=off kvm-arm.mode=protected bootconfig ioremap_guard"
+# CONFIG_DMI is not set
+CONFIG_PM_WAKELOCKS=y
+CONFIG_PM_WAKELOCKS_LIMIT=0
+# CONFIG_PM_WAKELOCKS_GC is not set
+CONFIG_ENERGY_MODEL=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_MENU=y
+CONFIG_CPU_IDLE_GOV_TEO=y
+CONFIG_ARM_PSCI_CPUIDLE=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_TIMES=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_ARM_SCPI_CPUFREQ=y
+CONFIG_ARM_SCMI_CPUFREQ=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_KPROBES=y
+CONFIG_JUMP_LABEL=y
+CONFIG_SHADOW_CALL_STACK=y
+CONFIG_CFI_CLANG=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_SIG=y
+CONFIG_MODULE_SIG_PROTECT=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
+CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y
+CONFIG_IOSCHED_BFQ=y
+CONFIG_BFQ_GROUP_IOSCHED=y
+CONFIG_GKI_HACKS_TO_FIX=y
+# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+CONFIG_BINFMT_MISC=y
+# CONFIG_SLAB_MERGE_DEFAULT is not set
+CONFIG_SLAB_FREELIST_RANDOM=y
+CONFIG_SLAB_FREELIST_HARDENED=y
+CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTREMOVE=y
+CONFIG_DEFAULT_MMAP_MIN_ADDR=32768
+CONFIG_TRANSPARENT_HUGEPAGE=y
+CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
+CONFIG_CMA=y
+CONFIG_CMA_DEBUGFS=y
+CONFIG_CMA_AREAS=16
+# CONFIG_ZONE_DMA is not set
+CONFIG_ANON_VMA_NAME=y
+CONFIG_USERFAULTFD=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_XFRM_USER=y
+CONFIG_XFRM_INTERFACE=y
+CONFIG_XFRM_MIGRATE=y
+CONFIG_XFRM_STATISTICS=y
+CONFIG_NET_KEY=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_NET_IPIP=y
+CONFIG_NET_IPGRE_DEMUX=y
+CONFIG_NET_IPGRE=y
+CONFIG_NET_IPVTI=y
+CONFIG_INET_ESP=y
+CONFIG_INET_UDP_DIAG=y
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_ESP=y
+CONFIG_INET6_IPCOMP=y
+CONFIG_IPV6_MIP6=y
+CONFIG_IPV6_VTI=y
+CONFIG_IPV6_GRE=y
+CONFIG_IPV6_MULTIPLE_TABLES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_NETFILTER=y
+CONFIG_NF_CONNTRACK=y
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_PROCFS=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_AMANDA=y
+CONFIG_NF_CONNTRACK_FTP=y
+CONFIG_NF_CONNTRACK_H323=y
+CONFIG_NF_CONNTRACK_IRC=y
+CONFIG_NF_CONNTRACK_NETBIOS_NS=y
+CONFIG_NF_CONNTRACK_PPTP=y
+CONFIG_NF_CONNTRACK_SANE=y
+CONFIG_NF_CONNTRACK_TFTP=y
+CONFIG_NF_CT_NETLINK=y
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=y
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y
+CONFIG_NETFILTER_XT_TARGET_DSCP=y
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y
+CONFIG_NETFILTER_XT_TARGET_MARK=y
+CONFIG_NETFILTER_XT_TARGET_NFLOG=y
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y
+CONFIG_NETFILTER_XT_TARGET_NOTRACK=y
+CONFIG_NETFILTER_XT_TARGET_TEE=y
+CONFIG_NETFILTER_XT_TARGET_TPROXY=y
+CONFIG_NETFILTER_XT_TARGET_TRACE=y
+CONFIG_NETFILTER_XT_TARGET_SECMARK=y
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=y
+CONFIG_NETFILTER_XT_MATCH_BPF=y
+CONFIG_NETFILTER_XT_MATCH_COMMENT=y
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
+CONFIG_NETFILTER_XT_MATCH_DSCP=y
+CONFIG_NETFILTER_XT_MATCH_ESP=y
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_HELPER=y
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=y
+CONFIG_NETFILTER_XT_MATCH_L2TP=y
+CONFIG_NETFILTER_XT_MATCH_LENGTH=y
+CONFIG_NETFILTER_XT_MATCH_LIMIT=y
+CONFIG_NETFILTER_XT_MATCH_MAC=y
+CONFIG_NETFILTER_XT_MATCH_MARK=y
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
+CONFIG_NETFILTER_XT_MATCH_OWNER=y
+CONFIG_NETFILTER_XT_MATCH_POLICY=y
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA2=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y
+CONFIG_NETFILTER_XT_MATCH_SOCKET=y
+CONFIG_NETFILTER_XT_MATCH_STATE=y
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
+CONFIG_NETFILTER_XT_MATCH_STRING=y
+CONFIG_NETFILTER_XT_MATCH_TIME=y
+CONFIG_NETFILTER_XT_MATCH_U32=y
+CONFIG_IP_NF_IPTABLES=y
+CONFIG_IP_NF_MATCH_ECN=y
+CONFIG_IP_NF_MATCH_TTL=y
+CONFIG_IP_NF_FILTER=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_NAT=y
+CONFIG_IP_NF_TARGET_MASQUERADE=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_RAW=y
+CONFIG_IP_NF_SECURITY=y
+CONFIG_IP_NF_ARPTABLES=y
+CONFIG_IP_NF_ARPFILTER=y
+CONFIG_IP_NF_ARP_MANGLE=y
+CONFIG_IP6_NF_IPTABLES=y
+CONFIG_IP6_NF_MATCH_RPFILTER=y
+CONFIG_IP6_NF_FILTER=y
+CONFIG_IP6_NF_TARGET_REJECT=y
+CONFIG_IP6_NF_MANGLE=y
+CONFIG_IP6_NF_RAW=y
+CONFIG_TIPC=m
+CONFIG_L2TP=m
+CONFIG_BRIDGE=y
+CONFIG_VLAN_8021Q=m
+CONFIG_6LOWPAN=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_HTB=y
+CONFIG_NET_SCH_PRIO=y
+CONFIG_NET_SCH_MULTIQ=y
+CONFIG_NET_SCH_SFQ=y
+CONFIG_NET_SCH_TBF=y
+CONFIG_NET_SCH_NETEM=y
+CONFIG_NET_SCH_CODEL=y
+CONFIG_NET_SCH_FQ_CODEL=y
+CONFIG_NET_SCH_FQ=y
+CONFIG_NET_SCH_INGRESS=y
+CONFIG_NET_CLS_BASIC=y
+CONFIG_NET_CLS_TCINDEX=y
+CONFIG_NET_CLS_FW=y
+CONFIG_NET_CLS_U32=y
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_FLOW=y
+CONFIG_NET_CLS_BPF=y
+CONFIG_NET_CLS_MATCHALL=y
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=y
+CONFIG_NET_EMATCH_NBYTE=y
+CONFIG_NET_EMATCH_U32=y
+CONFIG_NET_EMATCH_META=y
+CONFIG_NET_EMATCH_TEXT=y
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=y
+CONFIG_NET_ACT_GACT=y
+CONFIG_NET_ACT_MIRRED=y
+CONFIG_NET_ACT_SKBEDIT=y
+CONFIG_NET_ACT_BPF=y
+CONFIG_VSOCKETS=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_CAN=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_HIDP=m
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_LL=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_QCA=y
+CONFIG_CFG80211=m
+CONFIG_NL80211_TESTMODE=y
+# CONFIG_CFG80211_DEFAULT_PS is not set
+# CONFIG_CFG80211_CRDA_SUPPORT is not set
+CONFIG_MAC80211=m
+CONFIG_RFKILL=m
+CONFIG_NFC=m
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCI_IOV=y
+# CONFIG_VGA_ARB is not set
+CONFIG_PCI_HOST_GENERIC=y
+CONFIG_PCIE_DW_PLAT_EP=y
+CONFIG_PCIE_QCOM=y
+CONFIG_PCIE_KIRIN=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_FW_LOADER_USER_HELPER=y
+# CONFIG_FW_CACHE is not set
+# CONFIG_SUN50I_DE2_BUS is not set
+# CONFIG_SUNXI_RSB is not set
+CONFIG_ARM_SCMI_PROTOCOL=y
+# CONFIG_ARM_SCMI_POWER_DOMAIN is not set
+CONFIG_ARM_SCPI_PROTOCOL=y
+# CONFIG_ARM_SCPI_POWER_DOMAIN is not set
+# CONFIG_EFI_ARMSTUB_DTB_LOADER is not set
+CONFIG_GNSS=y
+CONFIG_ZRAM=m
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_LOOP_MIN_COUNT=16
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_SIZE=8192
+CONFIG_SRAM=y
+CONFIG_SCSI=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=y
+CONFIG_DM_CRYPT=y
+CONFIG_DM_DEFAULT_KEY=y
+CONFIG_DM_SNAPSHOT=y
+CONFIG_DM_UEVENT=y
+CONFIG_DM_VERITY=y
+CONFIG_DM_VERITY_FEC=y
+CONFIG_NETDEVICES=y
+CONFIG_DUMMY=y
+CONFIG_WIREGUARD=y
+CONFIG_IFB=y
+CONFIG_MACSEC=y
+CONFIG_TUN=y
+CONFIG_VETH=y
+CONFIG_CAN_VCAN=m
+CONFIG_CAN_SLCAN=m
+CONFIG_PPP=m
+CONFIG_PPP_BSDCOMP=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_MPPE=m
+CONFIG_PPTP=m
+CONFIG_PPPOL2TP=m
+CONFIG_USB_RTL8150=y
+CONFIG_USB_RTL8152=y
+CONFIG_USB_USBNET=y
+CONFIG_USB_NET_CDC_EEM=y
+# CONFIG_USB_NET_NET1080 is not set
+# CONFIG_USB_NET_CDC_SUBSET is not set
+# CONFIG_USB_NET_ZAURUS is not set
+CONFIG_USB_NET_AQC111=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+# CONFIG_WLAN_VENDOR_REALTEK is not set
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_GPIO=y
+# CONFIG_MOUSE_PS2 is not set
+CONFIG_INPUT_JOYSTICK=y
+CONFIG_JOYSTICK_XPAD=y
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_UINPUT=y
+# CONFIG_VT is not set
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+# CONFIG_SERIAL_8250_EXAR is not set
+CONFIG_SERIAL_8250_RUNTIME_UARTS=0
+CONFIG_SERIAL_8250_DW=y
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_AMBA_PL011=y
+CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
+CONFIG_SERIAL_SAMSUNG=y
+CONFIG_SERIAL_SAMSUNG_CONSOLE=y
+CONFIG_SERIAL_QCOM_GENI=y
+CONFIG_SERIAL_QCOM_GENI_CONSOLE=y
+CONFIG_SERIAL_SPRD=y
+CONFIG_SERIAL_SPRD_CONSOLE=y
+CONFIG_NULL_TTY=y
+CONFIG_HVC_DCC=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_HW_RANDOM=y
+# CONFIG_DEVMEM is not set
+# CONFIG_DEVPORT is not set
+# CONFIG_I2C_COMPAT is not set
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I3C=y
+CONFIG_SPI=y
+CONFIG_SPI_MEM=y
+# CONFIG_SPMI_MSM_PMIC_ARB is not set
+# CONFIG_PINCTRL_SUN8I_H3_R is not set
+# CONFIG_PINCTRL_SUN50I_A64 is not set
+# CONFIG_PINCTRL_SUN50I_A64_R is not set
+# CONFIG_PINCTRL_SUN50I_A100 is not set
+# CONFIG_PINCTRL_SUN50I_A100_R is not set
+# CONFIG_PINCTRL_SUN50I_H5 is not set
+# CONFIG_PINCTRL_SUN50I_H6 is not set
+# CONFIG_PINCTRL_SUN50I_H6_R is not set
+# CONFIG_PINCTRL_SUN50I_H616 is not set
+# CONFIG_PINCTRL_SUN50I_H616_R is not set
+CONFIG_GPIO_GENERIC_PLATFORM=y
+CONFIG_POWER_RESET_HISI=y
+CONFIG_POWER_RESET_SYSCON=y
+# CONFIG_HWMON is not set
+CONFIG_THERMAL=y
+CONFIG_THERMAL_NETLINK=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=100
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_GOV_USER_SPACE=y
+CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_CORE=y
+CONFIG_MFD_ACT8945A=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_RC_CORE=y
+CONFIG_BPF_LIRC_MODE2=y
+CONFIG_LIRC=y
+# CONFIG_RC_MAP is not set
+CONFIG_RC_DECODERS=y
+CONFIG_RC_DEVICES=y
+CONFIG_MEDIA_CEC_RC=y
+# CONFIG_MEDIA_ANALOG_TV_SUPPORT is not set
+# CONFIG_MEDIA_DIGITAL_TV_SUPPORT is not set
+# CONFIG_MEDIA_RADIO_SUPPORT is not set
+# CONFIG_MEDIA_SDR_SUPPORT is not set
+# CONFIG_MEDIA_TEST_SUPPORT is not set
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_GSPCA=y
+CONFIG_USB_VIDEO_CLASS=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_DRM=y
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HRTIMER=y
+# CONFIG_SND_SUPPORT_OLD_API is not set
+# CONFIG_SND_VERBOSE_PROCFS is not set
+# CONFIG_SND_DRIVERS is not set
+CONFIG_SND_USB_AUDIO=y
+CONFIG_SND_SOC=y
+CONFIG_HID_BATTERY_STRENGTH=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_APPLE=y
+CONFIG_HID_PRODIKEYS=y
+CONFIG_HID_ELECOM=y
+CONFIG_HID_UCLOGIC=y
+CONFIG_HID_LOGITECH=y
+CONFIG_HID_LOGITECH_DJ=y
+CONFIG_HID_MAGICMOUSE=y
+CONFIG_HID_MICROSOFT=y
+CONFIG_HID_MULTITOUCH=y
+CONFIG_HID_NINTENDO=y
+CONFIG_HID_PICOLCD=y
+CONFIG_HID_PLANTRONICS=y
+CONFIG_HID_PLAYSTATION=y
+CONFIG_PLAYSTATION_FF=y
+CONFIG_HID_ROCCAT=y
+CONFIG_HID_SONY=y
+CONFIG_SONY_FF=y
+CONFIG_HID_STEAM=y
+CONFIG_HID_WACOM=y
+CONFIG_HID_WIIMOTE=y
+CONFIG_USB_HIDDEV=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_OTG=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_GADGET=y
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_UEVENT=y
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_ACM=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_ECM=y
+CONFIG_USB_CONFIGFS_EEM=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_ACC=y
+CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y
+CONFIG_USB_CONFIGFS_F_UAC2=y
+CONFIG_USB_CONFIGFS_F_MIDI=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_TCPM=y
+CONFIG_TYPEC_TCPCI=y
+CONFIG_TYPEC_UCSI=y
+CONFIG_MMC=y
+# CONFIG_PWRSEQ_EMMC is not set
+# CONFIG_PWRSEQ_SIMPLE is not set
+CONFIG_MMC_CRYPTO=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFS_BSG=y
+CONFIG_SCSI_UFS_CRYPTO=y
+CONFIG_SCSI_UFSHCD_PCI=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_DWC_TC_PLATFORM=y
+CONFIG_SCSI_UFS_HISI=y
+CONFIG_LEDS_CLASS_FLASH=y
+CONFIG_LEDS_CLASS_MULTICOLOR=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_TRANSIENT=y
+CONFIG_EDAC=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_PL030=y
+CONFIG_RTC_DRV_PL031=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_UIO=y
+CONFIG_VHOST_VSOCK=y
+CONFIG_STAGING=y
+CONFIG_ASHMEM=y
+CONFIG_COMMON_CLK_SCPI=y
+# CONFIG_SUNXI_CCU is not set
+CONFIG_HWSPINLOCK=y
+# CONFIG_SUN50I_ERRATUM_UNKNOWN1 is not set
+CONFIG_MAILBOX=y
+CONFIG_IOMMU_IO_PGTABLE_ARMV7S=y
+CONFIG_REMOTEPROC=y
+CONFIG_REMOTEPROC_CDEV=y
+CONFIG_RPMSG_CHAR=y
+CONFIG_QCOM_GENI_SE=y
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_IIO=y
+CONFIG_IIO_BUFFER=y
+CONFIG_IIO_TRIGGER=y
+CONFIG_PWM=y
+CONFIG_GENERIC_PHY=y
+CONFIG_POWERCAP=y
+CONFIG_ANDROID_BINDER_IPC=y
+CONFIG_ANDROID_BINDERFS=y
+CONFIG_ANDROID_VENDOR_HOOKS=y
+CONFIG_ANDROID_DEBUG_KINFO=y
+CONFIG_LIBNVDIMM=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_F2FS_FS=y
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_COMPRESSION=y
+CONFIG_F2FS_UNFAIR_RWSEM=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y
+CONFIG_FS_VERITY=y
+CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y
+# CONFIG_DNOTIFY is not set
+CONFIG_QUOTA=y
+CONFIG_QFMT_V2=y
+CONFIG_FUSE_FS=y
+CONFIG_VIRTIO_FS=y
+CONFIG_OVERLAY_FS=y
+CONFIG_INCREMENTAL_FS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=y
+CONFIG_TMPFS=y
+# CONFIG_EFIVAR_FS is not set
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_RAM=y
+CONFIG_EROFS_FS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=y
+CONFIG_NLS_CODEPAGE_775=y
+CONFIG_NLS_CODEPAGE_850=y
+CONFIG_NLS_CODEPAGE_852=y
+CONFIG_NLS_CODEPAGE_855=y
+CONFIG_NLS_CODEPAGE_857=y
+CONFIG_NLS_CODEPAGE_860=y
+CONFIG_NLS_CODEPAGE_861=y
+CONFIG_NLS_CODEPAGE_862=y
+CONFIG_NLS_CODEPAGE_863=y
+CONFIG_NLS_CODEPAGE_864=y
+CONFIG_NLS_CODEPAGE_865=y
+CONFIG_NLS_CODEPAGE_866=y
+CONFIG_NLS_CODEPAGE_869=y
+CONFIG_NLS_CODEPAGE_936=y
+CONFIG_NLS_CODEPAGE_950=y
+CONFIG_NLS_CODEPAGE_932=y
+CONFIG_NLS_CODEPAGE_949=y
+CONFIG_NLS_CODEPAGE_874=y
+CONFIG_NLS_ISO8859_8=y
+CONFIG_NLS_CODEPAGE_1250=y
+CONFIG_NLS_CODEPAGE_1251=y
+CONFIG_NLS_ASCII=y
+CONFIG_NLS_ISO8859_1=y
+CONFIG_NLS_ISO8859_2=y
+CONFIG_NLS_ISO8859_3=y
+CONFIG_NLS_ISO8859_4=y
+CONFIG_NLS_ISO8859_5=y
+CONFIG_NLS_ISO8859_6=y
+CONFIG_NLS_ISO8859_7=y
+CONFIG_NLS_ISO8859_9=y
+CONFIG_NLS_ISO8859_13=y
+CONFIG_NLS_ISO8859_14=y
+CONFIG_NLS_ISO8859_15=y
+CONFIG_NLS_KOI8_R=y
+CONFIG_NLS_KOI8_U=y
+CONFIG_NLS_MAC_ROMAN=y
+CONFIG_NLS_MAC_CELTIC=y
+CONFIG_NLS_MAC_CENTEURO=y
+CONFIG_NLS_MAC_CROATIAN=y
+CONFIG_NLS_MAC_CYRILLIC=y
+CONFIG_NLS_MAC_GAELIC=y
+CONFIG_NLS_MAC_GREEK=y
+CONFIG_NLS_MAC_ICELAND=y
+CONFIG_NLS_MAC_INUIT=y
+CONFIG_NLS_MAC_ROMANIAN=y
+CONFIG_NLS_MAC_TURKISH=y
+CONFIG_NLS_UTF8=y
+CONFIG_UNICODE=y
+CONFIG_SECURITY=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_FORTIFY_SOURCE=y
+CONFIG_STATIC_USERMODEHELPER=y
+CONFIG_STATIC_USERMODEHELPER_PATH=""
+CONFIG_SECURITY_SELINUX=y
+CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y
+CONFIG_CRYPTO_ECDH=y
+CONFIG_CRYPTO_DES=y
+CONFIG_CRYPTO_ADIANTUM=y
+CONFIG_CRYPTO_HCTR2=y
+CONFIG_CRYPTO_CHACHA20POLY1305=y
+CONFIG_CRYPTO_CCM=y
+CONFIG_CRYPTO_BLAKE2B=y
+CONFIG_CRYPTO_CMAC=y
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_XCBC=y
+CONFIG_CRYPTO_LZO=y
+CONFIG_CRYPTO_LZ4=y
+CONFIG_CRYPTO_ZSTD=y
+CONFIG_CRYPTO_ANSI_CPRNG=y
+CONFIG_CRYPTO_SHA2_ARM64_CE=y
+CONFIG_CRYPTO_SHA512_ARM64_CE=y
+CONFIG_CRYPTO_POLYVAL_ARM64_CE=y
+CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
+CONFIG_TRACE_MMIO_ACCESS=y
+CONFIG_CRC_CCITT=y
+CONFIG_XZ_DEC=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_PRINTK_CALLER=y
+CONFIG_DYNAMIC_DEBUG_CORE=y
+CONFIG_DEBUG_INFO_DWARF4=y
+CONFIG_HEADERS_INSTALL=y
+# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_UBSAN=y
+CONFIG_UBSAN_TRAP=y
+CONFIG_UBSAN_LOCAL_BOUNDS=y
+# CONFIG_UBSAN_SHIFT is not set
+# CONFIG_UBSAN_BOOL is not set
+# CONFIG_UBSAN_ENUM is not set
+CONFIG_PAGE_OWNER=y
+CONFIG_DEBUG_STACK_USAGE=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_KASAN=y
+CONFIG_KASAN_HW_TAGS=y
+CONFIG_KFENCE=y
+CONFIG_KFENCE_SAMPLE_INTERVAL=500
+CONFIG_KFENCE_NUM_OBJECTS=63
+CONFIG_KFENCE_STATIC_KEYS=y
+CONFIG_PANIC_ON_OOPS=y
+CONFIG_PANIC_TIMEOUT=-1
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_SCHEDSTATS=y
+# CONFIG_DEBUG_PREEMPT is not set
+CONFIG_BUG_ON_DATA_CORRUPTION=y
+CONFIG_HIST_TRIGGERS=y
+CONFIG_PID_IN_CONTEXTIDR=y
+CONFIG_KUNIT=y
+CONFIG_KUNIT_DEBUGFS=y
+# CONFIG_KUNIT_DEFAULT_ENABLED is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
diff --git a/arch/arm64/configs/rockpi4_gki.fragment b/arch/arm64/configs/rockpi4_gki.fragment
new file mode 100644
index 0000000..2af01b8
--- /dev/null
+++ b/arch/arm64/configs/rockpi4_gki.fragment
@@ -0,0 +1,82 @@
+# Core features
+CONFIG_ARCH_ROCKCHIP=y
+# CONFIG_CLK_PX30 is not set
+# CONFIG_CLK_RV110X is not set
+# CONFIG_CLK_RK3036 is not set
+# CONFIG_CLK_RK312X is not set
+# CONFIG_CLK_RK3188 is not set
+# CONFIG_CLK_RK322X is not set
+# CONFIG_CLK_RK3288 is not set
+# CONFIG_CLK_RK3308 is not set
+# CONFIG_CLK_RK3328 is not set
+# CONFIG_CLK_RK3368 is not set
+CONFIG_COMMON_CLK_RK808=m
+CONFIG_CPUFREQ_DT=m
+CONFIG_MFD_RK808=m
+CONFIG_PCIE_ROCKCHIP_HOST=m
+CONFIG_PHY_ROCKCHIP_PCIE=m
+CONFIG_PL330_DMA=m
+CONFIG_PWM_ROCKCHIP=m
+CONFIG_PWRSEQ_SIMPLE=m
+CONFIG_REGULATOR_FAN53555=m
+CONFIG_REGULATOR_PWM=m
+CONFIG_REGULATOR_RK808=m
+CONFIG_ROCKCHIP_EFUSE=m
+CONFIG_ROCKCHIP_IOMMU=y
+CONFIG_ROCKCHIP_IODOMAIN=m
+CONFIG_ROCKCHIP_MBOX=y
+CONFIG_ROCKCHIP_PM_DOMAINS=y
+CONFIG_ROCKCHIP_THERMAL=m
+
+# Ethernet
+CONFIG_STMMAC_ETH=m
+# CONFIG_DWMAC_GENERIC is not set
+# CONFIG_DWMAC_IPQ806X is not set
+# CONFIG_DWMAC_QCOM_ETHQOS is not set
+# CONFIG_DWMAC_SUNXI is not set
+# CONFIG_DWMAC_SUN8I is not set
+
+# I2C
+CONFIG_I2C_RK3X=m
+
+# Watchdog
+CONFIG_DW_WATCHDOG=m
+
+# Display
+CONFIG_DRM_ROCKCHIP=m
+CONFIG_ROCKCHIP_ANALOGIX_DP=y
+CONFIG_ROCKCHIP_DW_HDMI=y
+CONFIG_ROCKCHIP_DW_MIPI_DSI=y
+
+# USB 2.x
+CONFIG_PHY_ROCKCHIP_INNO_USB2=m
+CONFIG_USB_OHCI_HCD=m
+# CONFIG_USB_OHCI_HCD_PCI is not set
+CONFIG_USB_OHCI_HCD_PLATFORM=m
+
+# eMMC / SD-Card
+CONFIG_MMC_SDHCI_OF_ARASAN=m
+CONFIG_MMC_DW=m
+CONFIG_MMC_DW_ROCKCHIP=m
+CONFIG_PHY_ROCKCHIP_EMMC=m
+
+# Real-time clock
+CONFIG_RTC_DRV_RK808=m
+
+# Type-C
+CONFIG_PHY_ROCKCHIP_TYPEC=m
+
+# SAR ADC
+CONFIG_ROCKCHIP_SARADC=m
+
+# Audio
+CONFIG_SND_SOC_ROCKCHIP_I2S=m
+
+# To boot Linux distributions like Debian
+CONFIG_DEVTMPFS=y
+
+# To bootstrap rootfs with QEMU
+CONFIG_HW_RANDOM_VIRTIO=m
+CONFIG_VIRTIO_PCI=m
+CONFIG_VIRTIO_BLK=m
+CONFIG_VIRTIO_NET=m
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index fc55f5a..b936377 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -87,6 +87,8 @@ extern void secondary_entry(void);
extern void arch_send_call_function_single_ipi(int cpu);
extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
+extern int nr_ipi_get(void);
+extern struct irq_desc **ipi_desc_get(void);
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask);
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 7e76e1f..3e4075c 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1460,6 +1460,7 @@ const struct cpumask *system_32bit_el0_cpumask(void)
return cpu_possible_mask;
}
+EXPORT_SYMBOL_GPL(system_32bit_el0_cpumask);
static int __init parse_32bit_el0_param(char *str)
{
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 3da0977..872218e 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -283,21 +283,25 @@ void register_user_break_hook(struct break_hook *hook)
{
register_debug_hook(&hook->node, &user_break_hook);
}
+EXPORT_SYMBOL_GPL(register_user_break_hook);
void unregister_user_break_hook(struct break_hook *hook)
{
unregister_debug_hook(&hook->node);
}
+EXPORT_SYMBOL_GPL(unregister_user_break_hook);
void register_kernel_break_hook(struct break_hook *hook)
{
register_debug_hook(&hook->node, &kernel_break_hook);
}
+EXPORT_SYMBOL_GPL(register_kernel_break_hook);
void unregister_kernel_break_hook(struct break_hook *hook)
{
unregister_debug_hook(&hook->node);
}
+EXPORT_SYMBOL_GPL(unregister_kernel_break_hook);
static int call_break_hook(struct pt_regs *regs, unsigned long esr)
{
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 269ac1c..faf68c9 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -41,6 +41,7 @@
#include <linux/thread_info.h>
#include <linux/prctl.h>
#include <linux/stacktrace.h>
+#include <trace/hooks/mpam.h>
#include <asm/alternative.h>
#include <asm/compat.h>
@@ -245,6 +246,7 @@ void show_regs(struct pt_regs *regs)
__show_regs(regs);
dump_backtrace(regs, NULL, KERN_DEFAULT);
}
+EXPORT_SYMBOL_GPL(show_regs);
static void tls_thread_flush(void)
{
@@ -534,6 +536,12 @@ struct task_struct *__switch_to(struct task_struct *prev,
ptrauth_thread_switch_user(next);
/*
+ * vendor hook is needed before the dsb(),
+ * because MPAM is related to cache maintenance.
+ */
+ trace_android_vh_mpam_set(prev, next);
+
+ /*
* Complete any pending TLB or cache maintenance on this CPU in case
* the thread migrates to a different CPU.
* This full barrier is also required by the membarrier system
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index ffc5d76..de8dcba 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -53,9 +53,14 @@
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/debug.h>
DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_raise);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_entry);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_exit);
/*
* as from 2.5, kernels no longer have an init_tasks structure
@@ -877,6 +882,7 @@ static void do_handle_IPI(int ipinr)
break;
case IPI_CPU_STOP:
+ trace_android_vh_ipi_stop(get_irq_regs());
local_cpu_stop();
break;
@@ -1097,3 +1103,15 @@ bool cpus_are_stuck_in_kernel(void)
return !!cpus_stuck_in_kernel || smp_spin_tables ||
is_protected_kvm_enabled();
}
+
+int nr_ipi_get(void)
+{
+ return nr_ipi;
+}
+EXPORT_SYMBOL_GPL(nr_ipi_get);
+
+struct irq_desc **ipi_desc_get(void)
+{
+ return ipi_desc;
+}
+EXPORT_SYMBOL_GPL(ipi_desc_get);
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 117e2c18..8073ed6 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -157,6 +157,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
put_task_stack(tsk);
}
+EXPORT_SYMBOL_GPL(dump_backtrace);
void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
{
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 5240f6a..8e84749 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -9,6 +9,7 @@
#include <linux/dma-map-ops.h>
#include <linux/iommu.h>
#include <xen/xen.h>
+#include <trace/hooks/iommu.h>
#include <asm/cacheflush.h>
#include <asm/xen/xen-ops.h>
@@ -73,8 +74,10 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
ARCH_DMA_MINALIGN, cls);
dev->dma_coherent = coherent;
- if (iommu)
+ if (iommu) {
iommu_setup_dma_ops(dev, dma_base, dma_base + size - 1);
+ trace_android_rvh_iommu_setup_dma_ops(dev, dma_base, dma_base + size - 1);
+ }
xen_setup_dma_ops(dev);
}
diff --git a/arch/riscv/configs/gki.config b/arch/riscv/configs/gki.config
new file mode 100644
index 0000000..5558fef
--- /dev/null
+++ b/arch/riscv/configs/gki.config
@@ -0,0 +1,2 @@
+CONFIG_CPU_FREQ=y
+CONFIG_GPIOLIB=y
diff --git a/arch/riscv/configs/gki_defconfig b/arch/riscv/configs/gki_defconfig
new file mode 120000
index 0000000..3388ea4
--- /dev/null
+++ b/arch/riscv/configs/gki_defconfig
@@ -0,0 +1 @@
+../../arm64/configs/gki_defconfig
\ No newline at end of file
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 415a5d1..291c71e 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -264,8 +264,11 @@
PHONY += bzImage $(BOOT_TARGETS)
+# Don't compile Image in mixed build with "all" target
+ifndef KBUILD_MIXED_TREE
# Default kernel to build
all: bzImage
+endif
# KBUILD_IMAGE specify target image being built
KBUILD_IMAGE := $(boot)/bzImage
diff --git a/arch/x86/OWNERS b/arch/x86/OWNERS
new file mode 100644
index 0000000..f59fa99
--- /dev/null
+++ b/arch/x86/OWNERS
@@ -0,0 +1,3 @@
+per-file crypto/**=file:/crypto/OWNERS
+per-file mm/**=file:/mm/OWNERS
+per-file net/**=file:/net/OWNERS
diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig
new file mode 100644
index 0000000..a601151
--- /dev/null
+++ b/arch/x86/configs/gki_defconfig
@@ -0,0 +1,635 @@
+CONFIG_UAPI_HEADER_TEST=y
+CONFIG_LOCALVERSION="-mainline"
+CONFIG_KERNEL_LZ4=y
+CONFIG_AUDIT=y
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
+CONFIG_PREEMPT=y
+# CONFIG_PREEMPT_DYNAMIC is not set
+CONFIG_IRQ_TIME_ACCOUNTING=y
+CONFIG_TASKSTATS=y
+CONFIG_TASK_XACCT=y
+CONFIG_TASK_IO_ACCOUNTING=y
+CONFIG_PSI=y
+CONFIG_RCU_EXPERT=y
+CONFIG_RCU_NOCB_CPU=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=y
+CONFIG_UCLAMP_TASK=y
+CONFIG_UCLAMP_BUCKETS_COUNT=20
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_UCLAMP_TASK_GROUP=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_BPF=y
+CONFIG_NAMESPACES=y
+# CONFIG_TIME_NS is not set
+# CONFIG_PID_NS is not set
+# CONFIG_RD_BZIP2 is not set
+# CONFIG_RD_LZMA is not set
+# CONFIG_RD_XZ is not set
+# CONFIG_RD_LZO is not set
+CONFIG_BOOT_CONFIG=y
+# CONFIG_SYSFS_SYSCALL is not set
+# CONFIG_FHANDLE is not set
+CONFIG_KALLSYMS_ALL=y
+# CONFIG_RSEQ is not set
+CONFIG_EMBEDDED=y
+CONFIG_PROFILING=y
+CONFIG_SMP=y
+CONFIG_X86_X2APIC=y
+CONFIG_HYPERVISOR_GUEST=y
+CONFIG_PARAVIRT=y
+CONFIG_PARAVIRT_TIME_ACCOUNTING=y
+CONFIG_NR_CPUS=32
+# CONFIG_X86_MCE is not set
+CONFIG_EFI=y
+CONFIG_CMDLINE_BOOL=y
+CONFIG_CMDLINE="console=ttynull stack_depot_disable=on cgroup_disable=pressure bootconfig"
+CONFIG_PM_WAKELOCKS=y
+CONFIG_PM_WAKELOCKS_LIMIT=0
+# CONFIG_PM_WAKELOCKS_GC is not set
+CONFIG_CPU_FREQ_STAT=y
+CONFIG_CPU_FREQ_TIMES=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_IA32_EMULATION=y
+CONFIG_KVM=y
+CONFIG_KVM_INTEL=y
+CONFIG_KVM_AMD=y
+CONFIG_KPROBES=y
+CONFIG_JUMP_LABEL=y
+CONFIG_CFI_CLANG=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODVERSIONS=y
+CONFIG_MODULE_SIG=y
+CONFIG_MODULE_SIG_PROTECT=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
+CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y
+CONFIG_IOSCHED_BFQ=y
+CONFIG_BFQ_GROUP_IOSCHED=y
+CONFIG_GKI_HACKS_TO_FIX=y
+# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+CONFIG_BINFMT_MISC=y
+# CONFIG_SLAB_MERGE_DEFAULT is not set
+CONFIG_SLAB_FREELIST_RANDOM=y
+CONFIG_SLAB_FREELIST_HARDENED=y
+CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTREMOVE=y
+CONFIG_DEFAULT_MMAP_MIN_ADDR=32768
+CONFIG_TRANSPARENT_HUGEPAGE=y
+CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
+CONFIG_CMA=y
+CONFIG_CMA_DEBUGFS=y
+CONFIG_CMA_AREAS=16
+# CONFIG_ZONE_DMA is not set
+CONFIG_ANON_VMA_NAME=y
+CONFIG_USERFAULTFD=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_XFRM_USER=y
+CONFIG_XFRM_INTERFACE=y
+CONFIG_XFRM_MIGRATE=y
+CONFIG_XFRM_STATISTICS=y
+CONFIG_NET_KEY=y
+CONFIG_XDP_SOCKETS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_NET_IPIP=y
+CONFIG_NET_IPGRE_DEMUX=y
+CONFIG_NET_IPGRE=y
+CONFIG_NET_IPVTI=y
+CONFIG_INET_ESP=y
+CONFIG_INET_UDP_DIAG=y
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
+CONFIG_INET6_ESP=y
+CONFIG_INET6_IPCOMP=y
+CONFIG_IPV6_MIP6=y
+CONFIG_IPV6_VTI=y
+CONFIG_IPV6_GRE=y
+CONFIG_IPV6_MULTIPLE_TABLES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_NETFILTER=y
+CONFIG_NF_CONNTRACK=y
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_PROCFS=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_AMANDA=y
+CONFIG_NF_CONNTRACK_FTP=y
+CONFIG_NF_CONNTRACK_H323=y
+CONFIG_NF_CONNTRACK_IRC=y
+CONFIG_NF_CONNTRACK_NETBIOS_NS=y
+CONFIG_NF_CONNTRACK_PPTP=y
+CONFIG_NF_CONNTRACK_SANE=y
+CONFIG_NF_CONNTRACK_TFTP=y
+CONFIG_NF_CT_NETLINK=y
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=y
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y
+CONFIG_NETFILTER_XT_TARGET_DSCP=y
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y
+CONFIG_NETFILTER_XT_TARGET_MARK=y
+CONFIG_NETFILTER_XT_TARGET_NFLOG=y
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y
+CONFIG_NETFILTER_XT_TARGET_NOTRACK=y
+CONFIG_NETFILTER_XT_TARGET_TEE=y
+CONFIG_NETFILTER_XT_TARGET_TPROXY=y
+CONFIG_NETFILTER_XT_TARGET_TRACE=y
+CONFIG_NETFILTER_XT_TARGET_SECMARK=y
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=y
+CONFIG_NETFILTER_XT_MATCH_BPF=y
+CONFIG_NETFILTER_XT_MATCH_COMMENT=y
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
+CONFIG_NETFILTER_XT_MATCH_DSCP=y
+CONFIG_NETFILTER_XT_MATCH_ESP=y
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_HELPER=y
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=y
+CONFIG_NETFILTER_XT_MATCH_L2TP=y
+CONFIG_NETFILTER_XT_MATCH_LENGTH=y
+CONFIG_NETFILTER_XT_MATCH_LIMIT=y
+CONFIG_NETFILTER_XT_MATCH_MAC=y
+CONFIG_NETFILTER_XT_MATCH_MARK=y
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
+CONFIG_NETFILTER_XT_MATCH_OWNER=y
+CONFIG_NETFILTER_XT_MATCH_POLICY=y
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA2=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y
+CONFIG_NETFILTER_XT_MATCH_SOCKET=y
+CONFIG_NETFILTER_XT_MATCH_STATE=y
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
+CONFIG_NETFILTER_XT_MATCH_STRING=y
+CONFIG_NETFILTER_XT_MATCH_TIME=y
+CONFIG_NETFILTER_XT_MATCH_U32=y
+CONFIG_IP_NF_IPTABLES=y
+CONFIG_IP_NF_MATCH_ECN=y
+CONFIG_IP_NF_MATCH_TTL=y
+CONFIG_IP_NF_FILTER=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_NAT=y
+CONFIG_IP_NF_TARGET_MASQUERADE=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_RAW=y
+CONFIG_IP_NF_SECURITY=y
+CONFIG_IP_NF_ARPTABLES=y
+CONFIG_IP_NF_ARPFILTER=y
+CONFIG_IP_NF_ARP_MANGLE=y
+CONFIG_IP6_NF_IPTABLES=y
+CONFIG_IP6_NF_MATCH_RPFILTER=y
+CONFIG_IP6_NF_FILTER=y
+CONFIG_IP6_NF_TARGET_REJECT=y
+CONFIG_IP6_NF_MANGLE=y
+CONFIG_IP6_NF_RAW=y
+CONFIG_TIPC=m
+CONFIG_L2TP=m
+CONFIG_BRIDGE=y
+CONFIG_VLAN_8021Q=m
+CONFIG_6LOWPAN=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_HTB=y
+CONFIG_NET_SCH_PRIO=y
+CONFIG_NET_SCH_MULTIQ=y
+CONFIG_NET_SCH_SFQ=y
+CONFIG_NET_SCH_TBF=y
+CONFIG_NET_SCH_NETEM=y
+CONFIG_NET_SCH_CODEL=y
+CONFIG_NET_SCH_FQ_CODEL=y
+CONFIG_NET_SCH_FQ=y
+CONFIG_NET_SCH_INGRESS=y
+CONFIG_NET_CLS_BASIC=y
+CONFIG_NET_CLS_TCINDEX=y
+CONFIG_NET_CLS_FW=y
+CONFIG_NET_CLS_U32=y
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_FLOW=y
+CONFIG_NET_CLS_BPF=y
+CONFIG_NET_CLS_MATCHALL=y
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=y
+CONFIG_NET_EMATCH_NBYTE=y
+CONFIG_NET_EMATCH_U32=y
+CONFIG_NET_EMATCH_META=y
+CONFIG_NET_EMATCH_TEXT=y
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=y
+CONFIG_NET_ACT_GACT=y
+CONFIG_NET_ACT_MIRRED=y
+CONFIG_NET_ACT_SKBEDIT=y
+CONFIG_NET_ACT_BPF=y
+CONFIG_VSOCKETS=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_CAN=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_HIDP=m
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_LL=y
+CONFIG_BT_HCIUART_BCM=y
+CONFIG_BT_HCIUART_QCA=y
+CONFIG_CFG80211=m
+CONFIG_NL80211_TESTMODE=y
+# CONFIG_CFG80211_DEFAULT_PS is not set
+# CONFIG_CFG80211_CRDA_SUPPORT is not set
+CONFIG_MAC80211=m
+CONFIG_RFKILL=m
+CONFIG_NFC=m
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIEAER=y
+CONFIG_PCI_MSI=y
+CONFIG_PCI_IOV=y
+CONFIG_PCIE_DW_PLAT_EP=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_FW_LOADER_USER_HELPER=y
+# CONFIG_FW_CACHE is not set
+CONFIG_GNSS=y
+CONFIG_OF=y
+CONFIG_ZRAM=m
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_LOOP_MIN_COUNT=16
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_SIZE=8192
+CONFIG_SRAM=y
+CONFIG_SCSI=y
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=y
+CONFIG_DM_CRYPT=y
+CONFIG_DM_DEFAULT_KEY=y
+CONFIG_DM_SNAPSHOT=y
+CONFIG_DM_UEVENT=y
+CONFIG_DM_VERITY=y
+CONFIG_DM_VERITY_FEC=y
+CONFIG_NETDEVICES=y
+CONFIG_DUMMY=y
+CONFIG_WIREGUARD=y
+CONFIG_IFB=y
+CONFIG_MACSEC=y
+CONFIG_TUN=y
+CONFIG_VETH=y
+CONFIG_CAN_VCAN=m
+CONFIG_CAN_SLCAN=m
+CONFIG_PPP=m
+CONFIG_PPP_BSDCOMP=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_MPPE=m
+CONFIG_PPTP=m
+CONFIG_PPPOL2TP=m
+CONFIG_USB_RTL8150=y
+CONFIG_USB_RTL8152=y
+CONFIG_USB_USBNET=y
+CONFIG_USB_NET_CDC_EEM=y
+# CONFIG_USB_NET_NET1080 is not set
+# CONFIG_USB_NET_CDC_SUBSET is not set
+# CONFIG_USB_NET_ZAURUS is not set
+CONFIG_USB_NET_AQC111=y
+# CONFIG_WLAN_VENDOR_ADMTEK is not set
+# CONFIG_WLAN_VENDOR_ATH is not set
+# CONFIG_WLAN_VENDOR_ATMEL is not set
+# CONFIG_WLAN_VENDOR_BROADCOM is not set
+# CONFIG_WLAN_VENDOR_CISCO is not set
+# CONFIG_WLAN_VENDOR_INTEL is not set
+# CONFIG_WLAN_VENDOR_INTERSIL is not set
+# CONFIG_WLAN_VENDOR_MARVELL is not set
+# CONFIG_WLAN_VENDOR_MEDIATEK is not set
+# CONFIG_WLAN_VENDOR_RALINK is not set
+# CONFIG_WLAN_VENDOR_REALTEK is not set
+# CONFIG_WLAN_VENDOR_RSI is not set
+# CONFIG_WLAN_VENDOR_ST is not set
+# CONFIG_WLAN_VENDOR_TI is not set
+# CONFIG_WLAN_VENDOR_ZYDAS is not set
+# CONFIG_WLAN_VENDOR_QUANTENNA is not set
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_GPIO=y
+# CONFIG_MOUSE_PS2 is not set
+CONFIG_INPUT_JOYSTICK=y
+CONFIG_JOYSTICK_XPAD=y
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_UINPUT=y
+# CONFIG_VT is not set
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_RUNTIME_UARTS=0
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_SAMSUNG=y
+CONFIG_SERIAL_SAMSUNG_CONSOLE=y
+CONFIG_NULL_TTY=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_HW_RANDOM=y
+# CONFIG_DEVMEM is not set
+# CONFIG_DEVPORT is not set
+CONFIG_HPET=y
+# CONFIG_I2C_COMPAT is not set
+# CONFIG_I2C_HELPER_AUTO is not set
+CONFIG_I3C=y
+CONFIG_SPI=y
+CONFIG_SPI_MEM=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_GENERIC_PLATFORM=y
+# CONFIG_HWMON is not set
+CONFIG_THERMAL_NETLINK=y
+CONFIG_THERMAL_STATISTICS=y
+CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=100
+CONFIG_THERMAL_WRITABLE_TRIPS=y
+CONFIG_THERMAL_GOV_USER_SPACE=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_THERMAL_EMULATION=y
+# CONFIG_X86_PKG_TEMP_THERMAL is not set
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_CORE=y
+CONFIG_MFD_SYSCON=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+CONFIG_RC_CORE=y
+CONFIG_BPF_LIRC_MODE2=y
+CONFIG_LIRC=y
+# CONFIG_RC_MAP is not set
+CONFIG_RC_DECODERS=y
+CONFIG_RC_DEVICES=y
+CONFIG_MEDIA_CEC_RC=y
+# CONFIG_MEDIA_ANALOG_TV_SUPPORT is not set
+# CONFIG_MEDIA_DIGITAL_TV_SUPPORT is not set
+# CONFIG_MEDIA_RADIO_SUPPORT is not set
+# CONFIG_MEDIA_SDR_SUPPORT is not set
+# CONFIG_MEDIA_TEST_SUPPORT is not set
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_GSPCA=y
+CONFIG_USB_VIDEO_CLASS=y
+CONFIG_V4L_PLATFORM_DRIVERS=y
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_DRM=y
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_HRTIMER=y
+# CONFIG_SND_SUPPORT_OLD_API is not set
+# CONFIG_SND_VERBOSE_PROCFS is not set
+# CONFIG_SND_DRIVERS is not set
+CONFIG_SND_USB_AUDIO=y
+CONFIG_SND_SOC=y
+CONFIG_HID_BATTERY_STRENGTH=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_APPLE=y
+CONFIG_HID_PRODIKEYS=y
+CONFIG_HID_ELECOM=y
+CONFIG_HID_UCLOGIC=y
+CONFIG_HID_LOGITECH=y
+CONFIG_HID_LOGITECH_DJ=y
+CONFIG_HID_MAGICMOUSE=y
+CONFIG_HID_MICROSOFT=y
+CONFIG_HID_MULTITOUCH=y
+CONFIG_HID_NINTENDO=y
+CONFIG_HID_PICOLCD=y
+CONFIG_HID_PLANTRONICS=y
+CONFIG_HID_PLAYSTATION=y
+CONFIG_PLAYSTATION_FF=y
+CONFIG_HID_ROCCAT=y
+CONFIG_HID_SONY=y
+CONFIG_SONY_FF=y
+CONFIG_HID_STEAM=y
+CONFIG_HID_WACOM=y
+CONFIG_HID_WIIMOTE=y
+CONFIG_USB_HIDDEV=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_EHCI_HCD=y
+CONFIG_USB_EHCI_ROOT_HUB_TT=y
+CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_ACM=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_GADGET=y
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_UEVENT=y
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_ACM=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_ECM=y
+CONFIG_USB_CONFIGFS_EEM=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_ACC=y
+CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y
+CONFIG_USB_CONFIGFS_F_MIDI=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_TCPM=y
+CONFIG_TYPEC_TCPCI=y
+CONFIG_TYPEC_UCSI=y
+CONFIG_MMC=y
+# CONFIG_PWRSEQ_EMMC is not set
+# CONFIG_PWRSEQ_SIMPLE is not set
+CONFIG_MMC_CRYPTO=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFS_BSG=y
+CONFIG_SCSI_UFS_CRYPTO=y
+CONFIG_SCSI_UFSHCD_PCI=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_DWC_TC_PLATFORM=y
+CONFIG_LEDS_CLASS_FLASH=y
+CONFIG_LEDS_CLASS_MULTICOLOR=y
+CONFIG_LEDS_TRIGGER_TIMER=y
+CONFIG_LEDS_TRIGGER_TRANSIENT=y
+CONFIG_EDAC=y
+CONFIG_RTC_CLASS=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_UIO=y
+CONFIG_VHOST_VSOCK=y
+CONFIG_STAGING=y
+CONFIG_ASHMEM=y
+CONFIG_REMOTEPROC=y
+CONFIG_REMOTEPROC_CDEV=y
+CONFIG_RPMSG_CHAR=y
+CONFIG_PM_DEVFREQ_EVENT=y
+CONFIG_IIO=y
+CONFIG_IIO_BUFFER=y
+CONFIG_IIO_TRIGGER=y
+CONFIG_POWERCAP=y
+CONFIG_ANDROID_BINDER_IPC=y
+CONFIG_ANDROID_BINDERFS=y
+CONFIG_ANDROID_VENDOR_HOOKS=y
+CONFIG_ANDROID_DEBUG_KINFO=y
+CONFIG_LIBNVDIMM=y
+CONFIG_INTERCONNECT=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_F2FS_FS=y
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_COMPRESSION=y
+CONFIG_F2FS_UNFAIR_RWSEM=y
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y
+CONFIG_FS_VERITY=y
+CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y
+# CONFIG_DNOTIFY is not set
+CONFIG_QUOTA=y
+CONFIG_QFMT_V2=y
+CONFIG_FUSE_FS=y
+CONFIG_VIRTIO_FS=y
+CONFIG_OVERLAY_FS=y
+CONFIG_INCREMENTAL_FS=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+# CONFIG_EFIVAR_FS is not set
+CONFIG_PSTORE=y
+CONFIG_PSTORE_CONSOLE=y
+CONFIG_PSTORE_PMSG=y
+CONFIG_PSTORE_RAM=y
+CONFIG_EROFS_FS=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=y
+CONFIG_NLS_CODEPAGE_775=y
+CONFIG_NLS_CODEPAGE_850=y
+CONFIG_NLS_CODEPAGE_852=y
+CONFIG_NLS_CODEPAGE_855=y
+CONFIG_NLS_CODEPAGE_857=y
+CONFIG_NLS_CODEPAGE_860=y
+CONFIG_NLS_CODEPAGE_861=y
+CONFIG_NLS_CODEPAGE_862=y
+CONFIG_NLS_CODEPAGE_863=y
+CONFIG_NLS_CODEPAGE_864=y
+CONFIG_NLS_CODEPAGE_865=y
+CONFIG_NLS_CODEPAGE_866=y
+CONFIG_NLS_CODEPAGE_869=y
+CONFIG_NLS_CODEPAGE_936=y
+CONFIG_NLS_CODEPAGE_950=y
+CONFIG_NLS_CODEPAGE_932=y
+CONFIG_NLS_CODEPAGE_949=y
+CONFIG_NLS_CODEPAGE_874=y
+CONFIG_NLS_ISO8859_8=y
+CONFIG_NLS_CODEPAGE_1250=y
+CONFIG_NLS_CODEPAGE_1251=y
+CONFIG_NLS_ASCII=y
+CONFIG_NLS_ISO8859_1=y
+CONFIG_NLS_ISO8859_2=y
+CONFIG_NLS_ISO8859_3=y
+CONFIG_NLS_ISO8859_4=y
+CONFIG_NLS_ISO8859_5=y
+CONFIG_NLS_ISO8859_6=y
+CONFIG_NLS_ISO8859_7=y
+CONFIG_NLS_ISO8859_9=y
+CONFIG_NLS_ISO8859_13=y
+CONFIG_NLS_ISO8859_14=y
+CONFIG_NLS_ISO8859_15=y
+CONFIG_NLS_KOI8_R=y
+CONFIG_NLS_KOI8_U=y
+CONFIG_NLS_MAC_ROMAN=y
+CONFIG_NLS_MAC_CELTIC=y
+CONFIG_NLS_MAC_CENTEURO=y
+CONFIG_NLS_MAC_CROATIAN=y
+CONFIG_NLS_MAC_CYRILLIC=y
+CONFIG_NLS_MAC_GAELIC=y
+CONFIG_NLS_MAC_GREEK=y
+CONFIG_NLS_MAC_ICELAND=y
+CONFIG_NLS_MAC_INUIT=y
+CONFIG_NLS_MAC_ROMANIAN=y
+CONFIG_NLS_MAC_TURKISH=y
+CONFIG_NLS_UTF8=y
+CONFIG_UNICODE=y
+CONFIG_SECURITY=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_FORTIFY_SOURCE=y
+CONFIG_STATIC_USERMODEHELPER=y
+CONFIG_STATIC_USERMODEHELPER_PATH=""
+CONFIG_SECURITY_SELINUX=y
+CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y
+CONFIG_CRYPTO_ECDH=y
+CONFIG_CRYPTO_DES=y
+CONFIG_CRYPTO_ADIANTUM=y
+CONFIG_CRYPTO_HCTR2=y
+CONFIG_CRYPTO_CHACHA20POLY1305=y
+CONFIG_CRYPTO_CCM=y
+CONFIG_CRYPTO_BLAKE2B=y
+CONFIG_CRYPTO_CMAC=y
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_XCBC=y
+CONFIG_CRYPTO_LZO=y
+CONFIG_CRYPTO_LZ4=y
+CONFIG_CRYPTO_ZSTD=y
+CONFIG_CRYPTO_ANSI_CPRNG=y
+CONFIG_CRYPTO_AES_NI_INTEL=y
+CONFIG_CRYPTO_POLYVAL_CLMUL_NI=y
+CONFIG_CRYPTO_SHA256_SSSE3=y
+CONFIG_CRYPTO_SHA512_SSSE3=y
+CONFIG_CRC_CCITT=y
+CONFIG_CRC8=y
+CONFIG_XZ_DEC=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_DYNAMIC_DEBUG_CORE=y
+CONFIG_DEBUG_INFO_DWARF4=y
+CONFIG_HEADERS_INSTALL=y
+# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_UBSAN=y
+CONFIG_UBSAN_TRAP=y
+CONFIG_UBSAN_LOCAL_BOUNDS=y
+# CONFIG_UBSAN_SHIFT is not set
+# CONFIG_UBSAN_BOOL is not set
+# CONFIG_UBSAN_ENUM is not set
+CONFIG_PAGE_OWNER=y
+CONFIG_DEBUG_STACK_USAGE=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_KFENCE=y
+CONFIG_KFENCE_SAMPLE_INTERVAL=500
+CONFIG_KFENCE_NUM_OBJECTS=63
+CONFIG_KFENCE_STATIC_KEYS=y
+CONFIG_PANIC_ON_OOPS=y
+CONFIG_PANIC_TIMEOUT=-1
+CONFIG_SOFTLOCKUP_DETECTOR=y
+CONFIG_WQ_WATCHDOG=y
+CONFIG_SCHEDSTATS=y
+CONFIG_BUG_ON_DATA_CORRUPTION=y
+CONFIG_HIST_TRIGGERS=y
+CONFIG_UNWINDER_FRAME_POINTER=y
+CONFIG_KUNIT=y
+CONFIG_KUNIT_DEBUGFS=y
+# CONFIG_KUNIT_DEFAULT_ENABLED is not set
diff --git a/block/OWNERS b/block/OWNERS
new file mode 100644
index 0000000..2641e06
--- /dev/null
+++ b/block/OWNERS
@@ -0,0 +1,2 @@
+bvanassche@google.com
+jaegeuk@google.com
diff --git a/block/bio.c b/block/bio.c
index 5f96fca..4559a37 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -270,6 +270,9 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
#endif
#ifdef CONFIG_BLK_INLINE_ENCRYPTION
bio->bi_crypt_context = NULL;
+#if IS_ENABLED(CONFIG_DM_DEFAULT_KEY)
+ bio->bi_skip_dm_default_key = false;
+#endif
#endif
#ifdef CONFIG_BLK_DEV_INTEGRITY
bio->bi_integrity = NULL;
diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c
index ad9844c..243953d 100644
--- a/block/blk-crypto-fallback.c
+++ b/block/blk-crypto-fallback.c
@@ -87,7 +87,7 @@ static struct bio_set crypto_bio_split;
* This is the key we set when evicting a keyslot. This *should* be the all 0's
* key, but AES-XTS rejects that key, so we use some random bytes instead.
*/
-static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
+static u8 blank_key[BLK_CRYPTO_MAX_STANDARD_KEY_SIZE];
static void blk_crypto_fallback_evict_keyslot(unsigned int slot)
{
@@ -180,6 +180,8 @@ static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src)
bio_clone_blkg_association(bio, bio_src);
+ bio_clone_skip_dm_default_key(bio, bio_src);
+
return bio;
}
@@ -539,7 +541,7 @@ static int blk_crypto_fallback_init(void)
if (blk_crypto_fallback_inited)
return 0;
- get_random_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
+ get_random_bytes(blank_key, sizeof(blank_key));
err = bioset_init(&crypto_bio_split, 64, 0, 0);
if (err)
@@ -552,6 +554,7 @@ static int blk_crypto_fallback_init(void)
profile->ll_ops = blk_crypto_fallback_ll_ops;
profile->max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE;
+ profile->key_types_supported = BLK_CRYPTO_KEY_TYPE_STANDARD;
/* All blk-crypto modes have a crypto API fallback. */
for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++)
diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h
index a8cdaf2..f26076f 100644
--- a/block/blk-crypto-internal.h
+++ b/block/blk-crypto-internal.h
@@ -14,6 +14,7 @@ struct blk_crypto_mode {
const char *name; /* name of this mode, shown in sysfs */
const char *cipher_str; /* crypto API name (for fallback case) */
unsigned int keysize; /* key size in bytes */
+ unsigned int security_strength; /* security strength in bytes */
unsigned int ivsize; /* iv size in bytes */
};
diff --git a/block/blk-crypto-profile.c b/block/blk-crypto-profile.c
index 0307fb0..757d0fc 100644
--- a/block/blk-crypto-profile.c
+++ b/block/blk-crypto-profile.c
@@ -351,6 +351,8 @@ bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile,
return false;
if (profile->max_dun_bytes_supported < cfg->dun_bytes)
return false;
+ if (!(profile->key_types_supported & cfg->key_type))
+ return false;
return true;
}
@@ -465,6 +467,45 @@ bool blk_crypto_register(struct blk_crypto_profile *profile,
EXPORT_SYMBOL_GPL(blk_crypto_register);
/**
+ * blk_crypto_derive_sw_secret() - Derive software secret from wrapped key
+ * @bdev: a block device that supports hardware-wrapped keys
+ * @eph_key: the hardware-wrapped key in ephemerally-wrapped form
+ * @eph_key_size: size of @eph_key in bytes
+ * @sw_secret: (output) the software secret
+ *
+ * Given a hardware-wrapped key in ephemerally-wrapped form (the same form that
+ * it is used for I/O), ask the hardware to derive the secret which software can
+ * use for cryptographic tasks other than inline encryption. This secret is
+ * guaranteed to be cryptographically isolated from the inline encryption key,
+ * i.e. derived with a different KDF context.
+ *
+ * Return: 0 on success, -EOPNOTSUPP if the block device doesn't support
+ * hardware-wrapped keys, -EBADMSG if the key isn't a valid
+ * hardware-wrapped key, or another -errno code.
+ */
+int blk_crypto_derive_sw_secret(struct block_device *bdev,
+ const u8 *eph_key, size_t eph_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
+{
+ struct blk_crypto_profile *profile =
+ bdev_get_queue(bdev)->crypto_profile;
+ int err;
+
+ if (!profile)
+ return -EOPNOTSUPP;
+ if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED))
+ return -EOPNOTSUPP;
+ if (!profile->ll_ops.derive_sw_secret)
+ return -EOPNOTSUPP;
+ blk_crypto_hw_enter(profile);
+ err = profile->ll_ops.derive_sw_secret(profile, eph_key, eph_key_size,
+ sw_secret);
+ blk_crypto_hw_exit(profile);
+ return err;
+}
+EXPORT_SYMBOL_GPL(blk_crypto_derive_sw_secret);
+
+/**
* blk_crypto_intersect_capabilities() - restrict supported crypto capabilities
* by child device
* @parent: the crypto profile for the parent device
@@ -487,10 +528,12 @@ void blk_crypto_intersect_capabilities(struct blk_crypto_profile *parent,
child->max_dun_bytes_supported);
for (i = 0; i < ARRAY_SIZE(child->modes_supported); i++)
parent->modes_supported[i] &= child->modes_supported[i];
+ parent->key_types_supported &= child->key_types_supported;
} else {
parent->max_dun_bytes_supported = 0;
memset(parent->modes_supported, 0,
sizeof(parent->modes_supported));
+ parent->key_types_supported = 0;
}
}
EXPORT_SYMBOL_GPL(blk_crypto_intersect_capabilities);
@@ -523,6 +566,9 @@ bool blk_crypto_has_capabilities(const struct blk_crypto_profile *target,
target->max_dun_bytes_supported)
return false;
+ if (reference->key_types_supported & ~target->key_types_supported)
+ return false;
+
return true;
}
EXPORT_SYMBOL_GPL(blk_crypto_has_capabilities);
@@ -557,5 +603,6 @@ void blk_crypto_update_capabilities(struct blk_crypto_profile *dst,
sizeof(dst->modes_supported));
dst->max_dun_bytes_supported = src->max_dun_bytes_supported;
+ dst->key_types_supported = src->key_types_supported;
}
EXPORT_SYMBOL_GPL(blk_crypto_update_capabilities);
diff --git a/block/blk-crypto.c b/block/blk-crypto.c
index 4537858..ceb6c8a 100644
--- a/block/blk-crypto.c
+++ b/block/blk-crypto.c
@@ -22,18 +22,21 @@ const struct blk_crypto_mode blk_crypto_modes[] = {
.name = "AES-256-XTS",
.cipher_str = "xts(aes)",
.keysize = 64,
+ .security_strength = 32,
.ivsize = 16,
},
[BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = {
.name = "AES-128-CBC-ESSIV",
.cipher_str = "essiv(cbc(aes),sha256)",
.keysize = 16,
+ .security_strength = 16,
.ivsize = 16,
},
[BLK_ENCRYPTION_MODE_ADIANTUM] = {
.name = "Adiantum",
.cipher_str = "adiantum(xchacha12,aes)",
.keysize = 32,
+ .security_strength = 32,
.ivsize = 32,
},
[BLK_ENCRYPTION_MODE_SM4_XTS] = {
@@ -75,9 +78,15 @@ static int __init bio_crypt_ctx_init(void)
/* This is assumed in various places. */
BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0);
- /* Sanity check that no algorithm exceeds the defined limits. */
+ /*
+ * Validate the crypto mode properties. This ideally would be done with
+ * static assertions, but boot-time checks are the next best thing.
+ */
for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) {
- BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE);
+ BUG_ON(blk_crypto_modes[i].keysize >
+ BLK_CRYPTO_MAX_STANDARD_KEY_SIZE);
+ BUG_ON(blk_crypto_modes[i].security_strength >
+ blk_crypto_modes[i].keysize);
BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE);
}
@@ -105,6 +114,7 @@ void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
bio->bi_crypt_context = bc;
}
+EXPORT_SYMBOL_GPL(bio_crypt_set_ctx);
void __bio_crypt_free_ctx(struct bio *bio)
{
@@ -314,8 +324,9 @@ int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
/**
* blk_crypto_init_key() - Prepare a key for use with blk-crypto
* @blk_key: Pointer to the blk_crypto_key to initialize.
- * @raw_key: Pointer to the raw key. Must be the correct length for the chosen
- * @crypto_mode; see blk_crypto_modes[].
+ * @raw_key: the raw bytes of the key
+ * @raw_key_size: size of the raw key in bytes
+ * @key_type: type of the key -- either standard or hardware-wrapped
* @crypto_mode: identifier for the encryption algorithm to use
* @dun_bytes: number of bytes that will be used to specify the DUN when this
* key is used
@@ -324,7 +335,9 @@ int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
* Return: 0 on success, -errno on failure. The caller is responsible for
* zeroizing both blk_key and raw_key when done with them.
*/
-int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
+int blk_crypto_init_key(struct blk_crypto_key *blk_key,
+ const u8 *raw_key, size_t raw_key_size,
+ enum blk_crypto_key_type key_type,
enum blk_crypto_mode_num crypto_mode,
unsigned int dun_bytes,
unsigned int data_unit_size)
@@ -337,8 +350,19 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
return -EINVAL;
mode = &blk_crypto_modes[crypto_mode];
- if (mode->keysize == 0)
+ switch (key_type) {
+ case BLK_CRYPTO_KEY_TYPE_STANDARD:
+ if (raw_key_size != mode->keysize)
+ return -EINVAL;
+ break;
+ case BLK_CRYPTO_KEY_TYPE_HW_WRAPPED:
+ if (raw_key_size < mode->security_strength ||
+ raw_key_size > BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE)
+ return -EINVAL;
+ break;
+ default:
return -EINVAL;
+ }
if (dun_bytes == 0 || dun_bytes > mode->ivsize)
return -EINVAL;
@@ -349,12 +373,14 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
blk_key->crypto_cfg.crypto_mode = crypto_mode;
blk_key->crypto_cfg.dun_bytes = dun_bytes;
blk_key->crypto_cfg.data_unit_size = data_unit_size;
+ blk_key->crypto_cfg.key_type = key_type;
blk_key->data_unit_size_bits = ilog2(data_unit_size);
- blk_key->size = mode->keysize;
- memcpy(blk_key->raw, raw_key, mode->keysize);
+ blk_key->size = raw_key_size;
+ memcpy(blk_key->raw, raw_key, raw_key_size);
return 0;
}
+EXPORT_SYMBOL_GPL(blk_crypto_init_key);
bool blk_crypto_config_supported_natively(struct block_device *bdev,
const struct blk_crypto_config *cfg)
@@ -371,8 +397,10 @@ bool blk_crypto_config_supported_natively(struct block_device *bdev,
bool blk_crypto_config_supported(struct block_device *bdev,
const struct blk_crypto_config *cfg)
{
- return IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) ||
- blk_crypto_config_supported_natively(bdev, cfg);
+ if (IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) &&
+ cfg->key_type == BLK_CRYPTO_KEY_TYPE_STANDARD)
+ return true;
+ return blk_crypto_config_supported_natively(bdev, cfg);
}
/**
@@ -395,8 +423,13 @@ int blk_crypto_start_using_key(struct block_device *bdev,
{
if (blk_crypto_config_supported_natively(bdev, &key->crypto_cfg))
return 0;
+ if (key->crypto_cfg.key_type != BLK_CRYPTO_KEY_TYPE_STANDARD) {
+ pr_warn_once("tried to use wrapped key, but hardware doesn't support it\n");
+ return -EOPNOTSUPP;
+ }
return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode);
}
+EXPORT_SYMBOL_GPL(blk_crypto_start_using_key);
/**
* blk_crypto_evict_key() - Evict a key from any inline encryption hardware
diff --git a/build.config.aarch64 b/build.config.aarch64
new file mode 100644
index 0000000..9fcbceb
--- /dev/null
+++ b/build.config.aarch64
@@ -0,0 +1,16 @@
+ARCH=arm64
+MAKE_GOALS="
+Image
+modules
+"
+
+FILES="
+arch/arm64/boot/Image
+vmlinux
+System.map
+vmlinux.symvers
+modules.builtin
+modules.builtin.modinfo
+"
+
+NDK_TRIPLE=aarch64-linux-android31
diff --git a/build.config.allmodconfig b/build.config.allmodconfig
new file mode 100644
index 0000000..8bd0d5f
--- /dev/null
+++ b/build.config.allmodconfig
@@ -0,0 +1,16 @@
+DEFCONFIG=allmodconfig
+
+POST_DEFCONFIG_CMDS="update_config"
+function update_config() {
+ ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \
+ -e UNWINDER_FRAME_POINTER \
+ -d WERROR \
+ -d SAMPLES \
+ -d BPFILTER \
+ -e RANDSTRUCT_NONE \
+ -d RANDSTRUCT_FULL \
+ -d RANDSTRUCT \
+
+ (cd ${OUT_DIR} && \
+ make O=${OUT_DIR} $archsubarch CROSS_COMPILE=${CROSS_COMPILE} ${TOOL_ARGS} ${MAKE_ARGS} olddefconfig)
+}
diff --git a/build.config.allmodconfig.aarch64 b/build.config.allmodconfig.aarch64
new file mode 100644
index 0000000..2fbe380
--- /dev/null
+++ b/build.config.allmodconfig.aarch64
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.allmodconfig
+
diff --git a/build.config.allmodconfig.arm b/build.config.allmodconfig.arm
new file mode 100644
index 0000000..e92744a
--- /dev/null
+++ b/build.config.allmodconfig.arm
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.arm
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.allmodconfig
+
diff --git a/build.config.allmodconfig.x86_64 b/build.config.allmodconfig.x86_64
new file mode 100644
index 0000000..f06b30c
--- /dev/null
+++ b/build.config.allmodconfig.x86_64
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.x86_64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.allmodconfig
+
diff --git a/build.config.amlogic b/build.config.amlogic
new file mode 100644
index 0000000..e04bd52
--- /dev/null
+++ b/build.config.amlogic
@@ -0,0 +1,34 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki.aarch64
+
+DEFCONFIG=amlogic_gki_defconfig
+FRAGMENT_CONFIG=${KERNEL_DIR}/arch/arm64/configs/amlogic_gki.fragment
+
+PRE_DEFCONFIG_CMDS="KCONFIG_CONFIG=${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG} ${ROOT_DIR}/${KERNEL_DIR}/scripts/kconfig/merge_config.sh -m -r ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${FRAGMENT_CONFIG}"
+POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG}"
+
+# needed for DT overlay support
+DTC_FLAGS="-@"
+
+MAKE_GOALS="${MAKE_GOALS}
+dtbs
+"
+
+FILES="${FILES}
+arch/arm64/boot/Image.lz4
+arch/arm64/boot/dts/amlogic/meson-g12a-sei510*.dtb
+arch/arm64/boot/dts/amlogic/meson-sm1-sei610*.dtb
+arch/arm64/boot/dts/amlogic/meson-sm1-khadas-vim3l*.dtb
+arch/arm64/boot/dts/amlogic/meson-g12b-a311d-khadas-vim3*.dtb
+"
+
+#
+# NOTE: Using Image.lz4 in MAKE_GOALS does not work because
+# kernel build passes legacy option (-l) to lz4 command
+# and u-boot fails to decompress. Instead, add custom
+# command to lz4 compress same options as kernel, but
+# without the -l.
+#
+EXTRA_CMDS="lz4_compress"
+function lz4_compress() {
+ lz4 -f -12 --favor-decSpeed ${OUT_DIR}/arch/arm64/boot/Image ${OUT_DIR}/arch/arm64/boot/Image.lz4
+}
diff --git a/build.config.arm b/build.config.arm
new file mode 100644
index 0000000..c7f114c
--- /dev/null
+++ b/build.config.arm
@@ -0,0 +1,13 @@
+ARCH=arm
+MAKE_GOALS="
+zImage
+modules
+"
+
+FILES="
+arch/arm/boot/zImage
+vmlinux
+System.map
+"
+
+NDK_TRIPLE=arm-linux-androideabi31
diff --git a/build.config.common b/build.config.common
new file mode 100644
index 0000000..3106a14
--- /dev/null
+++ b/build.config.common
@@ -0,0 +1,17 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.constants
+
+KMI_GENERATION=0
+
+LLVM=1
+DEPMOD=depmod
+CLANG_PREBUILT_BIN=prebuilts/clang/host/linux-x86/clang-${CLANG_VERSION}/bin
+BUILDTOOLS_PREBUILT_BIN=build/kernel/build-tools/path/linux-x86
+DTC=${ROOT_DIR}/${BUILDTOOLS_PREBUILT_BIN}/dtc
+
+KCFLAGS="${KCFLAGS} -D__ANDROID_COMMON_KERNEL__"
+EXTRA_CMDS=''
+STOP_SHIP_TRACEPRINTK=1
+IN_KERNEL_MODULES=1
+DO_NOT_STRIP_MODULES=1
+
+HERMETIC_TOOLCHAIN=${HERMETIC_TOOLCHAIN:-1}
diff --git a/build.config.constants b/build.config.constants
new file mode 100644
index 0000000..dbbfc07
--- /dev/null
+++ b/build.config.constants
@@ -0,0 +1,2 @@
+BRANCH=android-mainline
+CLANG_VERSION=r475365b
diff --git a/build.config.db845c b/build.config.db845c
new file mode 100644
index 0000000..9e1c48e
--- /dev/null
+++ b/build.config.db845c
@@ -0,0 +1,21 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+
+BUILD_INITRAMFS=1
+DEFCONFIG=db845c_gki_defconfig
+FRAGMENT_CONFIG=${KERNEL_DIR}/arch/arm64/configs/db845c_gki.fragment
+PRE_DEFCONFIG_CMDS="KCONFIG_CONFIG=${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG} ${ROOT_DIR}/${KERNEL_DIR}/scripts/kconfig/merge_config.sh -m -r ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${FRAGMENT_CONFIG}"
+POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG}"
+
+MAKE_GOALS="
+modules
+qcom/sdm845-db845c.dtb
+qcom/qrb5165-rb5.dtb
+qcom/sm8450-qrd.dtb
+"
+
+FILES="
+arch/arm64/boot/dts/qcom/sdm845-db845c.dtb
+arch/arm64/boot/dts/qcom/qrb5165-rb5.dtb
+arch/arm64/boot/dts/qcom/sm8450-qrd.dtb
+"
diff --git a/build.config.gki b/build.config.gki
new file mode 100644
index 0000000..4b931d9
--- /dev/null
+++ b/build.config.gki
@@ -0,0 +1,2 @@
+DEFCONFIG=gki_defconfig
+POST_DEFCONFIG_CMDS="check_defconfig"
diff --git a/build.config.gki-debug.aarch64 b/build.config.gki-debug.aarch64
new file mode 100644
index 0000000..c1fe2f0
--- /dev/null
+++ b/build.config.gki-debug.aarch64
@@ -0,0 +1,3 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki.aarch64
+TRIM_NONLISTED_KMI=""
+KMI_SYMBOL_LIST_STRICT_MODE=""
diff --git a/build.config.gki-debug.x86_64 b/build.config.gki-debug.x86_64
new file mode 100644
index 0000000..d89b7ad
--- /dev/null
+++ b/build.config.gki-debug.x86_64
@@ -0,0 +1,3 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki.x86_64
+TRIM_NONLISTED_KMI=""
+KMI_SYMBOL_LIST_STRICT_MODE=""
diff --git a/build.config.gki.aarch64 b/build.config.gki.aarch64
new file mode 100644
index 0000000..709af184
--- /dev/null
+++ b/build.config.gki.aarch64
@@ -0,0 +1,27 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki
+
+MAKE_GOALS="${MAKE_GOALS}
+Image.lz4
+Image.gz
+"
+
+FILES="${FILES}
+arch/arm64/boot/Image.lz4
+arch/arm64/boot/Image.gz
+"
+
+BUILD_SYSTEM_DLKM=1
+MODULES_LIST=${ROOT_DIR}/${KERNEL_DIR}/android/gki_system_dlkm_modules
+
+BUILD_GKI_CERTIFICATION_TOOLS=1
+
+BUILD_GKI_ARTIFACTS=1
+BUILD_GKI_BOOT_IMG_SIZE=67108864
+BUILD_GKI_BOOT_IMG_GZ_SIZE=47185920
+BUILD_GKI_BOOT_IMG_LZ4_SIZE=53477376
+
+if [ -n "${GKI_BUILD_CONFIG_FRAGMENT}" ]; then
+source ${GKI_BUILD_CONFIG_FRAGMENT}
+fi
diff --git a/build.config.gki.aarch64.16k b/build.config.gki.aarch64.16k
new file mode 100644
index 0000000..20be95d
--- /dev/null
+++ b/build.config.gki.aarch64.16k
@@ -0,0 +1,5 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki.aarch64
+
+DEFCONFIG=16k_gki_defconfig
+PRE_DEFCONFIG_CMDS="mkdir -p \${OUT_DIR}/arch/arm64/configs/ && cat ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/16k_gki.fragment > \${OUT_DIR}/arch/arm64/configs/${DEFCONFIG};"
+POST_DEFCONFIG_CMDS=""
diff --git a/build.config.gki.riscv64 b/build.config.gki.riscv64
new file mode 100644
index 0000000..8215d16
--- /dev/null
+++ b/build.config.gki.riscv64
@@ -0,0 +1,30 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.riscv64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki
+
+MAKE_GOALS="${MAKE_GOALS}
+Image.lz4
+Image.gz
+"
+
+FILES="${FILES}
+arch/riscv/boot/Image.lz4
+arch/riscv/boot/Image.gz
+"
+
+BUILD_SYSTEM_DLKM=1
+MODULES_LIST=${ROOT_DIR}/${KERNEL_DIR}/android/gki_system_dlkm_modules
+
+BUILD_GKI_CERTIFICATION_TOOLS=1
+
+BUILD_GKI_ARTIFACTS=1
+BUILD_GKI_BOOT_IMG_SIZE=67108864
+BUILD_GKI_BOOT_IMG_GZ_SIZE=47185920
+BUILD_GKI_BOOT_IMG_LZ4_SIZE=53477376
+
+PRE_DEFCONFIG_CMDS="mkdir -p \${OUT_DIR}/arch/riscv/configs/ && cat ${ROOT_DIR}/${KERNEL_DIR}/arch/riscv/configs/gki_defconfig ${ROOT_DIR}/${KERNEL_DIR}/arch/riscv/configs/64-bit.config ${ROOT_DIR}/${KERNEL_DIR}/arch/riscv/configs/gki.config > \${OUT_DIR}/arch/riscv/configs/${DEFCONFIG};"
+POST_DEFCONFIG_CMDS=""
+
+if [ -n "${GKI_BUILD_CONFIG_FRAGMENT}" ]; then
+source ${GKI_BUILD_CONFIG_FRAGMENT}
+fi
diff --git a/build.config.gki.x86_64 b/build.config.gki.x86_64
new file mode 100644
index 0000000..93f492c
--- /dev/null
+++ b/build.config.gki.x86_64
@@ -0,0 +1,15 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.x86_64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki
+
+BUILD_SYSTEM_DLKM=1
+MODULES_LIST=${ROOT_DIR}/${KERNEL_DIR}/android/gki_system_dlkm_modules
+
+BUILD_GKI_CERTIFICATION_TOOLS=1
+
+BUILD_GKI_ARTIFACTS=1
+BUILD_GKI_BOOT_IMG_SIZE=67108864
+
+if [ -n "${GKI_BUILD_CONFIG_FRAGMENT}" ]; then
+source ${GKI_BUILD_CONFIG_FRAGMENT}
+fi
diff --git a/build.config.gki_kasan b/build.config.gki_kasan
new file mode 100644
index 0000000..1aca62c
--- /dev/null
+++ b/build.config.gki_kasan
@@ -0,0 +1,22 @@
+DEFCONFIG=gki_defconfig
+POST_DEFCONFIG_CMDS="check_defconfig && update_kasan_config"
+KERNEL_DIR=common
+LTO=none
+
+function update_kasan_config() {
+ ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \
+ -e CONFIG_KASAN \
+ -e CONFIG_KASAN_INLINE \
+ -e CONFIG_KCOV \
+ -e CONFIG_PANIC_ON_WARN_DEFAULT_ENABLE \
+ -d CONFIG_RANDOMIZE_BASE \
+ -d CONFIG_KASAN_OUTLINE \
+ --set-val CONFIG_FRAME_WARN 0 \
+ -d CFI \
+ -d CFI_PERMISSIVE \
+ -d CFI_CLANG \
+ -d SHADOW_CALL_STACK
+ (cd ${OUT_DIR} && \
+ make ${TOOL_ARGS} O=${OUT_DIR} olddefconfig)
+}
+
diff --git a/build.config.gki_kasan.aarch64 b/build.config.gki_kasan.aarch64
new file mode 100644
index 0000000..9fd2560
--- /dev/null
+++ b/build.config.gki_kasan.aarch64
@@ -0,0 +1,3 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki_kasan
diff --git a/build.config.gki_kasan.x86_64 b/build.config.gki_kasan.x86_64
new file mode 100644
index 0000000..eec6458
--- /dev/null
+++ b/build.config.gki_kasan.x86_64
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.x86_64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki_kasan
+
diff --git a/build.config.gki_kprobes b/build.config.gki_kprobes
new file mode 100644
index 0000000..f1c6cf7
--- /dev/null
+++ b/build.config.gki_kprobes
@@ -0,0 +1,20 @@
+DEFCONFIG=gki_defconfig
+POST_DEFCONFIG_CMDS="check_defconfig && update_kprobes_config"
+function update_kprobes_config() {
+ ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \
+ -d LTO \
+ -d LTO_CLANG_THIN \
+ -d CFI \
+ -d CFI_PERMISSIVE \
+ -d CFI_CLANG \
+ -e CONFIG_DYNAMIC_FTRACE \
+ -e CONFIG_FUNCTION_TRACER \
+ -e CONFIG_IRQSOFF_TRACER \
+ -e CONFIG_FUNCTION_PROFILER \
+ -e CONFIG_PREEMPT_TRACER \
+ -e CONFIG_CHECKPOINT_RESTORE \
+ -d CONFIG_RANDOMIZE_BASE
+ (cd ${OUT_DIR} && \
+ make ${TOOL_ARGS} O=${OUT_DIR} olddefconfig)
+}
+
diff --git a/build.config.gki_kprobes.aarch64 b/build.config.gki_kprobes.aarch64
new file mode 100644
index 0000000..627c217
--- /dev/null
+++ b/build.config.gki_kprobes.aarch64
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki_kprobes
+
diff --git a/build.config.gki_kprobes.x86_64 b/build.config.gki_kprobes.x86_64
new file mode 100644
index 0000000..a1d3da7
--- /dev/null
+++ b/build.config.gki_kprobes.x86_64
@@ -0,0 +1,4 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.x86_64
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki_kprobes
+
diff --git a/build.config.khwasan b/build.config.khwasan
new file mode 100644
index 0000000..c8e0c5e
--- /dev/null
+++ b/build.config.khwasan
@@ -0,0 +1,17 @@
+append_cmd POST_DEFCONFIG_CMDS update_khwasan_config
+
+function update_khwasan_config() {
+ ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \
+ -e CONFIG_KASAN \
+ -d CONFIG_KASAN_HW_TAGS \
+ -e CONFIG_KASAN_SW_TAGS \
+ -e CONFIG_KASAN_OUTLINE \
+ -e CONFIG_KASAN_PANIC_ON_WARN \
+ -e CONFIG_KCOV \
+ -e CONFIG_PANIC_ON_WARN_DEFAULT_ENABLE \
+ -d CONFIG_RANDOMIZE_BASE \
+ --set-val CONFIG_FRAME_WARN 0 \
+ -d SHADOW_CALL_STACK
+ (cd ${OUT_DIR} && \
+ make O=${OUT_DIR} "${TOOL_ARGS[@]}" ${MAKE_ARGS} olddefconfig)
+}
diff --git a/build.config.riscv64 b/build.config.riscv64
new file mode 100644
index 0000000..0ef054b
--- /dev/null
+++ b/build.config.riscv64
@@ -0,0 +1,14 @@
+ARCH=riscv
+MAKE_GOALS="
+Image
+modules
+"
+
+FILES="
+arch/riscv/boot/Image
+vmlinux
+System.map
+vmlinux.symvers
+modules.builtin
+modules.builtin.modinfo
+"
diff --git a/build.config.rockpi4 b/build.config.rockpi4
new file mode 100644
index 0000000..c9c9776
--- /dev/null
+++ b/build.config.rockpi4
@@ -0,0 +1,21 @@
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.common
+. ${ROOT_DIR}/${KERNEL_DIR}/build.config.aarch64
+TRIM_NONLISTED_KMI=""
+KMI_SYMBOL_LIST_STRICT_MODE=""
+
+BUILD_INITRAMFS=1
+LZ4_RAMDISK=1
+DEFCONFIG=rockpi4_gki_defconfig
+FRAGMENT_CONFIG=${KERNEL_DIR}/arch/arm64/configs/rockpi4_gki.fragment
+PRE_DEFCONFIG_CMDS="KCONFIG_CONFIG=${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG} ${ROOT_DIR}/${KERNEL_DIR}/scripts/kconfig/merge_config.sh -m -r ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${FRAGMENT_CONFIG}"
+POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG}"
+
+DTS_EXT_DIR=common-modules/virtual-device
+DTC_INCLUDE=${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/boot/dts/rockchip
+MAKE_GOALS="${MAKE_GOALS}
+rk3399-rock-pi-4b.dtb
+"
+
+FILES="${FILES}
+../common-modules/virtual-device/rk3399-rock-pi-4b.dtb
+"
diff --git a/build.config.x86_64 b/build.config.x86_64
new file mode 100644
index 0000000..b5ac82b
--- /dev/null
+++ b/build.config.x86_64
@@ -0,0 +1,16 @@
+ARCH=x86_64
+MAKE_GOALS="
+bzImage
+modules
+"
+
+FILES="
+arch/x86/boot/bzImage
+vmlinux
+System.map
+vmlinux.symvers
+modules.builtin
+modules.builtin.modinfo
+"
+
+NDK_TRIPLE=x86_64-linux-android31
diff --git a/certs/extract-cert.c b/certs/extract-cert.c
index 8c1fb9a..5c66eda 100644
--- a/certs/extract-cert.c
+++ b/certs/extract-cert.c
@@ -56,6 +56,7 @@ static void display_openssl_errors(int l)
}
}
+#ifndef OPENSSL_IS_BORINGSSL
static void drain_openssl_errors(void)
{
const char *file;
@@ -65,6 +66,7 @@ static void drain_openssl_errors(void)
return;
while (ERR_get_error_line(&file, &line)) {}
}
+#endif
#define ERR(cond, fmt, ...) \
do { \
@@ -119,6 +121,10 @@ int main(int argc, char **argv)
fclose(f);
exit(0);
} else if (!strncmp(cert_src, "pkcs11:", 7)) {
+#ifdef OPENSSL_IS_BORINGSSL
+ ERR(1, "BoringSSL does not support extracting from PKCS#11");
+ exit(1);
+#else
ENGINE *e;
struct {
const char *cert_id;
@@ -141,6 +147,7 @@ int main(int argc, char **argv)
ENGINE_ctrl_cmd(e, "LOAD_CERT_CTRL", 0, &parms, NULL, 1);
ERR(!parms.cert, "Get X.509 from PKCS#11");
write_cert(parms.cert);
+#endif
} else {
BIO *b;
X509 *x509;
diff --git a/crypto/OWNERS b/crypto/OWNERS
new file mode 100644
index 0000000..4ed35a0
--- /dev/null
+++ b/crypto/OWNERS
@@ -0,0 +1 @@
+ardb@google.com
diff --git a/drivers/OWNERS b/drivers/OWNERS
new file mode 100644
index 0000000..c13fa05
--- /dev/null
+++ b/drivers/OWNERS
@@ -0,0 +1,6 @@
+per-file base/**=gregkh@google.com,saravanak@google.com
+per-file block/**=akailash@google.com
+per-file md/**=akailash@google.com,paullawrence@google.com
+per-file net/**=file:/net/OWNERS
+per-file scsi/**=bvanassche@google.com,jaegeuk@google.com
+per-file {tty,usb}/**=gregkh@google.com
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 07aa8ae..c5e063b 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -47,4 +47,59 @@
exhaustively with combinations of various buffer sizes and
alignments.
+config ANDROID_VENDOR_HOOKS
+ bool "Android Vendor Hooks"
+ depends on TRACEPOINTS
+ help
+ Enable vendor hooks implemented as tracepoints
+
+ Allow vendor modules to attach to tracepoint "hooks" defined via
+ DECLARE_HOOK or DECLARE_RESTRICTED_HOOK.
+
+config ANDROID_DEBUG_KINFO
+ bool "Android Debug Kernel Information Support"
+ depends on KALLSYMS
+ help
+ This supports kernel information backup for bootloader usage.
+ Specifics:
+ - The kallsyms symbols for unwind_backtrace
+ - Page directory pointer
+ - UTS_RELEASE
+ - BUILD_INFO(ro.build.fingerprint)
+
+config ANDROID_KABI_RESERVE
+ bool "Android KABI reserve padding"
+ default y
+ help
+ This option enables the padding that the Android GKI kernel adds
+ to many different kernel structures to support an in-kernel stable ABI
+ over the lifespan of support for the kernel.
+
+ Only disable this option if you have a system that needs the Android
+ kernel drivers, but is NOT an Android GKI kernel image. If disabled
+ it has the possibility to make the kernel static and runtime image
+ slightly smaller but will NOT be supported by the Google Android
+ kernel team.
+
+ If even slightly unsure, say Y.
+
+config ANDROID_VENDOR_OEM_DATA
+ bool "Android vendor and OEM data padding"
+ default y
+ help
+ This option enables the padding that the Android GKI kernel adds
+ to many different kernel structures to support an in-kernel stable ABI
+ over the lifespan of support for the kernel as well as OEM additional
+ fields that are needed by some of the Android kernel tracepoints. The
+ macros enabled by this option are used to enable padding in vendor modules
+ used for the above specified purposes.
+
+ Only disable this option if you have a system that needs the Android
+ kernel drivers, but is NOT an Android GKI kernel image and you do NOT
+ use the Android kernel tracepoints. If disabled it has the possibility
+ to make the kernel static and runtime image slightly smaller but will
+ NOT be supported by the Google Android kernel team.
+
+ If even slightly unsure, say Y.
+
endmenu
diff --git a/drivers/android/Makefile b/drivers/android/Makefile
index c9d3d0c9..9b89e4b 100644
--- a/drivers/android/Makefile
+++ b/drivers/android/Makefile
@@ -4,3 +4,5 @@
obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o
obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o
obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
+obj-$(CONFIG_ANDROID_VENDOR_HOOKS) += vendor_hooks.o
+obj-$(CONFIG_ANDROID_DEBUG_KINFO) += debug_kinfo.o
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index 880224e..3479308 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -66,13 +66,16 @@
#include <linux/syscalls.h>
#include <linux/task_work.h>
#include <linux/sizes.h>
+#include <linux/android_vendor.h>
+#include <uapi/linux/sched/types.h>
#include <uapi/linux/android/binder.h>
#include <linux/cacheflush.h>
#include "binder_internal.h"
#include "binder_trace.h"
+#include <trace/hooks/binder.h>
static HLIST_HEAD(binder_deferred_list);
static DEFINE_MUTEX(binder_deferred_lock);
@@ -574,6 +577,7 @@ static void binder_wakeup_poll_threads_ilocked(struct binder_proc *proc,
thread = rb_entry(n, struct binder_thread, rb_node);
if (thread->looper & BINDER_LOOPER_STATE_POLL &&
binder_available_for_proc_work_ilocked(thread)) {
+ trace_android_vh_binder_wakeup_ilocked(thread->task, sync, proc);
if (sync)
wake_up_interruptible_sync(&thread->wait);
else
@@ -633,6 +637,7 @@ static void binder_wakeup_thread_ilocked(struct binder_proc *proc,
assert_spin_locked(&proc->inner_lock);
if (thread) {
+ trace_android_vh_binder_wakeup_ilocked(thread->task, sync, proc);
if (sync)
wake_up_interruptible_sync(&thread->wait);
else
@@ -663,22 +668,190 @@ static void binder_wakeup_proc_ilocked(struct binder_proc *proc)
binder_wakeup_thread_ilocked(proc, thread, /* sync = */false);
}
-static void binder_set_nice(long nice)
+static bool is_rt_policy(int policy)
{
- long min_nice;
+ return policy == SCHED_FIFO || policy == SCHED_RR;
+}
- if (can_nice(current, nice)) {
- set_user_nice(current, nice);
+static bool is_fair_policy(int policy)
+{
+ return policy == SCHED_NORMAL || policy == SCHED_BATCH;
+}
+
+static bool binder_supported_policy(int policy)
+{
+ return is_fair_policy(policy) || is_rt_policy(policy);
+}
+
+static int to_userspace_prio(int policy, int kernel_priority)
+{
+ if (is_fair_policy(policy))
+ return PRIO_TO_NICE(kernel_priority);
+ else
+ return MAX_RT_PRIO - 1 - kernel_priority;
+}
+
+static int to_kernel_prio(int policy, int user_priority)
+{
+ if (is_fair_policy(policy))
+ return NICE_TO_PRIO(user_priority);
+ else
+ return MAX_RT_PRIO - 1 - user_priority;
+}
+
+static void binder_do_set_priority(struct binder_thread *thread,
+ const struct binder_priority *desired,
+ bool verify)
+{
+ struct task_struct *task = thread->task;
+ int priority; /* user-space prio value */
+ bool has_cap_nice;
+ unsigned int policy = desired->sched_policy;
+
+ if (task->policy == policy && task->normal_prio == desired->prio) {
+ spin_lock(&thread->prio_lock);
+ if (thread->prio_state == BINDER_PRIO_PENDING)
+ thread->prio_state = BINDER_PRIO_SET;
+ spin_unlock(&thread->prio_lock);
return;
}
- min_nice = rlimit_to_nice(rlimit(RLIMIT_NICE));
- binder_debug(BINDER_DEBUG_PRIORITY_CAP,
- "%d: nice value %ld not allowed use %ld instead\n",
- current->pid, nice, min_nice);
- set_user_nice(current, min_nice);
- if (min_nice <= MAX_NICE)
+
+ has_cap_nice = has_capability_noaudit(task, CAP_SYS_NICE);
+
+ priority = to_userspace_prio(policy, desired->prio);
+
+ if (verify && is_rt_policy(policy) && !has_cap_nice) {
+ long max_rtprio = task_rlimit(task, RLIMIT_RTPRIO);
+
+ if (max_rtprio == 0) {
+ policy = SCHED_NORMAL;
+ priority = MIN_NICE;
+ } else if (priority > max_rtprio) {
+ priority = max_rtprio;
+ }
+ }
+
+ if (verify && is_fair_policy(policy) && !has_cap_nice) {
+ long min_nice = rlimit_to_nice(task_rlimit(task, RLIMIT_NICE));
+
+ if (min_nice > MAX_NICE) {
+ binder_user_error("%d RLIMIT_NICE not set\n",
+ task->pid);
+ return;
+ } else if (priority < min_nice) {
+ priority = min_nice;
+ }
+ }
+
+ if (policy != desired->sched_policy ||
+ to_kernel_prio(policy, priority) != desired->prio)
+ binder_debug(BINDER_DEBUG_PRIORITY_CAP,
+ "%d: priority %d not allowed, using %d instead\n",
+ task->pid, desired->prio,
+ to_kernel_prio(policy, priority));
+
+ trace_binder_set_priority(task->tgid, task->pid, task->normal_prio,
+ to_kernel_prio(policy, priority),
+ desired->prio);
+
+ spin_lock(&thread->prio_lock);
+ if (!verify && thread->prio_state == BINDER_PRIO_ABORT) {
+ /*
+ * A new priority has been set by an incoming nested
+ * transaction. Abort this priority restore and allow
+ * the transaction to run at the new desired priority.
+ */
+ spin_unlock(&thread->prio_lock);
+ binder_debug(BINDER_DEBUG_PRIORITY_CAP,
+ "%d: %s: aborting priority restore\n",
+ thread->pid, __func__);
return;
- binder_user_error("%d RLIMIT_NICE not set\n", current->pid);
+ }
+
+ /* Set the actual priority */
+ if (task->policy != policy || is_rt_policy(policy)) {
+ struct sched_param params;
+
+ params.sched_priority = is_rt_policy(policy) ? priority : 0;
+
+ sched_setscheduler_nocheck(task,
+ policy | SCHED_RESET_ON_FORK,
+ ¶ms);
+ }
+ if (is_fair_policy(policy))
+ set_user_nice(task, priority);
+
+ thread->prio_state = BINDER_PRIO_SET;
+ spin_unlock(&thread->prio_lock);
+}
+
+static void binder_set_priority(struct binder_thread *thread,
+ const struct binder_priority *desired)
+{
+ binder_do_set_priority(thread, desired, /* verify = */ true);
+}
+
+static void binder_restore_priority(struct binder_thread *thread,
+ const struct binder_priority *desired)
+{
+ binder_do_set_priority(thread, desired, /* verify = */ false);
+}
+
+static void binder_transaction_priority(struct binder_thread *thread,
+ struct binder_transaction *t,
+ struct binder_node *node)
+{
+ struct task_struct *task = thread->task;
+ struct binder_priority desired = t->priority;
+ const struct binder_priority node_prio = {
+ .sched_policy = node->sched_policy,
+ .prio = node->min_priority,
+ };
+
+ if (t->set_priority_called)
+ return;
+
+ t->set_priority_called = true;
+
+ if (!node->inherit_rt && is_rt_policy(desired.sched_policy)) {
+ desired.prio = NICE_TO_PRIO(0);
+ desired.sched_policy = SCHED_NORMAL;
+ }
+
+ if (node_prio.prio < t->priority.prio ||
+ (node_prio.prio == t->priority.prio &&
+ node_prio.sched_policy == SCHED_FIFO)) {
+ /*
+ * In case the minimum priority on the node is
+ * higher (lower value), use that priority. If
+ * the priority is the same, but the node uses
+ * SCHED_FIFO, prefer SCHED_FIFO, since it can
+ * run unbounded, unlike SCHED_RR.
+ */
+ desired = node_prio;
+ }
+
+ spin_lock(&thread->prio_lock);
+ if (thread->prio_state == BINDER_PRIO_PENDING) {
+ /*
+ * Task is in the process of changing priorities
+ * saving its current values would be incorrect.
+ * Instead, save the pending priority and signal
+ * the task to abort the priority restore.
+ */
+ t->saved_priority = thread->prio_next;
+ thread->prio_state = BINDER_PRIO_ABORT;
+ binder_debug(BINDER_DEBUG_PRIORITY_CAP,
+ "%d: saved pending priority %d\n",
+ current->pid, thread->prio_next.prio);
+ } else {
+ t->saved_priority.sched_policy = task->policy;
+ t->saved_priority.prio = task->normal_prio;
+ }
+ spin_unlock(&thread->prio_lock);
+
+ binder_set_priority(thread, &desired);
+ trace_android_vh_binder_set_priority(t, task);
}
static struct binder_node *binder_get_node_ilocked(struct binder_proc *proc,
@@ -731,6 +904,7 @@ static struct binder_node *binder_init_node_ilocked(
binder_uintptr_t ptr = fp ? fp->binder : 0;
binder_uintptr_t cookie = fp ? fp->cookie : 0;
__u32 flags = fp ? fp->flags : 0;
+ s8 priority;
assert_spin_locked(&proc->inner_lock);
@@ -763,8 +937,12 @@ static struct binder_node *binder_init_node_ilocked(
node->ptr = ptr;
node->cookie = cookie;
node->work.type = BINDER_WORK_NODE;
- node->min_priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
+ priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
+ node->sched_policy = (flags & FLAT_BINDER_FLAG_SCHED_POLICY_MASK) >>
+ FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT;
+ node->min_priority = to_kernel_prio(node->sched_policy, priority);
node->accept_fds = !!(flags & FLAT_BINDER_FLAG_ACCEPTS_FDS);
+ node->inherit_rt = !!(flags & FLAT_BINDER_FLAG_INHERIT_RT);
node->txn_security_ctx = !!(flags & FLAT_BINDER_FLAG_TXN_SECURITY_CTX);
spin_lock_init(&node->lock);
INIT_LIST_HEAD(&node->work.entry);
@@ -2741,6 +2919,7 @@ static int binder_proc_transaction(struct binder_transaction *t,
BUG_ON(!node);
binder_node_lock(node);
+
if (oneway) {
BUG_ON(thread);
if (node->has_async_transaction)
@@ -2766,6 +2945,7 @@ static int binder_proc_transaction(struct binder_transaction *t,
thread = binder_select_thread_ilocked(proc);
if (thread) {
+ binder_transaction_priority(thread, t, node);
binder_enqueue_thread_work_ilocked(thread, &t->work);
} else if (!pending_async) {
binder_enqueue_work_ilocked(&t->work, &proc->todo);
@@ -2902,6 +3082,7 @@ static void binder_transaction(struct binder_proc *proc,
struct list_head pf_head;
const void __user *user_buffer = (const void __user *)
(uintptr_t)tr->data.ptr.buffer;
+ bool is_nested = false;
INIT_LIST_HEAD(&sgc_head);
INIT_LIST_HEAD(&pf_head);
@@ -2949,7 +3130,6 @@ static void binder_transaction(struct binder_proc *proc,
}
thread->transaction_stack = in_reply_to->to_parent;
binder_inner_proc_unlock(proc);
- binder_set_nice(in_reply_to->saved_priority);
target_thread = binder_get_txn_from_and_acq_inner(in_reply_to);
if (target_thread == NULL) {
/* annotation for sparse */
@@ -3099,6 +3279,7 @@ static void binder_transaction(struct binder_proc *proc,
atomic_inc(&from->tmp_ref);
target_thread = from;
spin_unlock(&tmp->lock);
+ is_nested = true;
break;
}
spin_unlock(&tmp->lock);
@@ -3124,6 +3305,7 @@ static void binder_transaction(struct binder_proc *proc,
INIT_LIST_HEAD(&t->fd_fixups);
binder_stats_created(BINDER_STAT_TRANSACTION);
spin_lock_init(&t->lock);
+ trace_android_vh_binder_transaction_init(t);
tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL);
if (tcomplete == NULL) {
@@ -3166,7 +3348,16 @@ static void binder_transaction(struct binder_proc *proc,
t->to_thread = target_thread;
t->code = tr->code;
t->flags = tr->flags;
- t->priority = task_nice(current);
+ t->is_nested = is_nested;
+ if (!(t->flags & TF_ONE_WAY) &&
+ binder_supported_policy(current->policy)) {
+ /* Inherit supported policies for synchronous transactions */
+ t->priority.sched_policy = current->policy;
+ t->priority.prio = current->normal_prio;
+ } else {
+ /* Otherwise, fall back to the default priority */
+ t->priority = target_proc->default_priority;
+ }
if (target_node && target_node->txn_security_ctx) {
u32 secid;
@@ -3579,7 +3770,15 @@ static void binder_transaction(struct binder_proc *proc,
binder_enqueue_thread_work_ilocked(target_thread, &t->work);
target_proc->outstanding_txns++;
binder_inner_proc_unlock(target_proc);
+ if (in_reply_to->is_nested) {
+ spin_lock(&thread->prio_lock);
+ thread->prio_state = BINDER_PRIO_PENDING;
+ thread->prio_next = in_reply_to->saved_priority;
+ spin_unlock(&thread->prio_lock);
+ }
wake_up_interruptible_sync(&target_thread->wait);
+ trace_android_vh_binder_restore_priority(in_reply_to, current);
+ binder_restore_priority(thread, &in_reply_to->saved_priority);
binder_free_transaction(in_reply_to);
} else if (!(t->flags & TF_ONE_WAY)) {
BUG_ON(t->buffer->async_transaction != 0);
@@ -3702,6 +3901,8 @@ static void binder_transaction(struct binder_proc *proc,
BUG_ON(thread->return_error.cmd != BR_OK);
if (in_reply_to) {
+ trace_android_vh_binder_restore_priority(in_reply_to, current);
+ binder_restore_priority(thread, &in_reply_to->saved_priority);
binder_set_txn_from_error(in_reply_to, t_debug_id,
return_error, return_error_param);
thread->return_error.cmd = BR_TRANSACTION_COMPLETE;
@@ -4267,6 +4468,7 @@ static int binder_wait_for_work(struct binder_thread *thread,
if (do_proc_work)
list_add(&thread->waiting_thread_node,
&proc->waiting_threads);
+ trace_android_vh_binder_wait_for_work(do_proc_work, thread, proc);
binder_inner_proc_unlock(proc);
schedule();
binder_inner_proc_lock(proc);
@@ -4372,7 +4574,8 @@ static int binder_thread_read(struct binder_proc *proc,
wait_event_interruptible(binder_user_error_wait,
binder_stop_on_user_error < 2);
}
- binder_set_nice(proc->default_priority);
+ trace_android_vh_binder_restore_priority(NULL, current);
+ binder_restore_priority(thread, &proc->default_priority);
}
if (non_block) {
@@ -4602,13 +4805,7 @@ static int binder_thread_read(struct binder_proc *proc,
trd->target.ptr = target_node->ptr;
trd->cookie = target_node->cookie;
- t->saved_priority = task_nice(current);
- if (t->priority < target_node->min_priority &&
- !(t->flags & TF_ONE_WAY))
- binder_set_nice(t->priority);
- else if (!(t->flags & TF_ONE_WAY) ||
- t->saved_priority > target_node->min_priority)
- binder_set_nice(target_node->min_priority);
+ binder_transaction_priority(thread, t, target_node);
cmd = BR_TRANSACTION;
} else {
trd->target.ptr = 0;
@@ -4626,6 +4823,7 @@ static int binder_thread_read(struct binder_proc *proc,
trd->sender_pid =
task_tgid_nr_ns(sender,
task_active_pid_ns(current));
+ trace_android_vh_sync_txn_recvd(thread->task, t_from->task);
} else {
trd->sender_pid = 0;
}
@@ -4826,6 +5024,8 @@ static struct binder_thread *binder_get_thread_ilocked(
binder_stats_created(BINDER_STAT_THREAD);
thread->proc = proc;
thread->pid = current->pid;
+ get_task_struct(current);
+ thread->task = current;
atomic_set(&thread->tmp_ref, 0);
init_waitqueue_head(&thread->wait);
INIT_LIST_HEAD(&thread->todo);
@@ -4836,6 +5036,8 @@ static struct binder_thread *binder_get_thread_ilocked(
thread->return_error.cmd = BR_OK;
thread->reply_error.work.type = BINDER_WORK_RETURN_ERROR;
thread->reply_error.cmd = BR_OK;
+ spin_lock_init(&thread->prio_lock);
+ thread->prio_state = BINDER_PRIO_SET;
thread->ee.command = BR_OK;
INIT_LIST_HEAD(&new_thread->waiting_thread_node);
return thread;
@@ -4888,6 +5090,7 @@ static void binder_free_thread(struct binder_thread *thread)
BUG_ON(!list_empty(&thread->todo));
binder_stats_deleted(BINDER_STAT_THREAD);
binder_proc_dec_tmpref(thread->proc);
+ put_task_struct(thread->task);
kfree(thread);
}
@@ -5602,7 +5805,14 @@ static int binder_open(struct inode *nodp, struct file *filp)
proc->cred = get_cred(filp->f_cred);
INIT_LIST_HEAD(&proc->todo);
init_waitqueue_head(&proc->freeze_wait);
- proc->default_priority = task_nice(current);
+ if (binder_supported_policy(current->policy)) {
+ proc->default_priority.sched_policy = current->policy;
+ proc->default_priority.prio = current->normal_prio;
+ } else {
+ proc->default_priority.sched_policy = SCHED_NORMAL;
+ proc->default_priority.prio = NICE_TO_PRIO(0);
+ }
+
/* binderfs stashes devices in i_private */
if (is_binderfs_device(nodp)) {
binder_dev = nodp->i_private;
@@ -5927,13 +6137,14 @@ static void print_binder_transaction_ilocked(struct seq_file *m,
spin_lock(&t->lock);
to_proc = t->to_proc;
seq_printf(m,
- "%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %ld r%d",
+ "%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %d:%d r%d",
prefix, t->debug_id, t,
t->from ? t->from->proc->pid : 0,
t->from ? t->from->pid : 0,
to_proc ? to_proc->pid : 0,
t->to_thread ? t->to_thread->pid : 0,
- t->code, t->flags, t->priority, t->need_reply);
+ t->code, t->flags, t->priority.sched_policy,
+ t->priority.prio, t->need_reply);
spin_unlock(&t->lock);
if (proc != to_proc) {
@@ -6051,8 +6262,9 @@ static void print_binder_node_nilocked(struct seq_file *m,
hlist_for_each_entry(ref, &node->refs, node_entry)
count++;
- seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d tr %d",
+ seq_printf(m, " node %d: u%016llx c%016llx pri %d:%d hs %d hw %d ls %d lw %d is %d iw %d tr %d",
node->debug_id, (u64)node->ptr, (u64)node->cookie,
+ node->sched_policy, node->min_priority,
node->has_strong_ref, node->has_weak_ref,
node->local_strong_refs, node->local_weak_refs,
node->internal_strong_refs, count, node->tmp_refs);
@@ -6596,5 +6808,6 @@ device_initcall(binder_init);
#define CREATE_TRACE_POINTS
#include "binder_trace.h"
+EXPORT_TRACEPOINT_SYMBOL_GPL(binder_transaction_received);
MODULE_LICENSE("GPL v2");
diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h
index abe19d8..9132d14 100644
--- a/drivers/android/binder_internal.h
+++ b/drivers/android/binder_internal.h
@@ -215,10 +215,13 @@ struct binder_error {
* and by @lock)
* @has_async_transaction: async transaction to node in progress
* (protected by @lock)
+ * @sched_policy: minimum scheduling policy for node
+ * (invariant after initialized)
* @accept_fds: file descriptor operations supported for node
* (invariant after initialized)
* @min_priority: minimum scheduling priority
* (invariant after initialized)
+ * @inherit_rt: inherit RT scheduling policy from caller
* @txn_security_ctx: require sender's security context
* (invariant after initialized)
* @async_todo: list of async work items
@@ -256,6 +259,8 @@ struct binder_node {
/*
* invariant after initialization
*/
+ u8 sched_policy:2;
+ u8 inherit_rt:1;
u8 accept_fds:1;
u8 txn_security_ctx:1;
u8 min_priority;
@@ -325,6 +330,28 @@ struct binder_ref {
};
/**
+ * struct binder_priority - scheduler policy and priority
+ * @sched_policy scheduler policy
+ * @prio [100..139] for SCHED_NORMAL, [0..99] for FIFO/RT
+ *
+ * The binder driver supports inheriting the following scheduler policies:
+ * SCHED_NORMAL
+ * SCHED_BATCH
+ * SCHED_FIFO
+ * SCHED_RR
+ */
+struct binder_priority {
+ unsigned int sched_policy;
+ int prio;
+};
+
+enum binder_prio_state {
+ BINDER_PRIO_SET, /* desired priority set */
+ BINDER_PRIO_PENDING, /* initiated a saved priority restore */
+ BINDER_PRIO_ABORT, /* abort the pending priority restore */
+};
+
+/**
* struct binder_proc - binder process bookkeeping
* @proc_node: element for binder_procs list
* @threads: rbtree of binder_threads in this proc
@@ -424,7 +451,7 @@ struct binder_proc {
int requested_threads;
int requested_threads_started;
int tmp_ref;
- long default_priority;
+ struct binder_priority default_priority;
struct dentry *debugfs_entry;
struct binder_alloc alloc;
struct binder_context *context;
@@ -469,6 +496,13 @@ struct binder_proc {
* @is_dead: thread is dead and awaiting free
* when outstanding transactions are cleaned up
* (protected by @proc->inner_lock)
+ * @task: struct task_struct for this thread
+ * @prio_lock: protects thread priority fields
+ * @prio_next: saved priority to be restored next
+ * (protected by @prio_lock)
+ * @prio_state: state of the priority restore process as
+ * defined by enum binder_prio_state
+ * (protected by @prio_lock)
*
* Bookkeeping structure for binder threads.
*/
@@ -489,6 +523,10 @@ struct binder_thread {
struct binder_stats stats;
atomic_t tmp_ref;
bool is_dead;
+ struct task_struct *task;
+ spinlock_t prio_lock;
+ struct binder_priority prio_next;
+ enum binder_prio_state prio_state;
};
/**
@@ -524,8 +562,10 @@ struct binder_transaction {
struct binder_buffer *buffer;
unsigned int code;
unsigned int flags;
- long priority;
- long saved_priority;
+ struct binder_priority priority;
+ struct binder_priority saved_priority;
+ bool set_priority_called;
+ bool is_nested;
kuid_t sender_euid;
struct list_head fd_fixups;
binder_uintptr_t security_ctx;
@@ -536,6 +576,7 @@ struct binder_transaction {
* during thread teardown
*/
spinlock_t lock;
+ ANDROID_VENDOR_DATA(1);
};
/**
diff --git a/drivers/android/binder_trace.h b/drivers/android/binder_trace.h
index 8cc07e6..5d82cf8 100644
--- a/drivers/android/binder_trace.h
+++ b/drivers/android/binder_trace.h
@@ -76,6 +76,30 @@ DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done);
DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done);
DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done);
+TRACE_EVENT(binder_set_priority,
+ TP_PROTO(int proc, int thread, unsigned int old_prio,
+ unsigned int desired_prio, unsigned int new_prio),
+ TP_ARGS(proc, thread, old_prio, new_prio, desired_prio),
+
+ TP_STRUCT__entry(
+ __field(int, proc)
+ __field(int, thread)
+ __field(unsigned int, old_prio)
+ __field(unsigned int, new_prio)
+ __field(unsigned int, desired_prio)
+ ),
+ TP_fast_assign(
+ __entry->proc = proc;
+ __entry->thread = thread;
+ __entry->old_prio = old_prio;
+ __entry->new_prio = new_prio;
+ __entry->desired_prio = desired_prio;
+ ),
+ TP_printk("proc=%d thread=%d old=%d => new=%d desired=%d",
+ __entry->proc, __entry->thread, __entry->old_prio,
+ __entry->new_prio, __entry->desired_prio)
+);
+
TRACE_EVENT(binder_wait_for_work,
TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo),
TP_ARGS(proc_work, transaction_stack, thread_todo),
diff --git a/drivers/android/debug_kinfo.c b/drivers/android/debug_kinfo.c
new file mode 100644
index 0000000..6a16232
--- /dev/null
+++ b/drivers/android/debug_kinfo.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * debug_kinfo.c - backup kernel information for bootloader usage
+ *
+ * Copyright 2002 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
+ * Copyright 2021 Google LLC
+ */
+
+#include <linux/platform_device.h>
+#include <linux/kallsyms.h>
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/pgtable.h>
+#include <asm/module.h>
+#include "debug_kinfo.h"
+
+/*
+ * These will be re-linked against their real values
+ * during the second link stage.
+ */
+extern const unsigned long kallsyms_addresses[] __weak;
+extern const int kallsyms_offsets[] __weak;
+extern const u8 kallsyms_names[] __weak;
+
+/*
+ * Tell the compiler that the count isn't in the small data section if the arch
+ * has one (eg: FRV).
+ */
+extern const unsigned int kallsyms_num_syms __weak
+__section(".rodata");
+
+extern const unsigned long kallsyms_relative_base __weak
+__section(".rodata");
+
+extern const u8 kallsyms_token_table[] __weak;
+extern const u16 kallsyms_token_index[] __weak;
+
+extern const unsigned int kallsyms_markers[] __weak;
+
+static void *all_info_addr;
+static u32 all_info_size;
+
+static void update_kernel_all_info(struct kernel_all_info *all_info)
+{
+ int index;
+ struct kernel_info *info;
+ u32 *checksum_info;
+
+ all_info->magic_number = DEBUG_KINFO_MAGIC;
+ all_info->combined_checksum = 0;
+
+ info = &(all_info->info);
+ checksum_info = (u32 *)info;
+ for (index = 0; index < sizeof(*info) / sizeof(u32); index++)
+ all_info->combined_checksum ^= checksum_info[index];
+}
+
+static int build_info_set(const char *str, const struct kernel_param *kp)
+{
+ struct kernel_all_info *all_info;
+ size_t build_info_size;
+ int ret = 0;
+
+ if (all_info_addr == 0 || all_info_size == 0) {
+ ret = -EPERM;
+ goto Exit;
+ }
+
+ all_info = (struct kernel_all_info *)all_info_addr;
+ build_info_size = sizeof(all_info->info.build_info);
+
+ memcpy(&all_info->info.build_info, str, min(build_info_size - 1, strlen(str)));
+ update_kernel_all_info(all_info);
+
+ if (strlen(str) > build_info_size) {
+ pr_warn("%s: Build info buffer (len: %zd) can't hold entire string '%s'\n",
+ __func__, build_info_size, str);
+ ret = -ENOMEM;
+ }
+
+Exit:
+ return ret;
+}
+
+static const struct kernel_param_ops build_info_op = {
+ .set = build_info_set,
+};
+
+module_param_cb(build_info, &build_info_op, NULL, 0200);
+MODULE_PARM_DESC(build_info, "Write build info to field 'build_info' of debug kinfo.");
+
+static int debug_kinfo_probe(struct platform_device *pdev)
+{
+ struct device_node *mem_region;
+ struct reserved_mem *rmem;
+ struct kernel_all_info *all_info;
+ struct kernel_info *info;
+
+ mem_region = of_parse_phandle(pdev->dev.of_node, "memory-region", 0);
+ if (!mem_region) {
+ dev_warn(&pdev->dev, "no such memory-region\n");
+ return -ENODEV;
+ }
+
+ rmem = of_reserved_mem_lookup(mem_region);
+ if (!rmem) {
+ dev_warn(&pdev->dev, "no such reserved mem of node name %s\n",
+ pdev->dev.of_node->name);
+ return -ENODEV;
+ }
+
+ /* Need to wait for reserved memory to be mapped */
+ if (!rmem->priv) {
+ return -EPROBE_DEFER;
+ }
+
+ if (!rmem->base || !rmem->size) {
+ dev_warn(&pdev->dev, "unexpected reserved memory\n");
+ return -EINVAL;
+ }
+
+ if (rmem->size < sizeof(struct kernel_all_info)) {
+ dev_warn(&pdev->dev, "unexpected reserved memory size\n");
+ return -EINVAL;
+ }
+
+ all_info_addr = rmem->priv;
+ all_info_size = rmem->size;
+
+ memset(all_info_addr, 0, sizeof(struct kernel_all_info));
+ all_info = (struct kernel_all_info *)all_info_addr;
+ info = &(all_info->info);
+ info->enabled_all = IS_ENABLED(CONFIG_KALLSYMS_ALL);
+ info->enabled_base_relative = IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE);
+ info->enabled_absolute_percpu = IS_ENABLED(CONFIG_KALLSYMS_ABSOLUTE_PERCPU);
+ info->enabled_cfi_clang = IS_ENABLED(CONFIG_CFI_CLANG);
+ info->num_syms = kallsyms_num_syms;
+ info->name_len = KSYM_NAME_LEN;
+ info->bit_per_long = BITS_PER_LONG;
+ info->module_name_len = MODULE_NAME_LEN;
+ info->symbol_len = KSYM_SYMBOL_LEN;
+ if (!info->enabled_base_relative)
+ info->_addresses_pa = (u64)__pa_symbol((volatile void *)kallsyms_addresses);
+ else {
+ info->_relative_pa = (u64)__pa_symbol((volatile void *)kallsyms_relative_base);
+ info->_offsets_pa = (u64)__pa_symbol((volatile void *)kallsyms_offsets);
+ }
+ info->_stext_pa = (u64)__pa_symbol(_stext);
+ info->_etext_pa = (u64)__pa_symbol(_etext);
+ info->_sinittext_pa = (u64)__pa_symbol(_sinittext);
+ info->_einittext_pa = (u64)__pa_symbol(_einittext);
+ info->_end_pa = (u64)__pa_symbol(_end);
+ info->_names_pa = (u64)__pa_symbol((volatile void *)kallsyms_names);
+ info->_token_table_pa = (u64)__pa_symbol((volatile void *)kallsyms_token_table);
+ info->_token_index_pa = (u64)__pa_symbol((volatile void *)kallsyms_token_index);
+ info->_markers_pa = (u64)__pa_symbol((volatile void *)kallsyms_markers);
+ info->thread_size = THREAD_SIZE;
+ info->swapper_pg_dir_pa = (u64)__pa_symbol(swapper_pg_dir);
+ strlcpy(info->last_uts_release, init_utsname()->release, sizeof(info->last_uts_release));
+ info->enabled_modules_tree_lookup = IS_ENABLED(CONFIG_MODULES_TREE_LOOKUP);
+ info->mod_core_layout_offset = offsetof(struct module, core_layout);
+ info->mod_init_layout_offset = offsetof(struct module, init_layout);
+ info->mod_kallsyms_offset = offsetof(struct module, kallsyms);
+#if defined(CONFIG_RANDOMIZE_BASE) && defined(MODULES_VSIZE)
+ info->module_start_va = module_alloc_base;
+ info->module_end_va = info->module_start_va + MODULES_VSIZE;
+#elif defined(CONFIG_MODULES) && defined(MODULES_VADDR)
+ info->module_start_va = MODULES_VADDR;
+ info->module_end_va = MODULES_END;
+#else
+ info->module_start_va = VMALLOC_START;
+ info->module_end_va = VMALLOC_END;
+#endif
+ update_kernel_all_info(all_info);
+
+ return 0;
+}
+
+static const struct of_device_id debug_kinfo_of_match[] = {
+ { .compatible = "google,debug-kinfo" },
+ {},
+};
+MODULE_DEVICE_TABLE(of, debug_kinfo_of_match);
+
+static struct platform_driver debug_kinfo_driver = {
+ .probe = debug_kinfo_probe,
+ .driver = {
+ .name = "debug-kinfo",
+ .of_match_table = of_match_ptr(debug_kinfo_of_match),
+ },
+};
+module_platform_driver(debug_kinfo_driver);
+
+MODULE_AUTHOR("Jone Chou <jonechou@google.com>");
+MODULE_DESCRIPTION("Debug Kinfo Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/android/debug_kinfo.h b/drivers/android/debug_kinfo.h
new file mode 100644
index 0000000..921f140
--- /dev/null
+++ b/drivers/android/debug_kinfo.h
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * debug_kinfo.h - backup kernel information for bootloader usage
+ *
+ * Copyright 2021 Google LLC
+ */
+
+#ifndef DEBUG_KINFO_H
+#define DEBUG_KINFO_H
+
+#include <linux/utsname.h>
+
+#define BUILD_INFO_LEN 256
+#define DEBUG_KINFO_MAGIC 0xCCEEDDFF
+
+/*
+ * Header structure must be byte-packed, since the table is provided to
+ * bootloader.
+ */
+struct kernel_info {
+ /* For kallsyms */
+ __u8 enabled_all;
+ __u8 enabled_base_relative;
+ __u8 enabled_absolute_percpu;
+ __u8 enabled_cfi_clang;
+ __u32 num_syms;
+ __u16 name_len;
+ __u16 bit_per_long;
+ __u16 module_name_len;
+ __u16 symbol_len;
+ __u64 _addresses_pa;
+ __u64 _relative_pa;
+ __u64 _stext_pa;
+ __u64 _etext_pa;
+ __u64 _sinittext_pa;
+ __u64 _einittext_pa;
+ __u64 _end_pa;
+ __u64 _offsets_pa;
+ __u64 _names_pa;
+ __u64 _token_table_pa;
+ __u64 _token_index_pa;
+ __u64 _markers_pa;
+
+ /* For frame pointer */
+ __u32 thread_size;
+
+ /* For virt_to_phys */
+ __u64 swapper_pg_dir_pa;
+
+ /* For linux banner */
+ __u8 last_uts_release[__NEW_UTS_LEN];
+
+ /* Info of running build */
+ __u8 build_info[BUILD_INFO_LEN];
+
+ /* For module kallsyms */
+ __u32 enabled_modules_tree_lookup;
+ __u32 mod_core_layout_offset;
+ __u32 mod_init_layout_offset;
+ __u32 mod_kallsyms_offset;
+ __u64 module_start_va;
+ __u64 module_end_va;
+} __packed;
+
+struct kernel_all_info {
+ __u32 magic_number;
+ __u32 combined_checksum;
+ struct kernel_info info;
+} __packed;
+
+#endif // DEBUG_KINFO_H
diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c
new file mode 100644
index 0000000..1f1a6b0
--- /dev/null
+++ b/drivers/android/vendor_hooks.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* vendor_hook.c
+ *
+ * Android Vendor Hook Support
+ *
+ * Copyright 2020 Google LLC
+ */
+
+#define CREATE_TRACE_POINTS
+#include <trace/hooks/vendor_hooks.h>
+#include <linux/tracepoint.h>
+
+#include <trace/hooks/binder.h>
+#include <trace/hooks/cpuidle.h>
+#include <trace/hooks/mpam.h>
+#include <trace/hooks/wqlockup.h>
+#include <trace/hooks/debug.h>
+#include <trace/hooks/sysrqcrash.h>
+#include <trace/hooks/printk.h>
+#include <trace/hooks/epoch.h>
+#include <trace/hooks/cpufreq.h>
+#include <trace/hooks/preemptirq.h>
+#include <trace/hooks/ftrace_dump.h>
+#include <trace/hooks/ufshcd.h>
+#include <trace/hooks/cgroup.h>
+#include <trace/hooks/sys.h>
+#include <trace/hooks/iommu.h>
+#include <trace/hooks/net.h>
+#include <trace/hooks/pm_domain.h>
+#include <trace/hooks/cpuidle_psci.h>
+#include <trace/hooks/vmscan.h>
+#include <trace/hooks/avc.h>
+#include <trace/hooks/creds.h>
+#include <trace/hooks/module.h>
+#include <trace/hooks/selinux.h>
+#include <trace/hooks/syscall_check.h>
+#include <trace/hooks/remoteproc.h>
+
+/*
+ * Export tracepoints that act as a bare tracehook (ie: have no trace event
+ * associated with them) to allow external modules to probe them.
+ */
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_transaction_init);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_set_priority);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_restore_priority);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_wakeup_ilocked);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_exit);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_wq_lockup_pool);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipi_stop);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sysrq_crash);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_printk_hotplug);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_suspend_epoch_val);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_resume_epoch_val);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_freq_table_limits);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_disable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_enable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_disable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_enable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_attach);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_online);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_enter);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_exit);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_size_check);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_format_check);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_dump_buffer);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_fill_prdt);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_prepare_command);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_update_sysfs);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_command);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_compl_command);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_uic_command);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_tm_command);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_check_int_errors);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_attach);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_iommu_setup_dma_ops);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_iommu_iovad_alloc_iova);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_iommu_iovad_free_iova);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ptype_head);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_allow_domain_state);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpuidle_psci_enter);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpuidle_psci_exit);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_wait_for_work);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sync_txn_recvd);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpufreq_transition);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_balance_anon_file_reclaim);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_show_max_freq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_insert);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_delete);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_replace);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_lookup);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_revert_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_core_rw_nx);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_init_rw_nx);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_before_init);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_after_init);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_is_initialized);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_mmap_file);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_file_open);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rproc_recovery);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rproc_recovery_set);
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index e7d6e665..1ef327b 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -23,6 +23,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/thermal_pressure.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/sched.h>
+
static DEFINE_PER_CPU(struct scale_freq_data __rcu *, sft_data);
static struct cpumask scale_freq_counters_mask;
static bool scale_freq_invariant;
@@ -159,6 +162,7 @@ void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity)
}
DEFINE_PER_CPU(unsigned long, thermal_pressure);
+EXPORT_PER_CPU_SYMBOL_GPL(thermal_pressure);
/**
* topology_update_thermal_pressure() - Update thermal pressure for CPUs
@@ -201,8 +205,11 @@ void topology_update_thermal_pressure(const struct cpumask *cpus,
trace_thermal_pressure_update(cpu, th_pressure);
- for_each_cpu(cpu, cpus)
+ for_each_cpu(cpu, cpus) {
WRITE_ONCE(per_cpu(thermal_pressure, cpu), th_pressure);
+ trace_android_rvh_update_thermal_stats(cpu);
+ }
+
}
EXPORT_SYMBOL_GPL(topology_update_thermal_pressure);
@@ -240,6 +247,8 @@ static int register_cpu_capacity_sysctl(void)
subsys_initcall(register_cpu_capacity_sysctl);
static int update_topology;
+bool topology_update_done;
+EXPORT_SYMBOL_GPL(topology_update_done);
int topology_update_cpu_topology(void)
{
@@ -254,6 +263,8 @@ static void update_topology_flags_workfn(struct work_struct *work)
{
update_topology = 1;
rebuild_sched_domains();
+ topology_update_done = true;
+ trace_android_vh_update_topology_flags_workfn(NULL);
pr_debug("sched_domain hierarchy rebuilt, flags updated\n");
update_topology = 0;
}
diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c
index cc2c3a5a6d3..fb183c4 100644
--- a/drivers/base/power/domain_governor.c
+++ b/drivers/base/power/domain_governor.c
@@ -12,6 +12,8 @@
#include <linux/cpumask.h>
#include <linux/ktime.h>
+#include <trace/hooks/pm_domain.h>
+
static int dev_update_qos_constraint(struct device *dev, void *data)
{
s64 *constraint_ns_p = data;
@@ -179,6 +181,11 @@ static bool __default_power_down_ok(struct dev_pm_domain *pd,
struct pm_domain_data *pdd;
s64 min_off_time_ns;
s64 off_on_time_ns;
+ bool allow = true;
+
+ trace_android_vh_allow_domain_state(genpd, state, &allow);
+ if (!allow)
+ return false;
off_on_time_ns = genpd->states[state].power_off_latency_ns +
genpd->states[state].power_on_latency_ns;
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index c501392..5ebfb40 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -34,6 +34,7 @@
#include <linux/cpufreq.h>
#include <linux/devfreq.h>
#include <linux/timer.h>
+#include <linux/wakeup_reason.h>
#include "../base.h"
#include "power.h"
@@ -1241,6 +1242,8 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
error = dpm_run_callback(callback, dev, state, info);
if (error) {
async_error = error;
+ log_suspend_abort_reason("Callback failed on %s in %pS returned %d",
+ dev_name(dev), callback, error);
goto Complete;
}
@@ -1435,6 +1438,8 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
error = dpm_run_callback(callback, dev, state, info);
if (error) {
async_error = error;
+ log_suspend_abort_reason("Callback failed on %s in %pS returned %d",
+ dev_name(dev), callback, error);
goto Complete;
}
dpm_propagate_wakeup_to_parent(dev);
@@ -1711,6 +1716,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
dpm_propagate_wakeup_to_parent(dev);
dpm_clear_superiors_direct_complete(dev);
+ } else {
+ log_suspend_abort_reason("Callback failed on %s in %pS returned %d",
+ dev_name(dev), callback, error);
}
device_unlock(dev);
@@ -1924,6 +1932,9 @@ int dpm_prepare(pm_message_t state)
} else {
dev_info(dev, "not prepared for power transition: code %d\n",
error);
+ log_suspend_abort_reason("Device %s not prepared for power transition: code %d",
+ dev_name(dev), error);
+ dpm_save_failed_dev(dev_name(dev));
}
mutex_unlock(&dpm_list_mtx);
diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c
index 8e93167..0489db3 100644
--- a/drivers/base/power/qos.c
+++ b/drivers/base/power/qos.c
@@ -137,6 +137,7 @@ s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type)
return ret;
}
+EXPORT_SYMBOL_GPL(dev_pm_qos_read_value);
/**
* apply_constraint - Add/modify/remove device PM QoS request.
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 7cc0c0c..441092f 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -15,6 +15,9 @@
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/pm_wakeirq.h>
+#include <linux/irq.h>
+#include <linux/irqdesc.h>
+#include <linux/wakeup_reason.h>
#include <trace/events/power.h>
#include "power.h"
@@ -844,6 +847,37 @@ void pm_wakeup_dev_event(struct device *dev, unsigned int msec, bool hard)
}
EXPORT_SYMBOL_GPL(pm_wakeup_dev_event);
+void pm_get_active_wakeup_sources(char *pending_wakeup_source, size_t max)
+{
+ struct wakeup_source *ws, *last_active_ws = NULL;
+ int len = 0;
+ bool active = false;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(ws, &wakeup_sources, entry) {
+ if (ws->active && len < max) {
+ if (!active)
+ len += scnprintf(pending_wakeup_source, max,
+ "Pending Wakeup Sources: ");
+ len += scnprintf(pending_wakeup_source + len, max - len,
+ "%s ", ws->name);
+ active = true;
+ } else if (!active &&
+ (!last_active_ws ||
+ ktime_to_ns(ws->last_time) >
+ ktime_to_ns(last_active_ws->last_time))) {
+ last_active_ws = ws;
+ }
+ }
+ if (!active && last_active_ws) {
+ scnprintf(pending_wakeup_source, max,
+ "Last active Wakeup Source: %s",
+ last_active_ws->name);
+ }
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(pm_get_active_wakeup_sources);
+
void pm_print_active_wakeup_sources(void)
{
struct wakeup_source *ws;
@@ -882,6 +916,7 @@ bool pm_wakeup_pending(void)
{
unsigned long flags;
bool ret = false;
+ char suspend_abort[MAX_SUSPEND_ABORT_LEN];
raw_spin_lock_irqsave(&events_lock, flags);
if (events_check_enabled) {
@@ -896,6 +931,10 @@ bool pm_wakeup_pending(void)
if (ret) {
pm_pr_dbg("Wakeup pending, aborting suspend\n");
pm_print_active_wakeup_sources();
+ pm_get_active_wakeup_sources(suspend_abort,
+ MAX_SUSPEND_ABORT_LEN);
+ log_suspend_abort_reason(suspend_abort);
+ pr_info("PM: %s\n", suspend_abort);
}
return ret || atomic_read(&pm_abort_suspend) > 0;
@@ -948,8 +987,21 @@ void pm_system_irq_wakeup(unsigned int irq_number)
raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags);
- if (irq_number)
+ if (irq_number) {
+ struct irq_desc *desc;
+ const char *name = "null";
+
+ desc = irq_to_desc(irq_number);
+ if (desc == NULL)
+ name = "stray irq";
+ else if (desc->action && desc->action->name)
+ name = desc->action->name;
+
+ log_irq_wakeup_reason(irq_number);
+ pr_warn("%s: %d triggered %s\n", __func__, irq_number, name);
+
pm_system_wakeup();
+ }
}
unsigned int pm_wakeup_irq(void)
diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c
index 13db1f78..3a9527d 100644
--- a/drivers/base/syscore.c
+++ b/drivers/base/syscore.c
@@ -10,6 +10,7 @@
#include <linux/module.h>
#include <linux/suspend.h>
#include <trace/events/power.h>
+#include <linux/wakeup_reason.h>
static LIST_HEAD(syscore_ops_list);
static DEFINE_MUTEX(syscore_ops_lock);
@@ -73,7 +74,9 @@ int syscore_suspend(void)
return 0;
err_out:
- pr_err("PM: System core suspend callback %pS failed.\n", ops->suspend);
+ log_suspend_abort_reason("System core suspend callback %pS failed",
+ ops->suspend);
+ pr_err("PM: System core suspend callback %pF failed.\n", ops->suspend);
list_for_each_entry_continue(ops, &syscore_ops_list, node)
if (ops->resume)
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 57b8366..05fe21f 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -72,6 +72,8 @@ struct clk_core {
unsigned long flags;
bool orphan;
bool rpm_enabled;
+ bool need_sync;
+ bool boot_enabled;
unsigned int enable_count;
unsigned int prepare_count;
unsigned int protect_count;
@@ -1301,6 +1303,10 @@ static void __init clk_unprepare_unused_subtree(struct clk_core *core)
hlist_for_each_entry(child, &core->children, child_node)
clk_unprepare_unused_subtree(child);
+ if (dev_has_sync_state(core->dev) &&
+ !(core->flags & CLK_DONT_HOLD_STATE))
+ return;
+
if (core->prepare_count)
return;
@@ -1332,6 +1338,10 @@ static void __init clk_disable_unused_subtree(struct clk_core *core)
hlist_for_each_entry(child, &core->children, child_node)
clk_disable_unused_subtree(child);
+ if (dev_has_sync_state(core->dev) &&
+ !(core->flags & CLK_DONT_HOLD_STATE))
+ return;
+
if (core->flags & CLK_OPS_PARENT_ENABLE)
clk_core_prepare_enable(core->parent);
@@ -1405,6 +1415,38 @@ static int __init clk_disable_unused(void)
}
late_initcall_sync(clk_disable_unused);
+static void clk_unprepare_disable_dev_subtree(struct clk_core *core,
+ struct device *dev)
+{
+ struct clk_core *child;
+
+ lockdep_assert_held(&prepare_lock);
+
+ hlist_for_each_entry(child, &core->children, child_node)
+ clk_unprepare_disable_dev_subtree(child, dev);
+
+ if (core->dev != dev || !core->need_sync)
+ return;
+
+ clk_core_disable_unprepare(core);
+}
+
+void clk_sync_state(struct device *dev)
+{
+ struct clk_core *core;
+
+ clk_prepare_lock();
+
+ hlist_for_each_entry(core, &clk_root_list, child_node)
+ clk_unprepare_disable_dev_subtree(core, dev);
+
+ hlist_for_each_entry(core, &clk_orphan_list, child_node)
+ clk_unprepare_disable_dev_subtree(core, dev);
+
+ clk_prepare_unlock();
+}
+EXPORT_SYMBOL_GPL(clk_sync_state);
+
static int clk_core_determine_round_nolock(struct clk_core *core,
struct clk_rate_request *req)
{
@@ -1895,6 +1937,33 @@ int clk_hw_get_parent_index(struct clk_hw *hw)
}
EXPORT_SYMBOL_GPL(clk_hw_get_parent_index);
+static void clk_core_hold_state(struct clk_core *core)
+{
+ if (core->need_sync || !core->boot_enabled)
+ return;
+
+ if (core->orphan || !dev_has_sync_state(core->dev))
+ return;
+
+ if (core->flags & CLK_DONT_HOLD_STATE)
+ return;
+
+ core->need_sync = !clk_core_prepare_enable(core);
+}
+
+static void __clk_core_update_orphan_hold_state(struct clk_core *core)
+{
+ struct clk_core *child;
+
+ if (core->orphan)
+ return;
+
+ clk_core_hold_state(core);
+
+ hlist_for_each_entry(child, &core->children, child_node)
+ __clk_core_update_orphan_hold_state(child);
+}
+
/*
* Update the orphan status of @core and all its children.
*/
@@ -2198,6 +2267,13 @@ static struct clk_core *clk_propagate_rate_change(struct clk_core *core,
fail_clk = core;
}
+ if (core->ops->pre_rate_change) {
+ ret = core->ops->pre_rate_change(core->hw, core->rate,
+ core->new_rate);
+ if (ret)
+ fail_clk = core;
+ }
+
hlist_for_each_entry(child, &core->children, child_node) {
/* Skip children who will be reparented to another clock */
if (child->new_parent && child->new_parent != core)
@@ -2292,6 +2368,9 @@ static void clk_change_rate(struct clk_core *core)
if (core->flags & CLK_RECALC_NEW_RATES)
(void)clk_calc_new_rates(core, core->new_rate);
+ if (core->ops->post_rate_change)
+ core->ops->post_rate_change(core->hw, old_rate, core->rate);
+
/*
* Use safe iteration, as change_rate can actually swap parents
* for certain clock types.
@@ -3231,7 +3310,7 @@ static int clk_dump_show(struct seq_file *s, void *data)
}
DEFINE_SHOW_ATTRIBUTE(clk_dump);
-#undef CLOCK_ALLOW_WRITE_DEBUGFS
+#define CLOCK_ALLOW_WRITE_DEBUGFS
#ifdef CLOCK_ALLOW_WRITE_DEBUGFS
/*
* This can be dangerous, therefore don't provide any real compile time
@@ -3560,24 +3639,6 @@ static int __init clk_debug_init(void)
{
struct clk_core *core;
-#ifdef CLOCK_ALLOW_WRITE_DEBUGFS
- pr_warn("\n");
- pr_warn("********************************************************************\n");
- pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
- pr_warn("** **\n");
- pr_warn("** WRITEABLE clk DebugFS SUPPORT HAS BEEN ENABLED IN THIS KERNEL **\n");
- pr_warn("** **\n");
- pr_warn("** This means that this kernel is built to expose clk operations **\n");
- pr_warn("** such as parent or rate setting, enabling, disabling, etc. **\n");
- pr_warn("** to userspace, which may compromise security on your system. **\n");
- pr_warn("** **\n");
- pr_warn("** If you see this message and you are not debugging the **\n");
- pr_warn("** kernel, report this immediately to your vendor! **\n");
- pr_warn("** **\n");
- pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
- pr_warn("********************************************************************\n");
-#endif
-
rootdir = debugfs_create_dir("clk", NULL);
debugfs_create_file("clk_summary", 0444, rootdir, &all_lists,
@@ -3630,6 +3691,7 @@ static void clk_core_reparent_orphans_nolock(void)
__clk_set_parent_after(orphan, parent, NULL);
__clk_recalc_accuracies(orphan);
__clk_recalc_rates(orphan, true, 0);
+ __clk_core_update_orphan_hold_state(orphan);
/*
* __clk_init_parent() will set the initial req_rate to
@@ -3806,6 +3868,8 @@ static int __clk_core_init(struct clk_core *core)
rate = 0;
core->rate = core->req_rate = rate;
+ core->boot_enabled = clk_core_is_enabled(core);
+
/*
* Enable CLK_IS_CRITICAL clocks so newly added critical clocks
* don't get accidentally disabled when walking the orphan tree and
@@ -3828,6 +3892,7 @@ static int __clk_core_init(struct clk_core *core)
}
}
+ clk_core_hold_state(core);
clk_core_reparent_orphans_nolock();
kref_init(&core->ref);
diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c
index 735adfe..414ffb6 100644
--- a/drivers/clk/qcom/dispcc-sdm845.c
+++ b/drivers/clk/qcom/dispcc-sdm845.c
@@ -3,6 +3,7 @@
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
+#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/module.h>
#include <linux/platform_device.h>
@@ -869,6 +870,7 @@ static struct platform_driver disp_cc_sdm845_driver = {
.driver = {
.name = "disp_cc-sdm845",
.of_match_table = disp_cc_sdm845_match_table,
+ .sync_state = clk_sync_state,
},
};
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
index 33473c5..8b4e4a3 100644
--- a/drivers/clk/qcom/gcc-msm8998.c
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -3261,6 +3261,7 @@ static struct platform_driver gcc_msm8998_driver = {
.driver = {
.name = "gcc-msm8998",
.of_match_table = gcc_msm8998_match_table,
+ .sync_state = clk_sync_state,
},
};
diff --git a/drivers/clk/qcom/gcc-sdm845.c b/drivers/clk/qcom/gcc-sdm845.c
index 6af08e0..0f2aadd 100644
--- a/drivers/clk/qcom/gcc-sdm845.c
+++ b/drivers/clk/qcom/gcc-sdm845.c
@@ -4020,6 +4020,7 @@ static struct platform_driver gcc_sdm845_driver = {
.driver = {
.name = "gcc-sdm845",
.of_match_table = gcc_sdm845_match_table,
+ .sync_state = clk_sync_state,
},
};
diff --git a/drivers/clk/qcom/gpucc-sdm845.c b/drivers/clk/qcom/gpucc-sdm845.c
index 110b544..8749aab 100644
--- a/drivers/clk/qcom/gpucc-sdm845.c
+++ b/drivers/clk/qcom/gpucc-sdm845.c
@@ -205,6 +205,7 @@ static struct platform_driver gpu_cc_sdm845_driver = {
.driver = {
.name = "sdm845-gpucc",
.of_match_table = gpu_cc_sdm845_match_table,
+ .sync_state = clk_sync_state,
},
};
diff --git a/drivers/clk/qcom/videocc-sdm845.c b/drivers/clk/qcom/videocc-sdm845.c
index c77a4dd..f678ade 100644
--- a/drivers/clk/qcom/videocc-sdm845.c
+++ b/drivers/clk/qcom/videocc-sdm845.c
@@ -337,6 +337,7 @@ static struct platform_driver video_cc_sdm845_driver = {
.driver = {
.name = "sdm845-videocc",
.of_match_table = video_cc_sdm845_match_table,
+ .sync_state = clk_sync_state,
},
};
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 4469e7f..aa859ca 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -135,7 +135,7 @@
Enables the support for the RDA Micro timer driver.
config SUN4I_TIMER
- bool "Sun4i timer driver" if COMPILE_TEST
+ bool "Sun4i timer driver"
depends on HAS_IOMEM
select CLKSRC_MMIO
select TIMER_OF
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 2a84fc6..c939546 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -35,6 +35,13 @@
If in doubt, say N.
+config CPU_FREQ_TIMES
+ bool "CPU frequency time-in-state statistics"
+ help
+ Export CPU time-in-state information through procfs.
+
+ If in doubt, say N.
+
choice
prompt "Default CPUFreq governor"
default CPU_FREQ_DEFAULT_GOV_USERSPACE if ARM_SA1100_CPUFREQ || ARM_SA1110_CPUFREQ
@@ -227,6 +234,15 @@
If in doubt, say N.
+config CPUFREQ_DUMMY
+ tristate "Dummy CPU frequency driver"
+ help
+ This option adds a generic dummy CPUfreq driver, which sets a fake
+ 2-frequency table when initializing each policy and otherwise does
+ nothing.
+
+ If in doubt, say N
+
if X86
source "drivers/cpufreq/Kconfig.x86"
endif
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 32a7029..ee47c6c 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -5,7 +5,10 @@
# CPUfreq stats
obj-$(CONFIG_CPU_FREQ_STAT) += cpufreq_stats.o
-# CPUfreq governors
+# CPUfreq times
+obj-$(CONFIG_CPU_FREQ_TIMES) += cpufreq_times.o
+
+# CPUfreq governors
obj-$(CONFIG_CPU_FREQ_GOV_PERFORMANCE) += cpufreq_performance.o
obj-$(CONFIG_CPU_FREQ_GOV_POWERSAVE) += cpufreq_powersave.o
obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE) += cpufreq_userspace.o
@@ -17,6 +20,8 @@
obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o
obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o
+obj-$(CONFIG_CPUFREQ_DUMMY) += dummy-cpufreq.o
+
# Traces
CFLAGS_amd-pstate-trace.o := -I$(src)
amd_pstate-y := amd-pstate.o amd-pstate-trace.o
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 7e56a42..d406cc7 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -16,6 +16,7 @@
#include <linux/cpu.h>
#include <linux/cpufreq.h>
+#include <linux/cpufreq_times.h>
#include <linux/cpu_cooling.h>
#include <linux/delay.h>
#include <linux/device.h>
@@ -30,6 +31,7 @@
#include <linux/tick.h>
#include <linux/units.h>
#include <trace/events/power.h>
+#include <trace/hooks/cpufreq.h>
static LIST_HEAD(cpufreq_policy_list);
@@ -387,7 +389,9 @@ static void cpufreq_notify_transition(struct cpufreq_policy *policy,
CPUFREQ_POSTCHANGE, freqs);
cpufreq_stats_record_transition(policy, freqs->new);
+ cpufreq_times_record_transition(policy, freqs->new);
policy->cur = freqs->new;
+ trace_android_rvh_cpufreq_transition(policy);
}
}
@@ -689,8 +693,15 @@ static ssize_t show_##file_name \
return sprintf(buf, "%u\n", policy->object); \
}
+static ssize_t show_cpuinfo_max_freq(struct cpufreq_policy *policy, char *buf)
+{
+ unsigned int max_freq = policy->cpuinfo.max_freq;
+
+ trace_android_rvh_show_max_freq(policy, &max_freq);
+ return sprintf(buf, "%u\n", max_freq);
+}
+
show_one(cpuinfo_min_freq, cpuinfo.min_freq);
-show_one(cpuinfo_max_freq, cpuinfo.max_freq);
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
@@ -1489,6 +1500,7 @@ static int cpufreq_online(unsigned int cpu)
goto out_destroy_policy;
cpufreq_stats_create_table(policy);
+ cpufreq_times_create_policy(policy);
write_lock_irqsave(&cpufreq_driver_lock, flags);
list_add(&policy->policy_list, &cpufreq_policy_list);
@@ -2128,6 +2140,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
arch_set_freq_scale(policy->related_cpus, freq,
policy->cpuinfo.max_freq);
cpufreq_stats_record_transition(policy, freq);
+ trace_android_rvh_cpufreq_transition(policy);
if (trace_cpu_frequency_enabled()) {
for_each_cpu(cpu, policy->cpus)
@@ -2604,7 +2617,6 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
ret = cpufreq_start_governor(policy);
if (!ret) {
pr_debug("governor change\n");
- sched_cpufreq_governor_change(policy, old_gov);
return 0;
}
cpufreq_exit_governor(policy);
@@ -2622,6 +2634,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
return ret;
}
+EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency_limits);
/**
* cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
diff --git a/drivers/cpufreq/cpufreq_times.c b/drivers/cpufreq/cpufreq_times.c
new file mode 100644
index 0000000..4df55b3
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_times.c
@@ -0,0 +1,211 @@
+/* drivers/cpufreq/cpufreq_times.c
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/cpufreq_times.h>
+#include <linux/jiffies.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/threads.h>
+
+static DEFINE_SPINLOCK(task_time_in_state_lock); /* task->time_in_state */
+
+/**
+ * struct cpu_freqs - per-cpu frequency information
+ * @offset: start of these freqs' stats in task time_in_state array
+ * @max_state: number of entries in freq_table
+ * @last_index: index in freq_table of last frequency switched to
+ * @freq_table: list of available frequencies
+ */
+struct cpu_freqs {
+ unsigned int offset;
+ unsigned int max_state;
+ unsigned int last_index;
+ unsigned int freq_table[0];
+};
+
+static struct cpu_freqs *all_freqs[NR_CPUS];
+
+static unsigned int next_offset;
+
+void cpufreq_task_times_init(struct task_struct *p)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&task_time_in_state_lock, flags);
+ p->time_in_state = NULL;
+ spin_unlock_irqrestore(&task_time_in_state_lock, flags);
+ p->max_state = 0;
+}
+
+void cpufreq_task_times_alloc(struct task_struct *p)
+{
+ void *temp;
+ unsigned long flags;
+ unsigned int max_state = READ_ONCE(next_offset);
+
+ /* We use one array to avoid multiple allocs per task */
+ temp = kcalloc(max_state, sizeof(p->time_in_state[0]), GFP_ATOMIC);
+ if (!temp)
+ return;
+
+ spin_lock_irqsave(&task_time_in_state_lock, flags);
+ p->time_in_state = temp;
+ spin_unlock_irqrestore(&task_time_in_state_lock, flags);
+ p->max_state = max_state;
+}
+
+/* Caller must hold task_time_in_state_lock */
+static int cpufreq_task_times_realloc_locked(struct task_struct *p)
+{
+ void *temp;
+ unsigned int max_state = READ_ONCE(next_offset);
+
+ temp = krealloc(p->time_in_state, max_state * sizeof(u64), GFP_ATOMIC);
+ if (!temp)
+ return -ENOMEM;
+ p->time_in_state = temp;
+ memset(p->time_in_state + p->max_state, 0,
+ (max_state - p->max_state) * sizeof(u64));
+ p->max_state = max_state;
+ return 0;
+}
+
+void cpufreq_task_times_exit(struct task_struct *p)
+{
+ unsigned long flags;
+ void *temp;
+
+ if (!p->time_in_state)
+ return;
+
+ spin_lock_irqsave(&task_time_in_state_lock, flags);
+ temp = p->time_in_state;
+ p->time_in_state = NULL;
+ spin_unlock_irqrestore(&task_time_in_state_lock, flags);
+ kfree(temp);
+}
+
+int proc_time_in_state_show(struct seq_file *m, struct pid_namespace *ns,
+ struct pid *pid, struct task_struct *p)
+{
+ unsigned int cpu, i;
+ u64 cputime;
+ unsigned long flags;
+ struct cpu_freqs *freqs;
+ struct cpu_freqs *last_freqs = NULL;
+
+ spin_lock_irqsave(&task_time_in_state_lock, flags);
+ for_each_possible_cpu(cpu) {
+ freqs = all_freqs[cpu];
+ if (!freqs || freqs == last_freqs)
+ continue;
+ last_freqs = freqs;
+
+ seq_printf(m, "cpu%u\n", cpu);
+ for (i = 0; i < freqs->max_state; i++) {
+ cputime = 0;
+ if (freqs->offset + i < p->max_state &&
+ p->time_in_state)
+ cputime = p->time_in_state[freqs->offset + i];
+ seq_printf(m, "%u %lu\n", freqs->freq_table[i],
+ (unsigned long)nsec_to_clock_t(cputime));
+ }
+ }
+ spin_unlock_irqrestore(&task_time_in_state_lock, flags);
+ return 0;
+}
+
+void cpufreq_acct_update_power(struct task_struct *p, u64 cputime)
+{
+ unsigned long flags;
+ unsigned int state;
+ struct cpu_freqs *freqs = all_freqs[task_cpu(p)];
+
+ if (!freqs || is_idle_task(p) || p->flags & PF_EXITING)
+ return;
+
+ state = freqs->offset + READ_ONCE(freqs->last_index);
+
+ spin_lock_irqsave(&task_time_in_state_lock, flags);
+ if ((state < p->max_state || !cpufreq_task_times_realloc_locked(p)) &&
+ p->time_in_state)
+ p->time_in_state[state] += cputime;
+ spin_unlock_irqrestore(&task_time_in_state_lock, flags);
+}
+
+static int cpufreq_times_get_index(struct cpu_freqs *freqs, unsigned int freq)
+{
+ int index;
+ for (index = 0; index < freqs->max_state; ++index) {
+ if (freqs->freq_table[index] == freq)
+ return index;
+ }
+ return -1;
+}
+
+void cpufreq_times_create_policy(struct cpufreq_policy *policy)
+{
+ int cpu, index = 0;
+ unsigned int count = 0;
+ struct cpufreq_frequency_table *pos, *table;
+ struct cpu_freqs *freqs;
+ void *tmp;
+
+ if (all_freqs[policy->cpu])
+ return;
+
+ table = policy->freq_table;
+ if (!table)
+ return;
+
+ cpufreq_for_each_valid_entry(pos, table)
+ count++;
+
+ tmp = kzalloc(sizeof(*freqs) + sizeof(freqs->freq_table[0]) * count,
+ GFP_KERNEL);
+ if (!tmp)
+ return;
+
+ freqs = tmp;
+ freqs->max_state = count;
+
+ cpufreq_for_each_valid_entry(pos, table)
+ freqs->freq_table[index++] = pos->frequency;
+
+ index = cpufreq_times_get_index(freqs, policy->cur);
+ if (index >= 0)
+ WRITE_ONCE(freqs->last_index, index);
+
+ freqs->offset = next_offset;
+ WRITE_ONCE(next_offset, freqs->offset + count);
+ for_each_cpu(cpu, policy->related_cpus)
+ all_freqs[cpu] = freqs;
+}
+
+void cpufreq_times_record_transition(struct cpufreq_policy *policy,
+ unsigned int new_freq)
+{
+ int index;
+ struct cpu_freqs *freqs = all_freqs[policy->cpu];
+ if (!freqs)
+ return;
+
+ index = cpufreq_times_get_index(freqs, new_freq);
+ if (index >= 0)
+ WRITE_ONCE(freqs->last_index, index);
+}
diff --git a/drivers/cpufreq/dummy-cpufreq.c b/drivers/cpufreq/dummy-cpufreq.c
new file mode 100644
index 0000000..e74ef67
--- /dev/null
+++ b/drivers/cpufreq/dummy-cpufreq.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Google, Inc.
+ */
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+
+static struct cpufreq_frequency_table freq_table[] = {
+ { .frequency = 1 },
+ { .frequency = 2 },
+ { .frequency = CPUFREQ_TABLE_END },
+};
+
+static int dummy_cpufreq_target_index(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ return 0;
+}
+
+static int dummy_cpufreq_driver_init(struct cpufreq_policy *policy)
+{
+ policy->freq_table = freq_table;
+ return 0;
+}
+
+static unsigned int dummy_cpufreq_get(unsigned int cpu)
+{
+ return 1;
+}
+
+static int dummy_cpufreq_verify(struct cpufreq_policy_data *data)
+{
+ return 0;
+}
+
+static struct cpufreq_driver dummy_cpufreq_driver = {
+ .name = "dummy",
+ .target_index = dummy_cpufreq_target_index,
+ .init = dummy_cpufreq_driver_init,
+ .get = dummy_cpufreq_get,
+ .verify = dummy_cpufreq_verify,
+ .attr = cpufreq_generic_attr,
+};
+
+static int __init dummy_cpufreq_init(void)
+{
+ return cpufreq_register_driver(&dummy_cpufreq_driver);
+}
+
+static void __exit dummy_cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&dummy_cpufreq_driver);
+}
+
+module_init(dummy_cpufreq_init);
+module_exit(dummy_cpufreq_exit);
+
+MODULE_AUTHOR("Connor O'Brien <connoro@google.com>");
+MODULE_DESCRIPTION("dummy cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c
index 67e56cf..24d3115 100644
--- a/drivers/cpufreq/freq_table.c
+++ b/drivers/cpufreq/freq_table.c
@@ -9,6 +9,7 @@
#include <linux/cpufreq.h>
#include <linux/module.h>
+#include <trace/hooks/cpufreq.h>
/*********************************************************************
* FREQUENCY TABLE HELPERS *
@@ -51,6 +52,7 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy,
max_freq = freq;
}
+ trace_android_vh_freq_table_limits(policy, min_freq, max_freq);
policy->min = policy->cpuinfo.min_freq = min_freq;
policy->max = max_freq;
/*
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
index 57bc3e3..396df7d 100644
--- a/drivers/cpuidle/cpuidle-psci.c
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -26,6 +26,7 @@
#include <linux/syscore_ops.h>
#include <asm/cpuidle.h>
+#include <trace/hooks/cpuidle_psci.h>
#include "cpuidle-psci.h"
#include "dt_idle_states.h"
@@ -68,6 +69,8 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
if (ret)
return -1;
+ trace_android_vh_cpuidle_psci_enter(dev, s2idle);
+
/* Do runtime PM to manage a hierarchical CPU toplogy. */
ct_irq_enter_irqson();
if (s2idle)
@@ -89,6 +92,8 @@ static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
pm_runtime_get_sync(pd_dev);
ct_irq_exit_irqson();
+ trace_android_vh_cpuidle_psci_exit(dev, s2idle);
+
cpu_pm_exit();
/* Clear the domain state to start fresh when back from idle. */
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 6eceb19..054449e 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -26,6 +26,7 @@
#include <linux/mmu_context.h>
#include <linux/context_tracking.h>
#include <trace/events/power.h>
+#include <trace/hooks/cpuidle.h>
#include "cpuidle.h"
@@ -204,11 +205,22 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
{
int entered_state;
- struct cpuidle_state *target_state = &drv->states[index];
- bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
+ struct cpuidle_state *target_state;
+ bool broadcast;
ktime_t time_start, time_end;
/*
+ * The vendor hook may modify index, which means target_state and
+ * broadcast must be assigned after the vendor hook.
+ */
+ trace_android_vh_cpu_idle_enter(&index, dev);
+ if (index < 0)
+ return index;
+
+ target_state = &drv->states[index];
+ broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
+
+ /*
* Tell the time framework to switch to a broadcast timer because our
* local timer will be shut down. If a local timer is used from another
* CPU as a broadcast timer, this call may fail if it is not available.
@@ -244,6 +256,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
sched_clock_idle_wakeup_event();
time_end = ns_to_ktime(local_clock());
trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);
+ trace_android_vh_cpu_idle_exit(entered_state, dev);
/* The cpu is no longer idle or about to enter idle. */
sched_idle_set_state(NULL);
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index 0d0f975..ca15755 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -101,6 +101,7 @@ int cpuidle_register_governor(struct cpuidle_governor *gov)
return ret;
}
+EXPORT_SYMBOL_GPL(cpuidle_register_governor);
/**
* cpuidle_governor_latency_req - Compute a latency constraint for CPU
@@ -117,3 +118,4 @@ s64 cpuidle_governor_latency_req(unsigned int cpu)
return (s64)device_req * NSEC_PER_USEC;
}
+EXPORT_SYMBOL_GPL(cpuidle_governor_latency_req);
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index b6c3691..981a5ac 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -1423,6 +1423,30 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
}
EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, DMA_BUF);
+int dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,
+ enum dma_data_direction direction,
+ unsigned int offset, unsigned int len)
+{
+ int ret = 0;
+
+ if (WARN_ON(!dmabuf))
+ return -EINVAL;
+
+ if (dmabuf->ops->begin_cpu_access_partial)
+ ret = dmabuf->ops->begin_cpu_access_partial(dmabuf, direction,
+ offset, len);
+
+ /* Ensure that all fences are waited upon - but we first allow
+ * the native handler the chance to do so more efficiently if it
+ * chooses. A double invocation here will be reasonably cheap no-op.
+ */
+ if (ret == 0)
+ ret = __dma_buf_begin_cpu_access(dmabuf, direction);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access_partial);
+
/**
* dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the
* cpu in the kernel context. Calls end_cpu_access to allow exporter-specific
@@ -1451,6 +1475,21 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf,
}
EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, DMA_BUF);
+int dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,
+ enum dma_data_direction direction,
+ unsigned int offset, unsigned int len)
+{
+ int ret = 0;
+
+ WARN_ON(!dmabuf);
+
+ if (dmabuf->ops->end_cpu_access_partial)
+ ret = dmabuf->ops->end_cpu_access_partial(dmabuf, direction,
+ offset, len);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access_partial);
/**
* dma_buf_mmap - Setup up a userspace mmap with the given vma
@@ -1619,6 +1658,20 @@ void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)
}
EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, DMA_BUF);
+int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags)
+{
+ int ret = 0;
+
+ if (WARN_ON(!dmabuf) || !flags)
+ return -EINVAL;
+
+ if (dmabuf->ops->get_flags)
+ ret = dmabuf->ops->get_flags(dmabuf, flags);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dma_buf_get_flags);
+
#ifdef CONFIG_DEBUG_FS
static int dma_buf_debug_show(struct seq_file *s, void *unused)
{
diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
index 59d1588..9277e55 100644
--- a/drivers/dma-buf/dma-heap.c
+++ b/drivers/dma-buf/dma-heap.c
@@ -31,6 +31,7 @@
* @heap_devt heap device node
* @list list head connecting to list of heaps
* @heap_cdev heap char device
+ * @heap_dev heap device struct
*
* Represents a heap of memory from which buffers can be made.
*/
@@ -41,6 +42,8 @@ struct dma_heap {
dev_t heap_devt;
struct list_head list;
struct cdev heap_cdev;
+ struct kref refcount;
+ struct device *heap_dev;
};
static LIST_HEAD(heap_list);
@@ -49,22 +52,60 @@ static dev_t dma_heap_devt;
static struct class *dma_heap_class;
static DEFINE_XARRAY_ALLOC(dma_heap_minors);
-static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
- unsigned int fd_flags,
- unsigned int heap_flags)
+struct dma_heap *dma_heap_find(const char *name)
{
- struct dma_buf *dmabuf;
- int fd;
+ struct dma_heap *h;
+ mutex_lock(&heap_list_lock);
+ list_for_each_entry(h, &heap_list, list) {
+ if (!strcmp(h->name, name)) {
+ kref_get(&h->refcount);
+ mutex_unlock(&heap_list_lock);
+ return h;
+ }
+ }
+ mutex_unlock(&heap_list_lock);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(dma_heap_find);
+
+
+void dma_heap_buffer_free(struct dma_buf *dmabuf)
+{
+ dma_buf_put(dmabuf);
+}
+EXPORT_SYMBOL_GPL(dma_heap_buffer_free);
+
+struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
+ unsigned int fd_flags,
+ unsigned int heap_flags)
+{
+ if (fd_flags & ~DMA_HEAP_VALID_FD_FLAGS)
+ return ERR_PTR(-EINVAL);
+
+ if (heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS)
+ return ERR_PTR(-EINVAL);
/*
* Allocations from all heaps have to begin
* and end on page boundaries.
*/
len = PAGE_ALIGN(len);
if (!len)
- return -EINVAL;
+ return ERR_PTR(-EINVAL);
- dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags);
+ return heap->ops->allocate(heap, len, fd_flags, heap_flags);
+}
+EXPORT_SYMBOL_GPL(dma_heap_buffer_alloc);
+
+int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len,
+ unsigned int fd_flags,
+ unsigned int heap_flags)
+{
+ struct dma_buf *dmabuf;
+ int fd;
+
+ dmabuf = dma_heap_buffer_alloc(heap, len, fd_flags, heap_flags);
+
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
@@ -74,7 +115,9 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
/* just return, as put will call release and that will free */
}
return fd;
+
}
+EXPORT_SYMBOL_GPL(dma_heap_bufferfd_alloc);
static int dma_heap_open(struct inode *inode, struct file *file)
{
@@ -102,15 +145,9 @@ static long dma_heap_ioctl_allocate(struct file *file, void *data)
if (heap_allocation->fd)
return -EINVAL;
- if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS)
- return -EINVAL;
-
- if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS)
- return -EINVAL;
-
- fd = dma_heap_buffer_alloc(heap, heap_allocation->len,
- heap_allocation->fd_flags,
- heap_allocation->heap_flags);
+ fd = dma_heap_bufferfd_alloc(heap, heap_allocation->len,
+ heap_allocation->fd_flags,
+ heap_allocation->heap_flags);
if (fd < 0)
return fd;
@@ -203,6 +240,47 @@ void *dma_heap_get_drvdata(struct dma_heap *heap)
{
return heap->priv;
}
+EXPORT_SYMBOL_GPL(dma_heap_get_drvdata);
+
+static void dma_heap_release(struct kref *ref)
+{
+ struct dma_heap *heap = container_of(ref, struct dma_heap, refcount);
+ int minor = MINOR(heap->heap_devt);
+
+ /* Note, we already holding the heap_list_lock here */
+ list_del(&heap->list);
+
+ device_destroy(dma_heap_class, heap->heap_devt);
+ cdev_del(&heap->heap_cdev);
+ xa_erase(&dma_heap_minors, minor);
+
+ kfree(heap);
+}
+
+void dma_heap_put(struct dma_heap *h)
+{
+ /*
+ * Take the heap_list_lock now to avoid racing with code
+ * scanning the list and then taking a kref.
+ */
+ mutex_lock(&heap_list_lock);
+ kref_put(&h->refcount, dma_heap_release);
+ mutex_unlock(&heap_list_lock);
+}
+EXPORT_SYMBOL_GPL(dma_heap_put);
+
+/**
+ * dma_heap_get_dev() - get device struct for the heap
+ * @heap: DMA-Heap to retrieve device struct from
+ *
+ * Returns:
+ * The device struct for the heap.
+ */
+struct device *dma_heap_get_dev(struct dma_heap *heap)
+{
+ return heap->heap_dev;
+}
+EXPORT_SYMBOL_GPL(dma_heap_get_dev);
/**
* dma_heap_get_name() - get heap name
@@ -215,11 +293,11 @@ const char *dma_heap_get_name(struct dma_heap *heap)
{
return heap->name;
}
+EXPORT_SYMBOL_GPL(dma_heap_get_name);
struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
{
struct dma_heap *heap, *h, *err_ret;
- struct device *dev_ret;
unsigned int minor;
int ret;
@@ -237,6 +315,7 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
if (!heap)
return ERR_PTR(-ENOMEM);
+ kref_init(&heap->refcount);
heap->name = exp_info->name;
heap->ops = exp_info->ops;
heap->priv = exp_info->priv;
@@ -261,17 +340,20 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
goto err1;
}
- dev_ret = device_create(dma_heap_class,
- NULL,
- heap->heap_devt,
- NULL,
- heap->name);
- if (IS_ERR(dev_ret)) {
+ heap->heap_dev = device_create(dma_heap_class,
+ NULL,
+ heap->heap_devt,
+ NULL,
+ heap->name);
+ if (IS_ERR(heap->heap_dev)) {
pr_err("dma_heap: Unable to create device\n");
- err_ret = ERR_CAST(dev_ret);
+ err_ret = ERR_CAST(heap->heap_dev);
goto err2;
}
+ /* Make sure it doesn't disappear on us */
+ heap->heap_dev = get_device(heap->heap_dev);
+
mutex_lock(&heap_list_lock);
/* check the name is unique */
list_for_each_entry(h, &heap_list, list) {
@@ -280,6 +362,7 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
pr_err("dma_heap: Already registered heap named %s\n",
exp_info->name);
err_ret = ERR_PTR(-EINVAL);
+ put_device(heap->heap_dev);
goto err3;
}
}
@@ -300,6 +383,7 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
kfree(heap);
return err_ret;
}
+EXPORT_SYMBOL_GPL(dma_heap_add);
static char *dma_heap_devnode(struct device *dev, umode_t *mode)
{
diff --git a/drivers/dma-buf/heaps/Kconfig b/drivers/dma-buf/heaps/Kconfig
index a5eef06..e273fb1 100644
--- a/drivers/dma-buf/heaps/Kconfig
+++ b/drivers/dma-buf/heaps/Kconfig
@@ -1,12 +1,12 @@
config DMABUF_HEAPS_SYSTEM
- bool "DMA-BUF System Heap"
+ tristate "DMA-BUF System Heap"
depends on DMABUF_HEAPS
help
Choose this option to enable the system dmabuf heap. The system heap
is backed by pages from the buddy allocator. If in doubt, say Y.
config DMABUF_HEAPS_CMA
- bool "DMA-BUF CMA Heap"
+ tristate "DMA-BUF CMA Heap"
depends on DMABUF_HEAPS && DMA_CMA
help
Choose this option to enable dma-buf CMA heap. This heap is backed
diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c
index 1131fb9..aeb4544 100644
--- a/drivers/dma-buf/heaps/cma_heap.c
+++ b/drivers/dma-buf/heaps/cma_heap.c
@@ -100,9 +100,10 @@ static struct sg_table *cma_heap_map_dma_buf(struct dma_buf_attachment *attachme
{
struct dma_heap_attachment *a = attachment->priv;
struct sg_table *table = &a->table;
+ int attrs = attachment->dma_map_attrs;
int ret;
- ret = dma_map_sgtable(attachment->dev, table, direction, 0);
+ ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
if (ret)
return ERR_PTR(-ENOMEM);
a->mapped = true;
@@ -114,9 +115,10 @@ static void cma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
+ int attrs = attachment->dma_map_attrs;
a->mapped = false;
- dma_unmap_sgtable(attachment->dev, table, direction, 0);
+ dma_unmap_sgtable(attachment->dev, table, direction, attrs);
}
static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
@@ -408,3 +410,4 @@ static int add_default_cma_heap(void)
module_init(add_default_cma_heap);
MODULE_DESCRIPTION("DMA-BUF CMA Heap");
MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(DMA_BUF);
diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c
index e8bd10e..3eb3ee3 100644
--- a/drivers/dma-buf/heaps/system_heap.c
+++ b/drivers/dma-buf/heaps/system_heap.c
@@ -23,6 +23,7 @@
#include <linux/vmalloc.h>
static struct dma_heap *sys_heap;
+static struct dma_heap *sys_uncached_heap;
struct system_heap_buffer {
struct dma_heap *heap;
@@ -32,6 +33,8 @@ struct system_heap_buffer {
struct sg_table sg_table;
int vmap_cnt;
void *vaddr;
+
+ bool uncached;
};
struct dma_heap_attachment {
@@ -39,6 +42,8 @@ struct dma_heap_attachment {
struct sg_table *table;
struct list_head list;
bool mapped;
+
+ bool uncached;
};
#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
@@ -102,7 +107,7 @@ static int system_heap_attach(struct dma_buf *dmabuf,
a->dev = attachment->dev;
INIT_LIST_HEAD(&a->list);
a->mapped = false;
-
+ a->uncached = buffer->uncached;
attachment->priv = a;
mutex_lock(&buffer->lock);
@@ -132,9 +137,13 @@ static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attac
{
struct dma_heap_attachment *a = attachment->priv;
struct sg_table *table = a->table;
+ int attr = attachment->dma_map_attrs;
int ret;
- ret = dma_map_sgtable(attachment->dev, table, direction, 0);
+ if (a->uncached)
+ attr |= DMA_ATTR_SKIP_CPU_SYNC;
+
+ ret = dma_map_sgtable(attachment->dev, table, direction, attr);
if (ret)
return ERR_PTR(ret);
@@ -147,9 +156,12 @@ static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
+ int attr = attachment->dma_map_attrs;
+ if (a->uncached)
+ attr |= DMA_ATTR_SKIP_CPU_SYNC;
a->mapped = false;
- dma_unmap_sgtable(attachment->dev, table, direction, 0);
+ dma_unmap_sgtable(attachment->dev, table, direction, attr);
}
static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
@@ -163,10 +175,12 @@ static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
if (buffer->vmap_cnt)
invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
- list_for_each_entry(a, &buffer->attachments, list) {
- if (!a->mapped)
- continue;
- dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
+ if (!buffer->uncached) {
+ list_for_each_entry(a, &buffer->attachments, list) {
+ if (!a->mapped)
+ continue;
+ dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
+ }
}
mutex_unlock(&buffer->lock);
@@ -184,10 +198,12 @@ static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
if (buffer->vmap_cnt)
flush_kernel_vmap_range(buffer->vaddr, buffer->len);
- list_for_each_entry(a, &buffer->attachments, list) {
- if (!a->mapped)
- continue;
- dma_sync_sgtable_for_device(a->dev, a->table, direction);
+ if (!buffer->uncached) {
+ list_for_each_entry(a, &buffer->attachments, list) {
+ if (!a->mapped)
+ continue;
+ dma_sync_sgtable_for_device(a->dev, a->table, direction);
+ }
}
mutex_unlock(&buffer->lock);
@@ -204,6 +220,9 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
dma_resv_assert_held(dmabuf->resv);
+ if (buffer->uncached)
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
struct page *page = sg_page_iter_page(&piter);
@@ -225,17 +244,21 @@ static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
struct page **pages = vmalloc(sizeof(struct page *) * npages);
struct page **tmp = pages;
struct sg_page_iter piter;
+ pgprot_t pgprot = PAGE_KERNEL;
void *vaddr;
if (!pages)
return ERR_PTR(-ENOMEM);
+ if (buffer->uncached)
+ pgprot = pgprot_writecombine(PAGE_KERNEL);
+
for_each_sgtable_page(table, &piter, 0) {
WARN_ON(tmp - pages >= npages);
*tmp++ = sg_page_iter_page(&piter);
}
- vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
+ vaddr = vmap(pages, npages, VM_MAP, pgprot);
vfree(pages);
if (!vaddr)
@@ -335,10 +358,11 @@ static struct page *alloc_largest_available(unsigned long size,
return NULL;
}
-static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
- unsigned long len,
- unsigned long fd_flags,
- unsigned long heap_flags)
+static struct dma_buf *system_heap_do_allocate(struct dma_heap *heap,
+ unsigned long len,
+ unsigned long fd_flags,
+ unsigned long heap_flags,
+ bool uncached)
{
struct system_heap_buffer *buffer;
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
@@ -359,6 +383,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
mutex_init(&buffer->lock);
buffer->heap = heap;
buffer->len = len;
+ buffer->uncached = uncached;
INIT_LIST_HEAD(&pages);
i = 0;
@@ -404,6 +429,18 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
ret = PTR_ERR(dmabuf);
goto free_pages;
}
+
+ /*
+ * For uncached buffers, we need to initially flush cpu cache, since
+ * the __GFP_ZERO on the allocation means the zeroing was done by the
+ * cpu and thus it is likely cached. Map (and implicitly flush) and
+ * unmap it now so we don't get corruption later on.
+ */
+ if (buffer->uncached) {
+ dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
+ dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
+ }
+
return dmabuf;
free_pages:
@@ -421,10 +458,40 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
return ERR_PTR(ret);
}
+static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
+ unsigned long len,
+ unsigned long fd_flags,
+ unsigned long heap_flags)
+{
+ return system_heap_do_allocate(heap, len, fd_flags, heap_flags, false);
+}
+
static const struct dma_heap_ops system_heap_ops = {
.allocate = system_heap_allocate,
};
+static struct dma_buf *system_uncached_heap_allocate(struct dma_heap *heap,
+ unsigned long len,
+ unsigned long fd_flags,
+ unsigned long heap_flags)
+{
+ return system_heap_do_allocate(heap, len, fd_flags, heap_flags, true);
+}
+
+/* Dummy function to be used until we can call coerce_mask_and_coherent */
+static struct dma_buf *system_uncached_heap_not_initialized(struct dma_heap *heap,
+ unsigned long len,
+ unsigned long fd_flags,
+ unsigned long heap_flags)
+{
+ return ERR_PTR(-EBUSY);
+}
+
+static struct dma_heap_ops system_uncached_heap_ops = {
+ /* After system_heap_create is complete, we will swap this */
+ .allocate = system_uncached_heap_not_initialized,
+};
+
static int system_heap_create(void)
{
struct dma_heap_export_info exp_info;
@@ -437,7 +504,20 @@ static int system_heap_create(void)
if (IS_ERR(sys_heap))
return PTR_ERR(sys_heap);
+ exp_info.name = "system-uncached";
+ exp_info.ops = &system_uncached_heap_ops;
+ exp_info.priv = NULL;
+
+ sys_uncached_heap = dma_heap_add(&exp_info);
+ if (IS_ERR(sys_uncached_heap))
+ return PTR_ERR(sys_uncached_heap);
+
+ dma_coerce_mask_and_coherent(dma_heap_get_dev(sys_uncached_heap), DMA_BIT_MASK(64));
+ mb(); /* make sure we only set allocate after dma_mask is set */
+ system_uncached_heap_ops.allocate = system_uncached_heap_allocate;
+
return 0;
}
module_init(system_heap_create);
MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(DMA_BUF);
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index ca2a6e6..8c69534 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -678,9 +678,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER),
- DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, 0),
- DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, 0),
- DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, 0),
+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, 0),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER),
DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER),
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 3c8034a..63a8b6d 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -2084,6 +2084,7 @@ void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out,
strncpy(out->name, in->name, DRM_DISPLAY_MODE_LEN);
out->name[DRM_DISPLAY_MODE_LEN-1] = 0;
}
+EXPORT_SYMBOL_GPL(drm_mode_convert_to_umode);
/**
* drm_mode_convert_umode - convert a modeinfo into a drm_display_mode
@@ -2160,6 +2161,7 @@ int drm_mode_convert_umode(struct drm_device *dev,
return 0;
}
+EXPORT_SYMBOL_GPL(drm_mode_convert_umode);
/**
* drm_mode_is_420_only - if a given videomode can be only supported in YCBCR420
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 9ea7611..2df9d83 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -301,10 +301,6 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev,
struct virtio_gpu_framebuffer *virtio_gpu_fb;
int ret;
- if (mode_cmd->pixel_format != DRM_FORMAT_HOST_XRGB8888 &&
- mode_cmd->pixel_format != DRM_FORMAT_HOST_ARGB8888)
- return ERR_PTR(-ENOENT);
-
/* lookup object associated with res handle */
obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]);
if (!obj)
@@ -340,7 +336,6 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
if (ret)
return ret;
- vgdev->ddev->mode_config.quirk_addfb_prefer_host_byte_order = true;
vgdev->ddev->mode_config.funcs = &virtio_gpu_mode_funcs;
/* modes will be validated against the framebuffer size */
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 4c09e31..b8fcbca4 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -30,7 +30,14 @@
#include "virtgpu_drv.h"
static const uint32_t virtio_gpu_formats[] = {
- DRM_FORMAT_HOST_XRGB8888,
+ DRM_FORMAT_XRGB8888,
+ DRM_FORMAT_ARGB8888,
+ DRM_FORMAT_BGRX8888,
+ DRM_FORMAT_BGRA8888,
+ DRM_FORMAT_RGBX8888,
+ DRM_FORMAT_RGBA8888,
+ DRM_FORMAT_XBGR8888,
+ DRM_FORMAT_ABGR8888,
};
static const uint32_t virtio_gpu_cursor_formats[] = {
@@ -42,6 +49,32 @@ uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
uint32_t format;
switch (drm_fourcc) {
+#ifdef __BIG_ENDIAN
+ case DRM_FORMAT_XRGB8888:
+ format = VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM;
+ break;
+ case DRM_FORMAT_ARGB8888:
+ format = VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM;
+ break;
+ case DRM_FORMAT_BGRX8888:
+ format = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
+ break;
+ case DRM_FORMAT_BGRA8888:
+ format = VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM;
+ break;
+ case DRM_FORMAT_RGBX8888:
+ format = VIRTIO_GPU_FORMAT_R8G8B8X8_UNORM;
+ break;
+ case DRM_FORMAT_RGBA8888:
+ format = VIRTIO_GPU_FORMAT_R8G8B8A8_UNORM;
+ break;
+ case DRM_FORMAT_XBGR8888:
+ format = VIRTIO_GPU_FORMAT_X8B8G8R8_UNORM;
+ break;
+ case DRM_FORMAT_ABGR8888:
+ format = VIRTIO_GPU_FORMAT_A8B8G8R8_UNORM;
+ break;
+#else
case DRM_FORMAT_XRGB8888:
format = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
break;
@@ -54,6 +87,19 @@ uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
case DRM_FORMAT_BGRA8888:
format = VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM;
break;
+ case DRM_FORMAT_RGBX8888:
+ format = VIRTIO_GPU_FORMAT_X8B8G8R8_UNORM;
+ break;
+ case DRM_FORMAT_RGBA8888:
+ format = VIRTIO_GPU_FORMAT_A8B8G8R8_UNORM;
+ break;
+ case DRM_FORMAT_XBGR8888:
+ format = VIRTIO_GPU_FORMAT_R8G8B8X8_UNORM;
+ break;
+ case DRM_FORMAT_ABGR8888:
+ format = VIRTIO_GPU_FORMAT_R8G8B8A8_UNORM;
+ break;
+#endif
default:
/*
* This should not happen, we handle everything listed
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index f798c44..7efce95 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -28,6 +28,7 @@
#include <linux/spinlock.h>
#include <linux/swiotlb.h>
#include <linux/vmalloc.h>
+#include <trace/hooks/iommu.h>
#include "dma-iommu.h"
@@ -651,6 +652,8 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
true);
+ trace_android_vh_iommu_iovad_alloc_iova(dev, iovad, (dma_addr_t)iova << shift, size);
+
return (dma_addr_t)iova << shift;
}
@@ -669,6 +672,8 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
else
free_iova_fast(iovad, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
+
+ trace_android_vh_iommu_iovad_free_iova(iovad, iova, size);
}
static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 998a5cf..dd70f54 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -290,6 +290,27 @@
If unsure, say N.
+config DM_DEFAULT_KEY
+ tristate "Default-key target support"
+ depends on BLK_DEV_DM
+ depends on BLK_INLINE_ENCRYPTION
+ # dm-default-key doesn't require -o inlinecrypt, but it does currently
+ # rely on the inline encryption hooks being built into the kernel.
+ depends on FS_ENCRYPTION_INLINE_CRYPT
+ help
+ This device-mapper target allows you to create a device that
+ assigns a default encryption key to bios that aren't for the
+ contents of an encrypted file.
+
+ This ensures that all blocks on-disk will be encrypted with
+ some key, without the performance hit of file contents being
+ encrypted twice when fscrypt (File-Based Encryption) is used.
+
+ It is only appropriate to use dm-default-key when key
+ configuration is tightly controlled, like it is in Android,
+ such that all fscrypt keys are at least as hard to compromise
+ as the default key.
+
config DM_SNAPSHOT
tristate "Snapshot target"
depends on BLK_DEV_DM
@@ -653,4 +674,18 @@
Enables audit logging of several security relevant events in the
particular device-mapper targets, especially the integrity target.
+config DM_USER
+ tristate "Block device in userspace"
+ depends on BLK_DEV_DM
+ default y
+ help
+ This device-mapper target allows a userspace daemon to provide the
+ contents of a block device. See
+ <file:Documentation/block/dm-user.rst> for more information.
+
+ To compile this code as a module, choose M here: the module will be
+ called dm-user.
+
+ If unsure, say N.
+
endif # MD
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index 84291e3..369f2ae 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -57,6 +57,7 @@
obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
obj-$(CONFIG_DM_BIO_PRISON) += dm-bio-prison.o
obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
+obj-$(CONFIG_DM_DEFAULT_KEY) += dm-default-key.o
obj-$(CONFIG_DM_DELAY) += dm-delay.o
obj-$(CONFIG_DM_DUST) += dm-dust.o
obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
@@ -84,6 +85,7 @@
obj-$(CONFIG_DM_ZONED) += dm-zoned.o
obj-$(CONFIG_DM_WRITECACHE) += dm-writecache.o
obj-$(CONFIG_SECURITY_LOADPIN_VERITY) += dm-verity-loadpin.o
+obj-$(CONFIG_DM_USER) += dm-user.o
ifeq ($(CONFIG_DM_INIT),y)
dm-mod-objs += dm-init.o
diff --git a/drivers/md/dm-default-key.c b/drivers/md/dm-default-key.c
new file mode 100644
index 0000000..577c5fd
--- /dev/null
+++ b/drivers/md/dm-default-key.c
@@ -0,0 +1,439 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Google, Inc.
+ */
+
+#include <linux/blk-crypto.h>
+#include <linux/device-mapper.h>
+#include <linux/module.h>
+
+#define DM_MSG_PREFIX "default-key"
+
+static const struct dm_default_key_cipher {
+ const char *name;
+ enum blk_crypto_mode_num mode_num;
+ int key_size;
+} dm_default_key_ciphers[] = {
+ {
+ .name = "aes-xts-plain64",
+ .mode_num = BLK_ENCRYPTION_MODE_AES_256_XTS,
+ .key_size = 64,
+ }, {
+ .name = "xchacha12,aes-adiantum-plain64",
+ .mode_num = BLK_ENCRYPTION_MODE_ADIANTUM,
+ .key_size = 32,
+ },
+};
+
+/**
+ * struct dm_default_c - private data of a default-key target
+ * @dev: the underlying device
+ * @start: starting sector of the range of @dev which this target actually maps.
+ * For this purpose a "sector" is 512 bytes.
+ * @cipher_string: the name of the encryption algorithm being used
+ * @iv_offset: starting offset for IVs. IVs are generated as if the target were
+ * preceded by @iv_offset 512-byte sectors.
+ * @sector_size: crypto sector size in bytes (usually 4096)
+ * @sector_bits: log2(sector_size)
+ * @key: the encryption key to use
+ * @max_dun: the maximum DUN that may be used (computed from other params)
+ */
+struct default_key_c {
+ struct dm_dev *dev;
+ sector_t start;
+ const char *cipher_string;
+ u64 iv_offset;
+ unsigned int sector_size;
+ unsigned int sector_bits;
+ struct blk_crypto_key key;
+ enum blk_crypto_key_type key_type;
+ u64 max_dun;
+};
+
+static const struct dm_default_key_cipher *
+lookup_cipher(const char *cipher_string)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(dm_default_key_ciphers); i++) {
+ if (strcmp(cipher_string, dm_default_key_ciphers[i].name) == 0)
+ return &dm_default_key_ciphers[i];
+ }
+ return NULL;
+}
+
+static void default_key_dtr(struct dm_target *ti)
+{
+ struct default_key_c *dkc = ti->private;
+ int err;
+
+ if (dkc->dev) {
+ err = blk_crypto_evict_key(dkc->dev->bdev, &dkc->key);
+ if (err && err != -ENOKEY)
+ DMWARN("Failed to evict crypto key: %d", err);
+ dm_put_device(ti, dkc->dev);
+ }
+ kfree_sensitive(dkc->cipher_string);
+ kfree_sensitive(dkc);
+}
+
+static int default_key_ctr_optional(struct dm_target *ti,
+ unsigned int argc, char **argv)
+{
+ struct default_key_c *dkc = ti->private;
+ struct dm_arg_set as;
+ static const struct dm_arg _args[] = {
+ {0, 4, "Invalid number of feature args"},
+ };
+ unsigned int opt_params;
+ const char *opt_string;
+ bool iv_large_sectors = false;
+ char dummy;
+ int err;
+
+ as.argc = argc;
+ as.argv = argv;
+
+ err = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
+ if (err)
+ return err;
+
+ while (opt_params--) {
+ opt_string = dm_shift_arg(&as);
+ if (!opt_string) {
+ ti->error = "Not enough feature arguments";
+ return -EINVAL;
+ }
+ if (!strcmp(opt_string, "allow_discards")) {
+ ti->num_discard_bios = 1;
+ } else if (sscanf(opt_string, "sector_size:%u%c",
+ &dkc->sector_size, &dummy) == 1) {
+ if (dkc->sector_size < SECTOR_SIZE ||
+ dkc->sector_size > 4096 ||
+ !is_power_of_2(dkc->sector_size)) {
+ ti->error = "Invalid sector_size";
+ return -EINVAL;
+ }
+ } else if (!strcmp(opt_string, "iv_large_sectors")) {
+ iv_large_sectors = true;
+ } else if (!strcmp(opt_string, "wrappedkey_v0")) {
+ dkc->key_type = BLK_CRYPTO_KEY_TYPE_HW_WRAPPED;
+ } else {
+ ti->error = "Invalid feature arguments";
+ return -EINVAL;
+ }
+ }
+
+ /* dm-default-key doesn't implement iv_large_sectors=false. */
+ if (dkc->sector_size != SECTOR_SIZE && !iv_large_sectors) {
+ ti->error = "iv_large_sectors must be specified";
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * Construct a default-key mapping:
+ * <cipher> <key> <iv_offset> <dev_path> <start>
+ *
+ * This syntax matches dm-crypt's, but lots of unneeded functionality has been
+ * removed. Also, dm-default-key requires that the "iv_large_sectors" option be
+ * given whenever a non-default sector size is used.
+ */
+static int default_key_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct default_key_c *dkc;
+ const struct dm_default_key_cipher *cipher;
+ u8 raw_key[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
+ unsigned int raw_key_size;
+ unsigned int dun_bytes;
+ unsigned long long tmpll;
+ char dummy;
+ int err;
+
+ if (argc < 5) {
+ ti->error = "Not enough arguments";
+ return -EINVAL;
+ }
+
+ dkc = kzalloc(sizeof(*dkc), GFP_KERNEL);
+ if (!dkc) {
+ ti->error = "Out of memory";
+ return -ENOMEM;
+ }
+ ti->private = dkc;
+ dkc->key_type = BLK_CRYPTO_KEY_TYPE_STANDARD;
+
+ /* <cipher> */
+ dkc->cipher_string = kstrdup(argv[0], GFP_KERNEL);
+ if (!dkc->cipher_string) {
+ ti->error = "Out of memory";
+ err = -ENOMEM;
+ goto bad;
+ }
+ cipher = lookup_cipher(dkc->cipher_string);
+ if (!cipher) {
+ ti->error = "Unsupported cipher";
+ err = -EINVAL;
+ goto bad;
+ }
+
+ /* <key> */
+ raw_key_size = strlen(argv[1]);
+ if (raw_key_size > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE ||
+ raw_key_size % 2) {
+ ti->error = "Invalid keysize";
+ err = -EINVAL;
+ goto bad;
+ }
+ raw_key_size /= 2;
+ if (hex2bin(raw_key, argv[1], raw_key_size) != 0) {
+ ti->error = "Malformed key string";
+ err = -EINVAL;
+ goto bad;
+ }
+
+ /* <iv_offset> */
+ if (sscanf(argv[2], "%llu%c", &dkc->iv_offset, &dummy) != 1) {
+ ti->error = "Invalid iv_offset sector";
+ err = -EINVAL;
+ goto bad;
+ }
+
+ /* <dev_path> */
+ err = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table),
+ &dkc->dev);
+ if (err) {
+ ti->error = "Device lookup failed";
+ goto bad;
+ }
+
+ /* <start> */
+ if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 ||
+ tmpll != (sector_t)tmpll) {
+ ti->error = "Invalid start sector";
+ err = -EINVAL;
+ goto bad;
+ }
+ dkc->start = tmpll;
+
+ /* optional arguments */
+ dkc->sector_size = SECTOR_SIZE;
+ if (argc > 5) {
+ err = default_key_ctr_optional(ti, argc - 5, &argv[5]);
+ if (err)
+ goto bad;
+ }
+ dkc->sector_bits = ilog2(dkc->sector_size);
+ if (ti->len & ((dkc->sector_size >> SECTOR_SHIFT) - 1)) {
+ ti->error = "Device size is not a multiple of sector_size";
+ err = -EINVAL;
+ goto bad;
+ }
+
+ dkc->max_dun = (dkc->iv_offset + ti->len - 1) >>
+ (dkc->sector_bits - SECTOR_SHIFT);
+ dun_bytes = DIV_ROUND_UP(fls64(dkc->max_dun), 8);
+
+ err = blk_crypto_init_key(&dkc->key, raw_key, raw_key_size,
+ dkc->key_type, cipher->mode_num,
+ dun_bytes, dkc->sector_size);
+ if (err) {
+ ti->error = "Error initializing blk-crypto key";
+ goto bad;
+ }
+
+ err = blk_crypto_start_using_key(dkc->dev->bdev, &dkc->key);
+ if (err) {
+ ti->error = "Error starting to use blk-crypto";
+ goto bad;
+ }
+
+ ti->num_flush_bios = 1;
+
+ err = 0;
+ goto out;
+
+bad:
+ default_key_dtr(ti);
+out:
+ memzero_explicit(raw_key, sizeof(raw_key));
+ return err;
+}
+
+static int default_key_map(struct dm_target *ti, struct bio *bio)
+{
+ const struct default_key_c *dkc = ti->private;
+ sector_t sector_in_target;
+ u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE] = { 0 };
+
+ bio_set_dev(bio, dkc->dev->bdev);
+
+ /*
+ * If the bio is a device-level request which doesn't target a specific
+ * sector, there's nothing more to do.
+ */
+ if (bio_sectors(bio) == 0)
+ return DM_MAPIO_REMAPPED;
+
+ /* Map the bio's sector to the underlying device. (512-byte sectors) */
+ sector_in_target = dm_target_offset(ti, bio->bi_iter.bi_sector);
+ bio->bi_iter.bi_sector = dkc->start + sector_in_target;
+
+ /*
+ * If the bio should skip dm-default-key (i.e. if it's for an encrypted
+ * file's contents), or if it doesn't have any data (e.g. if it's a
+ * DISCARD request), there's nothing more to do.
+ */
+ if (bio_should_skip_dm_default_key(bio) || !bio_has_data(bio))
+ return DM_MAPIO_REMAPPED;
+
+ /*
+ * Else, dm-default-key needs to set this bio's encryption context.
+ * It must not already have one.
+ */
+ if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
+ return DM_MAPIO_KILL;
+
+ /* Calculate the DUN and enforce data-unit (crypto sector) alignment. */
+ dun[0] = dkc->iv_offset + sector_in_target; /* 512-byte sectors */
+ if (dun[0] & ((dkc->sector_size >> SECTOR_SHIFT) - 1))
+ return DM_MAPIO_KILL;
+ dun[0] >>= dkc->sector_bits - SECTOR_SHIFT; /* crypto sectors */
+
+ /*
+ * This check isn't necessary as we should have calculated max_dun
+ * correctly, but be safe.
+ */
+ if (WARN_ON_ONCE(dun[0] > dkc->max_dun))
+ return DM_MAPIO_KILL;
+
+ bio_crypt_set_ctx(bio, &dkc->key, dun, GFP_NOIO);
+
+ return DM_MAPIO_REMAPPED;
+}
+
+static void default_key_status(struct dm_target *ti, status_type_t type,
+ unsigned int status_flags, char *result,
+ unsigned int maxlen)
+{
+ const struct default_key_c *dkc = ti->private;
+ unsigned int sz = 0;
+ int num_feature_args = 0;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ case STATUSTYPE_IMA:
+ result[0] = '\0';
+ break;
+
+ case STATUSTYPE_TABLE:
+ /* Omit the key for now. */
+ DMEMIT("%s - %llu %s %llu", dkc->cipher_string, dkc->iv_offset,
+ dkc->dev->name, (unsigned long long)dkc->start);
+
+ num_feature_args += !!ti->num_discard_bios;
+ if (dkc->sector_size != SECTOR_SIZE)
+ num_feature_args += 2;
+ if (dkc->key_type == BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)
+ num_feature_args += 1;
+ if (num_feature_args != 0) {
+ DMEMIT(" %d", num_feature_args);
+ if (ti->num_discard_bios)
+ DMEMIT(" allow_discards");
+ if (dkc->sector_size != SECTOR_SIZE) {
+ DMEMIT(" sector_size:%u", dkc->sector_size);
+ DMEMIT(" iv_large_sectors");
+ }
+ if (dkc->key_type == BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)
+ DMEMIT(" wrappedkey_v0");
+ }
+ break;
+ }
+}
+
+static int default_key_prepare_ioctl(struct dm_target *ti,
+ struct block_device **bdev)
+{
+ const struct default_key_c *dkc = ti->private;
+ const struct dm_dev *dev = dkc->dev;
+
+ *bdev = dev->bdev;
+
+ /* Only pass ioctls through if the device sizes match exactly. */
+ if (dkc->start != 0 ||
+ ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT)
+ return 1;
+ return 0;
+}
+
+static int default_key_iterate_devices(struct dm_target *ti,
+ iterate_devices_callout_fn fn,
+ void *data)
+{
+ const struct default_key_c *dkc = ti->private;
+
+ return fn(ti, dkc->dev, dkc->start, ti->len, data);
+}
+
+static void default_key_io_hints(struct dm_target *ti,
+ struct queue_limits *limits)
+{
+ const struct default_key_c *dkc = ti->private;
+ const unsigned int sector_size = dkc->sector_size;
+
+ limits->logical_block_size =
+ max_t(unsigned int, limits->logical_block_size, sector_size);
+ limits->physical_block_size =
+ max_t(unsigned int, limits->physical_block_size, sector_size);
+ limits->io_min = max_t(unsigned int, limits->io_min, sector_size);
+}
+
+#ifdef CONFIG_BLK_DEV_ZONED
+static int default_key_report_zones(struct dm_target *ti,
+ struct dm_report_zones_args *args, unsigned int nr_zones)
+{
+ struct default_key_c *dkc = ti->private;
+
+ return dm_report_zones(dkc->dev->bdev, dkc->start,
+ dkc->start + dm_target_offset(ti, args->next_sector),
+ args, nr_zones);
+}
+#else
+#define default_key_report_zones NULL
+#endif
+
+static struct target_type default_key_target = {
+ .name = "default-key",
+ .version = {2, 1, 0},
+ .features = DM_TARGET_PASSES_CRYPTO | DM_TARGET_ZONED_HM,
+ .report_zones = default_key_report_zones,
+ .module = THIS_MODULE,
+ .ctr = default_key_ctr,
+ .dtr = default_key_dtr,
+ .map = default_key_map,
+ .status = default_key_status,
+ .prepare_ioctl = default_key_prepare_ioctl,
+ .iterate_devices = default_key_iterate_devices,
+ .io_hints = default_key_io_hints,
+};
+
+static int __init dm_default_key_init(void)
+{
+ return dm_register_target(&default_key_target);
+}
+
+static void __exit dm_default_key_exit(void)
+{
+ dm_unregister_target(&default_key_target);
+}
+
+module_init(dm_default_key_init);
+module_exit(dm_default_key_exit);
+
+MODULE_AUTHOR("Paul Lawrence <paullawrence@google.com>");
+MODULE_AUTHOR("Paul Crowley <paulcrowley@google.com>");
+MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
+MODULE_DESCRIPTION(DM_NAME " target for encrypting filesystem metadata");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 8541d56..e91afd6 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -1251,6 +1251,68 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile,
return args.err;
}
+struct dm_derive_sw_secret_args {
+ const u8 *eph_key;
+ size_t eph_key_size;
+ u8 *sw_secret;
+ int err;
+};
+
+static int dm_derive_sw_secret_callback(struct dm_target *ti,
+ struct dm_dev *dev, sector_t start,
+ sector_t len, void *data)
+{
+ struct dm_derive_sw_secret_args *args = data;
+
+ if (!args->err)
+ return 0;
+
+ args->err = blk_crypto_derive_sw_secret(dev->bdev,
+ args->eph_key,
+ args->eph_key_size,
+ args->sw_secret);
+ /* Try another device in case this fails. */
+ return 0;
+}
+
+/*
+ * Retrieve the sw_secret from the underlying device. Given that only one
+ * sw_secret can exist for a particular wrapped key, retrieve it only from the
+ * first device that supports derive_sw_secret().
+ */
+static int dm_derive_sw_secret(struct blk_crypto_profile *profile,
+ const u8 *eph_key, size_t eph_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
+{
+ struct mapped_device *md =
+ container_of(profile, struct dm_crypto_profile, profile)->md;
+ struct dm_derive_sw_secret_args args = {
+ .eph_key = eph_key,
+ .eph_key_size = eph_key_size,
+ .sw_secret = sw_secret,
+ .err = -EOPNOTSUPP,
+ };
+ struct dm_table *t;
+ int srcu_idx;
+ int i;
+ struct dm_target *ti;
+
+ t = dm_get_live_table(md, &srcu_idx);
+ if (!t)
+ return -EOPNOTSUPP;
+ for (i = 0; i < t->num_targets; i++) {
+ ti = dm_table_get_target(t, i);
+ if (!ti->type->iterate_devices)
+ continue;
+ ti->type->iterate_devices(ti, dm_derive_sw_secret_callback,
+ &args);
+ if (!args.err)
+ break;
+ }
+ dm_put_live_table(md, srcu_idx);
+ return args.err;
+}
+
static int
device_intersect_crypto_capabilities(struct dm_target *ti, struct dm_dev *dev,
sector_t start, sector_t len, void *data)
@@ -1306,9 +1368,11 @@ static int dm_table_construct_crypto_profile(struct dm_table *t)
profile = &dmcp->profile;
blk_crypto_profile_init(profile, 0);
profile->ll_ops.keyslot_evict = dm_keyslot_evict;
+ profile->ll_ops.derive_sw_secret = dm_derive_sw_secret;
profile->max_dun_bytes_supported = UINT_MAX;
memset(profile->modes_supported, 0xFF,
sizeof(profile->modes_supported));
+ profile->key_types_supported = ~0;
for (i = 0; i < t->num_targets; i++) {
struct dm_target *ti = dm_table_get_target(t, i);
diff --git a/drivers/md/dm-user.c b/drivers/md/dm-user.c
new file mode 100644
index 0000000..b3a4e2e
--- /dev/null
+++ b/drivers/md/dm-user.c
@@ -0,0 +1,1286 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2020 Google, Inc
+ * Copyright (C) 2020 Palmer Dabbelt <palmerdabbelt@google.com>
+ */
+
+#include <linux/device-mapper.h>
+#include <uapi/linux/dm-user.h>
+
+#include <linux/bio.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/uio.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+#define DM_MSG_PREFIX "user"
+
+#define MAX_OUTSTANDING_MESSAGES 128
+
+static unsigned int daemon_timeout_msec = 4000;
+module_param_named(dm_user_daemon_timeout_msec, daemon_timeout_msec, uint,
+ 0644);
+MODULE_PARM_DESC(dm_user_daemon_timeout_msec,
+ "IO Timeout in msec if daemon does not process");
+
+/*
+ * dm-user uses four structures:
+ *
+ * - "struct target", the outermost structure, corresponds to a single device
+ * mapper target. This contains the set of outstanding BIOs that have been
+ * provided by DM and are not actively being processed by the user, along
+ * with a misc device that userspace can open to communicate with the
+ * kernel. Each time userspaces opens the misc device a new channel is
+ * created.
+ * - "struct channel", which represents a single active communication channel
+ * with userspace. Userspace may choose arbitrary read/write sizes to use
+ * when processing messages, channels form these into logical accesses.
+ * When userspace responds to a full message the channel completes the BIO
+ * and obtains a new message to process from the target.
+ * - "struct message", which wraps a BIO with the additional information
+ * required by the kernel to sort out what to do with BIOs when they return
+ * from userspace.
+ * - "struct dm_user_message", which is the exact message format that
+ * userspace sees.
+ *
+ * The hot path contains three distinct operations:
+ *
+ * - user_map(), which is provided a BIO from device mapper that is queued
+ * into the target. This allocates and enqueues a new message.
+ * - dev_read(), which dequeues a message, copies it to userspace.
+ * - dev_write(), which looks up a message (keyed by sequence number) and
+ * completes the corresponding BIO.
+ *
+ * Lock ordering (outer to inner)
+ *
+ * 1) miscdevice's global lock. This is held around dev_open, so it has to be
+ * the outermost lock.
+ * 2) target->lock
+ * 3) channel->lock
+ */
+
+struct message {
+ /*
+ * Messages themselves do not need a lock, they're protected by either
+ * the target or channel's lock, depending on which can reference them
+ * directly.
+ */
+ struct dm_user_message msg;
+ struct bio *bio;
+ size_t posn_to_user;
+ size_t total_to_user;
+ size_t posn_from_user;
+ size_t total_from_user;
+
+ struct list_head from_user;
+ struct list_head to_user;
+
+ /*
+ * These are written back from the user. They live in the same spot in
+ * the message, but we need to either keep the old values around or
+ * call a bunch more BIO helpers. These are only valid after write has
+ * adopted the message.
+ */
+ u64 return_type;
+ u64 return_flags;
+
+ struct delayed_work work;
+ bool delayed;
+ struct target *t;
+};
+
+struct target {
+ /*
+ * A target has a single lock, which protects everything in the target
+ * (but does not protect the channels associated with a target).
+ */
+ struct mutex lock;
+
+ /*
+ * There is only one point at which anything blocks: userspace blocks
+ * reading a new message, which is woken up by device mapper providing
+ * a new BIO to process (or tearing down the target). The
+ * corresponding write side doesn't block, instead we treat userspace's
+ * response containing a message that has yet to be mapped as an
+ * invalid operation.
+ */
+ struct wait_queue_head wq;
+
+ /*
+ * Messages are delivered to userspace in order, but may be returned
+ * out of order. This allows userspace to schedule IO if it wants to.
+ */
+ mempool_t message_pool;
+ u64 next_seq_to_map;
+ u64 next_seq_to_user;
+ struct list_head to_user;
+
+ /*
+ * There is a misc device per target. The name is selected by
+ * userspace (via a DM create ioctl argument), and each ends up in
+ * /dev/dm-user/. It looks like a better way to do this may be to have
+ * a filesystem to manage these, but this was more expedient. The
+ * current mechanism is functional, but does result in an arbitrary
+ * number of dynamically created misc devices.
+ */
+ struct miscdevice miscdev;
+
+ /*
+ * Device mapper's target destructor triggers tearing this all down,
+ * but we can't actually free until every channel associated with this
+ * target has been destroyed. Channels each have a reference to their
+ * target, and there is an additional single reference that corresponds
+ * to both DM and the misc device (both of which are destroyed by DM).
+ *
+ * In the common case userspace will be asleep waiting for a new
+ * message when device mapper decides to destroy the target, which
+ * means no new messages will appear. The destroyed flag triggers a
+ * wakeup, which will end up removing the reference.
+ */
+ struct kref references;
+ int dm_destroyed;
+ bool daemon_terminated;
+};
+
+struct channel {
+ struct target *target;
+
+ /*
+ * A channel has a single lock, which prevents multiple reads (or
+ * multiple writes) from conflicting with each other.
+ */
+ struct mutex lock;
+
+ struct message *cur_to_user;
+ struct message *cur_from_user;
+ ssize_t to_user_error;
+ ssize_t from_user_error;
+
+ /*
+ * Once a message has been forwarded to userspace on a channel it must
+ * be responded to on the same channel. This allows us to error out
+ * the messages that have not yet been responded to by a channel when
+ * that channel closes, which makes handling errors more reasonable for
+ * fault-tolerant userspace daemons. It also happens to make avoiding
+ * shared locks between user_map() and dev_read() a lot easier.
+ *
+ * This does preclude a multi-threaded work stealing userspace
+ * implementation (or at least, force a degree of head-of-line blocking
+ * on the response path).
+ */
+ struct list_head from_user;
+
+ /*
+ * Responses from userspace can arrive in arbitrarily small chunks.
+ * We need some place to buffer one up until we can find the
+ * corresponding kernel-side message to continue processing, so instead
+ * of allocating them we just keep one off to the side here. This can
+ * only ever be pointer to by from_user_cur, and will never have a BIO.
+ */
+ struct message scratch_message_from_user;
+};
+
+static void message_kill(struct message *m, mempool_t *pool)
+{
+ m->bio->bi_status = BLK_STS_IOERR;
+ bio_endio(m->bio);
+ mempool_free(m, pool);
+}
+
+static inline bool is_user_space_thread_present(struct target *t)
+{
+ lockdep_assert_held(&t->lock);
+ return (kref_read(&t->references) > 1);
+}
+
+static void process_delayed_work(struct work_struct *work)
+{
+ struct delayed_work *del_work = to_delayed_work(work);
+ struct message *msg = container_of(del_work, struct message, work);
+
+ struct target *t = msg->t;
+
+ mutex_lock(&t->lock);
+
+ /*
+ * There is at least one thread to process the IO.
+ */
+ if (is_user_space_thread_present(t)) {
+ mutex_unlock(&t->lock);
+ return;
+ }
+
+ /*
+ * Terminate the IO with an error
+ */
+ list_del(&msg->to_user);
+ pr_err("I/O error: sector %llu: no user-space daemon for %s target\n",
+ msg->bio->bi_iter.bi_sector,
+ t->miscdev.name);
+ message_kill(msg, &t->message_pool);
+ mutex_unlock(&t->lock);
+}
+
+static void enqueue_delayed_work(struct message *m, bool is_delay)
+{
+ unsigned long delay = 0;
+
+ m->delayed = true;
+ INIT_DELAYED_WORK(&m->work, process_delayed_work);
+
+ /*
+ * Snapuserd daemon is the user-space process
+ * which processes IO request from dm-user
+ * when OTA is applied. Per the current design,
+ * when a dm-user target is created, daemon
+ * attaches to target and starts processing
+ * the IO's. Daemon is terminated only when
+ * dm-user target is destroyed.
+ *
+ * If for some reason, daemon crashes or terminates early,
+ * without destroying the dm-user target; then
+ * there is no mechanism to restart the daemon
+ * and start processing the IO's from the same target.
+ * Theoretically, it is possible but that infrastructure
+ * doesn't exist in the android ecosystem.
+ *
+ * Thus, when the daemon terminates, there is no way the IO's
+ * issued on that target will be processed. Hence,
+ * we set the delay to 0 and fail the IO's immediately.
+ *
+ * On the other hand, when a new dm-user target is created,
+ * we wait for the daemon to get attached for the first time.
+ * This primarily happens when init first stage spins up
+ * the daemon. At this point, since the snapshot device is mounted
+ * of a root filesystem, dm-user target may receive IO request
+ * even though daemon is not fully launched. We don't want
+ * to fail those IO requests immediately. Thus, we queue these
+ * requests with a timeout so that daemon is ready to process
+ * those IO requests. Again, if the daemon fails to launch within
+ * the timeout period, then IO's will be failed.
+ */
+ if (is_delay)
+ delay = msecs_to_jiffies(daemon_timeout_msec);
+
+ queue_delayed_work(system_wq, &m->work, delay);
+}
+
+static inline struct target *target_from_target(struct dm_target *target)
+{
+ WARN_ON(target->private == NULL);
+ return target->private;
+}
+
+static inline struct target *target_from_miscdev(struct miscdevice *miscdev)
+{
+ return container_of(miscdev, struct target, miscdev);
+}
+
+static inline struct channel *channel_from_file(struct file *file)
+{
+ WARN_ON(file->private_data == NULL);
+ return file->private_data;
+}
+
+static inline struct target *target_from_channel(struct channel *c)
+{
+ WARN_ON(c->target == NULL);
+ return c->target;
+}
+
+static inline size_t bio_size(struct bio *bio)
+{
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+ size_t out = 0;
+
+ bio_for_each_segment (bvec, bio, iter)
+ out += bio_iter_len(bio, iter);
+ return out;
+}
+
+static inline size_t bio_bytes_needed_to_user(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_WRITE:
+ return sizeof(struct dm_user_message) + bio_size(bio);
+ case REQ_OP_READ:
+ case REQ_OP_FLUSH:
+ case REQ_OP_DISCARD:
+ case REQ_OP_SECURE_ERASE:
+ case REQ_OP_WRITE_ZEROES:
+ return sizeof(struct dm_user_message);
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline size_t bio_bytes_needed_from_user(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_READ:
+ return sizeof(struct dm_user_message) + bio_size(bio);
+ case REQ_OP_WRITE:
+ case REQ_OP_FLUSH:
+ case REQ_OP_DISCARD:
+ case REQ_OP_SECURE_ERASE:
+ case REQ_OP_WRITE_ZEROES:
+ return sizeof(struct dm_user_message);
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline long bio_type_to_user_type(struct bio *bio)
+{
+ switch (bio_op(bio)) {
+ case REQ_OP_READ:
+ return DM_USER_REQ_MAP_READ;
+ case REQ_OP_WRITE:
+ return DM_USER_REQ_MAP_WRITE;
+ case REQ_OP_FLUSH:
+ return DM_USER_REQ_MAP_FLUSH;
+ case REQ_OP_DISCARD:
+ return DM_USER_REQ_MAP_DISCARD;
+ case REQ_OP_SECURE_ERASE:
+ return DM_USER_REQ_MAP_SECURE_ERASE;
+ case REQ_OP_WRITE_ZEROES:
+ return DM_USER_REQ_MAP_WRITE_ZEROES;
+
+ /*
+ * These ops are not passed to userspace under the assumption that
+ * they're not going to be particularly useful in that context.
+ */
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static inline long bio_flags_to_user_flags(struct bio *bio)
+{
+ u64 out = 0;
+ typeof(bio->bi_opf) opf = bio->bi_opf & ~REQ_OP_MASK;
+
+ if (opf & REQ_FAILFAST_DEV) {
+ opf &= ~REQ_FAILFAST_DEV;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DEV;
+ }
+
+ if (opf & REQ_FAILFAST_TRANSPORT) {
+ opf &= ~REQ_FAILFAST_TRANSPORT;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_TRANSPORT;
+ }
+
+ if (opf & REQ_FAILFAST_DRIVER) {
+ opf &= ~REQ_FAILFAST_DRIVER;
+ out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DRIVER;
+ }
+
+ if (opf & REQ_SYNC) {
+ opf &= ~REQ_SYNC;
+ out |= DM_USER_REQ_MAP_FLAG_SYNC;
+ }
+
+ if (opf & REQ_META) {
+ opf &= ~REQ_META;
+ out |= DM_USER_REQ_MAP_FLAG_META;
+ }
+
+ if (opf & REQ_PRIO) {
+ opf &= ~REQ_PRIO;
+ out |= DM_USER_REQ_MAP_FLAG_PRIO;
+ }
+
+ if (opf & REQ_NOMERGE) {
+ opf &= ~REQ_NOMERGE;
+ out |= DM_USER_REQ_MAP_FLAG_NOMERGE;
+ }
+
+ if (opf & REQ_IDLE) {
+ opf &= ~REQ_IDLE;
+ out |= DM_USER_REQ_MAP_FLAG_IDLE;
+ }
+
+ if (opf & REQ_INTEGRITY) {
+ opf &= ~REQ_INTEGRITY;
+ out |= DM_USER_REQ_MAP_FLAG_INTEGRITY;
+ }
+
+ if (opf & REQ_FUA) {
+ opf &= ~REQ_FUA;
+ out |= DM_USER_REQ_MAP_FLAG_FUA;
+ }
+
+ if (opf & REQ_PREFLUSH) {
+ opf &= ~REQ_PREFLUSH;
+ out |= DM_USER_REQ_MAP_FLAG_PREFLUSH;
+ }
+
+ if (opf & REQ_RAHEAD) {
+ opf &= ~REQ_RAHEAD;
+ out |= DM_USER_REQ_MAP_FLAG_RAHEAD;
+ }
+
+ if (opf & REQ_BACKGROUND) {
+ opf &= ~REQ_BACKGROUND;
+ out |= DM_USER_REQ_MAP_FLAG_BACKGROUND;
+ }
+
+ if (opf & REQ_NOWAIT) {
+ opf &= ~REQ_NOWAIT;
+ out |= DM_USER_REQ_MAP_FLAG_NOWAIT;
+ }
+
+ if (opf & REQ_NOUNMAP) {
+ opf &= ~REQ_NOUNMAP;
+ out |= DM_USER_REQ_MAP_FLAG_NOUNMAP;
+ }
+
+ if (unlikely(opf)) {
+ pr_warn("unsupported BIO type %x\n", opf);
+ return -EOPNOTSUPP;
+ }
+ WARN_ON(out < 0);
+ return out;
+}
+
+/*
+ * Not quite what's in blk-map.c, but instead what I thought the functions in
+ * blk-map did. This one seems more generally useful and I think we could
+ * write the blk-map version in terms of this one. The differences are that
+ * this has a return value that counts, and blk-map uses the BIO _all iters.
+ * Neither advance the BIO iter but don't advance the IOV iter, which is a bit
+ * odd here.
+ */
+static ssize_t bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
+{
+ struct bio_vec bvec;
+ struct bvec_iter biter;
+ ssize_t out = 0;
+
+ bio_for_each_segment (bvec, bio, biter) {
+ ssize_t ret;
+
+ ret = copy_page_from_iter(bvec.bv_page, bvec.bv_offset,
+ bvec.bv_len, iter);
+
+ /*
+ * FIXME: I thought that IOV copies had a mechanism for
+ * terminating early, if for example a signal came in while
+ * sleeping waiting for a page to be mapped, but I don't see
+ * where that would happen.
+ */
+ WARN_ON(ret < 0);
+ out += ret;
+
+ if (!iov_iter_count(iter))
+ break;
+
+ if (ret < bvec.bv_len)
+ return ret;
+ }
+
+ return out;
+}
+
+static ssize_t bio_copy_to_iter(struct bio *bio, struct iov_iter *iter)
+{
+ struct bio_vec bvec;
+ struct bvec_iter biter;
+ ssize_t out = 0;
+
+ bio_for_each_segment (bvec, bio, biter) {
+ ssize_t ret;
+
+ ret = copy_page_to_iter(bvec.bv_page, bvec.bv_offset,
+ bvec.bv_len, iter);
+
+ /* as above */
+ WARN_ON(ret < 0);
+ out += ret;
+
+ if (!iov_iter_count(iter))
+ break;
+
+ if (ret < bvec.bv_len)
+ return ret;
+ }
+
+ return out;
+}
+
+static ssize_t msg_copy_to_iov(struct message *msg, struct iov_iter *to)
+{
+ ssize_t copied = 0;
+
+ if (!iov_iter_count(to))
+ return 0;
+
+ if (msg->posn_to_user < sizeof(msg->msg)) {
+ copied = copy_to_iter((char *)(&msg->msg) + msg->posn_to_user,
+ sizeof(msg->msg) - msg->posn_to_user, to);
+ } else {
+ copied = bio_copy_to_iter(msg->bio, to);
+ if (copied > 0)
+ bio_advance(msg->bio, copied);
+ }
+
+ if (copied < 0)
+ return copied;
+
+ msg->posn_to_user += copied;
+ return copied;
+}
+
+static ssize_t msg_copy_from_iov(struct message *msg, struct iov_iter *from)
+{
+ ssize_t copied = 0;
+
+ if (!iov_iter_count(from))
+ return 0;
+
+ if (msg->posn_from_user < sizeof(msg->msg)) {
+ copied = copy_from_iter(
+ (char *)(&msg->msg) + msg->posn_from_user,
+ sizeof(msg->msg) - msg->posn_from_user, from);
+ } else {
+ copied = bio_copy_from_iter(msg->bio, from);
+ if (copied > 0)
+ bio_advance(msg->bio, copied);
+ }
+
+ if (copied < 0)
+ return copied;
+
+ msg->posn_from_user += copied;
+ return copied;
+}
+
+static struct message *msg_get_map(struct target *t)
+{
+ struct message *m;
+
+ lockdep_assert_held(&t->lock);
+
+ m = mempool_alloc(&t->message_pool, GFP_NOIO);
+ m->msg.seq = t->next_seq_to_map++;
+ INIT_LIST_HEAD(&m->to_user);
+ INIT_LIST_HEAD(&m->from_user);
+ return m;
+}
+
+static struct message *msg_get_to_user(struct target *t)
+{
+ struct message *m;
+
+ lockdep_assert_held(&t->lock);
+
+ if (list_empty(&t->to_user))
+ return NULL;
+
+ m = list_first_entry(&t->to_user, struct message, to_user);
+
+ list_del(&m->to_user);
+
+ /*
+ * If the IO was queued to workqueue since there
+ * was no daemon to service the IO, then we
+ * will have to cancel the delayed work as the
+ * IO will be processed by this user-space thread.
+ *
+ * If the delayed work was already picked up for
+ * processing, then wait for it to complete. Note
+ * that the IO will not be terminated by the work
+ * queue thread.
+ */
+ if (unlikely(m->delayed)) {
+ mutex_unlock(&t->lock);
+ cancel_delayed_work_sync(&m->work);
+ mutex_lock(&t->lock);
+ }
+ return m;
+}
+
+static struct message *msg_get_from_user(struct channel *c, u64 seq)
+{
+ struct message *m;
+ struct list_head *cur, *tmp;
+
+ lockdep_assert_held(&c->lock);
+
+ list_for_each_safe (cur, tmp, &c->from_user) {
+ m = list_entry(cur, struct message, from_user);
+ if (m->msg.seq == seq) {
+ list_del(&m->from_user);
+ return m;
+ }
+ }
+
+ return NULL;
+}
+
+/*
+ * Returns 0 when there is no work left to do. This must be callable without
+ * holding the target lock, as it is part of the waitqueue's check expression.
+ * When called without the lock it may spuriously indicate there is remaining
+ * work, but when called with the lock it must be accurate.
+ */
+static int target_poll(struct target *t)
+{
+ return !list_empty(&t->to_user) || t->dm_destroyed;
+}
+
+static void target_release(struct kref *ref)
+{
+ struct target *t = container_of(ref, struct target, references);
+ struct list_head *cur, *tmp;
+
+ /*
+ * There may be outstanding BIOs that have not yet been given to
+ * userspace. At this point there's nothing we can do about them, as
+ * there are and will never be any channels.
+ */
+ list_for_each_safe (cur, tmp, &t->to_user) {
+ struct message *m = list_entry(cur, struct message, to_user);
+
+ if (unlikely(m->delayed)) {
+ bool ret;
+
+ mutex_unlock(&t->lock);
+ ret = cancel_delayed_work_sync(&m->work);
+ mutex_lock(&t->lock);
+ if (!ret)
+ continue;
+ }
+ message_kill(m, &t->message_pool);
+ }
+
+ mempool_exit(&t->message_pool);
+ mutex_unlock(&t->lock);
+ mutex_destroy(&t->lock);
+ kfree(t);
+}
+
+static void target_put(struct target *t)
+{
+ /*
+ * This both releases a reference to the target and the lock. We leave
+ * it up to the caller to hold the lock, as they probably needed it for
+ * something else.
+ */
+ lockdep_assert_held(&t->lock);
+
+ if (!kref_put(&t->references, target_release)) {
+ /*
+ * User-space thread is getting terminated.
+ * We need to scan the list for all those
+ * pending IO's which were not processed yet
+ * and put them back to work-queue for delayed
+ * processing.
+ */
+ if (!is_user_space_thread_present(t)) {
+ struct list_head *cur, *tmp;
+
+ list_for_each_safe(cur, tmp, &t->to_user) {
+ struct message *m = list_entry(cur,
+ struct message,
+ to_user);
+ if (!m->delayed)
+ enqueue_delayed_work(m, false);
+ }
+ /*
+ * Daemon attached to this target is terminated.
+ */
+ t->daemon_terminated = true;
+ }
+ mutex_unlock(&t->lock);
+ }
+}
+
+static struct channel *channel_alloc(struct target *t)
+{
+ struct channel *c;
+
+ lockdep_assert_held(&t->lock);
+
+ c = kzalloc(sizeof(*c), GFP_KERNEL);
+ if (c == NULL)
+ return NULL;
+
+ kref_get(&t->references);
+ c->target = t;
+ c->cur_from_user = &c->scratch_message_from_user;
+ mutex_init(&c->lock);
+ INIT_LIST_HEAD(&c->from_user);
+ return c;
+}
+
+static void channel_free(struct channel *c)
+{
+ struct list_head *cur, *tmp;
+
+ lockdep_assert_held(&c->lock);
+
+ /*
+ * There may be outstanding BIOs that have been given to userspace but
+ * have not yet been completed. The channel has been shut down so
+ * there's no way to process the rest of those messages, so we just go
+ * ahead and error out the BIOs. Hopefully whatever's on the other end
+ * can handle the errors. One could imagine splitting the BIOs and
+ * completing as much as we got, but that seems like overkill here.
+ *
+ * Our only other options would be to let the BIO hang around (which
+ * seems way worse) or to resubmit it to userspace in the hope there's
+ * another channel. I don't really like the idea of submitting a
+ * message twice.
+ */
+ if (c->cur_to_user != NULL)
+ message_kill(c->cur_to_user, &c->target->message_pool);
+ if (c->cur_from_user != &c->scratch_message_from_user)
+ message_kill(c->cur_from_user, &c->target->message_pool);
+ list_for_each_safe (cur, tmp, &c->from_user)
+ message_kill(list_entry(cur, struct message, from_user),
+ &c->target->message_pool);
+
+ mutex_lock(&c->target->lock);
+ target_put(c->target);
+ mutex_unlock(&c->lock);
+ mutex_destroy(&c->lock);
+ kfree(c);
+}
+
+static int dev_open(struct inode *inode, struct file *file)
+{
+ struct channel *c;
+ struct target *t;
+
+ /*
+ * This is called by miscdev, which sets private_data to point to the
+ * struct miscdevice that was opened. The rest of our file operations
+ * want to refer to the channel that's been opened, so we swap that
+ * pointer out with a fresh channel.
+ *
+ * This is called with the miscdev lock held, which is also held while
+ * registering/unregistering the miscdev. The miscdev must be
+ * registered for this to get called, which means there must be an
+ * outstanding reference to the target, which means it cannot be freed
+ * out from under us despite us not holding a reference yet.
+ */
+ t = container_of(file->private_data, struct target, miscdev);
+ mutex_lock(&t->lock);
+ file->private_data = c = channel_alloc(t);
+
+ if (c == NULL) {
+ mutex_unlock(&t->lock);
+ return -ENOMEM;
+ }
+
+ mutex_unlock(&t->lock);
+ return 0;
+}
+
+static ssize_t dev_read(struct kiocb *iocb, struct iov_iter *to)
+{
+ struct channel *c = channel_from_file(iocb->ki_filp);
+ ssize_t total_processed = 0;
+ ssize_t processed;
+
+ mutex_lock(&c->lock);
+
+ if (unlikely(c->to_user_error)) {
+ total_processed = c->to_user_error;
+ goto cleanup_unlock;
+ }
+
+ if (c->cur_to_user == NULL) {
+ struct target *t = target_from_channel(c);
+
+ mutex_lock(&t->lock);
+
+ while (!target_poll(t)) {
+ int e;
+
+ mutex_unlock(&t->lock);
+ mutex_unlock(&c->lock);
+ e = wait_event_interruptible(t->wq, target_poll(t));
+ mutex_lock(&c->lock);
+ mutex_lock(&t->lock);
+
+ if (unlikely(e != 0)) {
+ /*
+ * We haven't processed any bytes in either the
+ * BIO or the IOV, so we can just terminate
+ * right now. Elsewhere in the kernel handles
+ * restarting the syscall when appropriate.
+ */
+ total_processed = e;
+ mutex_unlock(&t->lock);
+ goto cleanup_unlock;
+ }
+ }
+
+ if (unlikely(t->dm_destroyed)) {
+ /*
+ * DM has destroyed this target, so just lock
+ * the user out. There's really nothing else
+ * we can do here. Note that we don't actually
+ * tear any thing down until userspace has
+ * closed the FD, as there may still be
+ * outstanding BIOs.
+ *
+ * This is kind of a wacky error code to
+ * return. My goal was really just to try and
+ * find something that wasn't likely to be
+ * returned by anything else in the miscdev
+ * path. The message "block device required"
+ * seems like a somewhat reasonable thing to
+ * say when the target has disappeared out from
+ * under us, but "not block" isn't sensible.
+ */
+ c->to_user_error = total_processed = -ENOTBLK;
+ mutex_unlock(&t->lock);
+ goto cleanup_unlock;
+ }
+
+ /*
+ * Ensures that accesses to the message data are not ordered
+ * before the remote accesses that produce that message data.
+ *
+ * This pairs with the barrier in user_map(), via the
+ * conditional within the while loop above. Also see the lack
+ * of barrier in user_dtr(), which is why this can be after the
+ * destroyed check.
+ */
+ smp_rmb();
+
+ c->cur_to_user = msg_get_to_user(t);
+ WARN_ON(c->cur_to_user == NULL);
+ mutex_unlock(&t->lock);
+ }
+
+ processed = msg_copy_to_iov(c->cur_to_user, to);
+ total_processed += processed;
+
+ WARN_ON(c->cur_to_user->posn_to_user > c->cur_to_user->total_to_user);
+ if (c->cur_to_user->posn_to_user == c->cur_to_user->total_to_user) {
+ struct message *m = c->cur_to_user;
+
+ c->cur_to_user = NULL;
+ list_add_tail(&m->from_user, &c->from_user);
+ }
+
+cleanup_unlock:
+ mutex_unlock(&c->lock);
+ return total_processed;
+}
+
+static ssize_t dev_write(struct kiocb *iocb, struct iov_iter *from)
+{
+ struct channel *c = channel_from_file(iocb->ki_filp);
+ ssize_t total_processed = 0;
+ ssize_t processed;
+
+ mutex_lock(&c->lock);
+
+ if (unlikely(c->from_user_error)) {
+ total_processed = c->from_user_error;
+ goto cleanup_unlock;
+ }
+
+ /*
+ * cur_from_user can never be NULL. If there's no real message it must
+ * point to the scratch space.
+ */
+ WARN_ON(c->cur_from_user == NULL);
+ if (c->cur_from_user->posn_from_user < sizeof(struct dm_user_message)) {
+ struct message *msg, *old;
+
+ processed = msg_copy_from_iov(c->cur_from_user, from);
+ if (processed <= 0) {
+ pr_warn("msg_copy_from_iov() returned %zu\n",
+ processed);
+ c->from_user_error = -EINVAL;
+ goto cleanup_unlock;
+ }
+ total_processed += processed;
+
+ /*
+ * In the unlikely event the user has provided us a very short
+ * write, not even big enough to fill a message, just succeed.
+ * We'll eventually build up enough bytes to do something.
+ */
+ if (unlikely(c->cur_from_user->posn_from_user <
+ sizeof(struct dm_user_message)))
+ goto cleanup_unlock;
+
+ old = c->cur_from_user;
+ mutex_lock(&c->target->lock);
+ msg = msg_get_from_user(c, c->cur_from_user->msg.seq);
+ if (msg == NULL) {
+ pr_info("user provided an invalid messag seq of %llx\n",
+ old->msg.seq);
+ mutex_unlock(&c->target->lock);
+ c->from_user_error = -EINVAL;
+ goto cleanup_unlock;
+ }
+ mutex_unlock(&c->target->lock);
+
+ WARN_ON(old->posn_from_user != sizeof(struct dm_user_message));
+ msg->posn_from_user = sizeof(struct dm_user_message);
+ msg->return_type = old->msg.type;
+ msg->return_flags = old->msg.flags;
+ WARN_ON(msg->posn_from_user > msg->total_from_user);
+ c->cur_from_user = msg;
+ WARN_ON(old != &c->scratch_message_from_user);
+ }
+
+ /*
+ * Userspace can signal an error for single requests by overwriting the
+ * seq field.
+ */
+ switch (c->cur_from_user->return_type) {
+ case DM_USER_RESP_SUCCESS:
+ c->cur_from_user->bio->bi_status = BLK_STS_OK;
+ break;
+ case DM_USER_RESP_ERROR:
+ case DM_USER_RESP_UNSUPPORTED:
+ default:
+ c->cur_from_user->bio->bi_status = BLK_STS_IOERR;
+ goto finish_bio;
+ }
+
+ /*
+ * The op was a success as far as userspace is concerned, so process
+ * whatever data may come along with it. The user may provide the BIO
+ * data in multiple chunks, in which case we don't need to finish the
+ * BIO.
+ */
+ processed = msg_copy_from_iov(c->cur_from_user, from);
+ total_processed += processed;
+
+ if (c->cur_from_user->posn_from_user <
+ c->cur_from_user->total_from_user)
+ goto cleanup_unlock;
+
+finish_bio:
+ /*
+ * When we set up this message the BIO's size matched the
+ * message size, if that's not still the case then something
+ * has gone off the rails.
+ */
+ WARN_ON(bio_size(c->cur_from_user->bio) != 0);
+ bio_endio(c->cur_from_user->bio);
+
+ /*
+ * We don't actually need to take the target lock here, as all
+ * we're doing is freeing the message and mempools have their
+ * own lock. Each channel has its ows scratch message.
+ */
+ WARN_ON(c->cur_from_user == &c->scratch_message_from_user);
+ mempool_free(c->cur_from_user, &c->target->message_pool);
+ c->scratch_message_from_user.posn_from_user = 0;
+ c->cur_from_user = &c->scratch_message_from_user;
+
+cleanup_unlock:
+ mutex_unlock(&c->lock);
+ return total_processed;
+}
+
+static int dev_release(struct inode *inode, struct file *file)
+{
+ struct channel *c;
+
+ c = channel_from_file(file);
+ mutex_lock(&c->lock);
+ channel_free(c);
+
+ return 0;
+}
+
+static const struct file_operations file_operations = {
+ .owner = THIS_MODULE,
+ .open = dev_open,
+ .llseek = no_llseek,
+ .read_iter = dev_read,
+ .write_iter = dev_write,
+ .release = dev_release,
+};
+
+static int user_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct target *t;
+ int r;
+
+ if (argc != 3) {
+ ti->error = "Invalid argument count";
+ r = -EINVAL;
+ goto cleanup_none;
+ }
+
+ t = kzalloc(sizeof(*t), GFP_KERNEL);
+ if (t == NULL) {
+ r = -ENOMEM;
+ goto cleanup_none;
+ }
+ ti->private = t;
+
+ /* Enable more BIO types. */
+ ti->num_discard_bios = 1;
+ ti->discards_supported = true;
+ ti->num_flush_bios = 1;
+ ti->flush_supported = true;
+
+ /*
+ * We begin with a single reference to the target, which is miscdev's
+ * reference. This ensures that the target won't be freed
+ * until after the miscdev has been unregistered and all extant
+ * channels have been closed.
+ */
+ kref_init(&t->references);
+
+ t->daemon_terminated = false;
+ mutex_init(&t->lock);
+ init_waitqueue_head(&t->wq);
+ INIT_LIST_HEAD(&t->to_user);
+ mempool_init_kmalloc_pool(&t->message_pool, MAX_OUTSTANDING_MESSAGES,
+ sizeof(struct message));
+
+ t->miscdev.minor = MISC_DYNAMIC_MINOR;
+ t->miscdev.fops = &file_operations;
+ t->miscdev.name = kasprintf(GFP_KERNEL, "dm-user/%s", argv[2]);
+ if (t->miscdev.name == NULL) {
+ r = -ENOMEM;
+ goto cleanup_message_pool;
+ }
+
+ /*
+ * Once the miscdev is registered it can be opened and therefor
+ * concurrent references to the channel can happen. Holding the target
+ * lock during misc_register() could deadlock. If registration
+ * succeeds then we will not access the target again so we just stick a
+ * barrier here, which pairs with taking the target lock everywhere
+ * else the target is accessed.
+ *
+ * I forgot where we ended up on the RCpc/RCsc locks. IIU RCsc locks
+ * would mean that we could take the target lock earlier and release it
+ * here instead of the memory barrier. I'm not sure that's any better,
+ * though, and this isn't on a hot path so it probably doesn't matter
+ * either way.
+ */
+ smp_mb();
+
+ r = misc_register(&t->miscdev);
+ if (r) {
+ DMERR("Unable to register miscdev %s for dm-user",
+ t->miscdev.name);
+ r = -ENOMEM;
+ goto cleanup_misc_name;
+ }
+
+ return 0;
+
+cleanup_misc_name:
+ kfree(t->miscdev.name);
+cleanup_message_pool:
+ mempool_exit(&t->message_pool);
+ kfree(t);
+cleanup_none:
+ return r;
+}
+
+static void user_dtr(struct dm_target *ti)
+{
+ struct target *t = target_from_target(ti);
+
+ /*
+ * Removes the miscdev. This must be called without the target lock
+ * held to avoid a possible deadlock because our open implementation is
+ * called holding the miscdev lock and must later take the target lock.
+ *
+ * There is no race here because only DM can register/unregister the
+ * miscdev, and DM ensures that doesn't happen twice. The internal
+ * miscdev lock is sufficient to ensure there are no races between
+ * deregistering the miscdev and open.
+ */
+ misc_deregister(&t->miscdev);
+
+ /*
+ * We are now free to take the target's lock and drop our reference to
+ * the target. There are almost certainly tasks sleeping in read on at
+ * least one of the channels associated with this target, this
+ * explicitly wakes them up and terminates the read.
+ */
+ mutex_lock(&t->lock);
+ /*
+ * No barrier here, as wait/wake ensures that the flag visibility is
+ * correct WRT the wake/sleep state of the target tasks.
+ */
+ t->dm_destroyed = true;
+ wake_up_all(&t->wq);
+ target_put(t);
+}
+
+/*
+ * Consumes a BIO from device mapper, queueing it up for userspace.
+ */
+static int user_map(struct dm_target *ti, struct bio *bio)
+{
+ struct target *t;
+ struct message *entry;
+
+ t = target_from_target(ti);
+ /*
+ * FIXME
+ *
+ * This seems like a bad idea. Specifically, here we're
+ * directly on the IO path when we take the target lock, which may also
+ * be taken from a user context. The user context doesn't actively
+ * trigger anything that may sleep while holding the lock, but this
+ * still seems like a bad idea.
+ *
+ * The obvious way to fix this would be to use a proper queue, which
+ * would result in no shared locks between the direct IO path and user
+ * tasks. I had a version that did this, but the head-of-line blocking
+ * from the circular buffer resulted in us needing a fairly large
+ * allocation in order to avoid situations in which the queue fills up
+ * and everything goes off the rails.
+ *
+ * I could jump through a some hoops to avoid a shared lock while still
+ * allowing for a large queue, but I'm not actually sure that allowing
+ * for very large queues is the right thing to do here. Intuitively it
+ * seems better to keep the queues small in here (essentially sized to
+ * the user latency for performance reasons only) and rely on returning
+ * DM_MAPIO_REQUEUE regularly, as that would give the rest of the
+ * kernel more information.
+ *
+ * I'll spend some time trying to figure out what's going on with
+ * DM_MAPIO_REQUEUE, but if someone has a better idea of how to fix
+ * this I'm all ears.
+ */
+ mutex_lock(&t->lock);
+
+ /*
+ * FIXME
+ *
+ * The assumption here is that there's no benefit to returning
+ * DM_MAPIO_KILL as opposed to just erroring out the BIO, but I'm not
+ * sure that's actually true -- for example, I could imagine users
+ * expecting that submitted BIOs are unlikely to fail and therefor
+ * relying on submission failure to indicate an unsupported type.
+ *
+ * There's two ways I can think of to fix this:
+ * - Add DM arguments that are parsed during the constructor that
+ * allow various dm_target flags to be set that indicate the op
+ * types supported by this target. This may make sense for things
+ * like discard, where DM can already transform the BIOs to a form
+ * that's likely to be supported.
+ * - Some sort of pre-filter that allows userspace to hook in here
+ * and kill BIOs before marking them as submitted. My guess would
+ * be that a userspace round trip is a bad idea here, but a BPF
+ * call seems resonable.
+ *
+ * My guess is that we'd likely want to do both. The first one is easy
+ * and gives DM the proper info, so it seems better. The BPF call
+ * seems overly complex for just this, but one could imagine wanting to
+ * sometimes return _MAPPED and a BPF filter would be the way to do
+ * that.
+ *
+ * For example, in Android we have an in-kernel DM device called
+ * "dm-bow" that takes advange of some portion of the space that has
+ * been discarded on a device to provide opportunistic block-level
+ * backups. While one could imagine just implementing this entirely in
+ * userspace, that would come with an appreciable performance penalty.
+ * Instead one could keep a BPF program that forwards most accesses
+ * directly to the backing block device while informing a userspace
+ * daemon of any discarded space and on writes to blocks that are to be
+ * backed up.
+ */
+ if (unlikely((bio_type_to_user_type(bio) < 0) ||
+ (bio_flags_to_user_flags(bio) < 0))) {
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_KILL;
+ }
+
+ entry = msg_get_map(t);
+ if (unlikely(entry == NULL)) {
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_REQUEUE;
+ }
+
+ entry->msg.type = bio_type_to_user_type(bio);
+ entry->msg.flags = bio_flags_to_user_flags(bio);
+ entry->msg.sector = bio->bi_iter.bi_sector;
+ entry->msg.len = bio_size(bio);
+ entry->bio = bio;
+ entry->posn_to_user = 0;
+ entry->total_to_user = bio_bytes_needed_to_user(bio);
+ entry->posn_from_user = 0;
+ entry->total_from_user = bio_bytes_needed_from_user(bio);
+ entry->delayed = false;
+ entry->t = t;
+ /* Pairs with the barrier in dev_read() */
+ smp_wmb();
+ list_add_tail(&entry->to_user, &t->to_user);
+
+ /*
+ * If there is no daemon to process the IO's,
+ * queue these messages into a workqueue with
+ * a timeout.
+ */
+ if (!is_user_space_thread_present(t))
+ enqueue_delayed_work(entry, !t->daemon_terminated);
+
+ wake_up_interruptible(&t->wq);
+ mutex_unlock(&t->lock);
+ return DM_MAPIO_SUBMITTED;
+}
+
+static struct target_type user_target = {
+ .name = "user",
+ .version = { 1, 0, 0 },
+ .module = THIS_MODULE,
+ .ctr = user_ctr,
+ .dtr = user_dtr,
+ .map = user_map,
+};
+
+static int __init dm_user_init(void)
+{
+ int r;
+
+ r = dm_register_target(&user_target);
+ if (r) {
+ DMERR("register failed %d", r);
+ goto error;
+ }
+
+ return 0;
+
+error:
+ return r;
+}
+
+static void __exit dm_user_exit(void)
+{
+ dm_unregister_target(&user_target);
+}
+
+module_init(dm_user_init);
+module_exit(dm_user_exit);
+MODULE_AUTHOR("Palmer Dabbelt <palmerdabbelt@google.com>");
+MODULE_DESCRIPTION(DM_NAME " target returning blocks from userspace");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index 6abc930..d251201 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -65,8 +65,7 @@
# Multimedia support - automatically enable V4L2 and DVB core
#
config MEDIA_CAMERA_SUPPORT
- bool
- prompt "Cameras and video grabbers" if MEDIA_SUPPORT_FILTER
+ bool "Cameras and video grabbers"
default y if !MEDIA_SUPPORT_FILTER
help
Enable support for webcams and video grabbers.
@@ -74,8 +73,7 @@
Say Y when you have a webcam or a video capture grabber board.
config MEDIA_ANALOG_TV_SUPPORT
- bool
- prompt "Analog TV" if MEDIA_SUPPORT_FILTER
+ bool "Analog TV"
default y if !MEDIA_SUPPORT_FILTER
help
Enable analog TV support.
@@ -88,8 +86,7 @@
will disable support for them.
config MEDIA_DIGITAL_TV_SUPPORT
- bool
- prompt "Digital TV" if MEDIA_SUPPORT_FILTER
+ bool "Digital TV"
default y if !MEDIA_SUPPORT_FILTER
help
Enable digital TV support.
@@ -98,8 +95,7 @@
hybrid digital TV and analog TV.
config MEDIA_RADIO_SUPPORT
- bool
- prompt "AM/FM radio receivers/transmitters" if MEDIA_SUPPORT_FILTER
+ bool "AM/FM radio receivers/transmitters"
default y if !MEDIA_SUPPORT_FILTER
help
Enable AM/FM radio support.
@@ -114,8 +110,7 @@
disable support for them.
config MEDIA_SDR_SUPPORT
- bool
- prompt "Software defined radio" if MEDIA_SUPPORT_FILTER
+ bool "Software defined radio"
default y if !MEDIA_SUPPORT_FILTER
help
Enable software defined radio support.
@@ -123,8 +118,7 @@
Say Y when you have a software defined radio device.
config MEDIA_PLATFORM_SUPPORT
- bool
- prompt "Platform-specific devices" if MEDIA_SUPPORT_FILTER
+ bool "Platform-specific devices"
default y if !MEDIA_SUPPORT_FILTER
help
Enable support for complex cameras, codecs, and other hardware
@@ -137,8 +131,7 @@
Say Y when you want to be able to see such devices.
config MEDIA_TEST_SUPPORT
- bool
- prompt "Test drivers" if MEDIA_SUPPORT_FILTER
+ bool "Test drivers"
default y if !MEDIA_SUPPORT_FILTER
help
These drivers should not be used on production kernels, but
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 358ad56..8834e33 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -461,6 +461,22 @@
tristate
default MISC_RTSX_PCI || MISC_RTSX_USB
+config UID_SYS_STATS
+ bool "Per-UID statistics"
+ depends on PROFILING && TASK_XACCT && TASK_IO_ACCOUNTING
+ depends on BROKEN
+ help
+ Per UID based cpu time statistics exported to /proc/uid_cputime
+ Per UID based io statistics exported to /proc/uid_io
+ Per UID based procstat control in /proc/uid_procstat
+
+config UID_SYS_STATS_DEBUG
+ bool "Per-TASK statistics"
+ depends on UID_SYS_STATS
+ default n
+ help
+ Per TASK based io statistics exported to /proc/uid_io
+
config HISI_HIKEY_USB
tristate "USB GPIO Hub on HiSilicon Hikey 960/970 Platform"
depends on (OF && GPIOLIB) || COMPILE_TEST
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index ac9b3e7..72b154b 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -62,3 +62,4 @@
obj-$(CONFIG_OPEN_DICE) += open-dice.o
obj-$(CONFIG_GP_PCI1XXXX) += mchp_pci1xxxx/
obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o
+obj-$(CONFIG_UID_SYS_STATS) += uid_sys_stats.o
diff --git a/drivers/misc/uid_sys_stats.c b/drivers/misc/uid_sys_stats.c
new file mode 100644
index 0000000..36a34b1
--- /dev/null
+++ b/drivers/misc/uid_sys_stats.c
@@ -0,0 +1,774 @@
+/* drivers/misc/uid_sys_stats.c
+ *
+ * Copyright (C) 2014 - 2015 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/atomic.h>
+#include <linux/err.h>
+#include <linux/hashtable.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/proc_fs.h>
+#include <linux/profile.h>
+#include <linux/rtmutex.h>
+#include <linux/sched/cputime.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+
+#define UID_HASH_BITS 10
+DECLARE_HASHTABLE(hash_table, UID_HASH_BITS);
+
+static DEFINE_RT_MUTEX(uid_lock);
+static struct proc_dir_entry *cpu_parent;
+static struct proc_dir_entry *io_parent;
+static struct proc_dir_entry *proc_parent;
+
+struct io_stats {
+ u64 read_bytes;
+ u64 write_bytes;
+ u64 rchar;
+ u64 wchar;
+ u64 fsync;
+};
+
+#define UID_STATE_FOREGROUND 0
+#define UID_STATE_BACKGROUND 1
+#define UID_STATE_BUCKET_SIZE 2
+
+#define UID_STATE_TOTAL_CURR 2
+#define UID_STATE_TOTAL_LAST 3
+#define UID_STATE_DEAD_TASKS 4
+#define UID_STATE_SIZE 5
+
+#define MAX_TASK_COMM_LEN 256
+
+struct task_entry {
+ char comm[MAX_TASK_COMM_LEN];
+ pid_t pid;
+ struct io_stats io[UID_STATE_SIZE];
+ struct hlist_node hash;
+};
+
+struct uid_entry {
+ uid_t uid;
+ u64 utime;
+ u64 stime;
+ u64 active_utime;
+ u64 active_stime;
+ int state;
+ struct io_stats io[UID_STATE_SIZE];
+ struct hlist_node hash;
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ DECLARE_HASHTABLE(task_entries, UID_HASH_BITS);
+#endif
+};
+
+static u64 compute_write_bytes(struct task_io_accounting *ioac)
+{
+ if (ioac->write_bytes <= ioac->cancelled_write_bytes)
+ return 0;
+
+ return ioac->write_bytes - ioac->cancelled_write_bytes;
+}
+
+static void compute_io_bucket_stats(struct io_stats *io_bucket,
+ struct io_stats *io_curr,
+ struct io_stats *io_last,
+ struct io_stats *io_dead)
+{
+ /* tasks could switch to another uid group, but its io_last in the
+ * previous uid group could still be positive.
+ * therefore before each update, do an overflow check first
+ */
+ int64_t delta;
+
+ delta = io_curr->read_bytes + io_dead->read_bytes -
+ io_last->read_bytes;
+ io_bucket->read_bytes += delta > 0 ? delta : 0;
+ delta = io_curr->write_bytes + io_dead->write_bytes -
+ io_last->write_bytes;
+ io_bucket->write_bytes += delta > 0 ? delta : 0;
+ delta = io_curr->rchar + io_dead->rchar - io_last->rchar;
+ io_bucket->rchar += delta > 0 ? delta : 0;
+ delta = io_curr->wchar + io_dead->wchar - io_last->wchar;
+ io_bucket->wchar += delta > 0 ? delta : 0;
+ delta = io_curr->fsync + io_dead->fsync - io_last->fsync;
+ io_bucket->fsync += delta > 0 ? delta : 0;
+
+ io_last->read_bytes = io_curr->read_bytes;
+ io_last->write_bytes = io_curr->write_bytes;
+ io_last->rchar = io_curr->rchar;
+ io_last->wchar = io_curr->wchar;
+ io_last->fsync = io_curr->fsync;
+
+ memset(io_dead, 0, sizeof(struct io_stats));
+}
+
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+static void get_full_task_comm(struct task_entry *task_entry,
+ struct task_struct *task)
+{
+ int i = 0, offset = 0, len = 0;
+ /* save one byte for terminating null character */
+ int unused_len = MAX_TASK_COMM_LEN - TASK_COMM_LEN - 1;
+ char buf[MAX_TASK_COMM_LEN - TASK_COMM_LEN - 1];
+ struct mm_struct *mm = task->mm;
+
+ /* fill the first TASK_COMM_LEN bytes with thread name */
+ __get_task_comm(task_entry->comm, TASK_COMM_LEN, task);
+ i = strlen(task_entry->comm);
+ while (i < TASK_COMM_LEN)
+ task_entry->comm[i++] = ' ';
+
+ /* next the executable file name */
+ if (mm) {
+ mmap_write_lock(mm);
+ if (mm->exe_file) {
+ char *pathname = d_path(&mm->exe_file->f_path, buf,
+ unused_len);
+
+ if (!IS_ERR(pathname)) {
+ len = strlcpy(task_entry->comm + i, pathname,
+ unused_len);
+ i += len;
+ task_entry->comm[i++] = ' ';
+ unused_len--;
+ }
+ }
+ mmap_write_unlock(mm);
+ }
+ unused_len -= len;
+
+ /* fill the rest with command line argument
+ * replace each null or new line character
+ * between args in argv with whitespace */
+ len = get_cmdline(task, buf, unused_len);
+ while (offset < len) {
+ if (buf[offset] != '\0' && buf[offset] != '\n')
+ task_entry->comm[i++] = buf[offset];
+ else
+ task_entry->comm[i++] = ' ';
+ offset++;
+ }
+
+ /* get rid of trailing whitespaces in case when arg is memset to
+ * zero before being reset in userspace
+ */
+ while (task_entry->comm[i-1] == ' ')
+ i--;
+ task_entry->comm[i] = '\0';
+}
+
+static struct task_entry *find_task_entry(struct uid_entry *uid_entry,
+ struct task_struct *task)
+{
+ struct task_entry *task_entry;
+
+ hash_for_each_possible(uid_entry->task_entries, task_entry, hash,
+ task->pid) {
+ if (task->pid == task_entry->pid) {
+ /* if thread name changed, update the entire command */
+ int len = strnchr(task_entry->comm, ' ', TASK_COMM_LEN)
+ - task_entry->comm;
+
+ if (strncmp(task_entry->comm, task->comm, len))
+ get_full_task_comm(task_entry, task);
+ return task_entry;
+ }
+ }
+ return NULL;
+}
+
+static struct task_entry *find_or_register_task(struct uid_entry *uid_entry,
+ struct task_struct *task)
+{
+ struct task_entry *task_entry;
+ pid_t pid = task->pid;
+
+ task_entry = find_task_entry(uid_entry, task);
+ if (task_entry)
+ return task_entry;
+
+ task_entry = kzalloc(sizeof(struct task_entry), GFP_ATOMIC);
+ if (!task_entry)
+ return NULL;
+
+ get_full_task_comm(task_entry, task);
+
+ task_entry->pid = pid;
+ hash_add(uid_entry->task_entries, &task_entry->hash, (unsigned int)pid);
+
+ return task_entry;
+}
+
+static void remove_uid_tasks(struct uid_entry *uid_entry)
+{
+ struct task_entry *task_entry;
+ unsigned long bkt_task;
+ struct hlist_node *tmp_task;
+
+ hash_for_each_safe(uid_entry->task_entries, bkt_task,
+ tmp_task, task_entry, hash) {
+ hash_del(&task_entry->hash);
+ kfree(task_entry);
+ }
+}
+
+static void set_io_uid_tasks_zero(struct uid_entry *uid_entry)
+{
+ struct task_entry *task_entry;
+ unsigned long bkt_task;
+
+ hash_for_each(uid_entry->task_entries, bkt_task, task_entry, hash) {
+ memset(&task_entry->io[UID_STATE_TOTAL_CURR], 0,
+ sizeof(struct io_stats));
+ }
+}
+
+static void add_uid_tasks_io_stats(struct task_entry *task_entry,
+ struct task_io_accounting *ioac, int slot)
+{
+ struct io_stats *task_io_slot = &task_entry->io[slot];
+
+ task_io_slot->read_bytes += ioac->read_bytes;
+ task_io_slot->write_bytes += compute_write_bytes(ioac);
+ task_io_slot->rchar += ioac->rchar;
+ task_io_slot->wchar += ioac->wchar;
+ task_io_slot->fsync += ioac->syscfs;
+}
+
+static void compute_io_uid_tasks(struct uid_entry *uid_entry)
+{
+ struct task_entry *task_entry;
+ unsigned long bkt_task;
+
+ hash_for_each(uid_entry->task_entries, bkt_task, task_entry, hash) {
+ compute_io_bucket_stats(&task_entry->io[uid_entry->state],
+ &task_entry->io[UID_STATE_TOTAL_CURR],
+ &task_entry->io[UID_STATE_TOTAL_LAST],
+ &task_entry->io[UID_STATE_DEAD_TASKS]);
+ }
+}
+
+static void show_io_uid_tasks(struct seq_file *m, struct uid_entry *uid_entry)
+{
+ struct task_entry *task_entry;
+ unsigned long bkt_task;
+
+ hash_for_each(uid_entry->task_entries, bkt_task, task_entry, hash) {
+ /* Separated by comma because space exists in task comm */
+ seq_printf(m, "task,%s,%lu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n",
+ task_entry->comm,
+ (unsigned long)task_entry->pid,
+ task_entry->io[UID_STATE_FOREGROUND].rchar,
+ task_entry->io[UID_STATE_FOREGROUND].wchar,
+ task_entry->io[UID_STATE_FOREGROUND].read_bytes,
+ task_entry->io[UID_STATE_FOREGROUND].write_bytes,
+ task_entry->io[UID_STATE_BACKGROUND].rchar,
+ task_entry->io[UID_STATE_BACKGROUND].wchar,
+ task_entry->io[UID_STATE_BACKGROUND].read_bytes,
+ task_entry->io[UID_STATE_BACKGROUND].write_bytes,
+ task_entry->io[UID_STATE_FOREGROUND].fsync,
+ task_entry->io[UID_STATE_BACKGROUND].fsync);
+ }
+}
+#else
+static void remove_uid_tasks(struct uid_entry *uid_entry) {};
+static void set_io_uid_tasks_zero(struct uid_entry *uid_entry) {};
+static void compute_io_uid_tasks(struct uid_entry *uid_entry) {};
+static void show_io_uid_tasks(struct seq_file *m,
+ struct uid_entry *uid_entry) {}
+#endif
+
+static struct uid_entry *find_uid_entry(uid_t uid)
+{
+ struct uid_entry *uid_entry;
+ hash_for_each_possible(hash_table, uid_entry, hash, uid) {
+ if (uid_entry->uid == uid)
+ return uid_entry;
+ }
+ return NULL;
+}
+
+static struct uid_entry *find_or_register_uid(uid_t uid)
+{
+ struct uid_entry *uid_entry;
+
+ uid_entry = find_uid_entry(uid);
+ if (uid_entry)
+ return uid_entry;
+
+ uid_entry = kzalloc(sizeof(struct uid_entry), GFP_ATOMIC);
+ if (!uid_entry)
+ return NULL;
+
+ uid_entry->uid = uid;
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ hash_init(uid_entry->task_entries);
+#endif
+ hash_add(hash_table, &uid_entry->hash, uid);
+
+ return uid_entry;
+}
+
+static int uid_cputime_show(struct seq_file *m, void *v)
+{
+ struct uid_entry *uid_entry = NULL;
+ struct task_struct *task, *temp;
+ struct user_namespace *user_ns = current_user_ns();
+ u64 utime;
+ u64 stime;
+ unsigned long bkt;
+ uid_t uid;
+
+ rt_mutex_lock(&uid_lock);
+
+ hash_for_each(hash_table, bkt, uid_entry, hash) {
+ uid_entry->active_stime = 0;
+ uid_entry->active_utime = 0;
+ }
+
+ rcu_read_lock();
+ do_each_thread(temp, task) {
+ uid = from_kuid_munged(user_ns, task_uid(task));
+ if (!uid_entry || uid_entry->uid != uid)
+ uid_entry = find_or_register_uid(uid);
+ if (!uid_entry) {
+ rcu_read_unlock();
+ rt_mutex_unlock(&uid_lock);
+ pr_err("%s: failed to find the uid_entry for uid %d\n",
+ __func__, uid);
+ return -ENOMEM;
+ }
+ /* avoid double accounting of dying threads */
+ if (!(task->flags & PF_EXITING)) {
+ task_cputime_adjusted(task, &utime, &stime);
+ uid_entry->active_utime += utime;
+ uid_entry->active_stime += stime;
+ }
+ } while_each_thread(temp, task);
+ rcu_read_unlock();
+
+ hash_for_each(hash_table, bkt, uid_entry, hash) {
+ u64 total_utime = uid_entry->utime +
+ uid_entry->active_utime;
+ u64 total_stime = uid_entry->stime +
+ uid_entry->active_stime;
+ seq_printf(m, "%d: %llu %llu\n", uid_entry->uid,
+ ktime_to_us(total_utime), ktime_to_us(total_stime));
+ }
+
+ rt_mutex_unlock(&uid_lock);
+ return 0;
+}
+
+static int uid_cputime_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, uid_cputime_show, pde_data(inode));
+}
+
+static const struct proc_ops uid_cputime_fops = {
+ .proc_open = uid_cputime_open,
+ .proc_read = seq_read,
+ .proc_lseek = seq_lseek,
+ .proc_release = single_release,
+};
+
+static int uid_remove_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, NULL, NULL);
+}
+
+static ssize_t uid_remove_write(struct file *file,
+ const char __user *buffer, size_t count, loff_t *ppos)
+{
+ struct uid_entry *uid_entry;
+ struct hlist_node *tmp;
+ char uids[128];
+ char *start_uid, *end_uid = NULL;
+ long int uid_start = 0, uid_end = 0;
+
+ if (count >= sizeof(uids))
+ count = sizeof(uids) - 1;
+
+ if (copy_from_user(uids, buffer, count))
+ return -EFAULT;
+
+ uids[count] = '\0';
+ end_uid = uids;
+ start_uid = strsep(&end_uid, "-");
+
+ if (!start_uid || !end_uid)
+ return -EINVAL;
+
+ if (kstrtol(start_uid, 10, &uid_start) != 0 ||
+ kstrtol(end_uid, 10, &uid_end) != 0) {
+ return -EINVAL;
+ }
+
+ rt_mutex_lock(&uid_lock);
+
+ for (; uid_start <= uid_end; uid_start++) {
+ hash_for_each_possible_safe(hash_table, uid_entry, tmp,
+ hash, (uid_t)uid_start) {
+ if (uid_start == uid_entry->uid) {
+ remove_uid_tasks(uid_entry);
+ hash_del(&uid_entry->hash);
+ kfree(uid_entry);
+ }
+ }
+ }
+
+ rt_mutex_unlock(&uid_lock);
+ return count;
+}
+
+static const struct proc_ops uid_remove_fops = {
+ .proc_open = uid_remove_open,
+ .proc_release = single_release,
+ .proc_write = uid_remove_write,
+};
+
+static void __add_uid_io_stats(struct uid_entry *uid_entry,
+ struct task_io_accounting *ioac, int slot)
+{
+ struct io_stats *io_slot = &uid_entry->io[slot];
+
+ io_slot->read_bytes += ioac->read_bytes;
+ io_slot->write_bytes += compute_write_bytes(ioac);
+ io_slot->rchar += ioac->rchar;
+ io_slot->wchar += ioac->wchar;
+ io_slot->fsync += ioac->syscfs;
+}
+
+static void add_uid_io_stats(struct uid_entry *uid_entry,
+ struct task_struct *task, int slot)
+{
+ struct task_entry *task_entry __maybe_unused;
+
+ /* avoid double accounting of dying threads */
+ if (slot != UID_STATE_DEAD_TASKS && (task->flags & PF_EXITING))
+ return;
+
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ task_entry = find_or_register_task(uid_entry, task);
+ add_uid_tasks_io_stats(task_entry, &task->ioac, slot);
+#endif
+ __add_uid_io_stats(uid_entry, &task->ioac, slot);
+}
+
+static void update_io_stats_all_locked(void)
+{
+ struct uid_entry *uid_entry = NULL;
+ struct task_struct *task, *temp;
+ struct user_namespace *user_ns = current_user_ns();
+ unsigned long bkt;
+ uid_t uid;
+
+ hash_for_each(hash_table, bkt, uid_entry, hash) {
+ memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0,
+ sizeof(struct io_stats));
+ set_io_uid_tasks_zero(uid_entry);
+ }
+
+ rcu_read_lock();
+ do_each_thread(temp, task) {
+ uid = from_kuid_munged(user_ns, task_uid(task));
+ if (!uid_entry || uid_entry->uid != uid)
+ uid_entry = find_or_register_uid(uid);
+ if (!uid_entry)
+ continue;
+ add_uid_io_stats(uid_entry, task, UID_STATE_TOTAL_CURR);
+ } while_each_thread(temp, task);
+ rcu_read_unlock();
+
+ hash_for_each(hash_table, bkt, uid_entry, hash) {
+ compute_io_bucket_stats(&uid_entry->io[uid_entry->state],
+ &uid_entry->io[UID_STATE_TOTAL_CURR],
+ &uid_entry->io[UID_STATE_TOTAL_LAST],
+ &uid_entry->io[UID_STATE_DEAD_TASKS]);
+ compute_io_uid_tasks(uid_entry);
+ }
+}
+
+static void update_io_stats_uid_locked(struct uid_entry *uid_entry)
+{
+ struct task_struct *task, *temp;
+ struct user_namespace *user_ns = current_user_ns();
+
+ memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0,
+ sizeof(struct io_stats));
+ set_io_uid_tasks_zero(uid_entry);
+
+ rcu_read_lock();
+ do_each_thread(temp, task) {
+ if (from_kuid_munged(user_ns, task_uid(task)) != uid_entry->uid)
+ continue;
+ add_uid_io_stats(uid_entry, task, UID_STATE_TOTAL_CURR);
+ } while_each_thread(temp, task);
+ rcu_read_unlock();
+
+ compute_io_bucket_stats(&uid_entry->io[uid_entry->state],
+ &uid_entry->io[UID_STATE_TOTAL_CURR],
+ &uid_entry->io[UID_STATE_TOTAL_LAST],
+ &uid_entry->io[UID_STATE_DEAD_TASKS]);
+ compute_io_uid_tasks(uid_entry);
+}
+
+
+static int uid_io_show(struct seq_file *m, void *v)
+{
+ struct uid_entry *uid_entry;
+ unsigned long bkt;
+
+ rt_mutex_lock(&uid_lock);
+
+ update_io_stats_all_locked();
+
+ hash_for_each(hash_table, bkt, uid_entry, hash) {
+ seq_printf(m, "%d %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
+ uid_entry->uid,
+ uid_entry->io[UID_STATE_FOREGROUND].rchar,
+ uid_entry->io[UID_STATE_FOREGROUND].wchar,
+ uid_entry->io[UID_STATE_FOREGROUND].read_bytes,
+ uid_entry->io[UID_STATE_FOREGROUND].write_bytes,
+ uid_entry->io[UID_STATE_BACKGROUND].rchar,
+ uid_entry->io[UID_STATE_BACKGROUND].wchar,
+ uid_entry->io[UID_STATE_BACKGROUND].read_bytes,
+ uid_entry->io[UID_STATE_BACKGROUND].write_bytes,
+ uid_entry->io[UID_STATE_FOREGROUND].fsync,
+ uid_entry->io[UID_STATE_BACKGROUND].fsync);
+
+ show_io_uid_tasks(m, uid_entry);
+ }
+
+ rt_mutex_unlock(&uid_lock);
+ return 0;
+}
+
+static int uid_io_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, uid_io_show, pde_data(inode));
+}
+
+static const struct proc_ops uid_io_fops = {
+ .proc_open = uid_io_open,
+ .proc_read = seq_read,
+ .proc_lseek = seq_lseek,
+ .proc_release = single_release,
+};
+
+static int uid_procstat_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, NULL, NULL);
+}
+
+static ssize_t uid_procstat_write(struct file *file,
+ const char __user *buffer, size_t count, loff_t *ppos)
+{
+ struct uid_entry *uid_entry;
+ uid_t uid;
+ int argc, state;
+ char input[128];
+
+ if (count >= sizeof(input))
+ return -EINVAL;
+
+ if (copy_from_user(input, buffer, count))
+ return -EFAULT;
+
+ input[count] = '\0';
+
+ argc = sscanf(input, "%u %d", &uid, &state);
+ if (argc != 2)
+ return -EINVAL;
+
+ if (state != UID_STATE_BACKGROUND && state != UID_STATE_FOREGROUND)
+ return -EINVAL;
+
+ rt_mutex_lock(&uid_lock);
+
+ uid_entry = find_or_register_uid(uid);
+ if (!uid_entry) {
+ rt_mutex_unlock(&uid_lock);
+ return -EINVAL;
+ }
+
+ if (uid_entry->state == state) {
+ rt_mutex_unlock(&uid_lock);
+ return count;
+ }
+
+ update_io_stats_uid_locked(uid_entry);
+
+ uid_entry->state = state;
+
+ rt_mutex_unlock(&uid_lock);
+
+ return count;
+}
+
+static const struct proc_ops uid_procstat_fops = {
+ .proc_open = uid_procstat_open,
+ .proc_release = single_release,
+ .proc_write = uid_procstat_write,
+};
+
+struct update_stats_work {
+ struct work_struct work;
+ uid_t uid;
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ struct task_struct *task;
+#endif
+ struct task_io_accounting ioac;
+ u64 utime;
+ u64 stime;
+};
+
+static void update_stats_workfn(struct work_struct *work)
+{
+ struct update_stats_work *usw =
+ container_of(work, struct update_stats_work, work);
+ struct uid_entry *uid_entry;
+ struct task_entry *task_entry __maybe_unused;
+
+ rt_mutex_lock(&uid_lock);
+ uid_entry = find_uid_entry(usw->uid);
+ if (!uid_entry)
+ goto exit;
+
+ uid_entry->utime += usw->utime;
+ uid_entry->stime += usw->stime;
+
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ task_entry = find_task_entry(uid_entry, usw->task);
+ if (!task_entry)
+ goto exit;
+ add_uid_tasks_io_stats(task_entry, &usw->ioac,
+ UID_STATE_DEAD_TASKS);
+#endif
+ __add_uid_io_stats(uid_entry, &usw->ioac, UID_STATE_DEAD_TASKS);
+exit:
+ rt_mutex_unlock(&uid_lock);
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ put_task_struct(usw->task);
+#endif
+ kfree(usw);
+}
+
+static int process_notifier(struct notifier_block *self,
+ unsigned long cmd, void *v)
+{
+ struct task_struct *task = v;
+ struct uid_entry *uid_entry;
+ u64 utime, stime;
+ uid_t uid;
+
+ if (!task)
+ return NOTIFY_OK;
+
+ uid = from_kuid_munged(current_user_ns(), task_uid(task));
+ if (!rt_mutex_trylock(&uid_lock)) {
+ struct update_stats_work *usw;
+
+ usw = kmalloc(sizeof(struct update_stats_work), GFP_KERNEL);
+ if (usw) {
+ INIT_WORK(&usw->work, update_stats_workfn);
+ usw->uid = uid;
+#ifdef CONFIG_UID_SYS_STATS_DEBUG
+ usw->task = get_task_struct(task);
+#endif
+ /*
+ * Copy task->ioac since task might be destroyed before
+ * the work is later performed.
+ */
+ usw->ioac = task->ioac;
+ task_cputime_adjusted(task, &usw->utime, &usw->stime);
+ schedule_work(&usw->work);
+ }
+ return NOTIFY_OK;
+ }
+
+ uid_entry = find_or_register_uid(uid);
+ if (!uid_entry) {
+ pr_err("%s: failed to find uid %d\n", __func__, uid);
+ goto exit;
+ }
+
+ task_cputime_adjusted(task, &utime, &stime);
+ uid_entry->utime += utime;
+ uid_entry->stime += stime;
+
+ add_uid_io_stats(uid_entry, task, UID_STATE_DEAD_TASKS);
+
+exit:
+ rt_mutex_unlock(&uid_lock);
+ return NOTIFY_OK;
+}
+
+static struct notifier_block process_notifier_block = {
+ .notifier_call = process_notifier,
+};
+
+static int __init proc_uid_sys_stats_init(void)
+{
+ hash_init(hash_table);
+
+ cpu_parent = proc_mkdir("uid_cputime", NULL);
+ if (!cpu_parent) {
+ pr_err("%s: failed to create uid_cputime proc entry\n",
+ __func__);
+ goto err;
+ }
+
+ proc_create_data("remove_uid_range", 0222, cpu_parent,
+ &uid_remove_fops, NULL);
+ proc_create_data("show_uid_stat", 0444, cpu_parent,
+ &uid_cputime_fops, NULL);
+
+ io_parent = proc_mkdir("uid_io", NULL);
+ if (!io_parent) {
+ pr_err("%s: failed to create uid_io proc entry\n",
+ __func__);
+ goto err;
+ }
+
+ proc_create_data("stats", 0444, io_parent,
+ &uid_io_fops, NULL);
+
+ proc_parent = proc_mkdir("uid_procstat", NULL);
+ if (!proc_parent) {
+ pr_err("%s: failed to create uid_procstat proc entry\n",
+ __func__);
+ goto err;
+ }
+
+ proc_create_data("set", 0222, proc_parent,
+ &uid_procstat_fops, NULL);
+
+ profile_event_register(PROFILE_TASK_EXIT, &process_notifier_block);
+
+ return 0;
+
+err:
+ remove_proc_subtree("uid_cputime", NULL);
+ remove_proc_subtree("uid_io", NULL);
+ remove_proc_subtree("uid_procstat", NULL);
+ return -ENOMEM;
+}
+
+early_initcall(proc_uid_sys_stats_init);
diff --git a/drivers/mmc/host/cqhci-crypto.c b/drivers/mmc/host/cqhci-crypto.c
index d5f4b69..6652982 100644
--- a/drivers/mmc/host/cqhci-crypto.c
+++ b/drivers/mmc/host/cqhci-crypto.c
@@ -210,6 +210,8 @@ int cqhci_crypto_init(struct cqhci_host *cq_host)
/* Unfortunately, CQHCI crypto only supports 32 DUN bits. */
profile->max_dun_bytes_supported = 4;
+ profile->key_types_supported = BLK_CRYPTO_KEY_TYPE_STANDARD;
+
/*
* Cache all the crypto capabilities and advertise the supported crypto
* modes and data unit sizes to the block layer.
diff --git a/drivers/of/unittest-data/Makefile b/drivers/of/unittest-data/Makefile
index ea5f4da..1bb8670 100644
--- a/drivers/of/unittest-data/Makefile
+++ b/drivers/of/unittest-data/Makefile
@@ -42,9 +42,7 @@
DTC_FLAGS_testcases += -@
# suppress warnings about intentional errors
-DTC_FLAGS_testcases += -Wno-interrupts_property \
- -Wno-node_name_vs_property_name \
- -Wno-interrupt_map
+DTC_FLAGS_testcases += -Wno-interrupts_property
# Apply overlays statically with fdtoverlay. This is a build time test that
# the overlays can be applied successfully by fdtoverlay. This does not
@@ -94,10 +92,6 @@
apply_static_overlay_2 := overlay.dtbo
-DTC_FLAGS_static_base_1 += -Wno-interrupts_property \
- -Wno-node_name_vs_property_name \
- -Wno-interrupt_map
-
static_test_1-dtbs := static_base_1.dtb $(apply_static_overlay_1)
static_test_2-dtbs := static_base_2.dtb $(apply_static_overlay_2)
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 39f3b37..6241d3f 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -83,6 +83,7 @@ irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
return ret;
}
+EXPORT_SYMBOL_GPL(dw_handle_msi_irq);
/* Chained MSI interrupt service routine */
static void dw_chained_msi_isr(struct irq_desc *desc)
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index 4b5fb17..229a95b 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -34,6 +34,13 @@ EXPORT_SYMBOL_GPL(power_supply_notifier);
static struct device_type power_supply_dev_type;
+struct match_device_node_array_param {
+ struct device_node *parent_of_node;
+ struct power_supply **psy;
+ ssize_t psy_size;
+ ssize_t psy_count;
+};
+
#define POWER_SUPPLY_DEFERRED_REGISTER_TIME msecs_to_jiffies(10)
static bool __power_supply_is_supplied_by(struct power_supply *supplier,
@@ -526,6 +533,77 @@ struct power_supply *power_supply_get_by_phandle(struct device_node *np,
}
EXPORT_SYMBOL_GPL(power_supply_get_by_phandle);
+static int power_supply_match_device_node_array(struct device *dev,
+ void *data)
+{
+ struct match_device_node_array_param *param =
+ (struct match_device_node_array_param *)data;
+ struct power_supply **psy = param->psy;
+ ssize_t size = param->psy_size;
+ ssize_t *count = ¶m->psy_count;
+
+ if (!dev->parent || dev->parent->of_node != param->parent_of_node)
+ return 0;
+
+ if (*count >= size)
+ return -EOVERFLOW;
+
+ psy[*count] = dev_get_drvdata(dev);
+ atomic_inc(&psy[*count]->use_cnt);
+ (*count)++;
+
+ return 0;
+}
+
+/**
+ * power_supply_get_by_phandle_array() - Similar to
+ * power_supply_get_by_phandle but returns an array of power supply
+ * objects which are associated with the phandle.
+ * @np: Pointer to device node holding phandle property.
+ * @property: Name of property holding a power supply name.
+ * @psy: Array of power_supply pointers provided by the client which is
+ * filled by power_supply_get_by_phandle_array.
+ * @size: size of power_supply pointer array.
+ *
+ * If power supply was found, it increases reference count for the
+ * internal power supply's device. The user should power_supply_put()
+ * after usage.
+ *
+ * Return: On success returns the number of power supply objects filled
+ * in the @psy array.
+ * -EOVERFLOW when size of @psy array is not suffice.
+ * -EINVAL when @psy is NULL or @size is 0.
+ * -ENODEV when matching device_node is not found.
+ */
+int power_supply_get_by_phandle_array(struct device_node *np,
+ const char *property,
+ struct power_supply **psy,
+ ssize_t size)
+{
+ struct device_node *power_supply_np;
+ int ret;
+ struct match_device_node_array_param param;
+
+ if (!psy || !size)
+ return -EINVAL;
+
+ power_supply_np = of_parse_phandle(np, property, 0);
+ if (!power_supply_np)
+ return -ENODEV;
+
+ param.parent_of_node = power_supply_np;
+ param.psy = psy;
+ param.psy_size = size;
+ param.psy_count = 0;
+ ret = class_for_each_device(power_supply_class, NULL, ¶m,
+ power_supply_match_device_node_array);
+
+ of_node_put(power_supply_np);
+
+ return param.psy_count;
+}
+EXPORT_SYMBOL_GPL(power_supply_get_by_phandle_array);
+
static void devm_power_supply_put(struct device *dev, void *res)
{
struct power_supply **psy = res;
diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index 5369aba..bab8049 100644
--- a/drivers/power/supply/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
@@ -90,6 +90,7 @@ static const char * const POWER_SUPPLY_CHARGE_TYPE_TEXT[] = {
[POWER_SUPPLY_CHARGE_TYPE_CUSTOM] = "Custom",
[POWER_SUPPLY_CHARGE_TYPE_LONGLIFE] = "Long Life",
[POWER_SUPPLY_CHARGE_TYPE_BYPASS] = "Bypass",
+ [POWER_SUPPLY_CHARGE_TYPE_TAPER_EXT] = "Taper",
};
static const char * const POWER_SUPPLY_HEALTH_TEXT[] = {
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index 8768cb6..e604a24 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -38,6 +38,7 @@
#include <linux/virtio_ring.h>
#include <asm/byteorder.h>
#include <linux/platform_device.h>
+#include <trace/hooks/remoteproc.h>
#include "remoteproc_internal.h"
@@ -1877,6 +1878,8 @@ static void rproc_crash_handler_work(struct work_struct *work)
if (!rproc->recovery_disabled)
rproc_trigger_recovery(rproc);
+ trace_android_vh_rproc_recovery(rproc);
+
pm_relax(rproc->dev.parent);
}
diff --git a/drivers/remoteproc/remoteproc_coredump.c b/drivers/remoteproc/remoteproc_coredump.c
index 4b09342..9c53c3d 100644
--- a/drivers/remoteproc/remoteproc_coredump.c
+++ b/drivers/remoteproc/remoteproc_coredump.c
@@ -32,6 +32,7 @@ void rproc_coredump_cleanup(struct rproc *rproc)
kfree(entry);
}
}
+EXPORT_SYMBOL_GPL(rproc_coredump_cleanup);
/**
* rproc_coredump_add_segment() - add segment of device memory to coredump
@@ -327,6 +328,7 @@ void rproc_coredump(struct rproc *rproc)
*/
wait_for_completion(&dump_state.dump_done);
}
+EXPORT_SYMBOL_GPL(rproc_coredump);
/**
* rproc_coredump_using_sections() - perform coredump using section headers
diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c
index 8c7ea89..22c316d 100644
--- a/drivers/remoteproc/remoteproc_sysfs.c
+++ b/drivers/remoteproc/remoteproc_sysfs.c
@@ -5,6 +5,7 @@
#include <linux/remoteproc.h>
#include <linux/slab.h>
+#include <trace/hooks/remoteproc.h>
#include "remoteproc_internal.h"
@@ -51,9 +52,11 @@ static ssize_t recovery_store(struct device *dev,
if (sysfs_streq(buf, "enabled")) {
/* change the flag and begin the recovery process if needed */
rproc->recovery_disabled = false;
+ trace_android_vh_rproc_recovery_set(rproc);
rproc_trigger_recovery(rproc);
} else if (sysfs_streq(buf, "disabled")) {
rproc->recovery_disabled = true;
+ trace_android_vh_rproc_recovery_set(rproc);
} else if (sysfs_streq(buf, "recover")) {
/* begin the recovery process without changing the flag */
rproc_trigger_recovery(rproc);
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index 5cfabd5..306a484 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -56,6 +56,8 @@
source "drivers/staging/media/Kconfig"
+source "drivers/staging/android/Kconfig"
+
source "drivers/staging/board/Kconfig"
source "drivers/staging/gdm724x/Kconfig"
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index f8c3aa9c..a375649 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -18,6 +18,7 @@
obj-$(CONFIG_FB_SM750) += sm750fb/
obj-$(CONFIG_USB_EMXX) += emxx_udc/
obj-$(CONFIG_MFD_NVEC) += nvec/
+obj-$(CONFIG_ASHMEM) += android/
obj-$(CONFIG_STAGING_BOARD) += board/
obj-$(CONFIG_LTE_GDM724X) += gdm724x/
obj-$(CONFIG_FB_TFT) += fbtft/
diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig
new file mode 100644
index 0000000..36f4413
--- /dev/null
+++ b/drivers/staging/android/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0
+menu "Android"
+
+config ASHMEM
+ bool "Enable the Anonymous Shared Memory Subsystem"
+ depends on SHMEM
+ help
+ The ashmem subsystem is a new shared memory allocator, similar to
+ POSIX SHM but with different behavior and sporting a simpler
+ file-based API.
+
+ It is, in theory, a good memory allocator for low-memory devices,
+ because it can discard shared memory units when under memory pressure.
+
+endmenu
diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile
new file mode 100644
index 0000000..e9a55a5
--- /dev/null
+++ b/drivers/staging/android/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I$(src) # needed for trace events
+
+obj-$(CONFIG_ASHMEM) += ashmem.o
diff --git a/drivers/staging/android/TODO b/drivers/staging/android/TODO
new file mode 100644
index 0000000..f74eb44
--- /dev/null
+++ b/drivers/staging/android/TODO
@@ -0,0 +1,8 @@
+TODO:
+ - sparse fixes
+ - rename files to be not so "generic"
+ - add proper arch dependencies as needed
+ - audit userspace interfaces to make sure they are sane
+
+Please send patches to Greg Kroah-Hartman <greg@kroah.com> and Cc:
+Arve Hjønnevåg <arve@android.com> and Riley Andrews <riandrews@android.com>
diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
new file mode 100644
index 0000000..feadf798
--- /dev/null
+++ b/drivers/staging/android/ashmem.c
@@ -0,0 +1,991 @@
+// SPDX-License-Identifier: GPL-2.0
+/* mm/ashmem.c
+ *
+ * Anonymous Shared Memory Subsystem, ashmem
+ *
+ * Copyright (C) 2008 Google, Inc.
+ *
+ * Robert Love <rlove@google.com>
+ */
+
+#define pr_fmt(fmt) "ashmem: " fmt
+
+#include <linux/init.h>
+#include <linux/export.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/falloc.h>
+#include <linux/miscdevice.h>
+#include <linux/security.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/uaccess.h>
+#include <linux/personality.h>
+#include <linux/bitops.h>
+#include <linux/mutex.h>
+#include <linux/shmem_fs.h>
+#include "ashmem.h"
+
+#define ASHMEM_NAME_PREFIX "dev/ashmem/"
+#define ASHMEM_NAME_PREFIX_LEN (sizeof(ASHMEM_NAME_PREFIX) - 1)
+#define ASHMEM_FULL_NAME_LEN (ASHMEM_NAME_LEN + ASHMEM_NAME_PREFIX_LEN)
+
+/**
+ * struct ashmem_area - The anonymous shared memory area
+ * @name: The optional name in /proc/pid/maps
+ * @unpinned_list: The list of all ashmem areas
+ * @file: The shmem-based backing file
+ * @size: The size of the mapping, in bytes
+ * @prot_mask: The allowed protection bits, as vm_flags
+ *
+ * The lifecycle of this structure is from our parent file's open() until
+ * its release(). It is also protected by 'ashmem_mutex'
+ *
+ * Warning: Mappings do NOT pin this structure; It dies on close()
+ */
+struct ashmem_area {
+ char name[ASHMEM_FULL_NAME_LEN];
+ struct list_head unpinned_list;
+ struct file *file;
+ size_t size;
+ unsigned long prot_mask;
+};
+
+/**
+ * struct ashmem_range - A range of unpinned/evictable pages
+ * @lru: The entry in the LRU list
+ * @unpinned: The entry in its area's unpinned list
+ * @asma: The associated anonymous shared memory area.
+ * @pgstart: The starting page (inclusive)
+ * @pgend: The ending page (inclusive)
+ * @purged: The purge status (ASHMEM_NOT or ASHMEM_WAS_PURGED)
+ *
+ * The lifecycle of this structure is from unpin to pin.
+ * It is protected by 'ashmem_mutex'
+ */
+struct ashmem_range {
+ struct list_head lru;
+ struct list_head unpinned;
+ struct ashmem_area *asma;
+ size_t pgstart;
+ size_t pgend;
+ unsigned int purged;
+};
+
+/* LRU list of unpinned pages, protected by ashmem_mutex */
+static LIST_HEAD(ashmem_lru_list);
+
+static atomic_t ashmem_shrink_inflight = ATOMIC_INIT(0);
+static DECLARE_WAIT_QUEUE_HEAD(ashmem_shrink_wait);
+
+/*
+ * long lru_count - The count of pages on our LRU list.
+ *
+ * This is protected by ashmem_mutex.
+ */
+static unsigned long lru_count;
+
+/*
+ * ashmem_mutex - protects the list of and each individual ashmem_area
+ *
+ * Lock Ordering: ashmex_mutex -> i_mutex -> i_alloc_sem
+ */
+static DEFINE_MUTEX(ashmem_mutex);
+
+static struct kmem_cache *ashmem_area_cachep __read_mostly;
+static struct kmem_cache *ashmem_range_cachep __read_mostly;
+
+/*
+ * A separate lockdep class for the backing shmem inodes to resolve the lockdep
+ * warning about the race between kswapd taking fs_reclaim before inode_lock
+ * and write syscall taking inode_lock and then fs_reclaim.
+ * Note that such race is impossible because ashmem does not support write
+ * syscalls operating on the backing shmem.
+ */
+static struct lock_class_key backing_shmem_inode_class;
+
+static inline unsigned long range_size(struct ashmem_range *range)
+{
+ return range->pgend - range->pgstart + 1;
+}
+
+static inline bool range_on_lru(struct ashmem_range *range)
+{
+ return range->purged == ASHMEM_NOT_PURGED;
+}
+
+static inline bool page_range_subsumes_range(struct ashmem_range *range,
+ size_t start, size_t end)
+{
+ return (range->pgstart >= start) && (range->pgend <= end);
+}
+
+static inline bool page_range_subsumed_by_range(struct ashmem_range *range,
+ size_t start, size_t end)
+{
+ return (range->pgstart <= start) && (range->pgend >= end);
+}
+
+static inline bool page_in_range(struct ashmem_range *range, size_t page)
+{
+ return (range->pgstart <= page) && (range->pgend >= page);
+}
+
+static inline bool page_range_in_range(struct ashmem_range *range,
+ size_t start, size_t end)
+{
+ return page_in_range(range, start) || page_in_range(range, end) ||
+ page_range_subsumes_range(range, start, end);
+}
+
+static inline bool range_before_page(struct ashmem_range *range,
+ size_t page)
+{
+ return range->pgend < page;
+}
+
+#define PROT_MASK (PROT_EXEC | PROT_READ | PROT_WRITE)
+
+/**
+ * lru_add() - Adds a range of memory to the LRU list
+ * @range: The memory range being added.
+ *
+ * The range is first added to the end (tail) of the LRU list.
+ * After this, the size of the range is added to @lru_count
+ */
+static inline void lru_add(struct ashmem_range *range)
+{
+ list_add_tail(&range->lru, &ashmem_lru_list);
+ lru_count += range_size(range);
+}
+
+/**
+ * lru_del() - Removes a range of memory from the LRU list
+ * @range: The memory range being removed
+ *
+ * The range is first deleted from the LRU list.
+ * After this, the size of the range is removed from @lru_count
+ */
+static inline void lru_del(struct ashmem_range *range)
+{
+ list_del(&range->lru);
+ lru_count -= range_size(range);
+}
+
+/**
+ * range_alloc() - Allocates and initializes a new ashmem_range structure
+ * @asma: The associated ashmem_area
+ * @prev_range: The previous ashmem_range in the sorted asma->unpinned list
+ * @purged: Initial purge status (ASMEM_NOT_PURGED or ASHMEM_WAS_PURGED)
+ * @start: The starting page (inclusive)
+ * @end: The ending page (inclusive)
+ * @new_range: The placeholder for the new range
+ *
+ * This function is protected by ashmem_mutex.
+ */
+static void range_alloc(struct ashmem_area *asma,
+ struct ashmem_range *prev_range, unsigned int purged,
+ size_t start, size_t end,
+ struct ashmem_range **new_range)
+{
+ struct ashmem_range *range = *new_range;
+
+ *new_range = NULL;
+ range->asma = asma;
+ range->pgstart = start;
+ range->pgend = end;
+ range->purged = purged;
+
+ list_add_tail(&range->unpinned, &prev_range->unpinned);
+
+ if (range_on_lru(range))
+ lru_add(range);
+}
+
+/**
+ * range_del() - Deletes and deallocates an ashmem_range structure
+ * @range: The associated ashmem_range that has previously been allocated
+ */
+static void range_del(struct ashmem_range *range)
+{
+ list_del(&range->unpinned);
+ if (range_on_lru(range))
+ lru_del(range);
+ kmem_cache_free(ashmem_range_cachep, range);
+}
+
+/**
+ * range_shrink() - Shrinks an ashmem_range
+ * @range: The associated ashmem_range being shrunk
+ * @start: The starting byte of the new range
+ * @end: The ending byte of the new range
+ *
+ * This does not modify the data inside the existing range in any way - It
+ * simply shrinks the boundaries of the range.
+ *
+ * Theoretically, with a little tweaking, this could eventually be changed
+ * to range_resize, and expand the lru_count if the new range is larger.
+ */
+static inline void range_shrink(struct ashmem_range *range,
+ size_t start, size_t end)
+{
+ size_t pre = range_size(range);
+
+ range->pgstart = start;
+ range->pgend = end;
+
+ if (range_on_lru(range))
+ lru_count -= pre - range_size(range);
+}
+
+/**
+ * ashmem_open() - Opens an Anonymous Shared Memory structure
+ * @inode: The backing file's index node(?)
+ * @file: The backing file
+ *
+ * Please note that the ashmem_area is not returned by this function - It is
+ * instead written to "file->private_data".
+ *
+ * Return: 0 if successful, or another code if unsuccessful.
+ */
+static int ashmem_open(struct inode *inode, struct file *file)
+{
+ struct ashmem_area *asma;
+ int ret;
+
+ ret = generic_file_open(inode, file);
+ if (ret)
+ return ret;
+
+ asma = kmem_cache_zalloc(ashmem_area_cachep, GFP_KERNEL);
+ if (!asma)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&asma->unpinned_list);
+ memcpy(asma->name, ASHMEM_NAME_PREFIX, ASHMEM_NAME_PREFIX_LEN);
+ asma->prot_mask = PROT_MASK;
+ file->private_data = asma;
+
+ return 0;
+}
+
+/**
+ * ashmem_release() - Releases an Anonymous Shared Memory structure
+ * @ignored: The backing file's Index Node(?) - It is ignored here.
+ * @file: The backing file
+ *
+ * Return: 0 if successful. If it is anything else, go have a coffee and
+ * try again.
+ */
+static int ashmem_release(struct inode *ignored, struct file *file)
+{
+ struct ashmem_area *asma = file->private_data;
+ struct ashmem_range *range, *next;
+
+ mutex_lock(&ashmem_mutex);
+ list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned)
+ range_del(range);
+ mutex_unlock(&ashmem_mutex);
+
+ if (asma->file)
+ fput(asma->file);
+ kmem_cache_free(ashmem_area_cachep, asma);
+
+ return 0;
+}
+
+static ssize_t ashmem_read_iter(struct kiocb *iocb, struct iov_iter *iter)
+{
+ struct ashmem_area *asma = iocb->ki_filp->private_data;
+ int ret = 0;
+
+ mutex_lock(&ashmem_mutex);
+
+ /* If size is not set, or set to 0, always return EOF. */
+ if (asma->size == 0)
+ goto out_unlock;
+
+ if (!asma->file) {
+ ret = -EBADF;
+ goto out_unlock;
+ }
+
+ /*
+ * asma and asma->file are used outside the lock here. We assume
+ * once asma->file is set it will never be changed, and will not
+ * be destroyed until all references to the file are dropped and
+ * ashmem_release is called.
+ */
+ mutex_unlock(&ashmem_mutex);
+ ret = vfs_iter_read(asma->file, iter, &iocb->ki_pos, 0);
+ mutex_lock(&ashmem_mutex);
+ if (ret > 0)
+ asma->file->f_pos = iocb->ki_pos;
+out_unlock:
+ mutex_unlock(&ashmem_mutex);
+ return ret;
+}
+
+static loff_t ashmem_llseek(struct file *file, loff_t offset, int origin)
+{
+ struct ashmem_area *asma = file->private_data;
+ loff_t ret;
+
+ mutex_lock(&ashmem_mutex);
+
+ if (asma->size == 0) {
+ mutex_unlock(&ashmem_mutex);
+ return -EINVAL;
+ }
+
+ if (!asma->file) {
+ mutex_unlock(&ashmem_mutex);
+ return -EBADF;
+ }
+
+ mutex_unlock(&ashmem_mutex);
+
+ ret = vfs_llseek(asma->file, offset, origin);
+ if (ret < 0)
+ return ret;
+
+ /** Copy f_pos from backing file, since f_ops->llseek() sets it */
+ file->f_pos = asma->file->f_pos;
+ return ret;
+}
+
+static inline vm_flags_t calc_vm_may_flags(unsigned long prot)
+{
+ return _calc_vm_trans(prot, PROT_READ, VM_MAYREAD) |
+ _calc_vm_trans(prot, PROT_WRITE, VM_MAYWRITE) |
+ _calc_vm_trans(prot, PROT_EXEC, VM_MAYEXEC);
+}
+
+static int ashmem_vmfile_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ /* do not allow to mmap ashmem backing shmem file directly */
+ return -EPERM;
+}
+
+static unsigned long
+ashmem_vmfile_get_unmapped_area(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long pgoff,
+ unsigned long flags)
+{
+ return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
+}
+
+static int ashmem_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ static struct file_operations vmfile_fops;
+ struct ashmem_area *asma = file->private_data;
+ int ret = 0;
+
+ mutex_lock(&ashmem_mutex);
+
+ /* user needs to SET_SIZE before mapping */
+ if (!asma->size) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* requested mapping size larger than object size */
+ if (vma->vm_end - vma->vm_start > PAGE_ALIGN(asma->size)) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* requested protection bits must match our allowed protection mask */
+ if ((vma->vm_flags & ~calc_vm_prot_bits(asma->prot_mask, 0)) &
+ calc_vm_prot_bits(PROT_MASK, 0)) {
+ ret = -EPERM;
+ goto out;
+ }
+ vma->vm_flags &= ~calc_vm_may_flags(~asma->prot_mask);
+
+ if (!asma->file) {
+ char *name = ASHMEM_NAME_DEF;
+ struct file *vmfile;
+ struct inode *inode;
+
+ if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0')
+ name = asma->name;
+
+ /* ... and allocate the backing shmem file */
+ vmfile = shmem_file_setup(name, asma->size, vma->vm_flags);
+ if (IS_ERR(vmfile)) {
+ ret = PTR_ERR(vmfile);
+ goto out;
+ }
+ vmfile->f_mode |= FMODE_LSEEK;
+ inode = file_inode(vmfile);
+ lockdep_set_class(&inode->i_rwsem, &backing_shmem_inode_class);
+ asma->file = vmfile;
+ /*
+ * override mmap operation of the vmfile so that it can't be
+ * remapped which would lead to creation of a new vma with no
+ * asma permission checks. Have to override get_unmapped_area
+ * as well to prevent VM_BUG_ON check for f_ops modification.
+ */
+ if (!vmfile_fops.mmap) {
+ vmfile_fops = *vmfile->f_op;
+ vmfile_fops.mmap = ashmem_vmfile_mmap;
+ vmfile_fops.get_unmapped_area =
+ ashmem_vmfile_get_unmapped_area;
+ }
+ vmfile->f_op = &vmfile_fops;
+ }
+ get_file(asma->file);
+
+ /*
+ * XXX - Reworked to use shmem_zero_setup() instead of
+ * shmem_set_file while we're in staging. -jstultz
+ */
+ if (vma->vm_flags & VM_SHARED) {
+ ret = shmem_zero_setup(vma);
+ if (ret) {
+ fput(asma->file);
+ goto out;
+ }
+ } else {
+ vma_set_anonymous(vma);
+ }
+
+ vma_set_file(vma, asma->file);
+ /* XXX: merge this with the get_file() above if possible */
+ fput(asma->file);
+
+out:
+ mutex_unlock(&ashmem_mutex);
+ return ret;
+}
+
+/*
+ * ashmem_shrink - our cache shrinker, called from mm/vmscan.c
+ *
+ * 'nr_to_scan' is the number of objects to scan for freeing.
+ *
+ * 'gfp_mask' is the mask of the allocation that got us into this mess.
+ *
+ * Return value is the number of objects freed or -1 if we cannot
+ * proceed without risk of deadlock (due to gfp_mask).
+ *
+ * We approximate LRU via least-recently-unpinned, jettisoning unpinned partial
+ * chunks of ashmem regions LRU-wise one-at-a-time until we hit 'nr_to_scan'
+ * pages freed.
+ */
+static unsigned long
+ashmem_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+ unsigned long freed = 0;
+
+ /* We might recurse into filesystem code, so bail out if necessary */
+ if (!(sc->gfp_mask & __GFP_FS))
+ return SHRINK_STOP;
+
+ if (!mutex_trylock(&ashmem_mutex))
+ return -1;
+
+ while (!list_empty(&ashmem_lru_list)) {
+ struct ashmem_range *range =
+ list_first_entry(&ashmem_lru_list, typeof(*range), lru);
+ loff_t start = range->pgstart * PAGE_SIZE;
+ loff_t end = (range->pgend + 1) * PAGE_SIZE;
+ struct file *f = range->asma->file;
+
+ get_file(f);
+ atomic_inc(&ashmem_shrink_inflight);
+ range->purged = ASHMEM_WAS_PURGED;
+ lru_del(range);
+
+ freed += range_size(range);
+ mutex_unlock(&ashmem_mutex);
+ f->f_op->fallocate(f,
+ FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+ start, end - start);
+ fput(f);
+ if (atomic_dec_and_test(&ashmem_shrink_inflight))
+ wake_up_all(&ashmem_shrink_wait);
+ if (!mutex_trylock(&ashmem_mutex))
+ goto out;
+ if (--sc->nr_to_scan <= 0)
+ break;
+ }
+ mutex_unlock(&ashmem_mutex);
+out:
+ return freed;
+}
+
+static unsigned long
+ashmem_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+ /*
+ * note that lru_count is count of pages on the lru, not a count of
+ * objects on the list. This means the scan function needs to return the
+ * number of pages freed, not the number of objects scanned.
+ */
+ return lru_count;
+}
+
+static struct shrinker ashmem_shrinker = {
+ .count_objects = ashmem_shrink_count,
+ .scan_objects = ashmem_shrink_scan,
+ /*
+ * XXX (dchinner): I wish people would comment on why they need on
+ * significant changes to the default value here
+ */
+ .seeks = DEFAULT_SEEKS * 4,
+};
+
+static int set_prot_mask(struct ashmem_area *asma, unsigned long prot)
+{
+ int ret = 0;
+
+ mutex_lock(&ashmem_mutex);
+
+ /* the user can only remove, not add, protection bits */
+ if ((asma->prot_mask & prot) != prot) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* does the application expect PROT_READ to imply PROT_EXEC? */
+ if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
+ prot |= PROT_EXEC;
+
+ asma->prot_mask = prot;
+
+out:
+ mutex_unlock(&ashmem_mutex);
+ return ret;
+}
+
+static int set_name(struct ashmem_area *asma, void __user *name)
+{
+ int len;
+ int ret = 0;
+ char local_name[ASHMEM_NAME_LEN];
+
+ /*
+ * Holding the ashmem_mutex while doing a copy_from_user might cause
+ * an data abort which would try to access mmap_lock. If another
+ * thread has invoked ashmem_mmap then it will be holding the
+ * semaphore and will be waiting for ashmem_mutex, there by leading to
+ * deadlock. We'll release the mutex and take the name to a local
+ * variable that does not need protection and later copy the local
+ * variable to the structure member with lock held.
+ */
+ len = strncpy_from_user(local_name, name, ASHMEM_NAME_LEN);
+ if (len < 0)
+ return len;
+
+ mutex_lock(&ashmem_mutex);
+ /* cannot change an existing mapping's name */
+ if (asma->file)
+ ret = -EINVAL;
+ else
+ strscpy(asma->name + ASHMEM_NAME_PREFIX_LEN, local_name,
+ ASHMEM_NAME_LEN);
+
+ mutex_unlock(&ashmem_mutex);
+ return ret;
+}
+
+static int get_name(struct ashmem_area *asma, void __user *name)
+{
+ int ret = 0;
+ size_t len;
+ /*
+ * Have a local variable to which we'll copy the content
+ * from asma with the lock held. Later we can copy this to the user
+ * space safely without holding any locks. So even if we proceed to
+ * wait for mmap_lock, it won't lead to deadlock.
+ */
+ char local_name[ASHMEM_NAME_LEN];
+
+ mutex_lock(&ashmem_mutex);
+ if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') {
+ /*
+ * Copying only `len', instead of ASHMEM_NAME_LEN, bytes
+ * prevents us from revealing one user's stack to another.
+ */
+ len = strlen(asma->name + ASHMEM_NAME_PREFIX_LEN) + 1;
+ memcpy(local_name, asma->name + ASHMEM_NAME_PREFIX_LEN, len);
+ } else {
+ len = sizeof(ASHMEM_NAME_DEF);
+ memcpy(local_name, ASHMEM_NAME_DEF, len);
+ }
+ mutex_unlock(&ashmem_mutex);
+
+ /*
+ * Now we are just copying from the stack variable to userland
+ * No lock held
+ */
+ if (copy_to_user(name, local_name, len))
+ ret = -EFAULT;
+ return ret;
+}
+
+/*
+ * ashmem_pin - pin the given ashmem region, returning whether it was
+ * previously purged (ASHMEM_WAS_PURGED) or not (ASHMEM_NOT_PURGED).
+ *
+ * Caller must hold ashmem_mutex.
+ */
+static int ashmem_pin(struct ashmem_area *asma, size_t pgstart, size_t pgend,
+ struct ashmem_range **new_range)
+{
+ struct ashmem_range *range, *next;
+ int ret = ASHMEM_NOT_PURGED;
+
+ list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) {
+ /* moved past last applicable page; we can short circuit */
+ if (range_before_page(range, pgstart))
+ break;
+
+ /*
+ * The user can ask us to pin pages that span multiple ranges,
+ * or to pin pages that aren't even unpinned, so this is messy.
+ *
+ * Four cases:
+ * 1. The requested range subsumes an existing range, so we
+ * just remove the entire matching range.
+ * 2. The requested range overlaps the start of an existing
+ * range, so we just update that range.
+ * 3. The requested range overlaps the end of an existing
+ * range, so we just update that range.
+ * 4. The requested range punches a hole in an existing range,
+ * so we have to update one side of the range and then
+ * create a new range for the other side.
+ */
+ if (page_range_in_range(range, pgstart, pgend)) {
+ ret |= range->purged;
+
+ /* Case #1: Easy. Just nuke the whole thing. */
+ if (page_range_subsumes_range(range, pgstart, pgend)) {
+ range_del(range);
+ continue;
+ }
+
+ /* Case #2: We overlap from the start, so adjust it */
+ if (range->pgstart >= pgstart) {
+ range_shrink(range, pgend + 1, range->pgend);
+ continue;
+ }
+
+ /* Case #3: We overlap from the rear, so adjust it */
+ if (range->pgend <= pgend) {
+ range_shrink(range, range->pgstart,
+ pgstart - 1);
+ continue;
+ }
+
+ /*
+ * Case #4: We eat a chunk out of the middle. A bit
+ * more complicated, we allocate a new range for the
+ * second half and adjust the first chunk's endpoint.
+ */
+ range_alloc(asma, range, range->purged,
+ pgend + 1, range->pgend, new_range);
+ range_shrink(range, range->pgstart, pgstart - 1);
+ break;
+ }
+ }
+
+ return ret;
+}
+
+/*
+ * ashmem_unpin - unpin the given range of pages. Returns zero on success.
+ *
+ * Caller must hold ashmem_mutex.
+ */
+static int ashmem_unpin(struct ashmem_area *asma, size_t pgstart, size_t pgend,
+ struct ashmem_range **new_range)
+{
+ struct ashmem_range *range = NULL, *iter, *next;
+ unsigned int purged = ASHMEM_NOT_PURGED;
+
+restart:
+ list_for_each_entry_safe(iter, next, &asma->unpinned_list, unpinned) {
+ /* short circuit: this is our insertion point */
+ if (range_before_page(iter, pgstart)) {
+ range = iter;
+ break;
+ }
+
+ /*
+ * The user can ask us to unpin pages that are already entirely
+ * or partially pinned. We handle those two cases here.
+ */
+ if (page_range_subsumed_by_range(iter, pgstart, pgend))
+ return 0;
+ if (page_range_in_range(iter, pgstart, pgend)) {
+ pgstart = min(iter->pgstart, pgstart);
+ pgend = max(iter->pgend, pgend);
+ purged |= iter->purged;
+ range_del(iter);
+ goto restart;
+ }
+ }
+
+ range = list_prepare_entry(range, &asma->unpinned_list, unpinned);
+ range_alloc(asma, range, purged, pgstart, pgend, new_range);
+ return 0;
+}
+
+/*
+ * ashmem_get_pin_status - Returns ASHMEM_IS_UNPINNED if _any_ pages in the
+ * given interval are unpinned and ASHMEM_IS_PINNED otherwise.
+ *
+ * Caller must hold ashmem_mutex.
+ */
+static int ashmem_get_pin_status(struct ashmem_area *asma, size_t pgstart,
+ size_t pgend)
+{
+ struct ashmem_range *range;
+ int ret = ASHMEM_IS_PINNED;
+
+ list_for_each_entry(range, &asma->unpinned_list, unpinned) {
+ if (range_before_page(range, pgstart))
+ break;
+ if (page_range_in_range(range, pgstart, pgend)) {
+ ret = ASHMEM_IS_UNPINNED;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static int ashmem_pin_unpin(struct ashmem_area *asma, unsigned long cmd,
+ void __user *p)
+{
+ struct ashmem_pin pin;
+ size_t pgstart, pgend;
+ int ret = -EINVAL;
+ struct ashmem_range *range = NULL;
+
+ if (copy_from_user(&pin, p, sizeof(pin)))
+ return -EFAULT;
+
+ if (cmd == ASHMEM_PIN || cmd == ASHMEM_UNPIN) {
+ range = kmem_cache_zalloc(ashmem_range_cachep, GFP_KERNEL);
+ if (!range)
+ return -ENOMEM;
+ }
+
+ mutex_lock(&ashmem_mutex);
+ wait_event(ashmem_shrink_wait, !atomic_read(&ashmem_shrink_inflight));
+
+ if (!asma->file)
+ goto out_unlock;
+
+ /* per custom, you can pass zero for len to mean "everything onward" */
+ if (!pin.len)
+ pin.len = PAGE_ALIGN(asma->size) - pin.offset;
+
+ if ((pin.offset | pin.len) & ~PAGE_MASK)
+ goto out_unlock;
+
+ if (((__u32)-1) - pin.offset < pin.len)
+ goto out_unlock;
+
+ if (PAGE_ALIGN(asma->size) < pin.offset + pin.len)
+ goto out_unlock;
+
+ pgstart = pin.offset / PAGE_SIZE;
+ pgend = pgstart + (pin.len / PAGE_SIZE) - 1;
+
+ switch (cmd) {
+ case ASHMEM_PIN:
+ ret = ashmem_pin(asma, pgstart, pgend, &range);
+ break;
+ case ASHMEM_UNPIN:
+ ret = ashmem_unpin(asma, pgstart, pgend, &range);
+ break;
+ case ASHMEM_GET_PIN_STATUS:
+ ret = ashmem_get_pin_status(asma, pgstart, pgend);
+ break;
+ }
+
+out_unlock:
+ mutex_unlock(&ashmem_mutex);
+ if (range)
+ kmem_cache_free(ashmem_range_cachep, range);
+
+ return ret;
+}
+
+static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+ struct ashmem_area *asma = file->private_data;
+ unsigned long ino;
+ long ret = -ENOTTY;
+
+ switch (cmd) {
+ case ASHMEM_SET_NAME:
+ ret = set_name(asma, (void __user *)arg);
+ break;
+ case ASHMEM_GET_NAME:
+ ret = get_name(asma, (void __user *)arg);
+ break;
+ case ASHMEM_SET_SIZE:
+ ret = -EINVAL;
+ mutex_lock(&ashmem_mutex);
+ if (!asma->file) {
+ ret = 0;
+ asma->size = (size_t)arg;
+ }
+ mutex_unlock(&ashmem_mutex);
+ break;
+ case ASHMEM_GET_SIZE:
+ ret = asma->size;
+ break;
+ case ASHMEM_SET_PROT_MASK:
+ ret = set_prot_mask(asma, arg);
+ break;
+ case ASHMEM_GET_PROT_MASK:
+ ret = asma->prot_mask;
+ break;
+ case ASHMEM_PIN:
+ case ASHMEM_UNPIN:
+ case ASHMEM_GET_PIN_STATUS:
+ ret = ashmem_pin_unpin(asma, cmd, (void __user *)arg);
+ break;
+ case ASHMEM_PURGE_ALL_CACHES:
+ ret = -EPERM;
+ if (capable(CAP_SYS_ADMIN)) {
+ struct shrink_control sc = {
+ .gfp_mask = GFP_KERNEL,
+ .nr_to_scan = LONG_MAX,
+ };
+ ret = ashmem_shrink_count(&ashmem_shrinker, &sc);
+ ashmem_shrink_scan(&ashmem_shrinker, &sc);
+ }
+ break;
+ case ASHMEM_GET_FILE_ID:
+ /* Lock around our check to avoid racing with ashmem_mmap(). */
+ mutex_lock(&ashmem_mutex);
+ if (!asma || !asma->file) {
+ mutex_unlock(&ashmem_mutex);
+ ret = -EINVAL;
+ break;
+ }
+ ino = file_inode(asma->file)->i_ino;
+ mutex_unlock(&ashmem_mutex);
+
+ if (copy_to_user((void __user *)arg, &ino, sizeof(ino))) {
+ ret = -EFAULT;
+ break;
+ }
+ ret = 0;
+ break;
+ }
+
+ return ret;
+}
+
+/* support of 32bit userspace on 64bit platforms */
+#ifdef CONFIG_COMPAT
+static long compat_ashmem_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ switch (cmd) {
+ case COMPAT_ASHMEM_SET_SIZE:
+ cmd = ASHMEM_SET_SIZE;
+ break;
+ case COMPAT_ASHMEM_SET_PROT_MASK:
+ cmd = ASHMEM_SET_PROT_MASK;
+ break;
+ }
+ return ashmem_ioctl(file, cmd, arg);
+}
+#endif
+#ifdef CONFIG_PROC_FS
+static void ashmem_show_fdinfo(struct seq_file *m, struct file *file)
+{
+ struct ashmem_area *asma = file->private_data;
+
+ mutex_lock(&ashmem_mutex);
+
+ if (asma->file)
+ seq_printf(m, "inode:\t%ld\n", file_inode(asma->file)->i_ino);
+
+ if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0')
+ seq_printf(m, "name:\t%s\n",
+ asma->name + ASHMEM_NAME_PREFIX_LEN);
+
+ seq_printf(m, "size:\t%zu\n", asma->size);
+
+ mutex_unlock(&ashmem_mutex);
+}
+#endif
+static const struct file_operations ashmem_fops = {
+ .owner = THIS_MODULE,
+ .open = ashmem_open,
+ .release = ashmem_release,
+ .read_iter = ashmem_read_iter,
+ .llseek = ashmem_llseek,
+ .mmap = ashmem_mmap,
+ .unlocked_ioctl = ashmem_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = compat_ashmem_ioctl,
+#endif
+#ifdef CONFIG_PROC_FS
+ .show_fdinfo = ashmem_show_fdinfo,
+#endif
+};
+
+static struct miscdevice ashmem_misc = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "ashmem",
+ .fops = &ashmem_fops,
+};
+
+static int __init ashmem_init(void)
+{
+ int ret = -ENOMEM;
+
+ ashmem_area_cachep = kmem_cache_create("ashmem_area_cache",
+ sizeof(struct ashmem_area),
+ 0, 0, NULL);
+ if (!ashmem_area_cachep) {
+ pr_err("failed to create slab cache\n");
+ goto out;
+ }
+
+ ashmem_range_cachep = kmem_cache_create("ashmem_range_cache",
+ sizeof(struct ashmem_range),
+ 0, SLAB_RECLAIM_ACCOUNT, NULL);
+ if (!ashmem_range_cachep) {
+ pr_err("failed to create slab cache\n");
+ goto out_free1;
+ }
+
+ ret = misc_register(&ashmem_misc);
+ if (ret) {
+ pr_err("failed to register misc device!\n");
+ goto out_free2;
+ }
+
+ ret = register_shrinker(&ashmem_shrinker, "android-ashmem");
+ if (ret) {
+ pr_err("failed to register shrinker!\n");
+ goto out_demisc;
+ }
+
+ pr_info("initialized\n");
+
+ return 0;
+
+out_demisc:
+ misc_deregister(&ashmem_misc);
+out_free2:
+ kmem_cache_destroy(ashmem_range_cachep);
+out_free1:
+ kmem_cache_destroy(ashmem_area_cachep);
+out:
+ return ret;
+}
+device_initcall(ashmem_init);
diff --git a/drivers/staging/android/ashmem.h b/drivers/staging/android/ashmem.h
new file mode 100644
index 0000000..1a47817
--- /dev/null
+++ b/drivers/staging/android/ashmem.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
+/*
+ * include/linux/ashmem.h
+ *
+ * Copyright 2008 Google Inc.
+ * Author: Robert Love
+ */
+
+#ifndef _LINUX_ASHMEM_H
+#define _LINUX_ASHMEM_H
+
+#include <linux/limits.h>
+#include <linux/ioctl.h>
+#include <linux/compat.h>
+
+#include "uapi/ashmem.h"
+
+/* support of 32bit userspace on 64bit platforms */
+#ifdef CONFIG_COMPAT
+#define COMPAT_ASHMEM_SET_SIZE _IOW(__ASHMEMIOC, 3, compat_size_t)
+#define COMPAT_ASHMEM_SET_PROT_MASK _IOW(__ASHMEMIOC, 5, unsigned int)
+#endif
+
+#endif /* _LINUX_ASHMEM_H */
diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h
new file mode 100644
index 0000000..f962a5f
--- /dev/null
+++ b/drivers/staging/android/uapi/ashmem.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
+/*
+ * Copyright 2008 Google Inc.
+ * Author: Robert Love
+ */
+
+#ifndef _UAPI_LINUX_ASHMEM_H
+#define _UAPI_LINUX_ASHMEM_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define ASHMEM_NAME_LEN 256
+
+#define ASHMEM_NAME_DEF "dev/ashmem"
+
+/* Return values from ASHMEM_PIN: Was the mapping purged while unpinned? */
+#define ASHMEM_NOT_PURGED 0
+#define ASHMEM_WAS_PURGED 1
+
+/* Return values from ASHMEM_GET_PIN_STATUS: Is the mapping pinned? */
+#define ASHMEM_IS_UNPINNED 0
+#define ASHMEM_IS_PINNED 1
+
+struct ashmem_pin {
+ __u32 offset; /* offset into region, in bytes, page-aligned */
+ __u32 len; /* length forward from offset, in bytes, page-aligned */
+};
+
+#define __ASHMEMIOC 0x77
+
+#define ASHMEM_SET_NAME _IOW(__ASHMEMIOC, 1, char[ASHMEM_NAME_LEN])
+#define ASHMEM_GET_NAME _IOR(__ASHMEMIOC, 2, char[ASHMEM_NAME_LEN])
+#define ASHMEM_SET_SIZE _IOW(__ASHMEMIOC, 3, size_t)
+#define ASHMEM_GET_SIZE _IO(__ASHMEMIOC, 4)
+#define ASHMEM_SET_PROT_MASK _IOW(__ASHMEMIOC, 5, unsigned long)
+#define ASHMEM_GET_PROT_MASK _IO(__ASHMEMIOC, 6)
+#define ASHMEM_PIN _IOW(__ASHMEMIOC, 7, struct ashmem_pin)
+#define ASHMEM_UNPIN _IOW(__ASHMEMIOC, 8, struct ashmem_pin)
+#define ASHMEM_GET_PIN_STATUS _IO(__ASHMEMIOC, 9)
+#define ASHMEM_PURGE_ALL_CACHES _IO(__ASHMEMIOC, 10)
+#define ASHMEM_GET_FILE_ID _IOR(__ASHMEMIOC, 11, unsigned long)
+
+#endif /* _UAPI_LINUX_ASHMEM_H */
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
index 56aa2e8..1368ccc 100644
--- a/drivers/thermal/thermal_helpers.c
+++ b/drivers/thermal/thermal_helpers.c
@@ -254,6 +254,7 @@ void thermal_cdev_update(struct thermal_cooling_device *cdev)
}
mutex_unlock(&cdev->lock);
}
+EXPORT_SYMBOL_GPL(thermal_cdev_update);
/**
* thermal_zone_get_slope - return the slope attribute of the thermal zone
diff --git a/drivers/tty/hvc/hvc_console.h b/drivers/tty/hvc/hvc_console.h
index 18d0058..97200fb 100644
--- a/drivers/tty/hvc/hvc_console.h
+++ b/drivers/tty/hvc/hvc_console.h
@@ -30,7 +30,7 @@
* for the tty device. Since this driver supports hotplug of vty adapters we
* need to make sure we have enough allocated.
*/
-#define HVC_ALLOC_TTY_ADAPTERS 8
+#define HVC_ALLOC_TTY_ADAPTERS 64
struct hvc_struct {
struct tty_port port;
diff --git a/drivers/tty/hvc/hvc_dcc.c b/drivers/tty/hvc/hvc_dcc.c
index 1751108..3cbd441 100644
--- a/drivers/tty/hvc/hvc_dcc.c
+++ b/drivers/tty/hvc/hvc_dcc.c
@@ -6,6 +6,7 @@
#include <linux/cpumask.h>
#include <linux/init.h>
#include <linux/kfifo.h>
+#include <linux/moduleparam.h>
#include <linux/serial.h>
#include <linux/serial_core.h>
#include <linux/smp.h>
@@ -16,6 +17,13 @@
#include "hvc_console.h"
+/*
+ * Disable DCC driver at runtime. Want driver enabled for GKI, but some devices
+ * do not support the registers and crash when driver pokes the registers
+ */
+static bool enable;
+module_param(enable, bool, 0444);
+
/* DCC Status Bits */
#define DCC_STATUS_RX (1 << 30)
#define DCC_STATUS_TX (1 << 29)
@@ -257,7 +265,7 @@ static int __init hvc_dcc_console_init(void)
{
int ret;
- if (!hvc_dcc_check())
+ if (!enable || !hvc_dcc_check())
return -ENODEV;
/* Returns -1 if error */
@@ -271,7 +279,7 @@ static int __init hvc_dcc_init(void)
{
struct hvc_struct *p;
- if (!hvc_dcc_check())
+ if (!enable || !hvc_dcc_check())
return -ENODEV;
if (IS_ENABLED(CONFIG_HVC_DCC_SERIALIZE_SMP)) {
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 434f831..a504c0e 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -238,7 +238,6 @@
config SERIAL_SAMSUNG
tristate "Samsung SoC serial support"
- depends on PLAT_SAMSUNG || ARCH_S5PV210 || ARCH_EXYNOS || ARCH_APPLE || ARCH_ARTPEC || COMPILE_TEST
select SERIAL_CORE
help
Support for the on-chip UARTs on the Samsung
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index d2b2720..a8e9de2 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -55,6 +55,8 @@
#include <asm/ptrace.h>
#include <asm/irq_regs.h>
+#include <trace/hooks/sysrqcrash.h>
+
/* Whether we react on sysrq keys or just ignore them */
static int __read_mostly sysrq_enabled = CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE;
static bool __read_mostly sysrq_always_enabled;
@@ -152,6 +154,8 @@ static void sysrq_handle_crash(int key)
/* release the RCU read lock before crashing */
rcu_read_unlock();
+ trace_android_vh_sysrq_crash(current);
+
panic("sysrq triggered crash\n");
}
static const struct sysrq_key_op sysrq_crash_op = {
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 53aea56..d64db8a 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -10,6 +10,8 @@
#include "ufs-sysfs.h"
#include "ufshcd-priv.h"
+#include <trace/hooks/ufshcd.h>
+
static const char *ufshcd_uic_link_state_to_string(
enum uic_link_state state)
{
@@ -1337,10 +1339,12 @@ void ufs_sysfs_add_nodes(struct device *dev)
int ret;
ret = sysfs_create_groups(&dev->kobj, ufs_sysfs_groups);
- if (ret)
+ if (ret) {
dev_err(dev,
"%s: sysfs groups creation failed (err = %d)\n",
__func__, ret);
+ return;
+ }
}
void ufs_sysfs_remove_nodes(struct device *dev)
diff --git a/drivers/ufs/core/ufshcd-crypto.c b/drivers/ufs/core/ufshcd-crypto.c
index 198360f..72a9400 100644
--- a/drivers/ufs/core/ufshcd-crypto.c
+++ b/drivers/ufs/core/ufshcd-crypto.c
@@ -123,6 +123,10 @@ bool ufshcd_crypto_enable(struct ufs_hba *hba)
/* Reset might clear all keys, so reprogram all the keys. */
blk_crypto_reprogram_all_keys(&hba->crypto_profile);
+
+ if (hba->quirks & UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE)
+ return false;
+
return true;
}
@@ -159,6 +163,9 @@ int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba)
int err = 0;
enum blk_crypto_mode_num blk_mode_num;
+ if (hba->quirks & UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE)
+ return 0;
+
/*
* Don't use crypto if either the hardware doesn't advertise the
* standard crypto capability bit *or* if the vendor specific driver
@@ -190,6 +197,7 @@ int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba)
hba->crypto_profile.ll_ops = ufshcd_crypto_ops;
/* UFS only supports 8 bytes for any DUN */
hba->crypto_profile.max_dun_bytes_supported = 8;
+ hba->crypto_profile.key_types_supported = BLK_CRYPTO_KEY_TYPE_STANDARD;
hba->crypto_profile.dev = hba->dev;
/*
@@ -228,9 +236,10 @@ void ufshcd_init_crypto(struct ufs_hba *hba)
if (!(hba->caps & UFSHCD_CAP_CRYPTO))
return;
- /* Clear all keyslots - the number of keyslots is (CFGC + 1) */
- for (slot = 0; slot < hba->crypto_capabilities.config_count + 1; slot++)
- ufshcd_clear_keyslot(hba, slot);
+ /* Clear all keyslots */
+ for (slot = 0; slot < hba->crypto_profile.num_slots; slot++)
+ hba->crypto_profile.ll_ops.keyslot_evict(&hba->crypto_profile,
+ NULL, slot);
}
void ufshcd_crypto_register(struct ufs_hba *hba, struct request_queue *q)
diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h
index 504cc84..55c2a5d 100644
--- a/drivers/ufs/core/ufshcd-crypto.h
+++ b/drivers/ufs/core/ufshcd-crypto.h
@@ -37,6 +37,19 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0,
}
}
+static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrbp)
+{
+ if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
+ return;
+
+ if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))
+ return;
+
+ memzero_explicit(lrbp->ucd_prdt_ptr,
+ hba->sg_entry_size * scsi_sg_count(lrbp->cmd));
+}
+
bool ufshcd_crypto_enable(struct ufs_hba *hba);
int ufshcd_hba_init_crypto_capabilities(struct ufs_hba *hba);
@@ -54,6 +67,9 @@ static inline void
ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0,
u32 *dword_1, u32 *dword_3) { }
+static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrbp) { }
+
static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
{
return false;
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index d2b11d5..49978d4 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -40,6 +40,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/ufs.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/ufshcd.h>
+
#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
UTP_TASK_REQ_COMPL |\
UFSHCD_ERROR_MASK)
@@ -346,6 +349,8 @@ static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
{
struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
+ trace_android_vh_ufs_send_tm_command(hba, tag, (int)str_t);
+
if (!trace_ufshcd_upiu_enabled())
return;
@@ -367,6 +372,8 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
{
u32 cmd;
+ trace_android_vh_ufs_send_uic_command(hba, ucmd, (int)str_t);
+
if (!trace_ufshcd_uic_command_enabled())
return;
@@ -525,7 +532,7 @@ void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
prdt_length = le16_to_cpu(
lrbp->utr_descriptor_ptr->prd_table_length);
if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
- prdt_length /= sizeof(struct ufshcd_sg_entry);
+ prdt_length /= hba->sg_entry_size;
dev_err(hba->dev,
"UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
@@ -534,7 +541,7 @@ void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
if (pr_prdt)
ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
- sizeof(struct ufshcd_sg_entry) * prdt_length);
+ hba->sg_entry_size * prdt_length);
}
}
@@ -2147,6 +2154,7 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
lrbp->issue_time_stamp_local_clock = local_clock();
lrbp->compl_time_stamp = ktime_set(0, 0);
lrbp->compl_time_stamp_local_clock = 0;
+ trace_android_vh_ufs_send_command(hba, lrbp);
ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
ufshcd_clk_scaling_start_busy(hba);
if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
@@ -2405,11 +2413,12 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
*/
static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
{
- struct ufshcd_sg_entry *prd_table;
+ struct ufshcd_sg_entry *prd;
struct scatterlist *sg;
struct scsi_cmnd *cmd;
int sg_segments;
int i;
+ int err;
cmd = lrbp->cmd;
sg_segments = scsi_dma_map(cmd);
@@ -2420,13 +2429,12 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
lrbp->utr_descriptor_ptr->prd_table_length =
- cpu_to_le16((sg_segments *
- sizeof(struct ufshcd_sg_entry)));
+ cpu_to_le16(sg_segments * hba->sg_entry_size);
else
lrbp->utr_descriptor_ptr->prd_table_length =
cpu_to_le16(sg_segments);
- prd_table = lrbp->ucd_prdt_ptr;
+ prd = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
scsi_for_each_sg(cmd, sg, sg_segments, i) {
const unsigned int len = sg_dma_len(sg);
@@ -2440,15 +2448,18 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
* indicates 4 bytes, '7' indicates 8 bytes, etc."
*/
WARN_ONCE(len > 256 * 1024, "len = %#x\n", len);
- prd_table[i].size = cpu_to_le32(len - 1);
- prd_table[i].addr = cpu_to_le64(sg->dma_address);
- prd_table[i].reserved = 0;
+ prd->size = cpu_to_le32(len - 1);
+ prd->addr = cpu_to_le64(sg->dma_address);
+ prd->reserved = 0;
+ prd = (void *)prd + hba->sg_entry_size;
}
} else {
lrbp->utr_descriptor_ptr->prd_table_length = 0;
}
- return 0;
+ err = 0;
+ trace_android_vh_ufs_fill_prdt(hba, lrbp, sg_segments, &err);
+ return err;
}
/**
@@ -2732,10 +2743,11 @@ static void ufshcd_map_queues(struct Scsi_Host *shost)
static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
{
- struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
+ struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
+ i * sizeof_utp_transfer_cmd_desc(hba);
struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
- i * sizeof(struct utp_transfer_cmd_desc);
+ i * sizeof_utp_transfer_cmd_desc(hba);
u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
response_upiu);
u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
@@ -2743,11 +2755,11 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
lrb->utr_descriptor_ptr = utrdlp + i;
lrb->utrd_dma_addr = hba->utrdl_dma_addr +
i * sizeof(struct utp_transfer_req_desc);
- lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
+ lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp;
lrb->ucd_req_dma_addr = cmd_desc_element_addr;
- lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
+ lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
- lrb->ucd_prdt_ptr = cmd_descp[i].prd_table;
+ lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
}
@@ -2835,6 +2847,14 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
+ trace_android_vh_ufs_prepare_command(hba, scsi_cmd_to_rq(cmd), lrbp,
+ &err);
+ if (err) {
+ lrbp->cmd = NULL;
+ ufshcd_release(hba);
+ goto out;
+ }
+
lrbp->req_abort_skip = false;
ufshpb_prep(hba, lrbp);
@@ -3080,7 +3100,7 @@ static inline void ufshcd_init_query(struct ufs_hba *hba,
(*request)->upiu_req.selector = selector;
}
-static int ufshcd_query_flag_retry(struct ufs_hba *hba,
+int ufshcd_query_flag_retry(struct ufs_hba *hba,
enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
{
int ret;
@@ -3102,6 +3122,7 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba,
__func__, opcode, idn, ret, retries);
return ret;
}
+EXPORT_SYMBOL_GPL(ufshcd_query_flag_retry);
/**
* ufshcd_query_flag() - API function for sending flag query requests
@@ -3170,6 +3191,7 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
ufshcd_release(hba);
return err;
}
+EXPORT_SYMBOL_GPL(ufshcd_query_flag);
/**
* ufshcd_query_attr - API function for sending attribute requests
@@ -3233,6 +3255,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
ufshcd_release(hba);
return err;
}
+EXPORT_SYMBOL_GPL(ufshcd_query_attr);
/**
* ufshcd_query_attr_retry() - API function for sending query
@@ -3270,6 +3293,7 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba,
__func__, idn, ret, QUERY_REQ_RETRIES);
return ret;
}
+EXPORT_SYMBOL_GPL(ufshcd_query_attr_retry);
static int __ufshcd_query_descriptor(struct ufs_hba *hba,
enum query_opcode opcode, enum desc_idn idn, u8 index,
@@ -3365,6 +3389,7 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
return err;
}
+EXPORT_SYMBOL_GPL(ufshcd_query_descriptor_retry);
/**
* ufshcd_map_desc_id_to_length - map descriptor IDN to its length
@@ -3483,6 +3508,7 @@ int ufshcd_read_desc_param(struct ufs_hba *hba,
kfree(desc_buf);
return ret;
}
+EXPORT_SYMBOL_GPL(ufshcd_read_desc_param);
/**
* struct uc_string_id - unicode string
@@ -3656,7 +3682,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
size_t utmrdl_size, utrdl_size, ucdl_size;
/* Allocate memory for UTP command descriptors */
- ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
+ ucdl_size = (sizeof_utp_transfer_cmd_desc(hba) * hba->nutrs);
hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
ucdl_size,
&hba->ucdl_dma_addr,
@@ -3750,7 +3776,7 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba)
prdt_offset =
offsetof(struct utp_transfer_cmd_desc, prd_table);
- cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
+ cmd_desc_size = sizeof_utp_transfer_cmd_desc(hba);
cmd_desc_dma_addr = hba->ucdl_dma_addr;
for (i = 0; i < hba->nutrs; i++) {
@@ -5340,6 +5366,7 @@ static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
struct scsi_cmnd *cmd = lrbp->cmd;
scsi_dma_unmap(cmd);
+ ufshcd_crypto_clear_prdt(hba, lrbp);
lrbp->cmd = NULL; /* Mark the command as completed. */
ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba);
@@ -5363,6 +5390,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
lrbp->compl_time_stamp_local_clock = local_clock();
cmd = lrbp->cmd;
if (cmd) {
+ trace_android_vh_ufs_compl_command(hba, lrbp);
if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
ufshcd_update_monitor(hba, lrbp);
ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
@@ -5373,6 +5401,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
} else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
if (hba->dev_cmd.complete) {
+ trace_android_vh_ufs_compl_command(hba, lrbp);
ufshcd_add_command_trace(hba, index,
UFS_DEV_COMP);
complete(hba->dev_cmd.complete);
@@ -5640,7 +5669,7 @@ static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
* to know whether auto bkops is enabled or disabled after this function
* returns control to it.
*/
-static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
+int ufshcd_bkops_ctrl(struct ufs_hba *hba,
enum bkops_status status)
{
int err;
@@ -5665,6 +5694,7 @@ static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
out:
return err;
}
+EXPORT_SYMBOL_GPL(ufshcd_bkops_ctrl);
/**
* ufshcd_urgent_bkops - handle urgent bkops exception event
@@ -6539,6 +6569,8 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
queue_eh_work = true;
}
+ trace_android_vh_ufs_check_int_errors(hba, queue_eh_work);
+
if (queue_eh_work) {
/*
* update the transfer error masks to sticky bits, let's do this
@@ -9612,6 +9644,7 @@ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
hba->dev = dev;
hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
hba->nop_out_timeout = NOP_OUT_TIMEOUT;
+ hba->sg_entry_size = sizeof(struct ufshcd_sg_entry);
INIT_LIST_HEAD(&hba->clk_list_head);
spin_lock_init(&hba->outstanding_lock);
@@ -9834,7 +9867,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
ufshcd_set_ufs_dev_active(hba);
async_schedule(ufshcd_async_scan, hba);
- ufs_sysfs_add_nodes(hba->dev);
+ ufs_sysfs_add_nodes(dev);
+ trace_android_vh_ufs_update_sysfs(hba);
device_enable_async_suspend(dev);
return 0;
@@ -9991,11 +10025,6 @@ static int __init ufshcd_core_init(void)
{
int ret;
- /* Verify that there are no gaps in struct utp_transfer_cmd_desc. */
- static_assert(sizeof(struct utp_transfer_cmd_desc) ==
- 2 * ALIGNED_UPIU_SIZE +
- SG_ALL * sizeof(struct ufshcd_sg_entry));
-
ufs_debugfs_init();
ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig
index 4fa2ddf3..87a97af 100644
--- a/drivers/usb/gadget/Kconfig
+++ b/drivers/usb/gadget/Kconfig
@@ -216,6 +216,12 @@
config USB_F_TCM
tristate
+config USB_F_ACC
+ tristate
+
+config USB_F_AUDIO_SRC
+ tristate
+
# this first set of drivers all depend on bulk-capable hardware.
config USB_CONFIGFS
@@ -230,6 +236,14 @@
appropriate symbolic links.
For more information see Documentation/usb/gadget_configfs.rst.
+config USB_CONFIGFS_UEVENT
+ bool "Uevent notification of Gadget state"
+ depends on USB_CONFIGFS
+ help
+ Enable uevent notifications to userspace when the gadget
+ state changes. The gadget can be in any of the following
+ three states: "CONNECTED/DISCONNECTED/CONFIGURED"
+
config USB_CONFIGFS_SERIAL
bool "Generic serial bulk in/out"
depends on USB_CONFIGFS
@@ -371,6 +385,23 @@
implemented in kernel space (for instance Ethernet, serial or
mass storage) and other are implemented in user space.
+config USB_CONFIGFS_F_ACC
+ bool "Accessory gadget"
+ depends on USB_CONFIGFS
+ depends on HID=y
+ select USB_F_ACC
+ help
+ USB gadget Accessory support
+
+config USB_CONFIGFS_F_AUDIO_SRC
+ bool "Audio Source gadget"
+ depends on USB_CONFIGFS
+ depends on SND
+ select SND_PCM
+ select USB_F_AUDIO_SRC
+ help
+ USB gadget Audio Source support
+
config USB_CONFIGFS_F_UAC1
bool "Audio Class 1.0"
depends on USB_CONFIGFS
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 3a6b492..dff14f9 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -10,6 +10,32 @@
#include "u_f.h"
#include "u_os_desc.h"
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+#include <linux/platform_device.h>
+#include <linux/kdev_t.h>
+#include <linux/usb/ch9.h>
+
+#ifdef CONFIG_USB_CONFIGFS_F_ACC
+extern int acc_ctrlrequest_composite(struct usb_composite_dev *cdev,
+ const struct usb_ctrlrequest *ctrl);
+void acc_disconnect(void);
+#endif
+static struct class *android_class;
+static struct device *android_device;
+static int index;
+static int gadget_index;
+
+struct device *create_function_device(char *name)
+{
+ if (android_device && !IS_ERR(android_device))
+ return device_create(android_class, android_device,
+ MKDEV(0, index++), NULL, name);
+ else
+ return ERR_PTR(-EINVAL);
+}
+EXPORT_SYMBOL_GPL(create_function_device);
+#endif
+
int check_user_usb_string(const char *name,
struct usb_gadget_strings *stringtab_dev)
{
@@ -51,6 +77,12 @@ struct gadget_info {
char qw_sign[OS_STRING_QW_SIGN_LEN];
spinlock_t spinlock;
bool unbind;
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+ bool connected;
+ bool sw_connected;
+ struct work_struct work;
+ struct device *dev;
+#endif
};
static inline struct gadget_info *to_gadget_info(struct config_item *item)
@@ -273,7 +305,7 @@ static ssize_t gadget_dev_desc_UDC_store(struct config_item *item,
mutex_lock(&gi->lock);
- if (!strlen(name)) {
+ if (!strlen(name) || strcmp(name, "none") == 0) {
ret = unregister_gadget(gi);
if (ret)
goto err;
@@ -1417,6 +1449,57 @@ static int configfs_composite_bind(struct usb_gadget *gadget,
return ret;
}
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+static void android_work(struct work_struct *data)
+{
+ struct gadget_info *gi = container_of(data, struct gadget_info, work);
+ struct usb_composite_dev *cdev = &gi->cdev;
+ char *disconnected[2] = { "USB_STATE=DISCONNECTED", NULL };
+ char *connected[2] = { "USB_STATE=CONNECTED", NULL };
+ char *configured[2] = { "USB_STATE=CONFIGURED", NULL };
+ /* 0-connected 1-configured 2-disconnected*/
+ bool status[3] = { false, false, false };
+ unsigned long flags;
+ bool uevent_sent = false;
+
+ spin_lock_irqsave(&cdev->lock, flags);
+ if (cdev->config)
+ status[1] = true;
+
+ if (gi->connected != gi->sw_connected) {
+ if (gi->connected)
+ status[0] = true;
+ else
+ status[2] = true;
+ gi->sw_connected = gi->connected;
+ }
+ spin_unlock_irqrestore(&cdev->lock, flags);
+
+ if (status[0]) {
+ kobject_uevent_env(&gi->dev->kobj, KOBJ_CHANGE, connected);
+ pr_info("%s: sent uevent %s\n", __func__, connected[0]);
+ uevent_sent = true;
+ }
+
+ if (status[1]) {
+ kobject_uevent_env(&gi->dev->kobj, KOBJ_CHANGE, configured);
+ pr_info("%s: sent uevent %s\n", __func__, configured[0]);
+ uevent_sent = true;
+ }
+
+ if (status[2]) {
+ kobject_uevent_env(&gi->dev->kobj, KOBJ_CHANGE, disconnected);
+ pr_info("%s: sent uevent %s\n", __func__, disconnected[0]);
+ uevent_sent = true;
+ }
+
+ if (!uevent_sent) {
+ pr_info("%s: did not send uevent (%d %d %p)\n", __func__,
+ gi->connected, gi->sw_connected, cdev->config);
+ }
+}
+#endif
+
static void configfs_composite_unbind(struct usb_gadget *gadget)
{
struct usb_composite_dev *cdev;
@@ -1444,6 +1527,50 @@ static void configfs_composite_unbind(struct usb_gadget *gadget)
spin_unlock_irqrestore(&gi->spinlock, flags);
}
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+static int android_setup(struct usb_gadget *gadget,
+ const struct usb_ctrlrequest *c)
+{
+ struct usb_composite_dev *cdev = get_gadget_data(gadget);
+ unsigned long flags;
+ struct gadget_info *gi = container_of(cdev, struct gadget_info, cdev);
+ int value = -EOPNOTSUPP;
+ struct usb_function_instance *fi;
+
+ spin_lock_irqsave(&cdev->lock, flags);
+ if (!gi->connected) {
+ gi->connected = 1;
+ schedule_work(&gi->work);
+ }
+ spin_unlock_irqrestore(&cdev->lock, flags);
+ list_for_each_entry(fi, &gi->available_func, cfs_list) {
+ if (fi != NULL && fi->f != NULL && fi->f->setup != NULL) {
+ value = fi->f->setup(fi->f, c);
+ if (value >= 0)
+ break;
+ }
+ }
+
+#ifdef CONFIG_USB_CONFIGFS_F_ACC
+ if (value < 0)
+ value = acc_ctrlrequest_composite(cdev, c);
+#endif
+
+ if (value < 0)
+ value = composite_setup(gadget, c);
+
+ spin_lock_irqsave(&cdev->lock, flags);
+ if (c->bRequest == USB_REQ_SET_CONFIGURATION &&
+ cdev->config) {
+ schedule_work(&gi->work);
+ }
+ spin_unlock_irqrestore(&cdev->lock, flags);
+
+ return value;
+}
+
+#else // CONFIG_USB_CONFIGFS_UEVENT
+
static int configfs_composite_setup(struct usb_gadget *gadget,
const struct usb_ctrlrequest *ctrl)
{
@@ -1469,6 +1596,8 @@ static int configfs_composite_setup(struct usb_gadget *gadget,
return ret;
}
+#endif // CONFIG_USB_CONFIGFS_UEVENT
+
static void configfs_composite_disconnect(struct usb_gadget *gadget)
{
struct usb_composite_dev *cdev;
@@ -1479,6 +1608,14 @@ static void configfs_composite_disconnect(struct usb_gadget *gadget)
if (!cdev)
return;
+#ifdef CONFIG_USB_CONFIGFS_F_ACC
+ /*
+ * accessory HID support can be active while the
+ * accessory function is not actually enabled,
+ * so we need to inform it when we are disconnected.
+ */
+ acc_disconnect();
+#endif
gi = container_of(cdev, struct gadget_info, cdev);
spin_lock_irqsave(&gi->spinlock, flags);
cdev = get_gadget_data(gadget);
@@ -1487,6 +1624,10 @@ static void configfs_composite_disconnect(struct usb_gadget *gadget)
return;
}
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+ gi->connected = 0;
+ schedule_work(&gi->work);
+#endif
composite_disconnect(gadget);
spin_unlock_irqrestore(&gi->spinlock, flags);
}
@@ -1561,10 +1702,13 @@ static const struct usb_gadget_driver configfs_driver_template = {
.bind = configfs_composite_bind,
.unbind = configfs_composite_unbind,
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+ .setup = android_setup,
+#else
.setup = configfs_composite_setup,
+#endif
.reset = configfs_composite_reset,
.disconnect = configfs_composite_disconnect,
-
.suspend = configfs_composite_suspend,
.resume = configfs_composite_resume,
@@ -1576,6 +1720,91 @@ static const struct usb_gadget_driver configfs_driver_template = {
.match_existing_only = 1,
};
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+static ssize_t state_show(struct device *pdev, struct device_attribute *attr,
+ char *buf)
+{
+ struct gadget_info *dev = dev_get_drvdata(pdev);
+ struct usb_composite_dev *cdev;
+ char *state = "DISCONNECTED";
+ unsigned long flags;
+
+ if (!dev)
+ goto out;
+
+ cdev = &dev->cdev;
+
+ if (!cdev)
+ goto out;
+
+ spin_lock_irqsave(&cdev->lock, flags);
+ if (cdev->config)
+ state = "CONFIGURED";
+ else if (dev->connected)
+ state = "CONNECTED";
+ spin_unlock_irqrestore(&cdev->lock, flags);
+out:
+ return sprintf(buf, "%s\n", state);
+}
+
+static DEVICE_ATTR(state, S_IRUGO, state_show, NULL);
+
+static struct device_attribute *android_usb_attributes[] = {
+ &dev_attr_state,
+ NULL
+};
+
+static int android_device_create(struct gadget_info *gi)
+{
+ struct device_attribute **attrs;
+ struct device_attribute *attr;
+
+ INIT_WORK(&gi->work, android_work);
+ gi->dev = device_create(android_class, NULL,
+ MKDEV(0, 0), NULL, "android%d", gadget_index++);
+ if (IS_ERR(gi->dev))
+ return PTR_ERR(gi->dev);
+
+ dev_set_drvdata(gi->dev, gi);
+ if (!android_device)
+ android_device = gi->dev;
+
+ attrs = android_usb_attributes;
+ while ((attr = *attrs++)) {
+ int err;
+
+ err = device_create_file(gi->dev, attr);
+ if (err) {
+ device_destroy(gi->dev->class,
+ gi->dev->devt);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static void android_device_destroy(struct gadget_info *gi)
+{
+ struct device_attribute **attrs;
+ struct device_attribute *attr;
+
+ attrs = android_usb_attributes;
+ while ((attr = *attrs++))
+ device_remove_file(gi->dev, attr);
+ device_destroy(gi->dev->class, gi->dev->devt);
+}
+#else
+static inline int android_device_create(struct gadget_info *gi)
+{
+ return 0;
+}
+
+static inline void android_device_destroy(struct gadget_info *gi)
+{
+}
+#endif
+
static struct config_group *gadgets_make(
struct config_group *group,
const char *name)
@@ -1628,7 +1857,11 @@ static struct config_group *gadgets_make(
if (!gi->composite.gadget_driver.function)
goto err;
+ if (android_device_create(gi) < 0)
+ goto err;
+
return &gi->group;
+
err:
kfree(gi);
return ERR_PTR(-ENOMEM);
@@ -1636,7 +1869,11 @@ static struct config_group *gadgets_make(
static void gadgets_drop(struct config_group *group, struct config_item *item)
{
+ struct gadget_info *gi;
+
+ gi = container_of(to_config_group(item), struct gadget_info, group);
config_item_put(item);
+ android_device_destroy(gi);
}
static struct configfs_group_operations gadgets_ops = {
@@ -1676,6 +1913,13 @@ static int __init gadget_cfs_init(void)
config_group_init(&gadget_subsys.su_group);
ret = configfs_register_subsystem(&gadget_subsys);
+
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+ android_class = class_create(THIS_MODULE, "android_usb");
+ if (IS_ERR(android_class))
+ return PTR_ERR(android_class);
+#endif
+
return ret;
}
module_init(gadget_cfs_init);
@@ -1683,5 +1927,10 @@ module_init(gadget_cfs_init);
static void __exit gadget_cfs_exit(void)
{
configfs_unregister_subsystem(&gadget_subsys);
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+ if (!IS_ERR(android_class))
+ class_destroy(android_class);
+#endif
+
}
module_exit(gadget_cfs_exit);
diff --git a/drivers/usb/gadget/function/Makefile b/drivers/usb/gadget/function/Makefile
index 5d3a6cf..dd33a12 100644
--- a/drivers/usb/gadget/function/Makefile
+++ b/drivers/usb/gadget/function/Makefile
@@ -50,3 +50,7 @@
obj-$(CONFIG_USB_F_PRINTER) += usb_f_printer.o
usb_f_tcm-y := f_tcm.o
obj-$(CONFIG_USB_F_TCM) += usb_f_tcm.o
+usb_f_accessory-y := f_accessory.o
+obj-$(CONFIG_USB_F_ACC) += usb_f_accessory.o
+usb_f_audio_source-y := f_audio_source.o
+obj-$(CONFIG_USB_F_AUDIO_SRC) += usb_f_audio_source.o
diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c
new file mode 100644
index 0000000..7d35d6c
--- /dev/null
+++ b/drivers/usb/gadget/function/f_accessory.c
@@ -0,0 +1,1574 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Gadget Function Driver for Android USB accessories
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Author: Mike Lockwood <lockwood@android.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+/* #define DEBUG */
+/* #define VERBOSE_DEBUG */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/poll.h>
+#include <linux/delay.h>
+#include <linux/wait.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+#include <linux/kref.h>
+
+#include <linux/types.h>
+#include <linux/file.h>
+#include <linux/device.h>
+#include <linux/miscdevice.h>
+
+#include <linux/hid.h>
+#include <linux/hiddev.h>
+#include <linux/usb.h>
+#include <linux/usb/ch9.h>
+#include <linux/usb/f_accessory.h>
+
+#include <linux/configfs.h>
+#include <linux/usb/composite.h>
+
+#define MAX_INST_NAME_LEN 40
+#define BULK_BUFFER_SIZE 16384
+#define ACC_STRING_SIZE 256
+
+#define PROTOCOL_VERSION 2
+
+/* String IDs */
+#define INTERFACE_STRING_INDEX 0
+
+/* number of tx and rx requests to allocate */
+#define TX_REQ_MAX 4
+#define RX_REQ_MAX 2
+
+struct acc_hid_dev {
+ struct list_head list;
+ struct hid_device *hid;
+ struct acc_dev *dev;
+ /* accessory defined ID */
+ int id;
+ /* HID report descriptor */
+ u8 *report_desc;
+ /* length of HID report descriptor */
+ int report_desc_len;
+ /* number of bytes of report_desc we have received so far */
+ int report_desc_offset;
+};
+
+struct acc_dev {
+ struct usb_function function;
+ struct usb_composite_dev *cdev;
+ spinlock_t lock;
+ struct acc_dev_ref *ref;
+
+ struct usb_ep *ep_in;
+ struct usb_ep *ep_out;
+
+ /* online indicates state of function_set_alt & function_unbind
+ * set to 1 when we connect
+ */
+ int online;
+
+ /* disconnected indicates state of open & release
+ * Set to 1 when we disconnect.
+ * Not cleared until our file is closed.
+ */
+ int disconnected;
+
+ /* strings sent by the host */
+ char manufacturer[ACC_STRING_SIZE];
+ char model[ACC_STRING_SIZE];
+ char description[ACC_STRING_SIZE];
+ char version[ACC_STRING_SIZE];
+ char uri[ACC_STRING_SIZE];
+ char serial[ACC_STRING_SIZE];
+
+ /* for acc_complete_set_string */
+ int string_index;
+
+ /* set to 1 if we have a pending start request */
+ int start_requested;
+
+ int audio_mode;
+
+ /* synchronize access to our device file */
+ atomic_t open_excl;
+
+ struct list_head tx_idle;
+
+ wait_queue_head_t read_wq;
+ wait_queue_head_t write_wq;
+ struct usb_request *rx_req[RX_REQ_MAX];
+ int rx_done;
+
+ /* delayed work for handling ACCESSORY_START */
+ struct delayed_work start_work;
+
+ /* work for handling ACCESSORY GET PROTOCOL */
+ struct work_struct getprotocol_work;
+
+ /* work for handling ACCESSORY SEND STRING */
+ struct work_struct sendstring_work;
+
+ /* worker for registering and unregistering hid devices */
+ struct work_struct hid_work;
+
+ /* list of active HID devices */
+ struct list_head hid_list;
+
+ /* list of new HID devices to register */
+ struct list_head new_hid_list;
+
+ /* list of dead HID devices to unregister */
+ struct list_head dead_hid_list;
+};
+
+static struct usb_interface_descriptor acc_interface_desc = {
+ .bLength = USB_DT_INTERFACE_SIZE,
+ .bDescriptorType = USB_DT_INTERFACE,
+ .bInterfaceNumber = 0,
+ .bNumEndpoints = 2,
+ .bInterfaceClass = USB_CLASS_VENDOR_SPEC,
+ .bInterfaceSubClass = USB_SUBCLASS_VENDOR_SPEC,
+ .bInterfaceProtocol = 0,
+};
+
+static struct usb_endpoint_descriptor acc_superspeedplus_in_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(1024),
+};
+
+static struct usb_endpoint_descriptor acc_superspeedplus_out_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_OUT,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor acc_superspeedplus_comp_desc = {
+ .bLength = sizeof(acc_superspeedplus_comp_desc),
+ .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
+
+ /* the following 2 values can be tweaked if necessary */
+ /* .bMaxBurst = 0, */
+ /* .bmAttributes = 0, */
+};
+
+static struct usb_endpoint_descriptor acc_superspeed_in_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(1024),
+};
+
+static struct usb_endpoint_descriptor acc_superspeed_out_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_OUT,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor acc_superspeed_comp_desc = {
+ .bLength = sizeof(acc_superspeed_comp_desc),
+ .bDescriptorType = USB_DT_SS_ENDPOINT_COMP,
+
+ /* the following 2 values can be tweaked if necessary */
+ /* .bMaxBurst = 0, */
+ /* .bmAttributes = 0, */
+};
+
+static struct usb_endpoint_descriptor acc_highspeed_in_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor acc_highspeed_out_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_OUT,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+ .wMaxPacketSize = cpu_to_le16(512),
+};
+
+static struct usb_endpoint_descriptor acc_fullspeed_in_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_endpoint_descriptor acc_fullspeed_out_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_OUT,
+ .bmAttributes = USB_ENDPOINT_XFER_BULK,
+};
+
+static struct usb_descriptor_header *fs_acc_descs[] = {
+ (struct usb_descriptor_header *) &acc_interface_desc,
+ (struct usb_descriptor_header *) &acc_fullspeed_in_desc,
+ (struct usb_descriptor_header *) &acc_fullspeed_out_desc,
+ NULL,
+};
+
+static struct usb_descriptor_header *hs_acc_descs[] = {
+ (struct usb_descriptor_header *) &acc_interface_desc,
+ (struct usb_descriptor_header *) &acc_highspeed_in_desc,
+ (struct usb_descriptor_header *) &acc_highspeed_out_desc,
+ NULL,
+};
+
+static struct usb_descriptor_header *ss_acc_descs[] = {
+ (struct usb_descriptor_header *) &acc_interface_desc,
+ (struct usb_descriptor_header *) &acc_superspeed_in_desc,
+ (struct usb_descriptor_header *) &acc_superspeed_comp_desc,
+ (struct usb_descriptor_header *) &acc_superspeed_out_desc,
+ (struct usb_descriptor_header *) &acc_superspeed_comp_desc,
+ NULL,
+};
+
+static struct usb_descriptor_header *ssp_acc_descs[] = {
+ (struct usb_descriptor_header *) &acc_interface_desc,
+ (struct usb_descriptor_header *) &acc_superspeedplus_in_desc,
+ (struct usb_descriptor_header *) &acc_superspeedplus_comp_desc,
+ (struct usb_descriptor_header *) &acc_superspeedplus_out_desc,
+ (struct usb_descriptor_header *) &acc_superspeedplus_comp_desc,
+ NULL,
+};
+
+static struct usb_string acc_string_defs[] = {
+ [INTERFACE_STRING_INDEX].s = "Android Accessory Interface",
+ { }, /* end of list */
+};
+
+static struct usb_gadget_strings acc_string_table = {
+ .language = 0x0409, /* en-US */
+ .strings = acc_string_defs,
+};
+
+static struct usb_gadget_strings *acc_strings[] = {
+ &acc_string_table,
+ NULL,
+};
+
+struct acc_dev_ref {
+ struct kref kref;
+ struct acc_dev *acc_dev;
+};
+
+static struct acc_dev_ref _acc_dev_ref = {
+ .kref = KREF_INIT(0),
+};
+
+struct acc_instance {
+ struct usb_function_instance func_inst;
+ const char *name;
+};
+
+static struct acc_dev *get_acc_dev(void)
+{
+ struct acc_dev_ref *ref = &_acc_dev_ref;
+
+ return kref_get_unless_zero(&ref->kref) ? ref->acc_dev : NULL;
+}
+
+static void __put_acc_dev(struct kref *kref)
+{
+ struct acc_dev_ref *ref = container_of(kref, struct acc_dev_ref, kref);
+ struct acc_dev *dev = ref->acc_dev;
+
+ /* Cancel any async work */
+ cancel_delayed_work_sync(&dev->start_work);
+ cancel_work_sync(&dev->getprotocol_work);
+ cancel_work_sync(&dev->sendstring_work);
+ cancel_work_sync(&dev->hid_work);
+
+ ref->acc_dev = NULL;
+ kfree(dev);
+}
+
+static void put_acc_dev(struct acc_dev *dev)
+{
+ struct acc_dev_ref *ref = dev->ref;
+
+ WARN_ON(ref->acc_dev != dev);
+ kref_put(&ref->kref, __put_acc_dev);
+}
+
+static inline struct acc_dev *func_to_dev(struct usb_function *f)
+{
+ return container_of(f, struct acc_dev, function);
+}
+
+static struct usb_request *acc_request_new(struct usb_ep *ep, int buffer_size)
+{
+ struct usb_request *req = usb_ep_alloc_request(ep, GFP_KERNEL);
+
+ if (!req)
+ return NULL;
+
+ /* now allocate buffers for the requests */
+ req->buf = kmalloc(buffer_size, GFP_KERNEL);
+ if (!req->buf) {
+ usb_ep_free_request(ep, req);
+ return NULL;
+ }
+
+ return req;
+}
+
+static void acc_request_free(struct usb_request *req, struct usb_ep *ep)
+{
+ if (req) {
+ kfree(req->buf);
+ usb_ep_free_request(ep, req);
+ }
+}
+
+/* add a request to the tail of a list */
+static void req_put(struct acc_dev *dev, struct list_head *head,
+ struct usb_request *req)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ list_add_tail(&req->list, head);
+ spin_unlock_irqrestore(&dev->lock, flags);
+}
+
+/* remove a request from the head of a list */
+static struct usb_request *req_get(struct acc_dev *dev, struct list_head *head)
+{
+ unsigned long flags;
+ struct usb_request *req;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ if (list_empty(head)) {
+ req = 0;
+ } else {
+ req = list_first_entry(head, struct usb_request, list);
+ list_del(&req->list);
+ }
+ spin_unlock_irqrestore(&dev->lock, flags);
+ return req;
+}
+
+static void acc_set_disconnected(struct acc_dev *dev)
+{
+ dev->disconnected = 1;
+}
+
+static void acc_complete_in(struct usb_ep *ep, struct usb_request *req)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
+
+ if (req->status == -ESHUTDOWN) {
+ pr_debug("acc_complete_in set disconnected");
+ acc_set_disconnected(dev);
+ }
+
+ req_put(dev, &dev->tx_idle, req);
+
+ wake_up(&dev->write_wq);
+ put_acc_dev(dev);
+}
+
+static void acc_complete_out(struct usb_ep *ep, struct usb_request *req)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
+
+ dev->rx_done = 1;
+ if (req->status == -ESHUTDOWN) {
+ pr_debug("acc_complete_out set disconnected");
+ acc_set_disconnected(dev);
+ }
+
+ wake_up(&dev->read_wq);
+ put_acc_dev(dev);
+}
+
+static void acc_complete_set_string(struct usb_ep *ep, struct usb_request *req)
+{
+ struct acc_dev *dev = ep->driver_data;
+ char *string_dest = NULL;
+ int length = req->actual;
+
+ if (req->status != 0) {
+ pr_err("acc_complete_set_string, err %d\n", req->status);
+ return;
+ }
+
+ switch (dev->string_index) {
+ case ACCESSORY_STRING_MANUFACTURER:
+ string_dest = dev->manufacturer;
+ break;
+ case ACCESSORY_STRING_MODEL:
+ string_dest = dev->model;
+ break;
+ case ACCESSORY_STRING_DESCRIPTION:
+ string_dest = dev->description;
+ break;
+ case ACCESSORY_STRING_VERSION:
+ string_dest = dev->version;
+ break;
+ case ACCESSORY_STRING_URI:
+ string_dest = dev->uri;
+ break;
+ case ACCESSORY_STRING_SERIAL:
+ string_dest = dev->serial;
+ break;
+ }
+ if (string_dest) {
+ unsigned long flags;
+
+ if (length >= ACC_STRING_SIZE)
+ length = ACC_STRING_SIZE - 1;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ memcpy(string_dest, req->buf, length);
+ /* ensure zero termination */
+ string_dest[length] = 0;
+ spin_unlock_irqrestore(&dev->lock, flags);
+ } else {
+ pr_err("unknown accessory string index %d\n",
+ dev->string_index);
+ }
+}
+
+static void acc_complete_set_hid_report_desc(struct usb_ep *ep,
+ struct usb_request *req)
+{
+ struct acc_hid_dev *hid = req->context;
+ struct acc_dev *dev = hid->dev;
+ int length = req->actual;
+
+ if (req->status != 0) {
+ pr_err("acc_complete_set_hid_report_desc, err %d\n",
+ req->status);
+ return;
+ }
+
+ memcpy(hid->report_desc + hid->report_desc_offset, req->buf, length);
+ hid->report_desc_offset += length;
+ if (hid->report_desc_offset == hid->report_desc_len) {
+ /* After we have received the entire report descriptor
+ * we schedule work to initialize the HID device
+ */
+ schedule_work(&dev->hid_work);
+ }
+}
+
+static void acc_complete_send_hid_event(struct usb_ep *ep,
+ struct usb_request *req)
+{
+ struct acc_hid_dev *hid = req->context;
+ int length = req->actual;
+
+ if (req->status != 0) {
+ pr_err("acc_complete_send_hid_event, err %d\n", req->status);
+ return;
+ }
+
+ hid_report_raw_event(hid->hid, HID_INPUT_REPORT, req->buf, length, 1);
+}
+
+static int acc_hid_parse(struct hid_device *hid)
+{
+ struct acc_hid_dev *hdev = hid->driver_data;
+
+ hid_parse_report(hid, hdev->report_desc, hdev->report_desc_len);
+ return 0;
+}
+
+static int acc_hid_start(struct hid_device *hid)
+{
+ return 0;
+}
+
+static void acc_hid_stop(struct hid_device *hid)
+{
+}
+
+static int acc_hid_open(struct hid_device *hid)
+{
+ return 0;
+}
+
+static void acc_hid_close(struct hid_device *hid)
+{
+}
+
+static int acc_hid_raw_request(struct hid_device *hid, unsigned char reportnum,
+ __u8 *buf, size_t len, unsigned char rtype, int reqtype)
+{
+ return 0;
+}
+
+static struct hid_ll_driver acc_hid_ll_driver = {
+ .parse = acc_hid_parse,
+ .start = acc_hid_start,
+ .stop = acc_hid_stop,
+ .open = acc_hid_open,
+ .close = acc_hid_close,
+ .raw_request = acc_hid_raw_request,
+};
+
+static struct acc_hid_dev *acc_hid_new(struct acc_dev *dev,
+ int id, int desc_len)
+{
+ struct acc_hid_dev *hdev;
+
+ hdev = kzalloc(sizeof(*hdev), GFP_ATOMIC);
+ if (!hdev)
+ return NULL;
+ hdev->report_desc = kzalloc(desc_len, GFP_ATOMIC);
+ if (!hdev->report_desc) {
+ kfree(hdev);
+ return NULL;
+ }
+ hdev->dev = dev;
+ hdev->id = id;
+ hdev->report_desc_len = desc_len;
+
+ return hdev;
+}
+
+static struct acc_hid_dev *acc_hid_get(struct list_head *list, int id)
+{
+ struct acc_hid_dev *hid;
+
+ list_for_each_entry(hid, list, list) {
+ if (hid->id == id)
+ return hid;
+ }
+ return NULL;
+}
+
+static int acc_register_hid(struct acc_dev *dev, int id, int desc_length)
+{
+ struct acc_hid_dev *hid;
+ unsigned long flags;
+
+ /* report descriptor length must be > 0 */
+ if (desc_length <= 0)
+ return -EINVAL;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ /* replace HID if one already exists with this ID */
+ hid = acc_hid_get(&dev->hid_list, id);
+ if (!hid)
+ hid = acc_hid_get(&dev->new_hid_list, id);
+ if (hid)
+ list_move(&hid->list, &dev->dead_hid_list);
+
+ hid = acc_hid_new(dev, id, desc_length);
+ if (!hid) {
+ spin_unlock_irqrestore(&dev->lock, flags);
+ return -ENOMEM;
+ }
+
+ list_add(&hid->list, &dev->new_hid_list);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ /* schedule work to register the HID device */
+ schedule_work(&dev->hid_work);
+ return 0;
+}
+
+static int acc_unregister_hid(struct acc_dev *dev, int id)
+{
+ struct acc_hid_dev *hid;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ hid = acc_hid_get(&dev->hid_list, id);
+ if (!hid)
+ hid = acc_hid_get(&dev->new_hid_list, id);
+ if (!hid) {
+ spin_unlock_irqrestore(&dev->lock, flags);
+ return -EINVAL;
+ }
+
+ list_move(&hid->list, &dev->dead_hid_list);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ schedule_work(&dev->hid_work);
+ return 0;
+}
+
+static int create_bulk_endpoints(struct acc_dev *dev,
+ struct usb_endpoint_descriptor *in_desc,
+ struct usb_endpoint_descriptor *out_desc)
+{
+ struct usb_composite_dev *cdev = dev->cdev;
+ struct usb_request *req;
+ struct usb_ep *ep;
+ int i;
+
+ DBG(cdev, "create_bulk_endpoints dev: %p\n", dev);
+
+ ep = usb_ep_autoconfig(cdev->gadget, in_desc);
+ if (!ep) {
+ DBG(cdev, "usb_ep_autoconfig for ep_in failed\n");
+ return -ENODEV;
+ }
+ DBG(cdev, "usb_ep_autoconfig for ep_in got %s\n", ep->name);
+ ep->driver_data = dev; /* claim the endpoint */
+ dev->ep_in = ep;
+
+ ep = usb_ep_autoconfig(cdev->gadget, out_desc);
+ if (!ep) {
+ DBG(cdev, "usb_ep_autoconfig for ep_out failed\n");
+ return -ENODEV;
+ }
+ DBG(cdev, "usb_ep_autoconfig for ep_out got %s\n", ep->name);
+ ep->driver_data = dev; /* claim the endpoint */
+ dev->ep_out = ep;
+
+ /* now allocate requests for our endpoints */
+ for (i = 0; i < TX_REQ_MAX; i++) {
+ req = acc_request_new(dev->ep_in, BULK_BUFFER_SIZE);
+ if (!req)
+ goto fail;
+ req->complete = acc_complete_in;
+ req_put(dev, &dev->tx_idle, req);
+ }
+ for (i = 0; i < RX_REQ_MAX; i++) {
+ req = acc_request_new(dev->ep_out, BULK_BUFFER_SIZE);
+ if (!req)
+ goto fail;
+ req->complete = acc_complete_out;
+ dev->rx_req[i] = req;
+ }
+
+ return 0;
+
+fail:
+ pr_err("acc_bind() could not allocate requests\n");
+ while ((req = req_get(dev, &dev->tx_idle)))
+ acc_request_free(req, dev->ep_in);
+ for (i = 0; i < RX_REQ_MAX; i++) {
+ acc_request_free(dev->rx_req[i], dev->ep_out);
+ dev->rx_req[i] = NULL;
+ }
+
+ return -1;
+}
+
+static ssize_t acc_read(struct file *fp, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct acc_dev *dev = fp->private_data;
+ struct usb_request *req;
+ ssize_t r = count;
+ ssize_t data_length;
+ unsigned xfer;
+ int ret = 0;
+
+ pr_debug("acc_read(%zu)\n", count);
+
+ if (dev->disconnected) {
+ pr_debug("acc_read disconnected");
+ return -ENODEV;
+ }
+
+ if (count > BULK_BUFFER_SIZE)
+ count = BULK_BUFFER_SIZE;
+
+ /* we will block until we're online */
+ pr_debug("acc_read: waiting for online\n");
+ ret = wait_event_interruptible(dev->read_wq, dev->online);
+ if (ret < 0) {
+ r = ret;
+ goto done;
+ }
+
+ if (!dev->rx_req[0]) {
+ pr_warn("acc_read: USB request already handled/freed");
+ r = -EINVAL;
+ goto done;
+ }
+
+ /*
+ * Calculate the data length by considering termination character.
+ * Then compansite the difference of rounding up to
+ * integer multiple of maxpacket size.
+ */
+ data_length = count;
+ data_length += dev->ep_out->maxpacket - 1;
+ data_length -= data_length % dev->ep_out->maxpacket;
+
+ if (dev->rx_done) {
+ // last req cancelled. try to get it.
+ req = dev->rx_req[0];
+ goto copy_data;
+ }
+
+requeue_req:
+ /* queue a request */
+ req = dev->rx_req[0];
+ req->length = data_length;
+ dev->rx_done = 0;
+ ret = usb_ep_queue(dev->ep_out, req, GFP_KERNEL);
+ if (ret < 0) {
+ r = -EIO;
+ goto done;
+ } else {
+ pr_debug("rx %p queue\n", req);
+ }
+
+ /* wait for a request to complete */
+ ret = wait_event_interruptible(dev->read_wq, dev->rx_done);
+ if (ret < 0) {
+ r = ret;
+ ret = usb_ep_dequeue(dev->ep_out, req);
+ if (ret != 0) {
+ // cancel failed. There can be a data already received.
+ // it will be retrieved in the next read.
+ pr_debug("acc_read: cancelling failed %d", ret);
+ }
+ goto done;
+ }
+
+copy_data:
+ dev->rx_done = 0;
+ if (dev->online) {
+ /* If we got a 0-len packet, throw it back and try again. */
+ if (req->actual == 0)
+ goto requeue_req;
+
+ pr_debug("rx %p %u\n", req, req->actual);
+ xfer = (req->actual < count) ? req->actual : count;
+ r = xfer;
+ if (copy_to_user(buf, req->buf, xfer))
+ r = -EFAULT;
+ } else
+ r = -EIO;
+
+done:
+ pr_debug("acc_read returning %zd\n", r);
+ return r;
+}
+
+static ssize_t acc_write(struct file *fp, const char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct acc_dev *dev = fp->private_data;
+ struct usb_request *req = 0;
+ ssize_t r = count;
+ unsigned xfer;
+ int ret;
+
+ pr_debug("acc_write(%zu)\n", count);
+
+ if (!dev->online || dev->disconnected) {
+ pr_debug("acc_write disconnected or not online");
+ return -ENODEV;
+ }
+
+ while (count > 0) {
+ /* get an idle tx request to use */
+ req = 0;
+ ret = wait_event_interruptible(dev->write_wq,
+ ((req = req_get(dev, &dev->tx_idle)) || !dev->online));
+ if (!dev->online || dev->disconnected) {
+ pr_debug("acc_write dev->error\n");
+ r = -EIO;
+ break;
+ }
+
+ if (!req) {
+ r = ret;
+ break;
+ }
+
+ if (count > BULK_BUFFER_SIZE) {
+ xfer = BULK_BUFFER_SIZE;
+ /* ZLP, They will be more TX requests so not yet. */
+ req->zero = 0;
+ } else {
+ xfer = count;
+ /* If the data length is a multple of the
+ * maxpacket size then send a zero length packet(ZLP).
+ */
+ req->zero = ((xfer % dev->ep_in->maxpacket) == 0);
+ }
+ if (copy_from_user(req->buf, buf, xfer)) {
+ r = -EFAULT;
+ break;
+ }
+
+ req->length = xfer;
+ ret = usb_ep_queue(dev->ep_in, req, GFP_KERNEL);
+ if (ret < 0) {
+ pr_debug("acc_write: xfer error %d\n", ret);
+ r = -EIO;
+ break;
+ }
+
+ buf += xfer;
+ count -= xfer;
+
+ /* zero this so we don't try to free it on error exit */
+ req = 0;
+ }
+
+ if (req)
+ req_put(dev, &dev->tx_idle, req);
+
+ pr_debug("acc_write returning %zd\n", r);
+ return r;
+}
+
+static long acc_ioctl(struct file *fp, unsigned code, unsigned long value)
+{
+ struct acc_dev *dev = fp->private_data;
+ char *src = NULL;
+ int ret;
+
+ switch (code) {
+ case ACCESSORY_GET_STRING_MANUFACTURER:
+ src = dev->manufacturer;
+ break;
+ case ACCESSORY_GET_STRING_MODEL:
+ src = dev->model;
+ break;
+ case ACCESSORY_GET_STRING_DESCRIPTION:
+ src = dev->description;
+ break;
+ case ACCESSORY_GET_STRING_VERSION:
+ src = dev->version;
+ break;
+ case ACCESSORY_GET_STRING_URI:
+ src = dev->uri;
+ break;
+ case ACCESSORY_GET_STRING_SERIAL:
+ src = dev->serial;
+ break;
+ case ACCESSORY_IS_START_REQUESTED:
+ return dev->start_requested;
+ case ACCESSORY_GET_AUDIO_MODE:
+ return dev->audio_mode;
+ }
+ if (!src)
+ return -EINVAL;
+
+ ret = strlen(src) + 1;
+ if (copy_to_user((void __user *)value, src, ret))
+ ret = -EFAULT;
+ return ret;
+}
+
+static int acc_open(struct inode *ip, struct file *fp)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return -ENODEV;
+
+ if (atomic_xchg(&dev->open_excl, 1)) {
+ put_acc_dev(dev);
+ return -EBUSY;
+ }
+
+ dev->disconnected = 0;
+ fp->private_data = dev;
+ return 0;
+}
+
+static int acc_release(struct inode *ip, struct file *fp)
+{
+ struct acc_dev *dev = fp->private_data;
+
+ if (!dev)
+ return -ENOENT;
+
+ /* indicate that we are disconnected
+ * still could be online so don't touch online flag
+ */
+ dev->disconnected = 1;
+
+ fp->private_data = NULL;
+ WARN_ON(!atomic_xchg(&dev->open_excl, 0));
+ put_acc_dev(dev);
+ return 0;
+}
+
+/* file operations for /dev/usb_accessory */
+static const struct file_operations acc_fops = {
+ .owner = THIS_MODULE,
+ .read = acc_read,
+ .write = acc_write,
+ .unlocked_ioctl = acc_ioctl,
+ .compat_ioctl = acc_ioctl,
+ .open = acc_open,
+ .release = acc_release,
+};
+
+static int acc_hid_probe(struct hid_device *hdev,
+ const struct hid_device_id *id)
+{
+ int ret;
+
+ ret = hid_parse(hdev);
+ if (ret)
+ return ret;
+ return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+}
+
+static struct miscdevice acc_device = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "usb_accessory",
+ .fops = &acc_fops,
+};
+
+static const struct hid_device_id acc_hid_table[] = {
+ { HID_USB_DEVICE(HID_ANY_ID, HID_ANY_ID) },
+ { }
+};
+
+static struct hid_driver acc_hid_driver = {
+ .name = "USB accessory",
+ .id_table = acc_hid_table,
+ .probe = acc_hid_probe,
+};
+
+static void acc_complete_setup_noop(struct usb_ep *ep, struct usb_request *req)
+{
+ /*
+ * Default no-op function when nothing needs to be done for the
+ * setup request
+ */
+}
+
+int acc_ctrlrequest(struct usb_composite_dev *cdev,
+ const struct usb_ctrlrequest *ctrl)
+{
+ struct acc_dev *dev = get_acc_dev();
+ int value = -EOPNOTSUPP;
+ struct acc_hid_dev *hid;
+ int offset;
+ u8 b_requestType = ctrl->bRequestType;
+ u8 b_request = ctrl->bRequest;
+ u16 w_index = le16_to_cpu(ctrl->wIndex);
+ u16 w_value = le16_to_cpu(ctrl->wValue);
+ u16 w_length = le16_to_cpu(ctrl->wLength);
+ unsigned long flags;
+
+ /*
+ * If instance is not created which is the case in power off charging
+ * mode, dev will be NULL. Hence return error if it is the case.
+ */
+ if (!dev)
+ return -ENODEV;
+
+ if (b_requestType == (USB_DIR_OUT | USB_TYPE_VENDOR)) {
+ if (b_request == ACCESSORY_START) {
+ dev->start_requested = 1;
+ schedule_delayed_work(
+ &dev->start_work, msecs_to_jiffies(10));
+ value = 0;
+ cdev->req->complete = acc_complete_setup_noop;
+ } else if (b_request == ACCESSORY_SEND_STRING) {
+ schedule_work(&dev->sendstring_work);
+ dev->string_index = w_index;
+ cdev->gadget->ep0->driver_data = dev;
+ cdev->req->complete = acc_complete_set_string;
+ value = w_length;
+ } else if (b_request == ACCESSORY_SET_AUDIO_MODE &&
+ w_index == 0 && w_length == 0) {
+ dev->audio_mode = w_value;
+ cdev->req->complete = acc_complete_setup_noop;
+ value = 0;
+ } else if (b_request == ACCESSORY_REGISTER_HID) {
+ cdev->req->complete = acc_complete_setup_noop;
+ value = acc_register_hid(dev, w_value, w_index);
+ } else if (b_request == ACCESSORY_UNREGISTER_HID) {
+ cdev->req->complete = acc_complete_setup_noop;
+ value = acc_unregister_hid(dev, w_value);
+ } else if (b_request == ACCESSORY_SET_HID_REPORT_DESC) {
+ spin_lock_irqsave(&dev->lock, flags);
+ hid = acc_hid_get(&dev->new_hid_list, w_value);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ if (!hid) {
+ value = -EINVAL;
+ goto err;
+ }
+ offset = w_index;
+ if (offset != hid->report_desc_offset
+ || offset + w_length > hid->report_desc_len) {
+ value = -EINVAL;
+ goto err;
+ }
+ cdev->req->context = hid;
+ cdev->req->complete = acc_complete_set_hid_report_desc;
+ value = w_length;
+ } else if (b_request == ACCESSORY_SEND_HID_EVENT) {
+ spin_lock_irqsave(&dev->lock, flags);
+ hid = acc_hid_get(&dev->hid_list, w_value);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ if (!hid) {
+ value = -EINVAL;
+ goto err;
+ }
+ cdev->req->context = hid;
+ cdev->req->complete = acc_complete_send_hid_event;
+ value = w_length;
+ }
+ } else if (b_requestType == (USB_DIR_IN | USB_TYPE_VENDOR)) {
+ if (b_request == ACCESSORY_GET_PROTOCOL) {
+ schedule_work(&dev->getprotocol_work);
+ *((u16 *)cdev->req->buf) = PROTOCOL_VERSION;
+ value = sizeof(u16);
+ cdev->req->complete = acc_complete_setup_noop;
+ /* clear any string left over from a previous session */
+ memset(dev->manufacturer, 0, sizeof(dev->manufacturer));
+ memset(dev->model, 0, sizeof(dev->model));
+ memset(dev->description, 0, sizeof(dev->description));
+ memset(dev->version, 0, sizeof(dev->version));
+ memset(dev->uri, 0, sizeof(dev->uri));
+ memset(dev->serial, 0, sizeof(dev->serial));
+ dev->start_requested = 0;
+ dev->audio_mode = 0;
+ }
+ }
+
+ if (value >= 0) {
+ cdev->req->zero = 0;
+ cdev->req->length = value;
+ value = usb_ep_queue(cdev->gadget->ep0, cdev->req, GFP_ATOMIC);
+ if (value < 0)
+ ERROR(cdev, "%s setup response queue error\n",
+ __func__);
+ }
+
+err:
+ if (value == -EOPNOTSUPP)
+ VDBG(cdev,
+ "unknown class-specific control req "
+ "%02x.%02x v%04x i%04x l%u\n",
+ ctrl->bRequestType, ctrl->bRequest,
+ w_value, w_index, w_length);
+ put_acc_dev(dev);
+ return value;
+}
+EXPORT_SYMBOL_GPL(acc_ctrlrequest);
+
+int acc_ctrlrequest_composite(struct usb_composite_dev *cdev,
+ const struct usb_ctrlrequest *ctrl)
+{
+ u16 w_length = le16_to_cpu(ctrl->wLength);
+
+ if (w_length > USB_COMP_EP0_BUFSIZ) {
+ if (ctrl->bRequestType & USB_DIR_IN) {
+ /* Cast away the const, we are going to overwrite on purpose. */
+ __le16 *temp = (__le16 *)&ctrl->wLength;
+
+ *temp = cpu_to_le16(USB_COMP_EP0_BUFSIZ);
+ w_length = USB_COMP_EP0_BUFSIZ;
+ } else {
+ return -EINVAL;
+ }
+ }
+ return acc_ctrlrequest(cdev, ctrl);
+}
+EXPORT_SYMBOL_GPL(acc_ctrlrequest_composite);
+
+static int
+__acc_function_bind(struct usb_configuration *c,
+ struct usb_function *f, bool configfs)
+{
+ struct usb_composite_dev *cdev = c->cdev;
+ struct acc_dev *dev = func_to_dev(f);
+ int id;
+ int ret;
+
+ DBG(cdev, "acc_function_bind dev: %p\n", dev);
+
+ if (configfs) {
+ if (acc_string_defs[INTERFACE_STRING_INDEX].id == 0) {
+ ret = usb_string_id(c->cdev);
+ if (ret < 0)
+ return ret;
+ acc_string_defs[INTERFACE_STRING_INDEX].id = ret;
+ acc_interface_desc.iInterface = ret;
+ }
+ dev->cdev = c->cdev;
+ }
+ ret = hid_register_driver(&acc_hid_driver);
+ if (ret)
+ return ret;
+
+ dev->start_requested = 0;
+
+ /* allocate interface ID(s) */
+ id = usb_interface_id(c, f);
+ if (id < 0)
+ return id;
+ acc_interface_desc.bInterfaceNumber = id;
+
+ /* allocate endpoints */
+ ret = create_bulk_endpoints(dev, &acc_fullspeed_in_desc,
+ &acc_fullspeed_out_desc);
+ if (ret)
+ return ret;
+
+ /* support high speed hardware */
+ acc_highspeed_in_desc.bEndpointAddress =
+ acc_fullspeed_in_desc.bEndpointAddress;
+ acc_highspeed_out_desc.bEndpointAddress =
+ acc_fullspeed_out_desc.bEndpointAddress;
+
+ /* support super speed hardware */
+ acc_superspeed_in_desc.bEndpointAddress =
+ acc_fullspeed_in_desc.bEndpointAddress;
+ acc_superspeed_out_desc.bEndpointAddress =
+ acc_fullspeed_out_desc.bEndpointAddress;
+
+ /* support super speed plus hardware */
+ acc_superspeedplus_in_desc.bEndpointAddress =
+ acc_fullspeed_in_desc.bEndpointAddress;
+ acc_superspeedplus_out_desc.bEndpointAddress =
+ acc_fullspeed_out_desc.bEndpointAddress;
+
+ DBG(cdev, "%s speed %s: IN/%s, OUT/%s\n",
+ gadget_is_dualspeed(c->cdev->gadget) ? "dual" : "full",
+ f->name, dev->ep_in->name, dev->ep_out->name);
+ return 0;
+}
+
+static int
+acc_function_bind_configfs(struct usb_configuration *c,
+ struct usb_function *f) {
+ return __acc_function_bind(c, f, true);
+}
+
+static void
+kill_all_hid_devices(struct acc_dev *dev)
+{
+ struct acc_hid_dev *hid;
+ struct list_head *entry, *temp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ list_for_each_safe(entry, temp, &dev->hid_list) {
+ hid = list_entry(entry, struct acc_hid_dev, list);
+ list_del(&hid->list);
+ list_add(&hid->list, &dev->dead_hid_list);
+ }
+ list_for_each_safe(entry, temp, &dev->new_hid_list) {
+ hid = list_entry(entry, struct acc_hid_dev, list);
+ list_del(&hid->list);
+ list_add(&hid->list, &dev->dead_hid_list);
+ }
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ schedule_work(&dev->hid_work);
+}
+
+static void
+acc_hid_unbind(struct acc_dev *dev)
+{
+ hid_unregister_driver(&acc_hid_driver);
+ kill_all_hid_devices(dev);
+}
+
+static void
+acc_function_unbind(struct usb_configuration *c, struct usb_function *f)
+{
+ struct acc_dev *dev = func_to_dev(f);
+ struct usb_request *req;
+ int i;
+
+ dev->online = 0; /* clear online flag */
+ wake_up(&dev->read_wq); /* unblock reads on closure */
+ wake_up(&dev->write_wq); /* likewise for writes */
+
+ while ((req = req_get(dev, &dev->tx_idle)))
+ acc_request_free(req, dev->ep_in);
+ for (i = 0; i < RX_REQ_MAX; i++) {
+ acc_request_free(dev->rx_req[i], dev->ep_out);
+ dev->rx_req[i] = NULL;
+ }
+
+ acc_hid_unbind(dev);
+}
+
+static void acc_getprotocol_work(struct work_struct *data)
+{
+ char *envp[2] = { "ACCESSORY=GETPROTOCOL", NULL };
+
+ kobject_uevent_env(&acc_device.this_device->kobj, KOBJ_CHANGE, envp);
+}
+
+static void acc_sendstring_work(struct work_struct *data)
+{
+ char *envp[2] = { "ACCESSORY=SENDSTRING", NULL };
+
+ kobject_uevent_env(&acc_device.this_device->kobj, KOBJ_CHANGE, envp);
+}
+
+static void acc_start_work(struct work_struct *data)
+{
+ char *envp[2] = { "ACCESSORY=START", NULL };
+
+ kobject_uevent_env(&acc_device.this_device->kobj, KOBJ_CHANGE, envp);
+}
+
+static int acc_hid_init(struct acc_hid_dev *hdev)
+{
+ struct hid_device *hid;
+ int ret;
+
+ hid = hid_allocate_device();
+ if (IS_ERR(hid))
+ return PTR_ERR(hid);
+
+ hid->ll_driver = &acc_hid_ll_driver;
+ hid->dev.parent = acc_device.this_device;
+
+ hid->bus = BUS_USB;
+ hid->vendor = HID_ANY_ID;
+ hid->product = HID_ANY_ID;
+ hid->driver_data = hdev;
+ ret = hid_add_device(hid);
+ if (ret) {
+ pr_err("can't add hid device: %d\n", ret);
+ hid_destroy_device(hid);
+ return ret;
+ }
+
+ hdev->hid = hid;
+ return 0;
+}
+
+static void acc_hid_delete(struct acc_hid_dev *hid)
+{
+ kfree(hid->report_desc);
+ kfree(hid);
+}
+
+static void acc_hid_work(struct work_struct *data)
+{
+ struct acc_dev *dev = get_acc_dev();
+ struct list_head *entry, *temp;
+ struct acc_hid_dev *hid;
+ struct list_head new_list, dead_list;
+ unsigned long flags;
+
+ if (!dev)
+ return;
+
+ INIT_LIST_HEAD(&new_list);
+
+ spin_lock_irqsave(&dev->lock, flags);
+
+ /* copy hids that are ready for initialization to new_list */
+ list_for_each_safe(entry, temp, &dev->new_hid_list) {
+ hid = list_entry(entry, struct acc_hid_dev, list);
+ if (hid->report_desc_offset == hid->report_desc_len)
+ list_move(&hid->list, &new_list);
+ }
+
+ if (list_empty(&dev->dead_hid_list)) {
+ INIT_LIST_HEAD(&dead_list);
+ } else {
+ /* move all of dev->dead_hid_list to dead_list */
+ dead_list.prev = dev->dead_hid_list.prev;
+ dead_list.next = dev->dead_hid_list.next;
+ dead_list.next->prev = &dead_list;
+ dead_list.prev->next = &dead_list;
+ INIT_LIST_HEAD(&dev->dead_hid_list);
+ }
+
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ /* register new HID devices */
+ list_for_each_safe(entry, temp, &new_list) {
+ hid = list_entry(entry, struct acc_hid_dev, list);
+ if (acc_hid_init(hid)) {
+ pr_err("can't add HID device %p\n", hid);
+ acc_hid_delete(hid);
+ } else {
+ spin_lock_irqsave(&dev->lock, flags);
+ list_move(&hid->list, &dev->hid_list);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ }
+ }
+
+ /* remove dead HID devices */
+ list_for_each_safe(entry, temp, &dead_list) {
+ hid = list_entry(entry, struct acc_hid_dev, list);
+ list_del(&hid->list);
+ if (hid->hid)
+ hid_destroy_device(hid->hid);
+ acc_hid_delete(hid);
+ }
+
+ put_acc_dev(dev);
+}
+
+static int acc_function_set_alt(struct usb_function *f,
+ unsigned intf, unsigned alt)
+{
+ struct acc_dev *dev = func_to_dev(f);
+ struct usb_composite_dev *cdev = f->config->cdev;
+ int ret;
+
+ DBG(cdev, "acc_function_set_alt intf: %d alt: %d\n", intf, alt);
+
+ ret = config_ep_by_speed(cdev->gadget, f, dev->ep_in);
+ if (ret)
+ return ret;
+
+ ret = usb_ep_enable(dev->ep_in);
+ if (ret)
+ return ret;
+
+ ret = config_ep_by_speed(cdev->gadget, f, dev->ep_out);
+ if (ret)
+ return ret;
+
+ ret = usb_ep_enable(dev->ep_out);
+ if (ret) {
+ usb_ep_disable(dev->ep_in);
+ return ret;
+ }
+
+ dev->online = 1;
+ dev->disconnected = 0; /* if online then not disconnected */
+
+ /* readers may be blocked waiting for us to go online */
+ wake_up(&dev->read_wq);
+ return 0;
+}
+
+static void acc_function_disable(struct usb_function *f)
+{
+ struct acc_dev *dev = func_to_dev(f);
+ struct usb_composite_dev *cdev = dev->cdev;
+
+ DBG(cdev, "acc_function_disable\n");
+ acc_set_disconnected(dev); /* this now only sets disconnected */
+ dev->online = 0; /* so now need to clear online flag here too */
+ usb_ep_disable(dev->ep_in);
+ usb_ep_disable(dev->ep_out);
+
+ /* readers may be blocked waiting for us to go online */
+ wake_up(&dev->read_wq);
+
+ VDBG(cdev, "%s disabled\n", dev->function.name);
+}
+
+static int acc_setup(void)
+{
+ struct acc_dev_ref *ref = &_acc_dev_ref;
+ struct acc_dev *dev;
+ int ret;
+
+ if (kref_read(&ref->kref))
+ return -EBUSY;
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return -ENOMEM;
+
+ spin_lock_init(&dev->lock);
+ init_waitqueue_head(&dev->read_wq);
+ init_waitqueue_head(&dev->write_wq);
+ atomic_set(&dev->open_excl, 0);
+ INIT_LIST_HEAD(&dev->tx_idle);
+ INIT_LIST_HEAD(&dev->hid_list);
+ INIT_LIST_HEAD(&dev->new_hid_list);
+ INIT_LIST_HEAD(&dev->dead_hid_list);
+ INIT_DELAYED_WORK(&dev->start_work, acc_start_work);
+ INIT_WORK(&dev->hid_work, acc_hid_work);
+ INIT_WORK(&dev->getprotocol_work, acc_getprotocol_work);
+ INIT_WORK(&dev->sendstring_work, acc_sendstring_work);
+
+ dev->ref = ref;
+ if (cmpxchg_relaxed(&ref->acc_dev, NULL, dev)) {
+ ret = -EBUSY;
+ goto err_free_dev;
+ }
+
+ ret = misc_register(&acc_device);
+ if (ret)
+ goto err_zap_ptr;
+
+ kref_init(&ref->kref);
+ return 0;
+
+err_zap_ptr:
+ ref->acc_dev = NULL;
+err_free_dev:
+ kfree(dev);
+ pr_err("USB accessory gadget driver failed to initialize\n");
+ return ret;
+}
+
+void acc_disconnect(void)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ if (!dev)
+ return;
+
+ /* unregister all HID devices if USB is disconnected */
+ kill_all_hid_devices(dev);
+ put_acc_dev(dev);
+}
+EXPORT_SYMBOL_GPL(acc_disconnect);
+
+static void acc_cleanup(void)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ misc_deregister(&acc_device);
+ put_acc_dev(dev);
+ put_acc_dev(dev); /* Pairs with kref_init() in acc_setup() */
+}
+static struct acc_instance *to_acc_instance(struct config_item *item)
+{
+ return container_of(to_config_group(item), struct acc_instance,
+ func_inst.group);
+}
+
+static void acc_attr_release(struct config_item *item)
+{
+ struct acc_instance *fi_acc = to_acc_instance(item);
+
+ usb_put_function_instance(&fi_acc->func_inst);
+}
+
+static struct configfs_item_operations acc_item_ops = {
+ .release = acc_attr_release,
+};
+
+static struct config_item_type acc_func_type = {
+ .ct_item_ops = &acc_item_ops,
+ .ct_owner = THIS_MODULE,
+};
+
+static struct acc_instance *to_fi_acc(struct usb_function_instance *fi)
+{
+ return container_of(fi, struct acc_instance, func_inst);
+}
+
+static int acc_set_inst_name(struct usb_function_instance *fi, const char *name)
+{
+ struct acc_instance *fi_acc;
+ char *ptr;
+ int name_len;
+
+ name_len = strlen(name) + 1;
+ if (name_len > MAX_INST_NAME_LEN)
+ return -ENAMETOOLONG;
+
+ ptr = kstrndup(name, name_len, GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+
+ fi_acc = to_fi_acc(fi);
+ fi_acc->name = ptr;
+ return 0;
+}
+
+static void acc_free_inst(struct usb_function_instance *fi)
+{
+ struct acc_instance *fi_acc;
+
+ fi_acc = to_fi_acc(fi);
+ kfree(fi_acc->name);
+ acc_cleanup();
+}
+
+static struct usb_function_instance *acc_alloc_inst(void)
+{
+ struct acc_instance *fi_acc;
+ int err;
+
+ fi_acc = kzalloc(sizeof(*fi_acc), GFP_KERNEL);
+ if (!fi_acc)
+ return ERR_PTR(-ENOMEM);
+ fi_acc->func_inst.set_inst_name = acc_set_inst_name;
+ fi_acc->func_inst.free_func_inst = acc_free_inst;
+
+ err = acc_setup();
+ if (err) {
+ kfree(fi_acc);
+ return ERR_PTR(err);
+ }
+
+ config_group_init_type_name(&fi_acc->func_inst.group,
+ "", &acc_func_type);
+ return &fi_acc->func_inst;
+}
+
+static void acc_free(struct usb_function *f)
+{
+ struct acc_dev *dev = func_to_dev(f);
+
+ put_acc_dev(dev);
+}
+
+int acc_ctrlrequest_configfs(struct usb_function *f,
+ const struct usb_ctrlrequest *ctrl) {
+ if (f->config != NULL && f->config->cdev != NULL)
+ return acc_ctrlrequest(f->config->cdev, ctrl);
+ else
+ return -1;
+}
+
+static struct usb_function *acc_alloc(struct usb_function_instance *fi)
+{
+ struct acc_dev *dev = get_acc_dev();
+
+ dev->function.name = "accessory";
+ dev->function.strings = acc_strings,
+ dev->function.fs_descriptors = fs_acc_descs;
+ dev->function.hs_descriptors = hs_acc_descs;
+ dev->function.ss_descriptors = ss_acc_descs;
+ dev->function.ssp_descriptors = ssp_acc_descs;
+ dev->function.bind = acc_function_bind_configfs;
+ dev->function.unbind = acc_function_unbind;
+ dev->function.set_alt = acc_function_set_alt;
+ dev->function.disable = acc_function_disable;
+ dev->function.free_func = acc_free;
+ dev->function.setup = acc_ctrlrequest_configfs;
+
+ return &dev->function;
+}
+DECLARE_USB_FUNCTION_INIT(accessory, acc_alloc_inst, acc_alloc);
+MODULE_LICENSE("GPL");
diff --git a/drivers/usb/gadget/function/f_audio_source.c b/drivers/usb/gadget/function/f_audio_source.c
new file mode 100644
index 0000000..c768a52
--- /dev/null
+++ b/drivers/usb/gadget/function/f_audio_source.c
@@ -0,0 +1,1071 @@
+/*
+ * Gadget Function Driver for USB audio source device
+ *
+ * Copyright (C) 2012 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/usb/audio.h>
+#include <linux/wait.h>
+#include <linux/pm_qos.h>
+#include <sound/core.h>
+#include <sound/initval.h>
+#include <sound/pcm.h>
+
+#include <linux/usb.h>
+#include <linux/usb_usual.h>
+#include <linux/usb/ch9.h>
+#include <linux/configfs.h>
+#include <linux/usb/composite.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#define SAMPLE_RATE 44100
+#define FRAMES_PER_MSEC (SAMPLE_RATE / 1000)
+
+#define IN_EP_MAX_PACKET_SIZE 256
+
+/* Number of requests to allocate */
+#define IN_EP_REQ_COUNT 4
+
+#define AUDIO_AC_INTERFACE 0
+#define AUDIO_AS_INTERFACE 1
+#define AUDIO_NUM_INTERFACES 2
+#define MAX_INST_NAME_LEN 40
+
+/* B.3.1 Standard AC Interface Descriptor */
+static struct usb_interface_descriptor ac_interface_desc = {
+ .bLength = USB_DT_INTERFACE_SIZE,
+ .bDescriptorType = USB_DT_INTERFACE,
+ .bNumEndpoints = 0,
+ .bInterfaceClass = USB_CLASS_AUDIO,
+ .bInterfaceSubClass = USB_SUBCLASS_AUDIOCONTROL,
+};
+
+DECLARE_UAC_AC_HEADER_DESCRIPTOR(2);
+
+#define UAC_DT_AC_HEADER_LENGTH UAC_DT_AC_HEADER_SIZE(AUDIO_NUM_INTERFACES)
+/* 1 input terminal, 1 output terminal and 1 feature unit */
+#define UAC_DT_TOTAL_LENGTH (UAC_DT_AC_HEADER_LENGTH \
+ + UAC_DT_INPUT_TERMINAL_SIZE + UAC_DT_OUTPUT_TERMINAL_SIZE \
+ + UAC_DT_FEATURE_UNIT_SIZE(0))
+/* B.3.2 Class-Specific AC Interface Descriptor */
+static struct uac1_ac_header_descriptor_2 ac_header_desc = {
+ .bLength = UAC_DT_AC_HEADER_LENGTH,
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_HEADER,
+ .bcdADC = __constant_cpu_to_le16(0x0100),
+ .wTotalLength = __constant_cpu_to_le16(UAC_DT_TOTAL_LENGTH),
+ .bInCollection = AUDIO_NUM_INTERFACES,
+ .baInterfaceNr = {
+ [0] = AUDIO_AC_INTERFACE,
+ [1] = AUDIO_AS_INTERFACE,
+ }
+};
+
+#define INPUT_TERMINAL_ID 1
+static struct uac_input_terminal_descriptor input_terminal_desc = {
+ .bLength = UAC_DT_INPUT_TERMINAL_SIZE,
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_INPUT_TERMINAL,
+ .bTerminalID = INPUT_TERMINAL_ID,
+ .wTerminalType = UAC_INPUT_TERMINAL_MICROPHONE,
+ .bAssocTerminal = 0,
+ .wChannelConfig = 0x3,
+};
+
+DECLARE_UAC_FEATURE_UNIT_DESCRIPTOR(0);
+
+#define FEATURE_UNIT_ID 2
+static struct uac_feature_unit_descriptor_0 feature_unit_desc = {
+ .bLength = UAC_DT_FEATURE_UNIT_SIZE(0),
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_FEATURE_UNIT,
+ .bUnitID = FEATURE_UNIT_ID,
+ .bSourceID = INPUT_TERMINAL_ID,
+ .bControlSize = 2,
+};
+
+#define OUTPUT_TERMINAL_ID 3
+static struct uac1_output_terminal_descriptor output_terminal_desc = {
+ .bLength = UAC_DT_OUTPUT_TERMINAL_SIZE,
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_OUTPUT_TERMINAL,
+ .bTerminalID = OUTPUT_TERMINAL_ID,
+ .wTerminalType = UAC_TERMINAL_STREAMING,
+ .bAssocTerminal = FEATURE_UNIT_ID,
+ .bSourceID = FEATURE_UNIT_ID,
+};
+
+/* B.4.1 Standard AS Interface Descriptor */
+static struct usb_interface_descriptor as_interface_alt_0_desc = {
+ .bLength = USB_DT_INTERFACE_SIZE,
+ .bDescriptorType = USB_DT_INTERFACE,
+ .bAlternateSetting = 0,
+ .bNumEndpoints = 0,
+ .bInterfaceClass = USB_CLASS_AUDIO,
+ .bInterfaceSubClass = USB_SUBCLASS_AUDIOSTREAMING,
+};
+
+static struct usb_interface_descriptor as_interface_alt_1_desc = {
+ .bLength = USB_DT_INTERFACE_SIZE,
+ .bDescriptorType = USB_DT_INTERFACE,
+ .bAlternateSetting = 1,
+ .bNumEndpoints = 1,
+ .bInterfaceClass = USB_CLASS_AUDIO,
+ .bInterfaceSubClass = USB_SUBCLASS_AUDIOSTREAMING,
+};
+
+/* B.4.2 Class-Specific AS Interface Descriptor */
+static struct uac1_as_header_descriptor as_header_desc = {
+ .bLength = UAC_DT_AS_HEADER_SIZE,
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_AS_GENERAL,
+ .bTerminalLink = INPUT_TERMINAL_ID,
+ .bDelay = 1,
+ .wFormatTag = UAC_FORMAT_TYPE_I_PCM,
+};
+
+DECLARE_UAC_FORMAT_TYPE_I_DISCRETE_DESC(1);
+
+static struct uac_format_type_i_discrete_descriptor_1 as_type_i_desc = {
+ .bLength = UAC_FORMAT_TYPE_I_DISCRETE_DESC_SIZE(1),
+ .bDescriptorType = USB_DT_CS_INTERFACE,
+ .bDescriptorSubtype = UAC_FORMAT_TYPE,
+ .bFormatType = UAC_FORMAT_TYPE_I,
+ .bSubframeSize = 2,
+ .bBitResolution = 16,
+ .bSamFreqType = 1,
+};
+
+/* Standard ISO IN Endpoint Descriptor for highspeed */
+static struct usb_endpoint_descriptor hs_as_in_ep_desc = {
+ .bLength = USB_DT_ENDPOINT_AUDIO_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_SYNC_SYNC
+ | USB_ENDPOINT_XFER_ISOC,
+ .wMaxPacketSize = __constant_cpu_to_le16(IN_EP_MAX_PACKET_SIZE),
+ .bInterval = 4, /* poll 1 per millisecond */
+};
+
+/* Standard ISO IN Endpoint Descriptor for highspeed */
+static struct usb_endpoint_descriptor fs_as_in_ep_desc = {
+ .bLength = USB_DT_ENDPOINT_AUDIO_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bEndpointAddress = USB_DIR_IN,
+ .bmAttributes = USB_ENDPOINT_SYNC_SYNC
+ | USB_ENDPOINT_XFER_ISOC,
+ .wMaxPacketSize = __constant_cpu_to_le16(IN_EP_MAX_PACKET_SIZE),
+ .bInterval = 1, /* poll 1 per millisecond */
+};
+
+/* Class-specific AS ISO OUT Endpoint Descriptor */
+static struct uac_iso_endpoint_descriptor as_iso_in_desc = {
+ .bLength = UAC_ISO_ENDPOINT_DESC_SIZE,
+ .bDescriptorType = USB_DT_CS_ENDPOINT,
+ .bDescriptorSubtype = UAC_EP_GENERAL,
+ .bmAttributes = 1,
+ .bLockDelayUnits = 1,
+ .wLockDelay = __constant_cpu_to_le16(1),
+};
+
+static struct usb_descriptor_header *hs_audio_desc[] = {
+ (struct usb_descriptor_header *)&ac_interface_desc,
+ (struct usb_descriptor_header *)&ac_header_desc,
+
+ (struct usb_descriptor_header *)&input_terminal_desc,
+ (struct usb_descriptor_header *)&output_terminal_desc,
+ (struct usb_descriptor_header *)&feature_unit_desc,
+
+ (struct usb_descriptor_header *)&as_interface_alt_0_desc,
+ (struct usb_descriptor_header *)&as_interface_alt_1_desc,
+ (struct usb_descriptor_header *)&as_header_desc,
+
+ (struct usb_descriptor_header *)&as_type_i_desc,
+
+ (struct usb_descriptor_header *)&hs_as_in_ep_desc,
+ (struct usb_descriptor_header *)&as_iso_in_desc,
+ NULL,
+};
+
+static struct usb_descriptor_header *fs_audio_desc[] = {
+ (struct usb_descriptor_header *)&ac_interface_desc,
+ (struct usb_descriptor_header *)&ac_header_desc,
+
+ (struct usb_descriptor_header *)&input_terminal_desc,
+ (struct usb_descriptor_header *)&output_terminal_desc,
+ (struct usb_descriptor_header *)&feature_unit_desc,
+
+ (struct usb_descriptor_header *)&as_interface_alt_0_desc,
+ (struct usb_descriptor_header *)&as_interface_alt_1_desc,
+ (struct usb_descriptor_header *)&as_header_desc,
+
+ (struct usb_descriptor_header *)&as_type_i_desc,
+
+ (struct usb_descriptor_header *)&fs_as_in_ep_desc,
+ (struct usb_descriptor_header *)&as_iso_in_desc,
+ NULL,
+};
+
+static struct snd_pcm_hardware audio_hw_info = {
+ .info = SNDRV_PCM_INFO_MMAP |
+ SNDRV_PCM_INFO_MMAP_VALID |
+ SNDRV_PCM_INFO_BATCH |
+ SNDRV_PCM_INFO_INTERLEAVED |
+ SNDRV_PCM_INFO_BLOCK_TRANSFER,
+
+ .formats = SNDRV_PCM_FMTBIT_S16_LE,
+ .channels_min = 2,
+ .channels_max = 2,
+ .rate_min = SAMPLE_RATE,
+ .rate_max = SAMPLE_RATE,
+
+ .buffer_bytes_max = 1024 * 1024,
+ .period_bytes_min = 64,
+ .period_bytes_max = 512 * 1024,
+ .periods_min = 2,
+ .periods_max = 1024,
+};
+
+/*-------------------------------------------------------------------------*/
+
+struct audio_source_config {
+ int card;
+ int device;
+};
+
+struct audio_dev {
+ struct usb_function func;
+ struct snd_card *card;
+ struct snd_pcm *pcm;
+ struct snd_pcm_substream *substream;
+
+ struct list_head idle_reqs;
+ struct usb_ep *in_ep;
+
+ spinlock_t lock;
+
+ /* beginning, end and current position in our buffer */
+ void *buffer_start;
+ void *buffer_end;
+ void *buffer_pos;
+
+ /* byte size of a "period" */
+ unsigned int period;
+ /* bytes sent since last call to snd_pcm_period_elapsed */
+ unsigned int period_offset;
+ /* time we started playing */
+ ktime_t start_time;
+ /* number of frames sent since start_time */
+ s64 frames_sent;
+ struct audio_source_config *config;
+ /* for creating and issuing QoS requests */
+ struct pm_qos_request pm_qos;
+};
+
+static inline struct audio_dev *func_to_audio(struct usb_function *f)
+{
+ return container_of(f, struct audio_dev, func);
+}
+
+/*-------------------------------------------------------------------------*/
+
+struct audio_source_instance {
+ struct usb_function_instance func_inst;
+ const char *name;
+ struct audio_source_config *config;
+ struct device *audio_device;
+};
+
+static void audio_source_attr_release(struct config_item *item);
+
+static struct configfs_item_operations audio_source_item_ops = {
+ .release = audio_source_attr_release,
+};
+
+static struct config_item_type audio_source_func_type = {
+ .ct_item_ops = &audio_source_item_ops,
+ .ct_owner = THIS_MODULE,
+};
+
+static ssize_t audio_source_pcm_show(struct device *dev,
+ struct device_attribute *attr, char *buf);
+
+static DEVICE_ATTR(pcm, S_IRUGO, audio_source_pcm_show, NULL);
+
+static struct device_attribute *audio_source_function_attributes[] = {
+ &dev_attr_pcm,
+ NULL
+};
+
+/*--------------------------------------------------------------------------*/
+
+static struct usb_request *audio_request_new(struct usb_ep *ep, int buffer_size)
+{
+ struct usb_request *req = usb_ep_alloc_request(ep, GFP_KERNEL);
+
+ if (!req)
+ return NULL;
+
+ req->buf = kmalloc(buffer_size, GFP_KERNEL);
+ if (!req->buf) {
+ usb_ep_free_request(ep, req);
+ return NULL;
+ }
+ req->length = buffer_size;
+ return req;
+}
+
+static void audio_request_free(struct usb_request *req, struct usb_ep *ep)
+{
+ if (req) {
+ kfree(req->buf);
+ usb_ep_free_request(ep, req);
+ }
+}
+
+static void audio_req_put(struct audio_dev *audio, struct usb_request *req)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&audio->lock, flags);
+ list_add_tail(&req->list, &audio->idle_reqs);
+ spin_unlock_irqrestore(&audio->lock, flags);
+}
+
+static struct usb_request *audio_req_get(struct audio_dev *audio)
+{
+ unsigned long flags;
+ struct usb_request *req;
+
+ spin_lock_irqsave(&audio->lock, flags);
+ if (list_empty(&audio->idle_reqs)) {
+ req = 0;
+ } else {
+ req = list_first_entry(&audio->idle_reqs, struct usb_request,
+ list);
+ list_del(&req->list);
+ }
+ spin_unlock_irqrestore(&audio->lock, flags);
+ return req;
+}
+
+/* send the appropriate number of packets to match our bitrate */
+static void audio_send(struct audio_dev *audio)
+{
+ struct snd_pcm_runtime *runtime;
+ struct usb_request *req;
+ int length, length1, length2, ret;
+ s64 msecs;
+ s64 frames;
+ ktime_t now;
+
+ /* audio->substream will be null if we have been closed */
+ if (!audio->substream)
+ return;
+ /* audio->buffer_pos will be null if we have been stopped */
+ if (!audio->buffer_pos)
+ return;
+
+ runtime = audio->substream->runtime;
+
+ /* compute number of frames to send */
+ now = ktime_get();
+ msecs = div_s64((ktime_to_ns(now) - ktime_to_ns(audio->start_time)),
+ 1000000);
+ frames = div_s64((msecs * SAMPLE_RATE), 1000);
+
+ /* Readjust our frames_sent if we fall too far behind.
+ * If we get too far behind it is better to drop some frames than
+ * to keep sending data too fast in an attempt to catch up.
+ */
+ if (frames - audio->frames_sent > 10 * FRAMES_PER_MSEC)
+ audio->frames_sent = frames - FRAMES_PER_MSEC;
+
+ frames -= audio->frames_sent;
+
+ /* We need to send something to keep the pipeline going */
+ if (frames <= 0)
+ frames = FRAMES_PER_MSEC;
+
+ while (frames > 0) {
+ req = audio_req_get(audio);
+ if (!req)
+ break;
+
+ length = frames_to_bytes(runtime, frames);
+ if (length > IN_EP_MAX_PACKET_SIZE)
+ length = IN_EP_MAX_PACKET_SIZE;
+
+ if (audio->buffer_pos + length > audio->buffer_end)
+ length1 = audio->buffer_end - audio->buffer_pos;
+ else
+ length1 = length;
+ memcpy(req->buf, audio->buffer_pos, length1);
+ if (length1 < length) {
+ /* Wrap around and copy remaining length
+ * at beginning of buffer.
+ */
+ length2 = length - length1;
+ memcpy(req->buf + length1, audio->buffer_start,
+ length2);
+ audio->buffer_pos = audio->buffer_start + length2;
+ } else {
+ audio->buffer_pos += length1;
+ if (audio->buffer_pos >= audio->buffer_end)
+ audio->buffer_pos = audio->buffer_start;
+ }
+
+ req->length = length;
+ ret = usb_ep_queue(audio->in_ep, req, GFP_ATOMIC);
+ if (ret < 0) {
+ pr_err("usb_ep_queue failed ret: %d\n", ret);
+ audio_req_put(audio, req);
+ break;
+ }
+
+ frames -= bytes_to_frames(runtime, length);
+ audio->frames_sent += bytes_to_frames(runtime, length);
+ }
+}
+
+static void audio_control_complete(struct usb_ep *ep, struct usb_request *req)
+{
+ /* nothing to do here */
+}
+
+static void audio_data_complete(struct usb_ep *ep, struct usb_request *req)
+{
+ struct audio_dev *audio = req->context;
+
+ pr_debug("audio_data_complete req->status %d req->actual %d\n",
+ req->status, req->actual);
+
+ audio_req_put(audio, req);
+
+ if (!audio->buffer_start || req->status)
+ return;
+
+ audio->period_offset += req->actual;
+ if (audio->period_offset >= audio->period) {
+ snd_pcm_period_elapsed(audio->substream);
+ audio->period_offset = 0;
+ }
+ audio_send(audio);
+}
+
+static int audio_set_endpoint_req(struct usb_function *f,
+ const struct usb_ctrlrequest *ctrl)
+{
+ int value = -EOPNOTSUPP;
+ u16 ep = le16_to_cpu(ctrl->wIndex);
+ u16 len = le16_to_cpu(ctrl->wLength);
+ u16 w_value = le16_to_cpu(ctrl->wValue);
+
+ pr_debug("bRequest 0x%x, w_value 0x%04x, len %d, endpoint %d\n",
+ ctrl->bRequest, w_value, len, ep);
+
+ switch (ctrl->bRequest) {
+ case UAC_SET_CUR:
+ case UAC_SET_MIN:
+ case UAC_SET_MAX:
+ case UAC_SET_RES:
+ value = len;
+ break;
+ default:
+ break;
+ }
+
+ return value;
+}
+
+static int audio_get_endpoint_req(struct usb_function *f,
+ const struct usb_ctrlrequest *ctrl)
+{
+ struct usb_composite_dev *cdev = f->config->cdev;
+ int value = -EOPNOTSUPP;
+ u8 ep = ((le16_to_cpu(ctrl->wIndex) >> 8) & 0xFF);
+ u16 len = le16_to_cpu(ctrl->wLength);
+ u16 w_value = le16_to_cpu(ctrl->wValue);
+ u8 *buf = cdev->req->buf;
+
+ pr_debug("bRequest 0x%x, w_value 0x%04x, len %d, endpoint %d\n",
+ ctrl->bRequest, w_value, len, ep);
+
+ if (w_value == UAC_EP_CS_ATTR_SAMPLE_RATE << 8) {
+ switch (ctrl->bRequest) {
+ case UAC_GET_CUR:
+ case UAC_GET_MIN:
+ case UAC_GET_MAX:
+ case UAC_GET_RES:
+ /* return our sample rate */
+ buf[0] = (u8)SAMPLE_RATE;
+ buf[1] = (u8)(SAMPLE_RATE >> 8);
+ buf[2] = (u8)(SAMPLE_RATE >> 16);
+ value = 3;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return value;
+}
+
+static int
+audio_setup(struct usb_function *f, const struct usb_ctrlrequest *ctrl)
+{
+ struct usb_composite_dev *cdev = f->config->cdev;
+ struct usb_request *req = cdev->req;
+ int value = -EOPNOTSUPP;
+ u16 w_index = le16_to_cpu(ctrl->wIndex);
+ u16 w_value = le16_to_cpu(ctrl->wValue);
+ u16 w_length = le16_to_cpu(ctrl->wLength);
+
+ /* composite driver infrastructure handles everything; interface
+ * activation uses set_alt().
+ */
+ switch (ctrl->bRequestType) {
+ case USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_ENDPOINT:
+ value = audio_set_endpoint_req(f, ctrl);
+ break;
+
+ case USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_ENDPOINT:
+ value = audio_get_endpoint_req(f, ctrl);
+ break;
+ }
+
+ /* respond with data transfer or status phase? */
+ if (value >= 0) {
+ pr_debug("audio req%02x.%02x v%04x i%04x l%d\n",
+ ctrl->bRequestType, ctrl->bRequest,
+ w_value, w_index, w_length);
+ req->zero = 0;
+ req->length = value;
+ req->complete = audio_control_complete;
+ value = usb_ep_queue(cdev->gadget->ep0, req, GFP_ATOMIC);
+ if (value < 0)
+ pr_err("audio response on err %d\n", value);
+ }
+
+ /* device either stalls (value < 0) or reports success */
+ return value;
+}
+
+static int audio_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
+{
+ struct audio_dev *audio = func_to_audio(f);
+ struct usb_composite_dev *cdev = f->config->cdev;
+ int ret;
+
+ pr_debug("audio_set_alt intf %d, alt %d\n", intf, alt);
+
+ ret = config_ep_by_speed(cdev->gadget, f, audio->in_ep);
+ if (ret)
+ return ret;
+
+ usb_ep_enable(audio->in_ep);
+ return 0;
+}
+
+static void audio_disable(struct usb_function *f)
+{
+ struct audio_dev *audio = func_to_audio(f);
+
+ pr_debug("audio_disable\n");
+ usb_ep_disable(audio->in_ep);
+}
+
+static void audio_free_func(struct usb_function *f)
+{
+ /* no-op */
+}
+
+/*-------------------------------------------------------------------------*/
+
+static void audio_build_desc(struct audio_dev *audio)
+{
+ u8 *sam_freq;
+ int rate;
+
+ /* Set channel numbers */
+ input_terminal_desc.bNrChannels = 2;
+ as_type_i_desc.bNrChannels = 2;
+
+ /* Set sample rates */
+ rate = SAMPLE_RATE;
+ sam_freq = as_type_i_desc.tSamFreq[0];
+ memcpy(sam_freq, &rate, 3);
+}
+
+
+static int snd_card_setup(struct usb_configuration *c,
+ struct audio_source_config *config);
+static struct audio_source_instance *to_fi_audio_source(
+ const struct usb_function_instance *fi);
+
+
+/* audio function driver setup/binding */
+static int
+audio_bind(struct usb_configuration *c, struct usb_function *f)
+{
+ struct usb_composite_dev *cdev = c->cdev;
+ struct audio_dev *audio = func_to_audio(f);
+ int status;
+ struct usb_ep *ep;
+ struct usb_request *req;
+ int i;
+ int err;
+
+ if (IS_ENABLED(CONFIG_USB_CONFIGFS)) {
+ struct audio_source_instance *fi_audio =
+ to_fi_audio_source(f->fi);
+ struct audio_source_config *config =
+ fi_audio->config;
+
+ err = snd_card_setup(c, config);
+ if (err)
+ return err;
+ }
+
+ audio_build_desc(audio);
+
+ /* allocate instance-specific interface IDs, and patch descriptors */
+ status = usb_interface_id(c, f);
+ if (status < 0)
+ goto fail;
+ ac_interface_desc.bInterfaceNumber = status;
+
+ /* AUDIO_AC_INTERFACE */
+ ac_header_desc.baInterfaceNr[0] = status;
+
+ status = usb_interface_id(c, f);
+ if (status < 0)
+ goto fail;
+ as_interface_alt_0_desc.bInterfaceNumber = status;
+ as_interface_alt_1_desc.bInterfaceNumber = status;
+
+ /* AUDIO_AS_INTERFACE */
+ ac_header_desc.baInterfaceNr[1] = status;
+
+ status = -ENODEV;
+
+ /* allocate our endpoint */
+ ep = usb_ep_autoconfig(cdev->gadget, &fs_as_in_ep_desc);
+ if (!ep)
+ goto fail;
+ audio->in_ep = ep;
+ ep->driver_data = audio; /* claim */
+
+ if (gadget_is_dualspeed(c->cdev->gadget))
+ hs_as_in_ep_desc.bEndpointAddress =
+ fs_as_in_ep_desc.bEndpointAddress;
+
+ f->fs_descriptors = fs_audio_desc;
+ f->hs_descriptors = hs_audio_desc;
+
+ for (i = 0, status = 0; i < IN_EP_REQ_COUNT && status == 0; i++) {
+ req = audio_request_new(ep, IN_EP_MAX_PACKET_SIZE);
+ if (req) {
+ req->context = audio;
+ req->complete = audio_data_complete;
+ audio_req_put(audio, req);
+ } else
+ status = -ENOMEM;
+ }
+
+fail:
+ return status;
+}
+
+static void
+audio_unbind(struct usb_configuration *c, struct usb_function *f)
+{
+ struct audio_dev *audio = func_to_audio(f);
+ struct usb_request *req;
+
+ while ((req = audio_req_get(audio)))
+ audio_request_free(req, audio->in_ep);
+
+ snd_card_free_when_closed(audio->card);
+ audio->card = NULL;
+ audio->pcm = NULL;
+ audio->substream = NULL;
+ audio->in_ep = NULL;
+
+ if (IS_ENABLED(CONFIG_USB_CONFIGFS)) {
+ struct audio_source_instance *fi_audio =
+ to_fi_audio_source(f->fi);
+ struct audio_source_config *config =
+ fi_audio->config;
+
+ config->card = -1;
+ config->device = -1;
+ }
+}
+
+static void audio_pcm_playback_start(struct audio_dev *audio)
+{
+ audio->start_time = ktime_get();
+ audio->frames_sent = 0;
+ audio_send(audio);
+}
+
+static void audio_pcm_playback_stop(struct audio_dev *audio)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&audio->lock, flags);
+ audio->buffer_start = 0;
+ audio->buffer_end = 0;
+ audio->buffer_pos = 0;
+ spin_unlock_irqrestore(&audio->lock, flags);
+}
+
+static int audio_pcm_open(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct audio_dev *audio = substream->private_data;
+
+ runtime->private_data = audio;
+ runtime->hw = audio_hw_info;
+ snd_pcm_limit_hw_rates(runtime);
+ runtime->hw.channels_max = 2;
+
+ audio->substream = substream;
+
+ /* Add the QoS request and set the latency to 0 */
+ cpu_latency_qos_add_request(&audio->pm_qos, 0);
+
+ return 0;
+}
+
+static int audio_pcm_close(struct snd_pcm_substream *substream)
+{
+ struct audio_dev *audio = substream->private_data;
+ unsigned long flags;
+
+ spin_lock_irqsave(&audio->lock, flags);
+
+ /* Remove the QoS request */
+ cpu_latency_qos_remove_request(&audio->pm_qos);
+
+ audio->substream = NULL;
+ spin_unlock_irqrestore(&audio->lock, flags);
+
+ return 0;
+}
+
+static int audio_pcm_hw_params(struct snd_pcm_substream *substream,
+ struct snd_pcm_hw_params *params)
+{
+ unsigned int channels = params_channels(params);
+ unsigned int rate = params_rate(params);
+
+ if (rate != SAMPLE_RATE)
+ return -EINVAL;
+ if (channels != 2)
+ return -EINVAL;
+
+ return snd_pcm_lib_alloc_vmalloc_buffer(substream,
+ params_buffer_bytes(params));
+}
+
+static int audio_pcm_hw_free(struct snd_pcm_substream *substream)
+{
+ return snd_pcm_lib_free_vmalloc_buffer(substream);
+}
+
+static int audio_pcm_prepare(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct audio_dev *audio = runtime->private_data;
+
+ audio->period = snd_pcm_lib_period_bytes(substream);
+ audio->period_offset = 0;
+ audio->buffer_start = runtime->dma_area;
+ audio->buffer_end = audio->buffer_start
+ + snd_pcm_lib_buffer_bytes(substream);
+ audio->buffer_pos = audio->buffer_start;
+
+ return 0;
+}
+
+static snd_pcm_uframes_t audio_pcm_pointer(struct snd_pcm_substream *substream)
+{
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ struct audio_dev *audio = runtime->private_data;
+ ssize_t bytes = audio->buffer_pos - audio->buffer_start;
+
+ /* return offset of next frame to fill in our buffer */
+ return bytes_to_frames(runtime, bytes);
+}
+
+static int audio_pcm_playback_trigger(struct snd_pcm_substream *substream,
+ int cmd)
+{
+ struct audio_dev *audio = substream->runtime->private_data;
+ int ret = 0;
+
+ switch (cmd) {
+ case SNDRV_PCM_TRIGGER_START:
+ case SNDRV_PCM_TRIGGER_RESUME:
+ audio_pcm_playback_start(audio);
+ break;
+
+ case SNDRV_PCM_TRIGGER_STOP:
+ case SNDRV_PCM_TRIGGER_SUSPEND:
+ audio_pcm_playback_stop(audio);
+ break;
+
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static struct audio_dev _audio_dev = {
+ .func = {
+ .name = "audio_source",
+ .bind = audio_bind,
+ .unbind = audio_unbind,
+ .set_alt = audio_set_alt,
+ .setup = audio_setup,
+ .disable = audio_disable,
+ .free_func = audio_free_func,
+ },
+ .lock = __SPIN_LOCK_UNLOCKED(_audio_dev.lock),
+ .idle_reqs = LIST_HEAD_INIT(_audio_dev.idle_reqs),
+};
+
+static struct snd_pcm_ops audio_playback_ops = {
+ .open = audio_pcm_open,
+ .close = audio_pcm_close,
+ .ioctl = snd_pcm_lib_ioctl,
+ .hw_params = audio_pcm_hw_params,
+ .hw_free = audio_pcm_hw_free,
+ .prepare = audio_pcm_prepare,
+ .trigger = audio_pcm_playback_trigger,
+ .pointer = audio_pcm_pointer,
+};
+
+int audio_source_bind_config(struct usb_configuration *c,
+ struct audio_source_config *config)
+{
+ struct audio_dev *audio;
+ int err;
+
+ config->card = -1;
+ config->device = -1;
+
+ audio = &_audio_dev;
+
+ err = snd_card_setup(c, config);
+ if (err)
+ return err;
+
+ err = usb_add_function(c, &audio->func);
+ if (err)
+ goto add_fail;
+
+ return 0;
+
+add_fail:
+ snd_card_free(audio->card);
+ return err;
+}
+
+static int snd_card_setup(struct usb_configuration *c,
+ struct audio_source_config *config)
+{
+ struct audio_dev *audio;
+ struct snd_card *card;
+ struct snd_pcm *pcm;
+ int err;
+
+ audio = &_audio_dev;
+
+ err = snd_card_new(&c->cdev->gadget->dev,
+ SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1,
+ THIS_MODULE, 0, &card);
+ if (err)
+ return err;
+
+ err = snd_pcm_new(card, "USB audio source", 0, 1, 0, &pcm);
+ if (err)
+ goto pcm_fail;
+
+ pcm->private_data = audio;
+ pcm->info_flags = 0;
+ audio->pcm = pcm;
+
+ strlcpy(pcm->name, "USB gadget audio", sizeof(pcm->name));
+
+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &audio_playback_ops);
+ snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_DEV,
+ NULL, 0, 64 * 1024);
+
+ strlcpy(card->driver, "audio_source", sizeof(card->driver));
+ strlcpy(card->shortname, card->driver, sizeof(card->shortname));
+ strlcpy(card->longname, "USB accessory audio source",
+ sizeof(card->longname));
+
+ err = snd_card_register(card);
+ if (err)
+ goto register_fail;
+
+ config->card = pcm->card->number;
+ config->device = pcm->device;
+ audio->card = card;
+ return 0;
+
+register_fail:
+pcm_fail:
+ snd_card_free(audio->card);
+ return err;
+}
+
+static struct audio_source_instance *to_audio_source_instance(
+ struct config_item *item)
+{
+ return container_of(to_config_group(item), struct audio_source_instance,
+ func_inst.group);
+}
+
+static struct audio_source_instance *to_fi_audio_source(
+ const struct usb_function_instance *fi)
+{
+ return container_of(fi, struct audio_source_instance, func_inst);
+}
+
+static void audio_source_attr_release(struct config_item *item)
+{
+ struct audio_source_instance *fi_audio = to_audio_source_instance(item);
+
+ usb_put_function_instance(&fi_audio->func_inst);
+}
+
+static int audio_source_set_inst_name(struct usb_function_instance *fi,
+ const char *name)
+{
+ struct audio_source_instance *fi_audio;
+ char *ptr;
+ int name_len;
+
+ name_len = strlen(name) + 1;
+ if (name_len > MAX_INST_NAME_LEN)
+ return -ENAMETOOLONG;
+
+ ptr = kstrndup(name, name_len, GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+
+ fi_audio = to_fi_audio_source(fi);
+ fi_audio->name = ptr;
+
+ return 0;
+}
+
+static void audio_source_free_inst(struct usb_function_instance *fi)
+{
+ struct audio_source_instance *fi_audio;
+
+ fi_audio = to_fi_audio_source(fi);
+ device_destroy(fi_audio->audio_device->class,
+ fi_audio->audio_device->devt);
+ kfree(fi_audio->name);
+ kfree(fi_audio->config);
+}
+
+static ssize_t audio_source_pcm_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct audio_source_instance *fi_audio = dev_get_drvdata(dev);
+ struct audio_source_config *config = fi_audio->config;
+
+ /* print PCM card and device numbers */
+ return sprintf(buf, "%d %d\n", config->card, config->device);
+}
+
+struct device *create_function_device(char *name);
+
+static struct usb_function_instance *audio_source_alloc_inst(void)
+{
+ struct audio_source_instance *fi_audio;
+ struct device_attribute **attrs;
+ struct device_attribute *attr;
+ struct device *dev;
+ void *err_ptr;
+ int err = 0;
+
+ fi_audio = kzalloc(sizeof(*fi_audio), GFP_KERNEL);
+ if (!fi_audio)
+ return ERR_PTR(-ENOMEM);
+
+ fi_audio->func_inst.set_inst_name = audio_source_set_inst_name;
+ fi_audio->func_inst.free_func_inst = audio_source_free_inst;
+
+ fi_audio->config = kzalloc(sizeof(struct audio_source_config),
+ GFP_KERNEL);
+ if (!fi_audio->config) {
+ err_ptr = ERR_PTR(-ENOMEM);
+ goto fail_audio;
+ }
+
+ config_group_init_type_name(&fi_audio->func_inst.group, "",
+ &audio_source_func_type);
+ dev = create_function_device("f_audio_source");
+
+ if (IS_ERR(dev)) {
+ err_ptr = dev;
+ goto fail_audio_config;
+ }
+
+ fi_audio->config->card = -1;
+ fi_audio->config->device = -1;
+ fi_audio->audio_device = dev;
+
+ attrs = audio_source_function_attributes;
+ if (attrs) {
+ while ((attr = *attrs++) && !err)
+ err = device_create_file(dev, attr);
+ if (err) {
+ err_ptr = ERR_PTR(-EINVAL);
+ goto fail_device;
+ }
+ }
+
+ dev_set_drvdata(dev, fi_audio);
+ _audio_dev.config = fi_audio->config;
+
+ return &fi_audio->func_inst;
+
+fail_device:
+ device_destroy(dev->class, dev->devt);
+fail_audio_config:
+ kfree(fi_audio->config);
+fail_audio:
+ kfree(fi_audio);
+ return err_ptr;
+
+}
+
+static struct usb_function *audio_source_alloc(struct usb_function_instance *fi)
+{
+ return &_audio_dev.func;
+}
+
+DECLARE_USB_FUNCTION_INIT(audio_source, audio_source_alloc_inst,
+ audio_source_alloc);
+MODULE_LICENSE("GPL");
diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
index fddf539..9511b22 100644
--- a/drivers/usb/gadget/function/f_midi.c
+++ b/drivers/usb/gadget/function/f_midi.c
@@ -1268,6 +1268,65 @@ static void f_midi_free_inst(struct usb_function_instance *f)
}
}
+#ifdef CONFIG_USB_CONFIGFS_UEVENT
+extern struct device *create_function_device(char *name);
+static ssize_t alsa_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct usb_function_instance *fi_midi = dev_get_drvdata(dev);
+ struct f_midi *midi;
+
+ if (!fi_midi->f)
+ dev_warn(dev, "f_midi: function not set\n");
+
+ if (fi_midi && fi_midi->f) {
+ midi = func_to_midi(fi_midi->f);
+ if (midi->rmidi && midi->card && midi->rmidi->card)
+ return sprintf(buf, "%d %d\n",
+ midi->rmidi->card->number, midi->rmidi->device);
+ }
+
+ /* print PCM card and device numbers */
+ return sprintf(buf, "%d %d\n", -1, -1);
+}
+
+static DEVICE_ATTR(alsa, S_IRUGO, alsa_show, NULL);
+
+static struct device_attribute *alsa_function_attributes[] = {
+ &dev_attr_alsa,
+ NULL
+};
+
+static int create_alsa_device(struct usb_function_instance *fi)
+{
+ struct device *dev;
+ struct device_attribute **attrs;
+ struct device_attribute *attr;
+ int err = 0;
+
+ dev = create_function_device("f_midi");
+ if (IS_ERR(dev))
+ return PTR_ERR(dev);
+
+ attrs = alsa_function_attributes;
+ if (attrs) {
+ while ((attr = *attrs++) && !err)
+ err = device_create_file(dev, attr);
+ if (err) {
+ device_destroy(dev->class, dev->devt);
+ return -EINVAL;
+ }
+ }
+ dev_set_drvdata(dev, fi);
+ return 0;
+}
+#else
+static int create_alsa_device(struct usb_function_instance *fi)
+{
+ return 0;
+}
+#endif
+
static struct usb_function_instance *f_midi_alloc_inst(void)
{
struct f_midi_opts *opts;
@@ -1286,6 +1345,11 @@ static struct usb_function_instance *f_midi_alloc_inst(void)
opts->out_ports = 1;
opts->refcnt = 1;
+ if (create_alsa_device(&opts->func_inst)) {
+ kfree(opts);
+ return ERR_PTR(-ENODEV);
+ }
+
config_group_init_type_name(&opts->func_inst.group, "",
&midi_func_type);
@@ -1306,6 +1370,7 @@ static void f_midi_free(struct usb_function *f)
kfifo_free(&midi->in_req_fifo);
kfree(midi);
free = true;
+ opts->func_inst.f = NULL;
}
mutex_unlock(&opts->lock);
@@ -1393,6 +1458,7 @@ static struct usb_function *f_midi_alloc(struct usb_function_instance *fi)
midi->func.disable = f_midi_disable;
midi->func.free_func = f_midi_free;
+ fi->f = &midi->func;
return &midi->func;
midi_free:
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 0a53a61..8f4730f 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -167,7 +167,7 @@
If unsure, say 'N'.
config VIRTIO_DMA_SHARED_BUFFER
- tristate
+ tristate "Virtio DMA shared buffer support"
depends on DMA_SHARED_BUFFER
help
This option adds a flavor of dma buffers that are backed by
diff --git a/drivers/virtio/virtio_dma_buf.c b/drivers/virtio/virtio_dma_buf.c
index 2521a750..19a843f 100644
--- a/drivers/virtio/virtio_dma_buf.c
+++ b/drivers/virtio/virtio_dma_buf.c
@@ -25,11 +25,14 @@ struct dma_buf *virtio_dma_buf_export
const struct virtio_dma_buf_ops, ops);
if (!exp_info->ops ||
- exp_info->ops->attach != &virtio_dma_buf_attach ||
!virtio_ops->get_uuid) {
return ERR_PTR(-EINVAL);
}
+ if (!(IS_ENABLED(CONFIG_CFI_CLANG) && IS_ENABLED(CONFIG_MODULES)) &&
+ exp_info->ops->attach != &virtio_dma_buf_attach)
+ return ERR_PTR(-EINVAL);
+
return dma_buf_export(exp_info);
}
EXPORT_SYMBOL(virtio_dma_buf_export);
@@ -60,6 +63,9 @@ EXPORT_SYMBOL(virtio_dma_buf_attach);
*/
bool is_virtio_dma_buf(struct dma_buf *dma_buf)
{
+ if (IS_ENABLED(CONFIG_CFI_CLANG) && IS_ENABLED(CONFIG_MODULES))
+ return true;
+
return dma_buf->ops->attach == &virtio_dma_buf_attach;
}
EXPORT_SYMBOL(is_virtio_dma_buf);
diff --git a/fs/Kconfig b/fs/Kconfig
index 2685a4d..6e109b4 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -127,6 +127,7 @@
source "fs/autofs/Kconfig"
source "fs/fuse/Kconfig"
source "fs/overlayfs/Kconfig"
+source "fs/incfs/Kconfig"
menu "Caches"
diff --git a/fs/Makefile b/fs/Makefile
index 4dea178..bfdaafb 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -114,6 +114,7 @@
obj-$(CONFIG_FUSE_FS) += fuse/
obj-$(CONFIG_OVERLAY_FS) += overlayfs/
obj-$(CONFIG_ORANGEFS_FS) += orangefs/
+obj-$(CONFIG_INCREMENTAL_FS) += incfs/
obj-$(CONFIG_UDF_FS) += udf/
obj-$(CONFIG_SUN_OPENPROMFS) += openpromfs/
obj-$(CONFIG_OMFS_FS) += omfs/
diff --git a/fs/OWNERS b/fs/OWNERS
new file mode 100644
index 0000000..7780f6b
--- /dev/null
+++ b/fs/OWNERS
@@ -0,0 +1 @@
+per-file {crypto,verity}/**=ebiggers@google.com
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 316a778..98d1901 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -27,6 +27,27 @@
*/
#define FSCRYPT_MIN_KEY_SIZE 16
+/* Maximum size of a standard fscrypt master key */
+#define FSCRYPT_MAX_STANDARD_KEY_SIZE 64
+
+/* Maximum size of a hardware-wrapped fscrypt master key */
+#define FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE
+
+/*
+ * Maximum size of an fscrypt master key across both key types.
+ * This should just use max(), but max() doesn't work in a struct definition.
+ */
+#define FSCRYPT_MAX_ANY_KEY_SIZE \
+ (FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE > FSCRYPT_MAX_STANDARD_KEY_SIZE ? \
+ FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE : FSCRYPT_MAX_STANDARD_KEY_SIZE)
+
+/*
+ * FSCRYPT_MAX_KEY_SIZE is defined in the UAPI header, but the addition of
+ * hardware-wrapped keys has made it misleading as it's only for standard keys.
+ * Don't use it in kernel code; use one of the above constants instead.
+ */
+#undef FSCRYPT_MAX_KEY_SIZE
+
#define FSCRYPT_CONTEXT_V1 1
#define FSCRYPT_CONTEXT_V2 2
@@ -332,7 +353,8 @@ void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf);
/* inline_crypt.c */
#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
-int fscrypt_select_encryption_impl(struct fscrypt_info *ci);
+int fscrypt_select_encryption_impl(struct fscrypt_info *ci,
+ bool is_hw_wrapped_key);
static inline bool
fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
@@ -341,12 +363,17 @@ fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
}
int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
- const u8 *raw_key,
+ const u8 *raw_key, size_t raw_key_size,
+ bool is_hw_wrapped,
const struct fscrypt_info *ci);
void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
struct fscrypt_prepared_key *prep_key);
+int fscrypt_derive_sw_secret(struct super_block *sb,
+ const u8 *wrapped_key, size_t wrapped_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);
+
/*
* Check whether the crypto transform or blk-crypto key has been allocated in
* @prep_key, depending on which encryption implementation the file will use.
@@ -370,7 +397,8 @@ fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
-static inline int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
+static inline int fscrypt_select_encryption_impl(struct fscrypt_info *ci,
+ bool is_hw_wrapped_key)
{
return 0;
}
@@ -383,7 +411,8 @@ fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
static inline int
fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
- const u8 *raw_key,
+ const u8 *raw_key, size_t raw_key_size,
+ bool is_hw_wrapped,
const struct fscrypt_info *ci)
{
WARN_ON(1);
@@ -396,6 +425,15 @@ fscrypt_destroy_inline_crypt_key(struct super_block *sb,
{
}
+static inline int
+fscrypt_derive_sw_secret(struct super_block *sb,
+ const u8 *wrapped_key, size_t wrapped_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
+{
+ fscrypt_warn(NULL, "kernel doesn't support hardware-wrapped keys");
+ return -EOPNOTSUPP;
+}
+
static inline bool
fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
const struct fscrypt_info *ci)
@@ -412,20 +450,38 @@ fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
struct fscrypt_master_key_secret {
/*
- * For v2 policy keys: HKDF context keyed by this master key.
- * For v1 policy keys: not set (hkdf.hmac_tfm == NULL).
+ * The KDF with which subkeys of this key can be derived.
+ *
+ * For v1 policy keys, this isn't applicable and won't be set.
+ * Otherwise, this KDF will be keyed by this master key if
+ * ->is_hw_wrapped=false, or by the "software secret" that hardware
+ * derived from this master key if ->is_hw_wrapped=true.
*/
struct fscrypt_hkdf hkdf;
/*
+ * True if this key is a hardware-wrapped key; false if this key is a
+ * standard key (i.e. a "software key"). For v1 policy keys this will
+ * always be false, as v1 policy support is a legacy feature which
+ * doesn't support newer functionality such as hardware-wrapped keys.
+ */
+ bool is_hw_wrapped;
+
+ /*
* Size of the raw key in bytes. This remains set even if ->raw was
* zeroized due to no longer being needed. I.e. we still remember the
* size of the key even if we don't need to remember the key itself.
*/
u32 size;
- /* For v1 policy keys: the raw key. Wiped for v2 policy keys. */
- u8 raw[FSCRYPT_MAX_KEY_SIZE];
+ /*
+ * The raw key which userspace provided, when still needed. This can be
+ * either a standard key or a hardware-wrapped key, as indicated by
+ * ->is_hw_wrapped. In the case of a standard, v2 policy key, there is
+ * no need to remember the raw key separately from ->hkdf so this field
+ * will be zeroized as soon as ->hkdf is initialized.
+ */
+ u8 raw[FSCRYPT_MAX_ANY_KEY_SIZE];
} __randomize_layout;
diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c
index 7607d18b..41e7c9b 100644
--- a/fs/crypto/hkdf.c
+++ b/fs/crypto/hkdf.c
@@ -4,7 +4,9 @@
* Function"), aka RFC 5869. See also the original paper (Krawczyk 2010):
* "Cryptographic Extraction and Key Derivation: The HKDF Scheme".
*
- * This is used to derive keys from the fscrypt master keys.
+ * This is used to derive keys from the fscrypt master keys (or from the
+ * "software secrets" which hardware derives from the fscrypt master keys, in
+ * the case that the fscrypt master keys are hardware-wrapped keys).
*
* Copyright 2019 Google LLC
*/
diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c
index 8bfb3ce..64fff38 100644
--- a/fs/crypto/inline_crypt.c
+++ b/fs/crypto/inline_crypt.c
@@ -90,7 +90,8 @@ static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode,
}
/* Enable inline encryption for this file if supported. */
-int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
+int fscrypt_select_encryption_impl(struct fscrypt_info *ci,
+ bool is_hw_wrapped_key)
{
const struct inode *inode = ci->ci_inode;
struct super_block *sb = inode->i_sb;
@@ -131,6 +132,9 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode;
crypto_cfg.data_unit_size = sb->s_blocksize;
crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
+ crypto_cfg.key_type =
+ is_hw_wrapped_key ? BLK_CRYPTO_KEY_TYPE_HW_WRAPPED :
+ BLK_CRYPTO_KEY_TYPE_STANDARD;
devs = fscrypt_get_devices(sb, &num_devs);
if (IS_ERR(devs))
@@ -151,12 +155,15 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
}
int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
- const u8 *raw_key,
+ const u8 *raw_key, size_t raw_key_size,
+ bool is_hw_wrapped,
const struct fscrypt_info *ci)
{
const struct inode *inode = ci->ci_inode;
struct super_block *sb = inode->i_sb;
enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode;
+ enum blk_crypto_key_type key_type = is_hw_wrapped ?
+ BLK_CRYPTO_KEY_TYPE_HW_WRAPPED : BLK_CRYPTO_KEY_TYPE_STANDARD;
struct blk_crypto_key *blk_key;
struct block_device **devs;
unsigned int num_devs;
@@ -167,8 +174,9 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
if (!blk_key)
return -ENOMEM;
- err = blk_crypto_init_key(blk_key, raw_key, crypto_mode,
- fscrypt_get_dun_bytes(ci), sb->s_blocksize);
+ err = blk_crypto_init_key(blk_key, raw_key, raw_key_size, key_type,
+ crypto_mode, fscrypt_get_dun_bytes(ci),
+ sb->s_blocksize);
if (err) {
fscrypt_err(inode, "error %d initializing blk-crypto key", err);
goto fail;
@@ -226,6 +234,34 @@ void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
kfree_sensitive(blk_key);
}
+/*
+ * Ask the inline encryption hardware to derive the software secret from a
+ * hardware-wrapped key. Returns -EOPNOTSUPP if hardware-wrapped keys aren't
+ * supported on this filesystem or hardware.
+ */
+int fscrypt_derive_sw_secret(struct super_block *sb,
+ const u8 *wrapped_key, size_t wrapped_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
+{
+ int err;
+
+ /* The filesystem must be mounted with -o inlinecrypt. */
+ if (!(sb->s_flags & SB_INLINECRYPT)) {
+ fscrypt_warn(NULL,
+ "%s: filesystem not mounted with inlinecrypt\n",
+ sb->s_id);
+ return -EOPNOTSUPP;
+ }
+
+ err = blk_crypto_derive_sw_secret(sb->s_bdev, wrapped_key,
+ wrapped_key_size, sw_secret);
+ if (err == -EOPNOTSUPP)
+ fscrypt_warn(NULL,
+ "%s: block device doesn't support hardware-wrapped keys\n",
+ sb->s_id);
+ return err;
+}
+
bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
{
return inode->i_crypt_info->ci_inlinecrypt;
@@ -261,6 +297,8 @@ static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num,
* otherwise fscrypt_mergeable_bio() won't work as intended.
*
* The encryption context will be freed automatically when the bio is freed.
+ *
+ * This function also handles setting bi_skip_dm_default_key when needed.
*/
void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
u64 first_lblk, gfp_t gfp_mask)
@@ -268,6 +306,9 @@ void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
const struct fscrypt_info *ci;
u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
+ if (fscrypt_inode_should_skip_dm_default_key(inode))
+ bio_set_skip_dm_default_key(bio);
+
if (!fscrypt_inode_uses_inline_crypto(inode))
return;
ci = inode->i_crypt_info;
@@ -342,6 +383,9 @@ EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh);
* another way, such as I/O targeting only a single file (and thus a single key)
* combined with fscrypt_limit_io_blocks() to ensure DUN contiguity.
*
+ * This function also returns false if the next part of the I/O would need to
+ * have a different value for the bi_skip_dm_default_key flag.
+ *
* Return: true iff the I/O is mergeable
*/
bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
@@ -352,6 +396,9 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
if (!!bc != fscrypt_inode_uses_inline_crypto(inode))
return false;
+ if (bio_should_skip_dm_default_key(bio) !=
+ fscrypt_inode_should_skip_dm_default_key(inode))
+ return false;
if (!bc)
return true;
@@ -385,7 +432,8 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio,
u64 next_lblk;
if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk))
- return !bio->bi_crypt_context;
+ return !bio->bi_crypt_context &&
+ !bio_should_skip_dm_default_key(bio);
return fscrypt_mergeable_bio(bio, inode, next_lblk);
}
diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c
index 78dd2ff..d43c7e3 100644
--- a/fs/crypto/keyring.c
+++ b/fs/crypto/keyring.c
@@ -130,11 +130,11 @@ static int fscrypt_user_key_instantiate(struct key *key,
struct key_preparsed_payload *prep)
{
/*
- * We just charge FSCRYPT_MAX_KEY_SIZE bytes to the user's key quota for
- * each key, regardless of the exact key size. The amount of memory
- * actually used is greater than the size of the raw key anyway.
+ * We just charge FSCRYPT_MAX_STANDARD_KEY_SIZE bytes to the user's key
+ * quota for each key, regardless of the exact key size. The amount of
+ * memory actually used is greater than the size of the raw key anyway.
*/
- return key_payload_reserve(key, FSCRYPT_MAX_KEY_SIZE);
+ return key_payload_reserve(key, FSCRYPT_MAX_STANDARD_KEY_SIZE);
}
static void fscrypt_user_key_describe(const struct key *key, struct seq_file *m)
@@ -535,20 +535,36 @@ static int add_master_key(struct super_block *sb,
int err;
if (key_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) {
- err = fscrypt_init_hkdf(&secret->hkdf, secret->raw,
- secret->size);
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE];
+ u8 *kdf_key = secret->raw;
+ unsigned int kdf_key_size = secret->size;
+ u8 keyid_kdf_ctx = HKDF_CONTEXT_KEY_IDENTIFIER;
+
+ /*
+ * For standard keys, the fscrypt master key is used directly as
+ * the fscrypt KDF key. For hardware-wrapped keys, we have to
+ * pass the master key to the hardware to derive the KDF key,
+ * which is then only used to derive non-file-contents subkeys.
+ */
+ if (secret->is_hw_wrapped) {
+ err = fscrypt_derive_sw_secret(sb, secret->raw,
+ secret->size, sw_secret);
+ if (err)
+ return err;
+ kdf_key = sw_secret;
+ kdf_key_size = sizeof(sw_secret);
+ }
+ err = fscrypt_init_hkdf(&secret->hkdf, kdf_key, kdf_key_size);
+ /*
+ * Now that the KDF context is initialized, the raw KDF key is
+ * no longer needed.
+ */
+ memzero_explicit(kdf_key, kdf_key_size);
if (err)
return err;
- /*
- * Now that the HKDF context is initialized, the raw key is no
- * longer needed.
- */
- memzero_explicit(secret->raw, secret->size);
-
/* Calculate the key identifier */
- err = fscrypt_hkdf_expand(&secret->hkdf,
- HKDF_CONTEXT_KEY_IDENTIFIER, NULL, 0,
+ err = fscrypt_hkdf_expand(&secret->hkdf, keyid_kdf_ctx, NULL, 0,
key_spec->u.identifier,
FSCRYPT_KEY_IDENTIFIER_SIZE);
if (err)
@@ -562,7 +578,7 @@ static int fscrypt_provisioning_key_preparse(struct key_preparsed_payload *prep)
const struct fscrypt_provisioning_key_payload *payload = prep->data;
if (prep->datalen < sizeof(*payload) + FSCRYPT_MIN_KEY_SIZE ||
- prep->datalen > sizeof(*payload) + FSCRYPT_MAX_KEY_SIZE)
+ prep->datalen > sizeof(*payload) + FSCRYPT_MAX_ANY_KEY_SIZE)
return -EINVAL;
if (payload->type != FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR &&
@@ -711,15 +727,30 @@ int fscrypt_ioctl_add_key(struct file *filp, void __user *_uarg)
return -EACCES;
memset(&secret, 0, sizeof(secret));
+
+ if (arg.__flags) {
+ if (arg.__flags & ~__FSCRYPT_ADD_KEY_FLAG_HW_WRAPPED)
+ return -EINVAL;
+ if (arg.key_spec.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER)
+ return -EINVAL;
+ secret.is_hw_wrapped = true;
+ }
+
if (arg.key_id) {
if (arg.raw_size != 0)
return -EINVAL;
err = get_keyring_key(arg.key_id, arg.key_spec.type, &secret);
if (err)
goto out_wipe_secret;
+ err = -EINVAL;
+ if (secret.size > FSCRYPT_MAX_STANDARD_KEY_SIZE &&
+ !secret.is_hw_wrapped)
+ goto out_wipe_secret;
} else {
if (arg.raw_size < FSCRYPT_MIN_KEY_SIZE ||
- arg.raw_size > FSCRYPT_MAX_KEY_SIZE)
+ arg.raw_size > (secret.is_hw_wrapped ?
+ FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE :
+ FSCRYPT_MAX_STANDARD_KEY_SIZE))
return -EINVAL;
secret.size = arg.raw_size;
err = -EFAULT;
@@ -747,13 +778,13 @@ EXPORT_SYMBOL_GPL(fscrypt_ioctl_add_key);
static void
fscrypt_get_test_dummy_secret(struct fscrypt_master_key_secret *secret)
{
- static u8 test_key[FSCRYPT_MAX_KEY_SIZE];
+ static u8 test_key[FSCRYPT_MAX_STANDARD_KEY_SIZE];
- get_random_once(test_key, FSCRYPT_MAX_KEY_SIZE);
+ get_random_once(test_key, sizeof(test_key));
memset(secret, 0, sizeof(*secret));
- secret->size = FSCRYPT_MAX_KEY_SIZE;
- memcpy(secret->raw, test_key, FSCRYPT_MAX_KEY_SIZE);
+ secret->size = sizeof(test_key);
+ memcpy(secret->raw, test_key, sizeof(test_key));
}
int fscrypt_get_test_dummy_key_identifier(
diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
index 94757cc..6d4ae17 100644
--- a/fs/crypto/keysetup.c
+++ b/fs/crypto/keysetup.c
@@ -153,7 +153,9 @@ int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
struct crypto_skcipher *tfm;
if (fscrypt_using_inline_encryption(ci))
- return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci);
+ return fscrypt_prepare_inline_crypt_key(prep_key, raw_key,
+ ci->ci_mode->keysize,
+ false, ci);
tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode);
if (IS_ERR(tfm))
@@ -194,14 +196,29 @@ static int setup_per_mode_enc_key(struct fscrypt_info *ci,
struct fscrypt_mode *mode = ci->ci_mode;
const u8 mode_num = mode - fscrypt_modes;
struct fscrypt_prepared_key *prep_key;
- u8 mode_key[FSCRYPT_MAX_KEY_SIZE];
+ u8 mode_key[FSCRYPT_MAX_STANDARD_KEY_SIZE];
u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)];
unsigned int hkdf_infolen = 0;
+ bool use_hw_wrapped_key = false;
int err;
if (WARN_ON(mode_num > FSCRYPT_MODE_MAX))
return -EINVAL;
+ if (mk->mk_secret.is_hw_wrapped && S_ISREG(inode->i_mode)) {
+ /* Using a hardware-wrapped key for file contents encryption */
+ if (!fscrypt_using_inline_encryption(ci)) {
+ if (sb->s_flags & SB_INLINECRYPT)
+ fscrypt_warn(ci->ci_inode,
+ "Hardware-wrapped key required, but no suitable inline encryption capabilities are available");
+ else
+ fscrypt_warn(ci->ci_inode,
+ "Hardware-wrapped keys require inline encryption (-o inlinecrypt)");
+ return -EINVAL;
+ }
+ use_hw_wrapped_key = true;
+ }
+
prep_key = &keys[mode_num];
if (fscrypt_is_key_prepared(prep_key, ci)) {
ci->ci_enc_key = *prep_key;
@@ -213,6 +230,16 @@ static int setup_per_mode_enc_key(struct fscrypt_info *ci,
if (fscrypt_is_key_prepared(prep_key, ci))
goto done_unlock;
+ if (use_hw_wrapped_key) {
+ err = fscrypt_prepare_inline_crypt_key(prep_key,
+ mk->mk_secret.raw,
+ mk->mk_secret.size, true,
+ ci);
+ if (err)
+ goto out_unlock;
+ goto done_unlock;
+ }
+
BUILD_BUG_ON(sizeof(mode_num) != 1);
BUILD_BUG_ON(sizeof(sb->s_uuid) != 16);
BUILD_BUG_ON(sizeof(hkdf_info) != 17);
@@ -335,6 +362,14 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci,
{
int err;
+ if (mk->mk_secret.is_hw_wrapped &&
+ !(ci->ci_policy.v2.flags & (FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 |
+ FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32))) {
+ fscrypt_warn(ci->ci_inode,
+ "Hardware-wrapped keys are only supported with IV_INO_LBLK policies");
+ return -EINVAL;
+ }
+
if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) {
/*
* DIRECT_KEY: instead of deriving per-file encryption keys, the
@@ -361,7 +396,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci,
FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) {
err = fscrypt_setup_iv_ino_lblk_32_key(ci, mk);
} else {
- u8 derived_key[FSCRYPT_MAX_KEY_SIZE];
+ u8 derived_key[FSCRYPT_MAX_STANDARD_KEY_SIZE];
err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf,
HKDF_CONTEXT_PER_FILE_ENC_KEY,
@@ -442,10 +477,6 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
struct fscrypt_master_key *mk;
int err;
- err = fscrypt_select_encryption_impl(ci);
- if (err)
- return err;
-
err = fscrypt_policy_to_key_spec(&ci->ci_policy, &mk_spec);
if (err)
return err;
@@ -455,6 +486,10 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
if (ci->ci_policy.version != FSCRYPT_POLICY_V1)
return -ENOKEY;
+ err = fscrypt_select_encryption_impl(ci, false);
+ if (err)
+ return err;
+
/*
* As a legacy fallback for v1 policies, search for the key in
* the current task's subscribed keyrings too. Don't move this
@@ -476,8 +511,20 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
goto out_release_key;
}
+ err = fscrypt_select_encryption_impl(ci, mk->mk_secret.is_hw_wrapped);
+ if (err)
+ goto out_release_key;
+
switch (ci->ci_policy.version) {
case FSCRYPT_POLICY_V1:
+ if (WARN_ON(mk->mk_secret.is_hw_wrapped)) {
+ /*
+ * This should never happen, as adding a v1 policy key
+ * that is hardware-wrapped isn't allowed.
+ */
+ err = -EINVAL;
+ goto out_release_key;
+ }
err = fscrypt_setup_v1_file_key(ci, mk->mk_secret.raw);
break;
case FSCRYPT_POLICY_V2:
diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c
index 75dabd9b..ee82463 100644
--- a/fs/crypto/keysetup_v1.c
+++ b/fs/crypto/keysetup_v1.c
@@ -118,7 +118,8 @@ find_and_lock_process_key(const char *prefix,
payload = (const struct fscrypt_key *)ukp->data;
if (ukp->datalen != sizeof(struct fscrypt_key) ||
- payload->size < 1 || payload->size > FSCRYPT_MAX_KEY_SIZE) {
+ payload->size < 1 ||
+ payload->size > FSCRYPT_MAX_STANDARD_KEY_SIZE) {
fscrypt_warn(NULL,
"key with description '%s' has invalid payload",
key->description);
@@ -149,7 +150,7 @@ struct fscrypt_direct_key {
const struct fscrypt_mode *dk_mode;
struct fscrypt_prepared_key dk_key;
u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE];
- u8 dk_raw[FSCRYPT_MAX_KEY_SIZE];
+ u8 dk_raw[FSCRYPT_MAX_STANDARD_KEY_SIZE];
};
static void free_direct_key(struct fscrypt_direct_key *dk)
diff --git a/fs/f2fs/OWNERS b/fs/f2fs/OWNERS
new file mode 100644
index 0000000..6a5c01163
--- /dev/null
+++ b/fs/f2fs/OWNERS
@@ -0,0 +1 @@
+jaegeuk@google.com
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 7af7504..51395a2 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -493,6 +493,8 @@ static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
*/
if (!fio || !fio->encrypted_page)
fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);
+ else if (fscrypt_inode_should_skip_dm_default_key(inode))
+ bio_set_skip_dm_default_key(bio);
}
static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
@@ -504,7 +506,9 @@ static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
* read/write raw data without encryption.
*/
if (fio && fio->encrypted_page)
- return !bio_has_crypt_ctx(bio);
+ return !bio_has_crypt_ctx(bio) &&
+ (bio_should_skip_dm_default_key(bio) ==
+ fscrypt_inode_should_skip_dm_default_key(inode));
return fscrypt_mergeable_bio(bio, inode, next_idx);
}
diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile
index 0c48b35..d9e1b47 100644
--- a/fs/fuse/Makefile
+++ b/fs/fuse/Makefile
@@ -8,6 +8,7 @@
obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o
+fuse-y += passthrough.o
fuse-$(CONFIG_FUSE_DAX) += dax.o
virtiofs-y := virtio_fs.o
diff --git a/fs/fuse/OWNERS b/fs/fuse/OWNERS
new file mode 100644
index 0000000..5ee6098
--- /dev/null
+++ b/fs/fuse/OWNERS
@@ -0,0 +1 @@
+balsini@google.com
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index c73d9c4..9f4fc65 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -14,6 +14,7 @@
#include <linux/sched/signal.h>
#include <linux/uio.h>
#include <linux/miscdevice.h>
+#include <linux/namei.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/slab.h>
@@ -207,10 +208,13 @@ static unsigned int fuse_req_hash(u64 unique)
/**
* A new request is available, wake fiq->waitq
*/
-static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
+static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock)
{
- wake_up(&fiq->waitq);
+ if (sync)
+ wake_up_sync(&fiq->waitq);
+ else
+ wake_up(&fiq->waitq);
kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
spin_unlock(&fiq->lock);
}
@@ -223,14 +227,14 @@ const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
static void queue_request_and_unlock(struct fuse_iqueue *fiq,
- struct fuse_req *req)
+ struct fuse_req *req, bool sync)
__releases(fiq->lock)
{
req->in.h.len = sizeof(struct fuse_in_header) +
fuse_len_args(req->args->in_numargs,
(struct fuse_arg *) req->args->in_args);
list_add_tail(&req->list, &fiq->pending);
- fiq->ops->wake_pending_and_unlock(fiq);
+ fiq->ops->wake_pending_and_unlock(fiq, sync);
}
void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
@@ -245,7 +249,7 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
if (fiq->connected) {
fiq->forget_list_tail->next = forget;
fiq->forget_list_tail = forget;
- fiq->ops->wake_forget_and_unlock(fiq);
+ fiq->ops->wake_forget_and_unlock(fiq, false);
} else {
kfree(forget);
spin_unlock(&fiq->lock);
@@ -265,7 +269,7 @@ static void flush_bg_queue(struct fuse_conn *fc)
fc->active_background++;
spin_lock(&fiq->lock);
req->in.h.unique = fuse_get_unique(fiq);
- queue_request_and_unlock(fiq, req);
+ queue_request_and_unlock(fiq, req, false);
}
}
@@ -354,7 +358,7 @@ static int queue_interrupt(struct fuse_req *req)
spin_unlock(&fiq->lock);
return 0;
}
- fiq->ops->wake_interrupt_and_unlock(fiq);
+ fiq->ops->wake_interrupt_and_unlock(fiq, false);
} else {
spin_unlock(&fiq->lock);
}
@@ -421,7 +425,7 @@ static void __fuse_request_send(struct fuse_req *req)
/* acquire extra reference, since request is still needed
after fuse_request_end() */
__fuse_get_request(req);
- queue_request_and_unlock(fiq, req);
+ queue_request_and_unlock(fiq, req, true);
request_wait_answer(req);
/* Pairs with smp_wmb() in fuse_request_end() */
@@ -592,7 +596,7 @@ static int fuse_simple_notify_reply(struct fuse_mount *fm,
spin_lock(&fiq->lock);
if (fiq->connected) {
- queue_request_and_unlock(fiq, req);
+ queue_request_and_unlock(fiq, req, false);
} else {
err = -ENODEV;
spin_unlock(&fiq->lock);
@@ -1922,6 +1926,14 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
err = copy_out_args(cs, req->args, nbytes);
fuse_copy_finish(cs);
+ if (!err && req->in.h.opcode == FUSE_CANONICAL_PATH) {
+ char *path = (char *)req->args->out_args[0].value;
+
+ path[req->args->out_args[0].size - 1] = 0;
+ req->out.h.error =
+ kern_path(path, 0, req->args->canonical_path);
+ }
+
spin_lock(&fpq->lock);
clear_bit(FR_LOCKED, &req->flags);
if (!fpq->connected)
@@ -2279,6 +2291,15 @@ static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
}
}
break;
+ case FUSE_DEV_IOC_PASSTHROUGH_OPEN:
+ res = -EFAULT;
+ if (!get_user(oldfd, (__u32 __user *)arg)) {
+ res = -EINVAL;
+ fud = fuse_get_dev(file);
+ if (fud)
+ res = fuse_passthrough_open(fud, oldfd);
+ }
+ break;
default:
res = -ENOTTY;
break;
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index cd1a071b..7297edf 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -337,6 +337,45 @@ static struct vfsmount *fuse_dentry_automount(struct path *path)
return mnt;
}
+/*
+ * Get the canonical path. Since we must translate to a path, this must be done
+ * in the context of the userspace daemon, however, the userspace daemon cannot
+ * look up paths on its own. Instead, we handle the lookup as a special case
+ * inside of the write request.
+ */
+static void fuse_dentry_canonical_path(const struct path *path,
+ struct path *canonical_path)
+{
+ struct inode *inode = d_inode(path->dentry);
+ //struct fuse_conn *fc = get_fuse_conn(inode);
+ struct fuse_mount *fm = get_fuse_mount_super(path->mnt->mnt_sb);
+ FUSE_ARGS(args);
+ char *path_name;
+ int err;
+
+ path_name = (char *)get_zeroed_page(GFP_KERNEL);
+ if (!path_name)
+ goto default_path;
+
+ args.opcode = FUSE_CANONICAL_PATH;
+ args.nodeid = get_node_id(inode);
+ args.in_numargs = 0;
+ args.out_numargs = 1;
+ args.out_args[0].size = PATH_MAX;
+ args.out_args[0].value = path_name;
+ args.canonical_path = canonical_path;
+ args.out_argvar = 1;
+
+ err = fuse_simple_request(fm, &args);
+ free_page((unsigned long)path_name);
+ if (err > 0)
+ return;
+default_path:
+ canonical_path->dentry = path->dentry;
+ canonical_path->mnt = path->mnt;
+ path_get(canonical_path);
+}
+
const struct dentry_operations fuse_dentry_operations = {
.d_revalidate = fuse_dentry_revalidate,
.d_delete = fuse_dentry_delete,
@@ -345,6 +384,7 @@ const struct dentry_operations fuse_dentry_operations = {
.d_release = fuse_dentry_release,
#endif
.d_automount = fuse_dentry_automount,
+ .d_canonical_path = fuse_dentry_canonical_path,
};
const struct dentry_operations fuse_root_dentry_operations = {
@@ -533,6 +573,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry,
{
int err;
struct inode *inode;
+ struct fuse_conn *fc = get_fuse_conn(dir);
struct fuse_mount *fm = get_fuse_mount(dir);
FUSE_ARGS(args);
struct fuse_forget_link *forget;
@@ -610,6 +651,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry,
ff->fh = outopen.fh;
ff->nodeid = outentry.nodeid;
ff->open_flags = outopen.open_flags;
+ fuse_passthrough_setup(fc, ff, &outopen);
inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation,
&outentry.attr, entry_attr_timeout(&outentry), 0);
if (!inode) {
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 875314e..d29f583 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -146,7 +146,7 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
if (!err) {
ff->fh = outarg.fh;
ff->open_flags = outarg.open_flags;
-
+ fuse_passthrough_setup(fc, ff, &outarg);
} else if (err != -ENOSYS) {
fuse_file_free(ff);
return ERR_PTR(err);
@@ -308,6 +308,8 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff,
struct fuse_release_args *ra = ff->release_args;
int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
+ fuse_passthrough_release(&ff->passthrough);
+
fuse_prepare_release(fi, ff, open_flags, opcode);
if (ff->flock) {
@@ -1634,7 +1636,9 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
if (FUSE_IS_DAX(inode))
return fuse_dax_read_iter(iocb, to);
- if (!(ff->open_flags & FOPEN_DIRECT_IO))
+ if (ff->passthrough.filp)
+ return fuse_passthrough_read_iter(iocb, to);
+ else if (!(ff->open_flags & FOPEN_DIRECT_IO))
return fuse_cache_read_iter(iocb, to);
else
return fuse_direct_read_iter(iocb, to);
@@ -1652,7 +1656,9 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
if (FUSE_IS_DAX(inode))
return fuse_dax_write_iter(iocb, from);
- if (!(ff->open_flags & FOPEN_DIRECT_IO))
+ if (ff->passthrough.filp)
+ return fuse_passthrough_write_iter(iocb, from);
+ else if (!(ff->open_flags & FOPEN_DIRECT_IO))
return fuse_cache_write_iter(iocb, from);
else
return fuse_direct_write_iter(iocb, from);
@@ -2469,6 +2475,9 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
if (FUSE_IS_DAX(file_inode(file)))
return fuse_dax_mmap(file, vma);
+ if (ff->passthrough.filp)
+ return fuse_passthrough_mmap(file, vma);
+
if (ff->open_flags & FOPEN_DIRECT_IO) {
/* Can't provide the coherency needed for MAP_SHARED */
if (vma->vm_flags & VM_MAYSHARE)
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index c673fae..96836ef 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -173,6 +173,17 @@ struct fuse_conn;
struct fuse_mount;
struct fuse_release_args;
+/**
+ * Reference to lower filesystem file for read/write operations handled in
+ * passthrough mode.
+ * This struct also tracks the credentials to be used for handling read/write
+ * operations.
+ */
+struct fuse_passthrough {
+ struct file *filp;
+ struct cred *cred;
+};
+
/** FUSE specific file data */
struct fuse_file {
/** Fuse connection for this file */
@@ -218,6 +229,9 @@ struct fuse_file {
} readdir;
+ /** Container for data related to the passthrough functionality */
+ struct fuse_passthrough passthrough;
+
/** RB node to be linked on fuse_conn->polled_files */
struct rb_node polled_node;
@@ -264,6 +278,9 @@ struct fuse_args {
struct fuse_in_arg in_args[3];
struct fuse_arg out_args[2];
void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
+
+ /* Path used for completing d_canonical_path */
+ struct path *canonical_path;
};
struct fuse_args_pages {
@@ -367,10 +384,8 @@ struct fuse_req {
/** Used to wake up the task waiting for completion of request*/
wait_queue_head_t waitq;
-#if IS_ENABLED(CONFIG_VIRTIO_FS)
/** virtio-fs's physically contiguous buffer for in and out args */
void *argbuf;
-#endif
/** fuse_mount this request belongs to */
struct fuse_mount *fm;
@@ -390,19 +405,19 @@ struct fuse_iqueue_ops {
/**
* Signal that a forget has been queued
*/
- void (*wake_forget_and_unlock)(struct fuse_iqueue *fiq)
+ void (*wake_forget_and_unlock)(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock);
/**
* Signal that an INTERRUPT request has been queued
*/
- void (*wake_interrupt_and_unlock)(struct fuse_iqueue *fiq)
+ void (*wake_interrupt_and_unlock)(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock);
/**
* Signal that a request has been queued
*/
- void (*wake_pending_and_unlock)(struct fuse_iqueue *fiq)
+ void (*wake_pending_and_unlock)(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock);
/**
@@ -775,6 +790,9 @@ struct fuse_conn {
/* Auto-mount submounts announced by the server */
unsigned int auto_submounts:1;
+ /** Passthrough mode for read/write IO */
+ unsigned int passthrough:1;
+
/* Propagate syncfs() to server */
unsigned int sync_fs:1;
@@ -836,6 +854,12 @@ struct fuse_conn {
/* New writepages go into this bucket */
struct fuse_sync_bucket __rcu *curr_bucket;
+
+ /** IDR for passthrough requests */
+ struct idr passthrough_req;
+
+ /** Protects passthrough_req */
+ spinlock_t passthrough_req_lock;
};
/*
@@ -1319,4 +1343,13 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
void fuse_file_release(struct inode *inode, struct fuse_file *ff,
unsigned int open_flags, fl_owner_t id, bool isdir);
+/* passthrough.c */
+int fuse_passthrough_open(struct fuse_dev *fud, u32 lower_fd);
+int fuse_passthrough_setup(struct fuse_conn *fc, struct fuse_file *ff,
+ struct fuse_open_out *openarg);
+void fuse_passthrough_release(struct fuse_passthrough *passthrough);
+ssize_t fuse_passthrough_read_iter(struct kiocb *iocb, struct iov_iter *to);
+ssize_t fuse_passthrough_write_iter(struct kiocb *iocb, struct iov_iter *from);
+ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma);
+
#endif /* _FS_FUSE_I_H */
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index 6b3beda..632f94b 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -825,6 +825,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
memset(fc, 0, sizeof(*fc));
spin_lock_init(&fc->lock);
spin_lock_init(&fc->bg_lock);
+ spin_lock_init(&fc->passthrough_req_lock);
init_rwsem(&fc->killsb);
refcount_set(&fc->count, 1);
atomic_set(&fc->dev_count, 1);
@@ -833,6 +834,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
INIT_LIST_HEAD(&fc->bg_queue);
INIT_LIST_HEAD(&fc->entry);
INIT_LIST_HEAD(&fc->devices);
+ idr_init(&fc->passthrough_req);
atomic_set(&fc->num_waiting, 0);
fc->max_background = FUSE_DEFAULT_MAX_BACKGROUND;
fc->congestion_threshold = FUSE_DEFAULT_CONGESTION_THRESHOLD;
@@ -1197,6 +1199,12 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
fc->handle_killpriv_v2 = 1;
fm->sb->s_flags |= SB_NOSEC;
}
+ if (flags & FUSE_PASSTHROUGH) {
+ fc->passthrough = 1;
+ /* Prevent further stacking */
+ fm->sb->s_stack_depth =
+ FILESYSTEM_MAX_STACK_DEPTH;
+ }
if (flags & FUSE_SETXATTR_EXT)
fc->setxattr_ext = 1;
if (flags & FUSE_SECURITY_CTX)
@@ -1256,6 +1264,8 @@ void fuse_send_init(struct fuse_mount *fm)
if (fm->fc->auto_submounts)
flags |= FUSE_SUBMOUNTS;
+ flags |= FUSE_PASSTHROUGH;
+
ia->in.flags = flags;
ia->in.flags2 = flags >> 32;
@@ -1279,9 +1289,21 @@ void fuse_send_init(struct fuse_mount *fm)
}
EXPORT_SYMBOL_GPL(fuse_send_init);
+static int free_fuse_passthrough(int id, void *p, void *data)
+{
+ struct fuse_passthrough *passthrough = (struct fuse_passthrough *)p;
+
+ fuse_passthrough_release(passthrough);
+ kfree(p);
+
+ return 0;
+}
+
void fuse_free_conn(struct fuse_conn *fc)
{
WARN_ON(!list_empty(&fc->devices));
+ idr_for_each(&fc->passthrough_req, free_fuse_passthrough, NULL);
+ idr_destroy(&fc->passthrough_req);
kfree_rcu(fc, rcu);
}
EXPORT_SYMBOL_GPL(fuse_free_conn);
diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c
new file mode 100644
index 0000000..630fa28
--- /dev/null
+++ b/fs/fuse/passthrough.c
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "fuse_i.h"
+
+#include <linux/file.h>
+#include <linux/fuse.h>
+#include <linux/idr.h>
+#include <linux/uio.h>
+
+#define PASSTHROUGH_IOCB_MASK \
+ (IOCB_APPEND | IOCB_DSYNC | IOCB_HIPRI | IOCB_NOWAIT | IOCB_SYNC)
+
+struct fuse_aio_req {
+ struct kiocb iocb;
+ struct kiocb *iocb_fuse;
+};
+
+static void fuse_file_accessed(struct file *dst_file, struct file *src_file)
+{
+ struct inode *dst_inode;
+ struct inode *src_inode;
+
+ if (dst_file->f_flags & O_NOATIME)
+ return;
+
+ dst_inode = file_inode(dst_file);
+ src_inode = file_inode(src_file);
+
+ if ((!timespec64_equal(&dst_inode->i_mtime, &src_inode->i_mtime) ||
+ !timespec64_equal(&dst_inode->i_ctime, &src_inode->i_ctime))) {
+ dst_inode->i_mtime = src_inode->i_mtime;
+ dst_inode->i_ctime = src_inode->i_ctime;
+ }
+
+ touch_atime(&dst_file->f_path);
+}
+
+static void fuse_copyattr(struct file *dst_file, struct file *src_file)
+{
+ struct inode *dst = file_inode(dst_file);
+ struct inode *src = file_inode(src_file);
+
+ dst->i_atime = src->i_atime;
+ dst->i_mtime = src->i_mtime;
+ dst->i_ctime = src->i_ctime;
+ i_size_write(dst, i_size_read(src));
+}
+
+static void fuse_aio_cleanup_handler(struct fuse_aio_req *aio_req)
+{
+ struct kiocb *iocb = &aio_req->iocb;
+ struct kiocb *iocb_fuse = aio_req->iocb_fuse;
+
+ if (iocb->ki_flags & IOCB_WRITE) {
+ __sb_writers_acquired(file_inode(iocb->ki_filp)->i_sb,
+ SB_FREEZE_WRITE);
+ file_end_write(iocb->ki_filp);
+ fuse_copyattr(iocb_fuse->ki_filp, iocb->ki_filp);
+ }
+
+ iocb_fuse->ki_pos = iocb->ki_pos;
+ kfree(aio_req);
+}
+
+static void fuse_aio_rw_complete(struct kiocb *iocb, long res)
+{
+ struct fuse_aio_req *aio_req =
+ container_of(iocb, struct fuse_aio_req, iocb);
+ struct kiocb *iocb_fuse = aio_req->iocb_fuse;
+
+ fuse_aio_cleanup_handler(aio_req);
+ iocb_fuse->ki_complete(iocb_fuse, res);
+}
+
+ssize_t fuse_passthrough_read_iter(struct kiocb *iocb_fuse,
+ struct iov_iter *iter)
+{
+ ssize_t ret;
+ const struct cred *old_cred;
+ struct file *fuse_filp = iocb_fuse->ki_filp;
+ struct fuse_file *ff = fuse_filp->private_data;
+ struct file *passthrough_filp = ff->passthrough.filp;
+
+ if (!iov_iter_count(iter))
+ return 0;
+
+ old_cred = override_creds(ff->passthrough.cred);
+ if (is_sync_kiocb(iocb_fuse)) {
+ ret = vfs_iter_read(passthrough_filp, iter, &iocb_fuse->ki_pos,
+ iocb_to_rw_flags(iocb_fuse->ki_flags,
+ PASSTHROUGH_IOCB_MASK));
+ } else {
+ struct fuse_aio_req *aio_req;
+
+ aio_req = kmalloc(sizeof(struct fuse_aio_req), GFP_KERNEL);
+ if (!aio_req) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ aio_req->iocb_fuse = iocb_fuse;
+ kiocb_clone(&aio_req->iocb, iocb_fuse, passthrough_filp);
+ aio_req->iocb.ki_complete = fuse_aio_rw_complete;
+ ret = call_read_iter(passthrough_filp, &aio_req->iocb, iter);
+ if (ret != -EIOCBQUEUED)
+ fuse_aio_cleanup_handler(aio_req);
+ }
+out:
+ revert_creds(old_cred);
+
+ fuse_file_accessed(fuse_filp, passthrough_filp);
+
+ return ret;
+}
+
+ssize_t fuse_passthrough_write_iter(struct kiocb *iocb_fuse,
+ struct iov_iter *iter)
+{
+ ssize_t ret;
+ const struct cred *old_cred;
+ struct file *fuse_filp = iocb_fuse->ki_filp;
+ struct fuse_file *ff = fuse_filp->private_data;
+ struct inode *fuse_inode = file_inode(fuse_filp);
+ struct file *passthrough_filp = ff->passthrough.filp;
+ struct inode *passthrough_inode = file_inode(passthrough_filp);
+
+ if (!iov_iter_count(iter))
+ return 0;
+
+ inode_lock(fuse_inode);
+
+ fuse_copyattr(fuse_filp, passthrough_filp);
+
+ old_cred = override_creds(ff->passthrough.cred);
+ if (is_sync_kiocb(iocb_fuse)) {
+ file_start_write(passthrough_filp);
+ ret = vfs_iter_write(passthrough_filp, iter, &iocb_fuse->ki_pos,
+ iocb_to_rw_flags(iocb_fuse->ki_flags,
+ PASSTHROUGH_IOCB_MASK));
+ file_end_write(passthrough_filp);
+ if (ret > 0)
+ fuse_copyattr(fuse_filp, passthrough_filp);
+ } else {
+ struct fuse_aio_req *aio_req;
+
+ aio_req = kmalloc(sizeof(struct fuse_aio_req), GFP_KERNEL);
+ if (!aio_req) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ file_start_write(passthrough_filp);
+ __sb_writers_release(passthrough_inode->i_sb, SB_FREEZE_WRITE);
+
+ aio_req->iocb_fuse = iocb_fuse;
+ kiocb_clone(&aio_req->iocb, iocb_fuse, passthrough_filp);
+ aio_req->iocb.ki_complete = fuse_aio_rw_complete;
+ ret = call_write_iter(passthrough_filp, &aio_req->iocb, iter);
+ if (ret != -EIOCBQUEUED)
+ fuse_aio_cleanup_handler(aio_req);
+ }
+out:
+ revert_creds(old_cred);
+ inode_unlock(fuse_inode);
+
+ return ret;
+}
+
+ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ int ret;
+ const struct cred *old_cred;
+ struct fuse_file *ff = file->private_data;
+ struct file *passthrough_filp = ff->passthrough.filp;
+
+ if (!passthrough_filp->f_op->mmap)
+ return -ENODEV;
+
+ if (WARN_ON(file != vma->vm_file))
+ return -EIO;
+
+ vma->vm_file = get_file(passthrough_filp);
+
+ old_cred = override_creds(ff->passthrough.cred);
+ ret = call_mmap(vma->vm_file, vma);
+ revert_creds(old_cred);
+
+ if (ret)
+ fput(passthrough_filp);
+ else
+ fput(file);
+
+ fuse_file_accessed(file, passthrough_filp);
+
+ return ret;
+}
+
+int fuse_passthrough_open(struct fuse_dev *fud, u32 lower_fd)
+{
+ int res;
+ struct file *passthrough_filp;
+ struct fuse_conn *fc = fud->fc;
+ struct inode *passthrough_inode;
+ struct super_block *passthrough_sb;
+ struct fuse_passthrough *passthrough;
+
+ if (!fc->passthrough)
+ return -EPERM;
+
+ passthrough_filp = fget(lower_fd);
+ if (!passthrough_filp) {
+ pr_err("FUSE: invalid file descriptor for passthrough.\n");
+ return -EBADF;
+ }
+
+ if (!passthrough_filp->f_op->read_iter ||
+ !passthrough_filp->f_op->write_iter) {
+ pr_err("FUSE: passthrough file misses file operations.\n");
+ res = -EBADF;
+ goto err_free_file;
+ }
+
+ passthrough_inode = file_inode(passthrough_filp);
+ passthrough_sb = passthrough_inode->i_sb;
+ if (passthrough_sb->s_stack_depth >= FILESYSTEM_MAX_STACK_DEPTH) {
+ pr_err("FUSE: fs stacking depth exceeded for passthrough\n");
+ res = -EINVAL;
+ goto err_free_file;
+ }
+
+ passthrough = kmalloc(sizeof(struct fuse_passthrough), GFP_KERNEL);
+ if (!passthrough) {
+ res = -ENOMEM;
+ goto err_free_file;
+ }
+
+ passthrough->filp = passthrough_filp;
+ passthrough->cred = prepare_creds();
+
+ idr_preload(GFP_KERNEL);
+ spin_lock(&fc->passthrough_req_lock);
+ res = idr_alloc(&fc->passthrough_req, passthrough, 1, 0, GFP_ATOMIC);
+ spin_unlock(&fc->passthrough_req_lock);
+ idr_preload_end();
+
+ if (res > 0)
+ return res;
+
+ fuse_passthrough_release(passthrough);
+ kfree(passthrough);
+
+err_free_file:
+ fput(passthrough_filp);
+
+ return res;
+}
+
+int fuse_passthrough_setup(struct fuse_conn *fc, struct fuse_file *ff,
+ struct fuse_open_out *openarg)
+{
+ struct fuse_passthrough *passthrough;
+ int passthrough_fh = openarg->passthrough_fh;
+
+ if (!fc->passthrough)
+ return -EPERM;
+
+ /* Default case, passthrough is not requested */
+ if (passthrough_fh <= 0)
+ return -EINVAL;
+
+ spin_lock(&fc->passthrough_req_lock);
+ passthrough = idr_remove(&fc->passthrough_req, passthrough_fh);
+ spin_unlock(&fc->passthrough_req_lock);
+
+ if (!passthrough)
+ return -EINVAL;
+
+ ff->passthrough = *passthrough;
+ kfree(passthrough);
+
+ return 0;
+}
+
+void fuse_passthrough_release(struct fuse_passthrough *passthrough)
+{
+ if (passthrough->filp) {
+ fput(passthrough->filp);
+ passthrough->filp = NULL;
+ }
+ if (passthrough->cred) {
+ put_cred(passthrough->cred);
+ passthrough->cred = NULL;
+ }
+}
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index 4d8d4f1..23a42d8 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -970,7 +970,7 @@ static struct virtio_driver virtio_fs_driver = {
#endif
};
-static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
+static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock)
{
struct fuse_forget_link *link;
@@ -1005,7 +1005,8 @@ __releases(fiq->lock)
kfree(link);
}
-static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
+static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq,
+ bool sync)
__releases(fiq->lock)
{
/*
@@ -1220,7 +1221,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
return ret;
}
-static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
+static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq,
+ bool sync)
__releases(fiq->lock)
{
unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
diff --git a/fs/incfs/Kconfig b/fs/incfs/Kconfig
new file mode 100644
index 0000000..5f15000
--- /dev/null
+++ b/fs/incfs/Kconfig
@@ -0,0 +1,13 @@
+config INCREMENTAL_FS
+ tristate "Incremental file system support"
+ depends on BLOCK
+ select DECOMPRESS_LZ4
+ select DECOMPRESS_ZSTD
+ select CRYPTO_SHA256
+ help
+ Incremental FS is a read-only virtual file system that facilitates execution
+ of programs while their binaries are still being lazily downloaded over the
+ network, USB or pigeon post.
+
+ To compile this file system support as a module, choose M here: the
+ module will be called incrementalfs.
diff --git a/fs/incfs/Makefile b/fs/incfs/Makefile
new file mode 100644
index 0000000..05795d1
--- /dev/null
+++ b/fs/incfs/Makefile
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_INCREMENTAL_FS) += incrementalfs.o
+
+incrementalfs-y := \
+ data_mgmt.o \
+ format.o \
+ integrity.o \
+ main.o \
+ pseudo_files.o \
+ sysfs.o \
+ vfs.o
+
+incrementalfs-$(CONFIG_FS_VERITY) += verity.o
diff --git a/fs/incfs/OWNERS b/fs/incfs/OWNERS
new file mode 100644
index 0000000..1b97669
--- /dev/null
+++ b/fs/incfs/OWNERS
@@ -0,0 +1,2 @@
+akailash@google.com
+paullawrence@google.com
diff --git a/fs/incfs/data_mgmt.c b/fs/incfs/data_mgmt.c
new file mode 100644
index 0000000..01ee5d2
--- /dev/null
+++ b/fs/incfs/data_mgmt.c
@@ -0,0 +1,1892 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 Google LLC
+ */
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/file.h>
+#include <linux/fsverity.h>
+#include <linux/gfp.h>
+#include <linux/kobject.h>
+#include <linux/ktime.h>
+#include <linux/lz4.h>
+#include <linux/mm.h>
+#include <linux/namei.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+
+#include "data_mgmt.h"
+#include "format.h"
+#include "integrity.h"
+#include "sysfs.h"
+#include "verity.h"
+
+static int incfs_scan_metadata_chain(struct data_file *df);
+
+static void log_wake_up_all(struct work_struct *work)
+{
+ struct delayed_work *dw = container_of(work, struct delayed_work, work);
+ struct read_log *rl = container_of(dw, struct read_log, ml_wakeup_work);
+ wake_up_all(&rl->ml_notif_wq);
+}
+
+static void zstd_free_workspace(struct work_struct *work)
+{
+ struct delayed_work *dw = container_of(work, struct delayed_work, work);
+ struct mount_info *mi =
+ container_of(dw, struct mount_info, mi_zstd_cleanup_work);
+
+ mutex_lock(&mi->mi_zstd_workspace_mutex);
+ kvfree(mi->mi_zstd_workspace);
+ mi->mi_zstd_workspace = NULL;
+ mi->mi_zstd_stream = NULL;
+ mutex_unlock(&mi->mi_zstd_workspace_mutex);
+}
+
+struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
+ struct mount_options *options,
+ struct path *backing_dir_path)
+{
+ struct mount_info *mi = NULL;
+ int error = 0;
+ struct incfs_sysfs_node *node;
+
+ mi = kzalloc(sizeof(*mi), GFP_NOFS);
+ if (!mi)
+ return ERR_PTR(-ENOMEM);
+
+ mi->mi_sb = sb;
+ mi->mi_backing_dir_path = *backing_dir_path;
+ mi->mi_owner = get_current_cred();
+ path_get(&mi->mi_backing_dir_path);
+ mutex_init(&mi->mi_dir_struct_mutex);
+ init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
+ init_waitqueue_head(&mi->mi_log.ml_notif_wq);
+ init_waitqueue_head(&mi->mi_blocks_written_notif_wq);
+ atomic_set(&mi->mi_blocks_written, 0);
+ INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
+ spin_lock_init(&mi->mi_log.rl_lock);
+ spin_lock_init(&mi->pending_read_lock);
+ INIT_LIST_HEAD(&mi->mi_reads_list_head);
+ spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);
+ mutex_init(&mi->mi_zstd_workspace_mutex);
+ INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);
+ mutex_init(&mi->mi_le_mutex);
+
+ node = incfs_add_sysfs_node(options->sysfs_name, mi);
+ if (IS_ERR(node)) {
+ error = PTR_ERR(node);
+ goto err;
+ }
+ mi->mi_sysfs_node = node;
+
+ error = incfs_realloc_mount_info(mi, options);
+ if (error)
+ goto err;
+
+ return mi;
+
+err:
+ incfs_free_mount_info(mi);
+ return ERR_PTR(error);
+}
+
+int incfs_realloc_mount_info(struct mount_info *mi,
+ struct mount_options *options)
+{
+ void *new_buffer = NULL;
+ void *old_buffer;
+ size_t new_buffer_size = 0;
+
+ if (options->read_log_pages != mi->mi_options.read_log_pages) {
+ struct read_log_state log_state;
+ /*
+ * Even though having two buffers allocated at once isn't
+ * usually good, allocating a multipage buffer under a spinlock
+ * is even worse, so let's optimize for the shorter lock
+ * duration. It's not end of the world if we fail to increase
+ * the buffer size anyway.
+ */
+ if (options->read_log_pages > 0) {
+ new_buffer_size = PAGE_SIZE * options->read_log_pages;
+ new_buffer = kzalloc(new_buffer_size, GFP_NOFS);
+ if (!new_buffer)
+ return -ENOMEM;
+ }
+
+ spin_lock(&mi->mi_log.rl_lock);
+ old_buffer = mi->mi_log.rl_ring_buf;
+ mi->mi_log.rl_ring_buf = new_buffer;
+ mi->mi_log.rl_size = new_buffer_size;
+ log_state = (struct read_log_state){
+ .generation_id = mi->mi_log.rl_head.generation_id + 1,
+ };
+ mi->mi_log.rl_head = log_state;
+ mi->mi_log.rl_tail = log_state;
+ spin_unlock(&mi->mi_log.rl_lock);
+
+ kfree(old_buffer);
+ }
+
+ if (options->sysfs_name && !mi->mi_sysfs_node)
+ mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
+ mi);
+ else if (!options->sysfs_name && mi->mi_sysfs_node) {
+ incfs_free_sysfs_node(mi->mi_sysfs_node);
+ mi->mi_sysfs_node = NULL;
+ } else if (options->sysfs_name &&
+ strcmp(options->sysfs_name,
+ kobject_name(&mi->mi_sysfs_node->isn_sysfs_node))) {
+ incfs_free_sysfs_node(mi->mi_sysfs_node);
+ mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
+ mi);
+ }
+
+ if (IS_ERR(mi->mi_sysfs_node)) {
+ int err = PTR_ERR(mi->mi_sysfs_node);
+
+ mi->mi_sysfs_node = NULL;
+ return err;
+ }
+
+ mi->mi_options = *options;
+ return 0;
+}
+
+void incfs_free_mount_info(struct mount_info *mi)
+{
+ int i;
+ if (!mi)
+ return;
+
+ flush_delayed_work(&mi->mi_log.ml_wakeup_work);
+ flush_delayed_work(&mi->mi_zstd_cleanup_work);
+
+ dput(mi->mi_index_dir);
+ dput(mi->mi_incomplete_dir);
+ path_put(&mi->mi_backing_dir_path);
+ mutex_destroy(&mi->mi_dir_struct_mutex);
+ mutex_destroy(&mi->mi_zstd_workspace_mutex);
+ put_cred(mi->mi_owner);
+ kfree(mi->mi_log.rl_ring_buf);
+ for (i = 0; i < ARRAY_SIZE(mi->pseudo_file_xattr); ++i)
+ kfree(mi->pseudo_file_xattr[i].data);
+ kfree(mi->mi_per_uid_read_timeouts);
+ incfs_free_sysfs_node(mi->mi_sysfs_node);
+ kfree(mi);
+}
+
+static void data_file_segment_init(struct data_file_segment *segment)
+{
+ init_waitqueue_head(&segment->new_data_arrival_wq);
+ init_rwsem(&segment->rwsem);
+ INIT_LIST_HEAD(&segment->reads_list_head);
+}
+
+char *file_id_to_str(incfs_uuid_t id)
+{
+ char *result = kmalloc(1 + sizeof(id.bytes) * 2, GFP_NOFS);
+ char *end;
+
+ if (!result)
+ return NULL;
+
+ end = bin2hex(result, id.bytes, sizeof(id.bytes));
+ *end = 0;
+ return result;
+}
+
+struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name)
+{
+ struct inode *inode;
+ struct dentry *result = NULL;
+
+ if (!parent)
+ return ERR_PTR(-EFAULT);
+
+ inode = d_inode(parent);
+ inode_lock_nested(inode, I_MUTEX_PARENT);
+ result = lookup_one_len(name, parent, strlen(name));
+ inode_unlock(inode);
+
+ if (IS_ERR(result))
+ pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
+
+ return result;
+}
+
+static struct data_file *handle_mapped_file(struct mount_info *mi,
+ struct data_file *df)
+{
+ char *file_id_str;
+ struct dentry *index_file_dentry;
+ struct path path;
+ struct file *bf;
+ struct data_file *result = NULL;
+ const struct cred *old_cred;
+
+ file_id_str = file_id_to_str(df->df_id);
+ if (!file_id_str)
+ return ERR_PTR(-ENOENT);
+
+ index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
+ file_id_str);
+ kfree(file_id_str);
+ if (!index_file_dentry)
+ return ERR_PTR(-ENOENT);
+ if (IS_ERR(index_file_dentry))
+ return ERR_CAST(index_file_dentry);
+ if (!d_really_is_positive(index_file_dentry)) {
+ result = ERR_PTR(-ENOENT);
+ goto out;
+ }
+
+ path = (struct path) {
+ .mnt = mi->mi_backing_dir_path.mnt,
+ .dentry = index_file_dentry
+ };
+
+ old_cred = override_creds(mi->mi_owner);
+ bf = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
+ current_cred());
+ revert_creds(old_cred);
+
+ if (IS_ERR(bf)) {
+ result = ERR_CAST(bf);
+ goto out;
+ }
+
+ result = incfs_open_data_file(mi, bf);
+ fput(bf);
+ if (IS_ERR(result))
+ goto out;
+
+ result->df_mapped_offset = df->df_metadata_off;
+
+out:
+ dput(index_file_dentry);
+ return result;
+}
+
+struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf)
+{
+ struct data_file *df = NULL;
+ struct backing_file_context *bfc = NULL;
+ int md_records;
+ u64 size;
+ int error = 0;
+ int i;
+
+ if (!bf || !mi)
+ return ERR_PTR(-EFAULT);
+
+ if (!S_ISREG(bf->f_inode->i_mode))
+ return ERR_PTR(-EBADF);
+
+ bfc = incfs_alloc_bfc(mi, bf);
+ if (IS_ERR(bfc))
+ return ERR_CAST(bfc);
+
+ df = kzalloc(sizeof(*df), GFP_NOFS);
+ if (!df) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ mutex_init(&df->df_enable_verity);
+
+ df->df_backing_file_context = bfc;
+ df->df_mount_info = mi;
+ for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
+ data_file_segment_init(&df->df_segments[i]);
+
+ error = incfs_read_file_header(bfc, &df->df_metadata_off, &df->df_id,
+ &size, &df->df_header_flags);
+
+ if (error)
+ goto out;
+
+ df->df_size = size;
+ if (size > 0)
+ df->df_data_block_count = get_blocks_count_for_size(size);
+
+ if (df->df_header_flags & INCFS_FILE_MAPPED) {
+ struct data_file *mapped_df = handle_mapped_file(mi, df);
+
+ incfs_free_data_file(df);
+ return mapped_df;
+ }
+
+ md_records = incfs_scan_metadata_chain(df);
+ if (md_records < 0)
+ error = md_records;
+
+out:
+ if (error) {
+ incfs_free_bfc(bfc);
+ if (df)
+ df->df_backing_file_context = NULL;
+ incfs_free_data_file(df);
+ return ERR_PTR(error);
+ }
+ return df;
+}
+
+void incfs_free_data_file(struct data_file *df)
+{
+ u32 data_blocks_written, hash_blocks_written;
+
+ if (!df)
+ return;
+
+ data_blocks_written = atomic_read(&df->df_data_blocks_written);
+ hash_blocks_written = atomic_read(&df->df_hash_blocks_written);
+
+ if (data_blocks_written != df->df_initial_data_blocks_written ||
+ hash_blocks_written != df->df_initial_hash_blocks_written) {
+ struct backing_file_context *bfc = df->df_backing_file_context;
+ int error = -1;
+
+ if (bfc && !mutex_lock_interruptible(&bfc->bc_mutex)) {
+ error = incfs_write_status_to_backing_file(
+ df->df_backing_file_context,
+ df->df_status_offset,
+ data_blocks_written,
+ hash_blocks_written);
+ mutex_unlock(&bfc->bc_mutex);
+ }
+
+ if (error)
+ /* Nothing can be done, just warn */
+ pr_warn("incfs: failed to write status to backing file\n");
+ }
+
+ incfs_free_mtree(df->df_hash_tree);
+ incfs_free_bfc(df->df_backing_file_context);
+ kfree(df->df_signature);
+ kfree(df->df_verity_file_digest.data);
+ kfree(df->df_verity_signature);
+ mutex_destroy(&df->df_enable_verity);
+ kfree(df);
+}
+
+int make_inode_ready_for_data_ops(struct mount_info *mi,
+ struct inode *inode,
+ struct file *backing_file)
+{
+ struct inode_info *node = get_incfs_node(inode);
+ struct data_file *df = NULL;
+ int err = 0;
+
+ inode_lock(inode);
+ if (S_ISREG(inode->i_mode)) {
+ if (!node->n_file) {
+ df = incfs_open_data_file(mi, backing_file);
+
+ if (IS_ERR(df))
+ err = PTR_ERR(df);
+ else
+ node->n_file = df;
+ }
+ } else
+ err = -EBADF;
+ inode_unlock(inode);
+ return err;
+}
+
+struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf)
+{
+ struct dir_file *dir = NULL;
+
+ if (!S_ISDIR(bf->f_inode->i_mode))
+ return ERR_PTR(-EBADF);
+
+ dir = kzalloc(sizeof(*dir), GFP_NOFS);
+ if (!dir)
+ return ERR_PTR(-ENOMEM);
+
+ dir->backing_dir = get_file(bf);
+ dir->mount_info = mi;
+ return dir;
+}
+
+void incfs_free_dir_file(struct dir_file *dir)
+{
+ if (!dir)
+ return;
+ if (dir->backing_dir)
+ fput(dir->backing_dir);
+ kfree(dir);
+}
+
+static ssize_t zstd_decompress_safe(struct mount_info *mi,
+ struct mem_range src, struct mem_range dst)
+{
+ ssize_t result;
+ ZSTD_inBuffer inbuf = {.src = src.data, .size = src.len};
+ ZSTD_outBuffer outbuf = {.dst = dst.data, .size = dst.len};
+
+ result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);
+ if (result)
+ return result;
+
+ if (!mi->mi_zstd_stream) {
+ unsigned int workspace_size = zstd_dstream_workspace_bound(
+ INCFS_DATA_FILE_BLOCK_SIZE);
+ void *workspace = kvmalloc(workspace_size, GFP_NOFS);
+ ZSTD_DStream *stream;
+
+ if (!workspace) {
+ result = -ENOMEM;
+ goto out;
+ }
+
+ stream = zstd_init_dstream(INCFS_DATA_FILE_BLOCK_SIZE, workspace,
+ workspace_size);
+ if (!stream) {
+ kvfree(workspace);
+ result = -EIO;
+ goto out;
+ }
+
+ mi->mi_zstd_workspace = workspace;
+ mi->mi_zstd_stream = stream;
+ }
+
+ result = zstd_decompress_stream(mi->mi_zstd_stream, &outbuf, &inbuf) ?
+ -EBADMSG : outbuf.pos;
+
+ mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,
+ msecs_to_jiffies(5000));
+
+out:
+ mutex_unlock(&mi->mi_zstd_workspace_mutex);
+ return result;
+}
+
+static ssize_t decompress(struct mount_info *mi,
+ struct mem_range src, struct mem_range dst, int alg)
+{
+ int result;
+
+ switch (alg) {
+ case INCFS_BLOCK_COMPRESSED_LZ4:
+ result = LZ4_decompress_safe(src.data, dst.data, src.len,
+ dst.len);
+ if (result < 0)
+ return -EBADMSG;
+ return result;
+
+ case INCFS_BLOCK_COMPRESSED_ZSTD:
+ return zstd_decompress_safe(mi, src, dst);
+
+ default:
+ WARN_ON(true);
+ return -EOPNOTSUPP;
+ }
+}
+
+static void log_read_one_record(struct read_log *rl, struct read_log_state *rs)
+{
+ union log_record *record =
+ (union log_record *)((u8 *)rl->rl_ring_buf + rs->next_offset);
+ size_t record_size;
+
+ switch (record->full_record.type) {
+ case FULL:
+ rs->base_record = record->full_record;
+ record_size = sizeof(record->full_record);
+ break;
+
+ case SAME_FILE:
+ rs->base_record.block_index =
+ record->same_file.block_index;
+ rs->base_record.absolute_ts_us +=
+ record->same_file.relative_ts_us;
+ rs->base_record.uid = record->same_file.uid;
+ record_size = sizeof(record->same_file);
+ break;
+
+ case SAME_FILE_CLOSE_BLOCK:
+ rs->base_record.block_index +=
+ record->same_file_close_block.block_index_delta;
+ rs->base_record.absolute_ts_us +=
+ record->same_file_close_block.relative_ts_us;
+ record_size = sizeof(record->same_file_close_block);
+ break;
+
+ case SAME_FILE_CLOSE_BLOCK_SHORT:
+ rs->base_record.block_index +=
+ record->same_file_close_block_short.block_index_delta;
+ rs->base_record.absolute_ts_us +=
+ record->same_file_close_block_short.relative_ts_tens_us * 10;
+ record_size = sizeof(record->same_file_close_block_short);
+ break;
+
+ case SAME_FILE_NEXT_BLOCK:
+ ++rs->base_record.block_index;
+ rs->base_record.absolute_ts_us +=
+ record->same_file_next_block.relative_ts_us;
+ record_size = sizeof(record->same_file_next_block);
+ break;
+
+ case SAME_FILE_NEXT_BLOCK_SHORT:
+ ++rs->base_record.block_index;
+ rs->base_record.absolute_ts_us +=
+ record->same_file_next_block_short.relative_ts_tens_us * 10;
+ record_size = sizeof(record->same_file_next_block_short);
+ break;
+ }
+
+ rs->next_offset += record_size;
+ if (rs->next_offset > rl->rl_size - sizeof(*record)) {
+ rs->next_offset = 0;
+ ++rs->current_pass_no;
+ }
+ ++rs->current_record_no;
+}
+
+static void log_block_read(struct mount_info *mi, incfs_uuid_t *id,
+ int block_index)
+{
+ struct read_log *log = &mi->mi_log;
+ struct read_log_state *head, *tail;
+ s64 now_us;
+ s64 relative_us;
+ union log_record record;
+ size_t record_size;
+ uid_t uid = current_uid().val;
+ int block_delta;
+ bool same_file, same_uid;
+ bool next_block, close_block, very_close_block;
+ bool close_time, very_close_time, very_very_close_time;
+
+ /*
+ * This may read the old value, but it's OK to delay the logging start
+ * right after the configuration update.
+ */
+ if (READ_ONCE(log->rl_size) == 0)
+ return;
+
+ now_us = ktime_to_us(ktime_get());
+
+ spin_lock(&log->rl_lock);
+ if (log->rl_size == 0) {
+ spin_unlock(&log->rl_lock);
+ return;
+ }
+
+ head = &log->rl_head;
+ tail = &log->rl_tail;
+ relative_us = now_us - head->base_record.absolute_ts_us;
+
+ same_file = !memcmp(id, &head->base_record.file_id,
+ sizeof(incfs_uuid_t));
+ same_uid = uid == head->base_record.uid;
+
+ block_delta = block_index - head->base_record.block_index;
+ next_block = block_delta == 1;
+ very_close_block = block_delta >= S8_MIN && block_delta <= S8_MAX;
+ close_block = block_delta >= S16_MIN && block_delta <= S16_MAX;
+
+ very_very_close_time = relative_us < (1 << 5) * 10;
+ very_close_time = relative_us < (1 << 13);
+ close_time = relative_us < (1 << 16);
+
+ if (same_file && same_uid && next_block && very_very_close_time) {
+ record.same_file_next_block_short =
+ (struct same_file_next_block_short){
+ .type = SAME_FILE_NEXT_BLOCK_SHORT,
+ .relative_ts_tens_us = div_s64(relative_us, 10),
+ };
+ record_size = sizeof(struct same_file_next_block_short);
+ } else if (same_file && same_uid && next_block && very_close_time) {
+ record.same_file_next_block = (struct same_file_next_block){
+ .type = SAME_FILE_NEXT_BLOCK,
+ .relative_ts_us = relative_us,
+ };
+ record_size = sizeof(struct same_file_next_block);
+ } else if (same_file && same_uid && very_close_block &&
+ very_very_close_time) {
+ record.same_file_close_block_short =
+ (struct same_file_close_block_short){
+ .type = SAME_FILE_CLOSE_BLOCK_SHORT,
+ .relative_ts_tens_us = div_s64(relative_us, 10),
+ .block_index_delta = block_delta,
+ };
+ record_size = sizeof(struct same_file_close_block_short);
+ } else if (same_file && same_uid && close_block && very_close_time) {
+ record.same_file_close_block = (struct same_file_close_block){
+ .type = SAME_FILE_CLOSE_BLOCK,
+ .relative_ts_us = relative_us,
+ .block_index_delta = block_delta,
+ };
+ record_size = sizeof(struct same_file_close_block);
+ } else if (same_file && close_time) {
+ record.same_file = (struct same_file){
+ .type = SAME_FILE,
+ .block_index = block_index,
+ .relative_ts_us = relative_us,
+ .uid = uid,
+ };
+ record_size = sizeof(struct same_file);
+ } else {
+ record.full_record = (struct full_record){
+ .type = FULL,
+ .block_index = block_index,
+ .file_id = *id,
+ .absolute_ts_us = now_us,
+ .uid = uid,
+ };
+ head->base_record.file_id = *id;
+ record_size = sizeof(struct full_record);
+ }
+
+ head->base_record.block_index = block_index;
+ head->base_record.absolute_ts_us = now_us;
+
+ /* Advance tail beyond area we are going to overwrite */
+ while (tail->current_pass_no < head->current_pass_no &&
+ tail->next_offset < head->next_offset + record_size)
+ log_read_one_record(log, tail);
+
+ memcpy(((u8 *)log->rl_ring_buf) + head->next_offset, &record,
+ record_size);
+ head->next_offset += record_size;
+ if (head->next_offset > log->rl_size - sizeof(record)) {
+ head->next_offset = 0;
+ ++head->current_pass_no;
+ }
+ ++head->current_record_no;
+
+ spin_unlock(&log->rl_lock);
+ schedule_delayed_work(&log->ml_wakeup_work, msecs_to_jiffies(16));
+}
+
+static int validate_hash_tree(struct backing_file_context *bfc, struct file *f,
+ int block_index, struct mem_range data, u8 *buf)
+{
+ struct data_file *df = get_incfs_data_file(f);
+ u8 stored_digest[INCFS_MAX_HASH_SIZE] = {};
+ u8 calculated_digest[INCFS_MAX_HASH_SIZE] = {};
+ struct mtree *tree = NULL;
+ struct incfs_df_signature *sig = NULL;
+ int digest_size;
+ int hash_block_index = block_index;
+ int lvl;
+ int res;
+ loff_t hash_block_offset[INCFS_MAX_MTREE_LEVELS];
+ size_t hash_offset_in_block[INCFS_MAX_MTREE_LEVELS];
+ int hash_per_block;
+ pgoff_t file_pages;
+
+ /*
+ * Memory barrier to make sure tree is fully present if added via enable
+ * verity
+ */
+ tree = smp_load_acquire(&df->df_hash_tree);
+ sig = df->df_signature;
+ if (!tree || !sig)
+ return 0;
+
+ digest_size = tree->alg->digest_size;
+ hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
+ for (lvl = 0; lvl < tree->depth; lvl++) {
+ loff_t lvl_off = tree->hash_level_suboffset[lvl];
+
+ hash_block_offset[lvl] =
+ lvl_off + round_down(hash_block_index * digest_size,
+ INCFS_DATA_FILE_BLOCK_SIZE);
+ hash_offset_in_block[lvl] = hash_block_index * digest_size %
+ INCFS_DATA_FILE_BLOCK_SIZE;
+ hash_block_index /= hash_per_block;
+ }
+
+ memcpy(stored_digest, tree->root_hash, digest_size);
+
+ file_pages = DIV_ROUND_UP(df->df_size, INCFS_DATA_FILE_BLOCK_SIZE);
+ for (lvl = tree->depth - 1; lvl >= 0; lvl--) {
+ pgoff_t hash_page =
+ file_pages +
+ hash_block_offset[lvl] / INCFS_DATA_FILE_BLOCK_SIZE;
+ struct page *page = find_get_page_flags(
+ f->f_inode->i_mapping, hash_page, FGP_ACCESSED);
+
+ if (page && PageChecked(page)) {
+ u8 *addr = kmap_atomic(page);
+
+ memcpy(stored_digest, addr + hash_offset_in_block[lvl],
+ digest_size);
+
+ kunmap_atomic(addr);
+ put_page(page);
+ continue;
+ }
+
+ if (page)
+ put_page(page);
+
+ res = incfs_kread(bfc, buf, INCFS_DATA_FILE_BLOCK_SIZE,
+ hash_block_offset[lvl] + sig->hash_offset);
+ if (res < 0)
+ return res;
+ if (res != INCFS_DATA_FILE_BLOCK_SIZE)
+ return -EIO;
+ res = incfs_calc_digest(tree->alg,
+ range(buf, INCFS_DATA_FILE_BLOCK_SIZE),
+ range(calculated_digest, digest_size));
+ if (res)
+ return res;
+
+ if (memcmp(stored_digest, calculated_digest, digest_size)) {
+ int i;
+ bool zero = true;
+
+ pr_warn("incfs: Hash mismatch lvl:%d blk:%d\n",
+ lvl, block_index);
+ for (i = 0; i < digest_size; i++)
+ if (stored_digest[i]) {
+ zero = false;
+ break;
+ }
+
+ if (zero)
+ pr_debug("Note saved_digest all zero - did you forget to load the hashes?\n");
+ return -EBADMSG;
+ }
+
+ memcpy(stored_digest, buf + hash_offset_in_block[lvl],
+ digest_size);
+
+ page = grab_cache_page(f->f_inode->i_mapping, hash_page);
+ if (page) {
+ u8 *addr = kmap_atomic(page);
+
+ memcpy(addr, buf, INCFS_DATA_FILE_BLOCK_SIZE);
+ kunmap_atomic(addr);
+ SetPageChecked(page);
+ SetPageUptodate(page);
+ unlock_page(page);
+ put_page(page);
+ }
+ }
+
+ res = incfs_calc_digest(tree->alg, data,
+ range(calculated_digest, digest_size));
+ if (res)
+ return res;
+
+ if (memcmp(stored_digest, calculated_digest, digest_size)) {
+ pr_debug("Leaf hash mismatch blk:%d\n", block_index);
+ return -EBADMSG;
+ }
+
+ return 0;
+}
+
+static struct data_file_segment *get_file_segment(struct data_file *df,
+ int block_index)
+{
+ int seg_idx = block_index % ARRAY_SIZE(df->df_segments);
+
+ return &df->df_segments[seg_idx];
+}
+
+static bool is_data_block_present(struct data_file_block *block)
+{
+ return (block->db_backing_file_data_offset != 0) &&
+ (block->db_stored_size != 0);
+}
+
+static void convert_data_file_block(struct incfs_blockmap_entry *bme,
+ struct data_file_block *res_block)
+{
+ u16 flags = le16_to_cpu(bme->me_flags);
+
+ res_block->db_backing_file_data_offset =
+ le16_to_cpu(bme->me_data_offset_hi);
+ res_block->db_backing_file_data_offset <<= 32;
+ res_block->db_backing_file_data_offset |=
+ le32_to_cpu(bme->me_data_offset_lo);
+ res_block->db_stored_size = le16_to_cpu(bme->me_data_size);
+ res_block->db_comp_alg = flags & INCFS_BLOCK_COMPRESSED_MASK;
+}
+
+static int get_data_file_block(struct data_file *df, int index,
+ struct data_file_block *res_block)
+{
+ struct incfs_blockmap_entry bme = {};
+ struct backing_file_context *bfc = NULL;
+ loff_t blockmap_off = 0;
+ int error = 0;
+
+ if (!df || !res_block)
+ return -EFAULT;
+
+ blockmap_off = df->df_blockmap_off;
+ bfc = df->df_backing_file_context;
+
+ if (index < 0 || blockmap_off == 0)
+ return -EINVAL;
+
+ error = incfs_read_blockmap_entry(bfc, index, blockmap_off, &bme);
+ if (error)
+ return error;
+
+ convert_data_file_block(&bme, res_block);
+ return 0;
+}
+
+static int check_room_for_one_range(u32 size, u32 size_out)
+{
+ if (size_out + sizeof(struct incfs_filled_range) > size)
+ return -ERANGE;
+ return 0;
+}
+
+static int copy_one_range(struct incfs_filled_range *range, void __user *buffer,
+ u32 size, u32 *size_out)
+{
+ int error = check_room_for_one_range(size, *size_out);
+ if (error)
+ return error;
+
+ if (copy_to_user(((char __user *)buffer) + *size_out, range,
+ sizeof(*range)))
+ return -EFAULT;
+
+ *size_out += sizeof(*range);
+ return 0;
+}
+
+#define READ_BLOCKMAP_ENTRIES 512
+int incfs_get_filled_blocks(struct data_file *df,
+ struct incfs_file_data *fd,
+ struct incfs_get_filled_blocks_args *arg)
+{
+ int error = 0;
+ bool in_range = false;
+ struct incfs_filled_range range;
+ void __user *buffer = u64_to_user_ptr(arg->range_buffer);
+ u32 size = arg->range_buffer_size;
+ u32 end_index =
+ arg->end_index ? arg->end_index : df->df_total_block_count;
+ u32 *size_out = &arg->range_buffer_size_out;
+ int i = READ_BLOCKMAP_ENTRIES - 1;
+ int entries_read = 0;
+ struct incfs_blockmap_entry *bme;
+ int data_blocks_filled = 0;
+ int hash_blocks_filled = 0;
+
+ *size_out = 0;
+ if (end_index > df->df_total_block_count)
+ end_index = df->df_total_block_count;
+ arg->total_blocks_out = df->df_total_block_count;
+ arg->data_blocks_out = df->df_data_block_count;
+
+ if (atomic_read(&df->df_data_blocks_written) ==
+ df->df_data_block_count) {
+ pr_debug("File marked full, fast get_filled_blocks");
+ if (arg->start_index > end_index) {
+ arg->index_out = arg->start_index;
+ return 0;
+ }
+ arg->index_out = arg->start_index;
+
+ error = check_room_for_one_range(size, *size_out);
+ if (error)
+ return error;
+
+ range = (struct incfs_filled_range){
+ .begin = arg->start_index,
+ .end = end_index,
+ };
+
+ error = copy_one_range(&range, buffer, size, size_out);
+ if (error)
+ return error;
+ arg->index_out = end_index;
+ return 0;
+ }
+
+ bme = kzalloc(sizeof(*bme) * READ_BLOCKMAP_ENTRIES,
+ GFP_NOFS | __GFP_COMP);
+ if (!bme)
+ return -ENOMEM;
+
+ for (arg->index_out = arg->start_index; arg->index_out < end_index;
+ ++arg->index_out) {
+ struct data_file_block dfb;
+
+ if (++i == READ_BLOCKMAP_ENTRIES) {
+ entries_read = incfs_read_blockmap_entries(
+ df->df_backing_file_context, bme,
+ arg->index_out, READ_BLOCKMAP_ENTRIES,
+ df->df_blockmap_off);
+ if (entries_read < 0) {
+ error = entries_read;
+ break;
+ }
+
+ i = 0;
+ }
+
+ if (i >= entries_read) {
+ error = -EIO;
+ break;
+ }
+
+ convert_data_file_block(bme + i, &dfb);
+
+ if (is_data_block_present(&dfb)) {
+ if (arg->index_out >= df->df_data_block_count)
+ ++hash_blocks_filled;
+ else
+ ++data_blocks_filled;
+ }
+
+ if (is_data_block_present(&dfb) == in_range)
+ continue;
+
+ if (!in_range) {
+ error = check_room_for_one_range(size, *size_out);
+ if (error)
+ break;
+ in_range = true;
+ range.begin = arg->index_out;
+ } else {
+ range.end = arg->index_out;
+ error = copy_one_range(&range, buffer, size, size_out);
+ if (error) {
+ /* there will be another try out of the loop,
+ * it will reset the index_out if it fails too
+ */
+ break;
+ }
+ in_range = false;
+ }
+ }
+
+ if (in_range) {
+ range.end = arg->index_out;
+ error = copy_one_range(&range, buffer, size, size_out);
+ if (error)
+ arg->index_out = range.begin;
+ }
+
+ if (arg->start_index == 0) {
+ fd->fd_get_block_pos = 0;
+ fd->fd_filled_data_blocks = 0;
+ fd->fd_filled_hash_blocks = 0;
+ }
+
+ if (arg->start_index == fd->fd_get_block_pos) {
+ fd->fd_get_block_pos = arg->index_out + 1;
+ fd->fd_filled_data_blocks += data_blocks_filled;
+ fd->fd_filled_hash_blocks += hash_blocks_filled;
+ }
+
+ if (fd->fd_get_block_pos == df->df_total_block_count + 1) {
+ if (fd->fd_filled_data_blocks >
+ atomic_read(&df->df_data_blocks_written))
+ atomic_set(&df->df_data_blocks_written,
+ fd->fd_filled_data_blocks);
+
+ if (fd->fd_filled_hash_blocks >
+ atomic_read(&df->df_hash_blocks_written))
+ atomic_set(&df->df_hash_blocks_written,
+ fd->fd_filled_hash_blocks);
+ }
+
+ kfree(bme);
+ return error;
+}
+
+static bool is_read_done(struct pending_read *read)
+{
+ return atomic_read_acquire(&read->done) != 0;
+}
+
+static void set_read_done(struct pending_read *read)
+{
+ atomic_set_release(&read->done, 1);
+}
+
+/*
+ * Notifies a given data file about pending read from a given block.
+ * Returns a new pending read entry.
+ */
+static struct pending_read *add_pending_read(struct data_file *df,
+ int block_index)
+{
+ struct pending_read *result = NULL;
+ struct data_file_segment *segment = NULL;
+ struct mount_info *mi = NULL;
+
+ segment = get_file_segment(df, block_index);
+ mi = df->df_mount_info;
+
+ result = kzalloc(sizeof(*result), GFP_NOFS);
+ if (!result)
+ return NULL;
+
+ result->file_id = df->df_id;
+ result->block_index = block_index;
+ result->timestamp_us = ktime_to_us(ktime_get());
+ result->uid = current_uid().val;
+
+ spin_lock(&mi->pending_read_lock);
+
+ result->serial_number = ++mi->mi_last_pending_read_number;
+ mi->mi_pending_reads_count++;
+
+ list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
+ list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
+
+ spin_unlock(&mi->pending_read_lock);
+
+ wake_up_all(&mi->mi_pending_reads_notif_wq);
+ return result;
+}
+
+static void free_pending_read_entry(struct rcu_head *entry)
+{
+ struct pending_read *read;
+
+ read = container_of(entry, struct pending_read, rcu);
+
+ kfree(read);
+}
+
+/* Notifies a given data file that pending read is completed. */
+static void remove_pending_read(struct data_file *df, struct pending_read *read)
+{
+ struct mount_info *mi = NULL;
+
+ if (!df || !read) {
+ WARN_ON(!df);
+ WARN_ON(!read);
+ return;
+ }
+
+ mi = df->df_mount_info;
+
+ spin_lock(&mi->pending_read_lock);
+
+ list_del_rcu(&read->mi_reads_list);
+ list_del_rcu(&read->segment_reads_list);
+
+ mi->mi_pending_reads_count--;
+
+ spin_unlock(&mi->pending_read_lock);
+
+ /* Don't free. Wait for readers */
+ call_rcu(&read->rcu, free_pending_read_entry);
+}
+
+static void notify_pending_reads(struct mount_info *mi,
+ struct data_file_segment *segment,
+ int index)
+{
+ struct pending_read *entry = NULL;
+
+ /* Notify pending reads waiting for this block. */
+ rcu_read_lock();
+ list_for_each_entry_rcu(entry, &segment->reads_list_head,
+ segment_reads_list) {
+ if (entry->block_index == index)
+ set_read_done(entry);
+ }
+ rcu_read_unlock();
+ wake_up_all(&segment->new_data_arrival_wq);
+
+ atomic_inc(&mi->mi_blocks_written);
+ wake_up_all(&mi->mi_blocks_written_notif_wq);
+}
+
+static int usleep_interruptible(u32 us)
+{
+ /* See:
+ * https://www.kernel.org/doc/Documentation/timers/timers-howto.txt
+ * for explanation
+ */
+ if (us < 10) {
+ udelay(us);
+ return 0;
+ } else if (us < 20000) {
+ usleep_range(us, us + us / 10);
+ return 0;
+ } else
+ return msleep_interruptible(us / 1000);
+}
+
+static int wait_for_data_block(struct data_file *df, int block_index,
+ struct data_file_block *res_block,
+ struct incfs_read_data_file_timeouts *timeouts)
+{
+ struct data_file_block block = {};
+ struct data_file_segment *segment = NULL;
+ struct pending_read *read = NULL;
+ struct mount_info *mi = NULL;
+ int error;
+ int wait_res = 0;
+ unsigned int delayed_pending_us = 0, delayed_min_us = 0;
+ bool delayed_pending = false;
+
+ if (!df || !res_block)
+ return -EFAULT;
+
+ if (block_index < 0 || block_index >= df->df_data_block_count)
+ return -EINVAL;
+
+ if (df->df_blockmap_off <= 0 || !df->df_mount_info)
+ return -ENODATA;
+
+ mi = df->df_mount_info;
+ segment = get_file_segment(df, block_index);
+
+ error = down_read_killable(&segment->rwsem);
+ if (error)
+ return error;
+
+ /* Look up the given block */
+ error = get_data_file_block(df, block_index, &block);
+
+ up_read(&segment->rwsem);
+
+ if (error)
+ return error;
+
+ /* If the block was found, just return it. No need to wait. */
+ if (is_data_block_present(&block)) {
+ *res_block = block;
+ if (timeouts && timeouts->min_time_us) {
+ delayed_min_us = timeouts->min_time_us;
+ error = usleep_interruptible(delayed_min_us);
+ goto out;
+ }
+ return 0;
+ } else {
+ /* If it's not found, create a pending read */
+ if (timeouts && timeouts->max_pending_time_us) {
+ read = add_pending_read(df, block_index);
+ if (!read)
+ return -ENOMEM;
+ } else {
+ log_block_read(mi, &df->df_id, block_index);
+ return -ETIME;
+ }
+ }
+
+ /* Rest of function only applies if timeouts != NULL */
+ if (!timeouts) {
+ pr_warn("incfs: timeouts unexpectedly NULL\n");
+ return -EFSCORRUPTED;
+ }
+
+ /* Wait for notifications about block's arrival */
+ wait_res =
+ wait_event_interruptible_timeout(segment->new_data_arrival_wq,
+ (is_read_done(read)),
+ usecs_to_jiffies(timeouts->max_pending_time_us));
+
+ /* Woke up, the pending read is no longer needed. */
+ remove_pending_read(df, read);
+
+ if (wait_res == 0) {
+ /* Wait has timed out */
+ log_block_read(mi, &df->df_id, block_index);
+ return -ETIME;
+ }
+ if (wait_res < 0) {
+ /*
+ * Only ERESTARTSYS is really expected here when a signal
+ * comes while we wait.
+ */
+ return wait_res;
+ }
+
+ delayed_pending = true;
+ delayed_pending_us = timeouts->max_pending_time_us -
+ jiffies_to_usecs(wait_res);
+ if (timeouts->min_pending_time_us > delayed_pending_us) {
+ delayed_min_us = timeouts->min_pending_time_us -
+ delayed_pending_us;
+ error = usleep_interruptible(delayed_min_us);
+ if (error)
+ return error;
+ }
+
+ error = down_read_killable(&segment->rwsem);
+ if (error)
+ return error;
+
+ /*
+ * Re-read blocks info now, it has just arrived and
+ * should be available.
+ */
+ error = get_data_file_block(df, block_index, &block);
+ if (!error) {
+ if (is_data_block_present(&block))
+ *res_block = block;
+ else {
+ /*
+ * Somehow wait finished successfully but block still
+ * can't be found. It's not normal.
+ */
+ pr_warn("incfs: Wait succeeded but block not found.\n");
+ error = -ENODATA;
+ }
+ }
+ up_read(&segment->rwsem);
+
+out:
+ if (error)
+ return error;
+
+ if (delayed_pending) {
+ mi->mi_reads_delayed_pending++;
+ mi->mi_reads_delayed_pending_us +=
+ delayed_pending_us;
+ }
+
+ if (delayed_min_us) {
+ mi->mi_reads_delayed_min++;
+ mi->mi_reads_delayed_min_us += delayed_min_us;
+ }
+
+ return 0;
+}
+
+static int incfs_update_sysfs_error(struct file *file, int index, int result,
+ struct mount_info *mi, struct data_file *df)
+{
+ int error;
+
+ if (result >= 0)
+ return 0;
+
+ error = mutex_lock_interruptible(&mi->mi_le_mutex);
+ if (error)
+ return error;
+
+ mi->mi_le_file_id = df->df_id;
+ mi->mi_le_time_us = ktime_to_us(ktime_get());
+ mi->mi_le_page = index;
+ mi->mi_le_errno = result;
+ mi->mi_le_uid = current_uid().val;
+ mutex_unlock(&mi->mi_le_mutex);
+
+ return 0;
+}
+
+ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
+ int index, struct mem_range tmp,
+ struct incfs_read_data_file_timeouts *timeouts)
+{
+ loff_t pos;
+ ssize_t result;
+ size_t bytes_to_read;
+ struct mount_info *mi = NULL;
+ struct backing_file_context *bfc = NULL;
+ struct data_file_block block = {};
+ struct data_file *df = get_incfs_data_file(f);
+
+ if (!dst.data || !df || !tmp.data)
+ return -EFAULT;
+
+ if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
+ return -ERANGE;
+
+ mi = df->df_mount_info;
+ bfc = df->df_backing_file_context;
+
+ result = wait_for_data_block(df, index, &block, timeouts);
+ if (result < 0)
+ goto out;
+
+ pos = block.db_backing_file_data_offset;
+ if (block.db_comp_alg == COMPRESSION_NONE) {
+ bytes_to_read = min(dst.len, block.db_stored_size);
+ result = incfs_kread(bfc, dst.data, bytes_to_read, pos);
+
+ /* Some data was read, but not enough */
+ if (result >= 0 && result != bytes_to_read)
+ result = -EIO;
+ } else {
+ bytes_to_read = min(tmp.len, block.db_stored_size);
+ result = incfs_kread(bfc, tmp.data, bytes_to_read, pos);
+ if (result == bytes_to_read) {
+ result =
+ decompress(mi, range(tmp.data, bytes_to_read),
+ dst, block.db_comp_alg);
+ if (result < 0) {
+ const char *name =
+ bfc->bc_file->f_path.dentry->d_name.name;
+
+ pr_warn_once("incfs: Decompression error. %s",
+ name);
+ }
+ } else if (result >= 0) {
+ /* Some data was read, but not enough */
+ result = -EIO;
+ }
+ }
+
+ if (result > 0) {
+ int err = validate_hash_tree(bfc, f, index, dst, tmp.data);
+
+ if (err < 0)
+ result = err;
+ }
+
+ if (result >= 0)
+ log_block_read(mi, &df->df_id, index);
+
+out:
+ if (result == -ETIME)
+ mi->mi_reads_failed_timed_out++;
+ else if (result == -EBADMSG)
+ mi->mi_reads_failed_hash_verification++;
+ else if (result < 0)
+ mi->mi_reads_failed_other++;
+
+ incfs_update_sysfs_error(f, index, result, mi, df);
+
+ return result;
+}
+
+ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
+ struct data_file *df, size_t offset)
+{
+ struct backing_file_context *bfc = NULL;
+ struct incfs_df_signature *sig = NULL;
+ size_t to_read = dst.len;
+
+ if (!dst.data || !df)
+ return -EFAULT;
+
+ sig = df->df_signature;
+ bfc = df->df_backing_file_context;
+
+ if (offset > sig->hash_size)
+ return -ERANGE;
+
+ if (offset + to_read > sig->hash_size)
+ to_read = sig->hash_size - offset;
+
+ return incfs_kread(bfc, dst.data, to_read, sig->hash_offset + offset);
+}
+
+int incfs_process_new_data_block(struct data_file *df,
+ struct incfs_fill_block *block, u8 *data)
+{
+ struct mount_info *mi = NULL;
+ struct backing_file_context *bfc = NULL;
+ struct data_file_segment *segment = NULL;
+ struct data_file_block existing_block = {};
+ u16 flags = 0;
+ int error = 0;
+
+ if (!df || !block)
+ return -EFAULT;
+
+ bfc = df->df_backing_file_context;
+ mi = df->df_mount_info;
+
+ if (block->block_index >= df->df_data_block_count)
+ return -ERANGE;
+
+ segment = get_file_segment(df, block->block_index);
+ if (!segment)
+ return -EFAULT;
+
+ if (block->compression == COMPRESSION_LZ4)
+ flags |= INCFS_BLOCK_COMPRESSED_LZ4;
+ else if (block->compression == COMPRESSION_ZSTD)
+ flags |= INCFS_BLOCK_COMPRESSED_ZSTD;
+ else if (block->compression)
+ return -EINVAL;
+
+ error = down_read_killable(&segment->rwsem);
+ if (error)
+ return error;
+
+ error = get_data_file_block(df, block->block_index, &existing_block);
+
+ up_read(&segment->rwsem);
+
+ if (error)
+ return error;
+ if (is_data_block_present(&existing_block)) {
+ /* Block is already present, nothing to do here */
+ return 0;
+ }
+
+ error = down_write_killable(&segment->rwsem);
+ if (error)
+ return error;
+
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (!error) {
+ error = incfs_write_data_block_to_backing_file(
+ bfc, range(data, block->data_len), block->block_index,
+ df->df_blockmap_off, flags);
+ mutex_unlock(&bfc->bc_mutex);
+ }
+ if (!error) {
+ notify_pending_reads(mi, segment, block->block_index);
+ atomic_inc(&df->df_data_blocks_written);
+ }
+
+ up_write(&segment->rwsem);
+
+ if (error)
+ pr_debug("%d error: %d\n", block->block_index, error);
+ return error;
+}
+
+int incfs_read_file_signature(struct data_file *df, struct mem_range dst)
+{
+ struct backing_file_context *bfc = df->df_backing_file_context;
+ struct incfs_df_signature *sig;
+ int read_res = 0;
+
+ if (!dst.data)
+ return -EFAULT;
+
+ sig = df->df_signature;
+ if (!sig)
+ return 0;
+
+ if (dst.len < sig->sig_size)
+ return -E2BIG;
+
+ read_res = incfs_kread(bfc, dst.data, sig->sig_size, sig->sig_offset);
+
+ if (read_res < 0)
+ return read_res;
+
+ if (read_res != sig->sig_size)
+ return -EIO;
+
+ return read_res;
+}
+
+int incfs_process_new_hash_block(struct data_file *df,
+ struct incfs_fill_block *block, u8 *data)
+{
+ struct backing_file_context *bfc = NULL;
+ struct mount_info *mi = NULL;
+ struct mtree *hash_tree = NULL;
+ struct incfs_df_signature *sig = NULL;
+ loff_t hash_area_base = 0;
+ loff_t hash_area_size = 0;
+ int error = 0;
+
+ if (!df || !block)
+ return -EFAULT;
+
+ if (!(block->flags & INCFS_BLOCK_FLAGS_HASH))
+ return -EINVAL;
+
+ bfc = df->df_backing_file_context;
+ mi = df->df_mount_info;
+
+ if (!df)
+ return -ENOENT;
+
+ hash_tree = df->df_hash_tree;
+ sig = df->df_signature;
+ if (!hash_tree || !sig || sig->hash_offset == 0)
+ return -ENOTSUPP;
+
+ hash_area_base = sig->hash_offset;
+ hash_area_size = sig->hash_size;
+ if (hash_area_size < block->block_index * INCFS_DATA_FILE_BLOCK_SIZE
+ + block->data_len) {
+ /* Hash block goes beyond dedicated hash area of this file. */
+ return -ERANGE;
+ }
+
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (!error) {
+ error = incfs_write_hash_block_to_backing_file(
+ bfc, range(data, block->data_len), block->block_index,
+ hash_area_base, df->df_blockmap_off, df->df_size);
+ mutex_unlock(&bfc->bc_mutex);
+ }
+ if (!error)
+ atomic_inc(&df->df_hash_blocks_written);
+
+ return error;
+}
+
+static int process_blockmap_md(struct incfs_blockmap *bm,
+ struct metadata_handler *handler)
+{
+ struct data_file *df = handler->context;
+ int error = 0;
+ loff_t base_off = le64_to_cpu(bm->m_base_offset);
+ u32 block_count = le32_to_cpu(bm->m_block_count);
+
+ if (!df)
+ return -EFAULT;
+
+ if (df->df_data_block_count > block_count)
+ return -EBADMSG;
+
+ df->df_total_block_count = block_count;
+ df->df_blockmap_off = base_off;
+ return error;
+}
+
+static int process_file_signature_md(struct incfs_file_signature *sg,
+ struct metadata_handler *handler)
+{
+ struct data_file *df = handler->context;
+ struct mtree *hash_tree = NULL;
+ int error = 0;
+ struct incfs_df_signature *signature =
+ kzalloc(sizeof(*signature), GFP_NOFS);
+ void *buf = NULL;
+ ssize_t read;
+
+ if (!signature)
+ return -ENOMEM;
+
+ if (!df || !df->df_backing_file_context ||
+ !df->df_backing_file_context->bc_file) {
+ error = -ENOENT;
+ goto out;
+ }
+
+ signature->hash_offset = le64_to_cpu(sg->sg_hash_tree_offset);
+ signature->hash_size = le32_to_cpu(sg->sg_hash_tree_size);
+ signature->sig_offset = le64_to_cpu(sg->sg_sig_offset);
+ signature->sig_size = le32_to_cpu(sg->sg_sig_size);
+
+ buf = kzalloc(signature->sig_size, GFP_NOFS);
+ if (!buf) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ read = incfs_kread(df->df_backing_file_context, buf,
+ signature->sig_size, signature->sig_offset);
+ if (read < 0) {
+ error = read;
+ goto out;
+ }
+
+ if (read != signature->sig_size) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ hash_tree = incfs_alloc_mtree(range(buf, signature->sig_size),
+ df->df_data_block_count);
+ if (IS_ERR(hash_tree)) {
+ error = PTR_ERR(hash_tree);
+ hash_tree = NULL;
+ goto out;
+ }
+ if (hash_tree->hash_tree_area_size != signature->hash_size) {
+ error = -EINVAL;
+ goto out;
+ }
+ if (signature->hash_size > 0 &&
+ handler->md_record_offset <= signature->hash_offset) {
+ error = -EINVAL;
+ goto out;
+ }
+ if (handler->md_record_offset <= signature->sig_offset) {
+ error = -EINVAL;
+ goto out;
+ }
+ df->df_hash_tree = hash_tree;
+ hash_tree = NULL;
+ df->df_signature = signature;
+ signature = NULL;
+out:
+ incfs_free_mtree(hash_tree);
+ kfree(signature);
+ kfree(buf);
+
+ return error;
+}
+
+static int process_status_md(struct incfs_status *is,
+ struct metadata_handler *handler)
+{
+ struct data_file *df = handler->context;
+
+ df->df_initial_data_blocks_written =
+ le32_to_cpu(is->is_data_blocks_written);
+ atomic_set(&df->df_data_blocks_written,
+ df->df_initial_data_blocks_written);
+
+ df->df_initial_hash_blocks_written =
+ le32_to_cpu(is->is_hash_blocks_written);
+ atomic_set(&df->df_hash_blocks_written,
+ df->df_initial_hash_blocks_written);
+
+ df->df_status_offset = handler->md_record_offset;
+ return 0;
+}
+
+static int process_file_verity_signature_md(
+ struct incfs_file_verity_signature *vs,
+ struct metadata_handler *handler)
+{
+ struct data_file *df = handler->context;
+ struct incfs_df_verity_signature *verity_signature;
+
+ if (!df)
+ return -EFAULT;
+
+ verity_signature = kzalloc(sizeof(*verity_signature), GFP_NOFS);
+ if (!verity_signature)
+ return -ENOMEM;
+
+ verity_signature->offset = le64_to_cpu(vs->vs_offset);
+ verity_signature->size = le32_to_cpu(vs->vs_size);
+ if (verity_signature->size > FS_VERITY_MAX_SIGNATURE_SIZE) {
+ kfree(verity_signature);
+ return -EFAULT;
+ }
+
+ df->df_verity_signature = verity_signature;
+ return 0;
+}
+
+static int incfs_scan_metadata_chain(struct data_file *df)
+{
+ struct metadata_handler *handler = NULL;
+ int result = 0;
+ int records_count = 0;
+ int error = 0;
+ struct backing_file_context *bfc = NULL;
+ int nondata_block_count;
+
+ if (!df || !df->df_backing_file_context)
+ return -EFAULT;
+
+ bfc = df->df_backing_file_context;
+
+ handler = kzalloc(sizeof(*handler), GFP_NOFS);
+ if (!handler)
+ return -ENOMEM;
+
+ handler->md_record_offset = df->df_metadata_off;
+ handler->context = df;
+ handler->handle_blockmap = process_blockmap_md;
+ handler->handle_signature = process_file_signature_md;
+ handler->handle_status = process_status_md;
+ handler->handle_verity_signature = process_file_verity_signature_md;
+
+ while (handler->md_record_offset > 0) {
+ error = incfs_read_next_metadata_record(bfc, handler);
+ if (error) {
+ pr_warn("incfs: Error during reading incfs-metadata record. Offset: %lld Record #%d Error code: %d\n",
+ handler->md_record_offset, records_count + 1,
+ -error);
+ break;
+ }
+ records_count++;
+ }
+ if (error) {
+ pr_warn("incfs: Error %d after reading %d incfs-metadata records.\n",
+ -error, records_count);
+ result = error;
+ } else
+ result = records_count;
+
+ nondata_block_count = df->df_total_block_count -
+ df->df_data_block_count;
+ if (df->df_hash_tree) {
+ int hash_block_count = get_blocks_count_for_size(
+ df->df_hash_tree->hash_tree_area_size);
+
+ /*
+ * Files that were created with a hash tree have the hash tree
+ * included in the block map, i.e. nondata_block_count ==
+ * hash_block_count. Files whose hash tree was added by
+ * FS_IOC_ENABLE_VERITY will still have the original block
+ * count, i.e. nondata_block_count == 0.
+ */
+ if (nondata_block_count != hash_block_count &&
+ nondata_block_count != 0)
+ result = -EINVAL;
+ } else if (nondata_block_count != 0) {
+ result = -EINVAL;
+ }
+
+ kfree(handler);
+ return result;
+}
+
+/*
+ * Quickly checks if there are pending reads with a serial number larger
+ * than a given one.
+ */
+bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
+{
+ bool result = false;
+
+ spin_lock(&mi->pending_read_lock);
+ result = (mi->mi_last_pending_read_number > last_number) &&
+ (mi->mi_pending_reads_count > 0);
+ spin_unlock(&mi->pending_read_lock);
+ return result;
+}
+
+int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
+ struct incfs_pending_read_info *reads,
+ struct incfs_pending_read_info2 *reads2,
+ int reads_size, int *new_max_sn)
+{
+ int reported_reads = 0;
+ struct pending_read *entry = NULL;
+
+ if (!mi)
+ return -EFAULT;
+
+ if (reads_size <= 0)
+ return 0;
+
+ if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))
+ return 0;
+
+ rcu_read_lock();
+
+ list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
+ if (entry->serial_number <= sn_lowerbound)
+ continue;
+
+ if (reads) {
+ reads[reported_reads].file_id = entry->file_id;
+ reads[reported_reads].block_index = entry->block_index;
+ reads[reported_reads].serial_number =
+ entry->serial_number;
+ reads[reported_reads].timestamp_us =
+ entry->timestamp_us;
+ }
+
+ if (reads2) {
+ reads2[reported_reads].file_id = entry->file_id;
+ reads2[reported_reads].block_index = entry->block_index;
+ reads2[reported_reads].serial_number =
+ entry->serial_number;
+ reads2[reported_reads].timestamp_us =
+ entry->timestamp_us;
+ reads2[reported_reads].uid = entry->uid;
+ }
+
+ if (entry->serial_number > *new_max_sn)
+ *new_max_sn = entry->serial_number;
+
+ reported_reads++;
+ if (reported_reads >= reads_size)
+ break;
+ }
+
+ rcu_read_unlock();
+
+ return reported_reads;
+}
+
+struct read_log_state incfs_get_log_state(struct mount_info *mi)
+{
+ struct read_log *log = &mi->mi_log;
+ struct read_log_state result;
+
+ spin_lock(&log->rl_lock);
+ result = log->rl_head;
+ spin_unlock(&log->rl_lock);
+ return result;
+}
+
+int incfs_get_uncollected_logs_count(struct mount_info *mi,
+ const struct read_log_state *state)
+{
+ struct read_log *log = &mi->mi_log;
+ u32 generation;
+ u64 head_no, tail_no;
+
+ spin_lock(&log->rl_lock);
+ tail_no = log->rl_tail.current_record_no;
+ head_no = log->rl_head.current_record_no;
+ generation = log->rl_head.generation_id;
+ spin_unlock(&log->rl_lock);
+
+ if (generation != state->generation_id)
+ return head_no - tail_no;
+ else
+ return head_no - max_t(u64, tail_no, state->current_record_no);
+}
+
+int incfs_collect_logged_reads(struct mount_info *mi,
+ struct read_log_state *state,
+ struct incfs_pending_read_info *reads,
+ struct incfs_pending_read_info2 *reads2,
+ int reads_size)
+{
+ int dst_idx;
+ struct read_log *log = &mi->mi_log;
+ struct read_log_state *head, *tail;
+
+ spin_lock(&log->rl_lock);
+ head = &log->rl_head;
+ tail = &log->rl_tail;
+
+ if (state->generation_id != head->generation_id) {
+ pr_debug("read ptr is wrong generation: %u/%u",
+ state->generation_id, head->generation_id);
+
+ *state = (struct read_log_state){
+ .generation_id = head->generation_id,
+ };
+ }
+
+ if (state->current_record_no < tail->current_record_no) {
+ pr_debug("read ptr is behind, moving: %u/%u -> %u/%u\n",
+ (u32)state->next_offset,
+ (u32)state->current_pass_no,
+ (u32)tail->next_offset, (u32)tail->current_pass_no);
+
+ *state = *tail;
+ }
+
+ for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
+ if (state->current_record_no == head->current_record_no)
+ break;
+
+ log_read_one_record(log, state);
+
+ if (reads)
+ reads[dst_idx] = (struct incfs_pending_read_info) {
+ .file_id = state->base_record.file_id,
+ .block_index = state->base_record.block_index,
+ .serial_number = state->current_record_no,
+ .timestamp_us =
+ state->base_record.absolute_ts_us,
+ };
+
+ if (reads2)
+ reads2[dst_idx] = (struct incfs_pending_read_info2) {
+ .file_id = state->base_record.file_id,
+ .block_index = state->base_record.block_index,
+ .serial_number = state->current_record_no,
+ .timestamp_us =
+ state->base_record.absolute_ts_us,
+ .uid = state->base_record.uid,
+ };
+ }
+
+ spin_unlock(&log->rl_lock);
+ return dst_idx;
+}
+
diff --git a/fs/incfs/data_mgmt.h b/fs/incfs/data_mgmt.h
new file mode 100644
index 0000000..2227913
--- /dev/null
+++ b/fs/incfs/data_mgmt.h
@@ -0,0 +1,549 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2019 Google LLC
+ */
+#ifndef _INCFS_DATA_MGMT_H
+#define _INCFS_DATA_MGMT_H
+
+#include <linux/cred.h>
+#include <linux/fs.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/rcupdate.h>
+#include <linux/completion.h>
+#include <linux/wait.h>
+#include <linux/zstd.h>
+#include <crypto/hash.h>
+#include <linux/rwsem.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "internal.h"
+#include "pseudo_files.h"
+
+#define SEGMENTS_PER_FILE 3
+
+enum LOG_RECORD_TYPE {
+ FULL,
+ SAME_FILE,
+ SAME_FILE_CLOSE_BLOCK,
+ SAME_FILE_CLOSE_BLOCK_SHORT,
+ SAME_FILE_NEXT_BLOCK,
+ SAME_FILE_NEXT_BLOCK_SHORT,
+};
+
+struct full_record {
+ enum LOG_RECORD_TYPE type : 3; /* FULL */
+ u32 block_index : 29;
+ incfs_uuid_t file_id;
+ u64 absolute_ts_us;
+ uid_t uid;
+} __packed; /* 32 bytes */
+
+struct same_file {
+ enum LOG_RECORD_TYPE type : 3; /* SAME_FILE */
+ u32 block_index : 29;
+ uid_t uid;
+ u16 relative_ts_us; /* max 2^16 us ~= 64 ms */
+} __packed; /* 10 bytes */
+
+struct same_file_close_block {
+ enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_CLOSE_BLOCK */
+ u16 relative_ts_us : 13; /* max 2^13 us ~= 8 ms */
+ s16 block_index_delta;
+} __packed; /* 4 bytes */
+
+struct same_file_close_block_short {
+ enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_CLOSE_BLOCK_SHORT */
+ u8 relative_ts_tens_us : 5; /* max 2^5*10 us ~= 320 us */
+ s8 block_index_delta;
+} __packed; /* 2 bytes */
+
+struct same_file_next_block {
+ enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_NEXT_BLOCK */
+ u16 relative_ts_us : 13; /* max 2^13 us ~= 8 ms */
+} __packed; /* 2 bytes */
+
+struct same_file_next_block_short {
+ enum LOG_RECORD_TYPE type : 3; /* SAME_FILE_NEXT_BLOCK_SHORT */
+ u8 relative_ts_tens_us : 5; /* max 2^5*10 us ~= 320 us */
+} __packed; /* 1 byte */
+
+union log_record {
+ struct full_record full_record;
+ struct same_file same_file;
+ struct same_file_close_block same_file_close_block;
+ struct same_file_close_block_short same_file_close_block_short;
+ struct same_file_next_block same_file_next_block;
+ struct same_file_next_block_short same_file_next_block_short;
+};
+
+struct read_log_state {
+ /* Log buffer generation id, incremented on configuration changes */
+ u32 generation_id;
+
+ /* Offset in rl_ring_buf to write into. */
+ u32 next_offset;
+
+ /* Current number of writer passes over rl_ring_buf */
+ u32 current_pass_no;
+
+ /* Current full_record to diff against */
+ struct full_record base_record;
+
+ /* Current record number counting from configuration change */
+ u64 current_record_no;
+};
+
+/* A ring buffer to save records about data blocks which were recently read. */
+struct read_log {
+ void *rl_ring_buf;
+
+ int rl_size;
+
+ struct read_log_state rl_head;
+
+ struct read_log_state rl_tail;
+
+ /* A lock to protect the above fields */
+ spinlock_t rl_lock;
+
+ /* A queue of waiters who want to be notified about reads */
+ wait_queue_head_t ml_notif_wq;
+
+ /* A work item to wake up those waiters without slowing down readers */
+ struct delayed_work ml_wakeup_work;
+};
+
+struct mount_options {
+ unsigned int read_timeout_ms;
+ unsigned int readahead_pages;
+ unsigned int read_log_pages;
+ unsigned int read_log_wakeup_count;
+ bool report_uid;
+ char *sysfs_name;
+};
+
+struct mount_info {
+ struct super_block *mi_sb;
+
+ struct path mi_backing_dir_path;
+
+ struct dentry *mi_index_dir;
+ /* For stacking mounts, if true, this indicates if the index dir needs
+ * to be freed for this SB otherwise it was created by lower level SB */
+ bool mi_index_free;
+
+ struct dentry *mi_incomplete_dir;
+ /* For stacking mounts, if true, this indicates if the incomplete dir
+ * needs to be freed for this SB. Similar to mi_index_free */
+ bool mi_incomplete_free;
+
+ const struct cred *mi_owner;
+
+ struct mount_options mi_options;
+
+ /* This mutex is to be taken before create, rename, delete */
+ struct mutex mi_dir_struct_mutex;
+
+ /*
+ * A queue of waiters who want to be notified about new pending reads.
+ */
+ wait_queue_head_t mi_pending_reads_notif_wq;
+
+ /*
+ * Protects - RCU safe:
+ * - reads_list_head
+ * - mi_pending_reads_count
+ * - mi_last_pending_read_number
+ * - data_file_segment.reads_list_head
+ */
+ spinlock_t pending_read_lock;
+
+ /* List of active pending_read objects */
+ struct list_head mi_reads_list_head;
+
+ /* Total number of items in reads_list_head */
+ int mi_pending_reads_count;
+
+ /*
+ * Last serial number that was assigned to a pending read.
+ * 0 means no pending reads have been seen yet.
+ */
+ int mi_last_pending_read_number;
+
+ /* Temporary buffer for read logger. */
+ struct read_log mi_log;
+
+ /* SELinux needs special xattrs on our pseudo files */
+ struct mem_range pseudo_file_xattr[PSEUDO_FILE_COUNT];
+
+ /* A queue of waiters who want to be notified about blocks_written */
+ wait_queue_head_t mi_blocks_written_notif_wq;
+
+ /* Number of blocks written since mount */
+ atomic_t mi_blocks_written;
+
+ /* Per UID read timeouts */
+ spinlock_t mi_per_uid_read_timeouts_lock;
+ struct incfs_per_uid_read_timeouts *mi_per_uid_read_timeouts;
+ int mi_per_uid_read_timeouts_size;
+
+ /* zstd workspace */
+ struct mutex mi_zstd_workspace_mutex;
+ void *mi_zstd_workspace;
+ ZSTD_DStream *mi_zstd_stream;
+ struct delayed_work mi_zstd_cleanup_work;
+
+ /* sysfs node */
+ struct incfs_sysfs_node *mi_sysfs_node;
+
+ /* Last error information */
+ struct mutex mi_le_mutex;
+ incfs_uuid_t mi_le_file_id;
+ u64 mi_le_time_us;
+ u32 mi_le_page;
+ u32 mi_le_errno;
+ uid_t mi_le_uid;
+
+ /* Number of reads timed out */
+ u32 mi_reads_failed_timed_out;
+
+ /* Number of reads failed because hash verification failed */
+ u32 mi_reads_failed_hash_verification;
+
+ /* Number of reads failed for another reason */
+ u32 mi_reads_failed_other;
+
+ /* Number of reads delayed because page had to be fetched */
+ u32 mi_reads_delayed_pending;
+
+ /* Total time waiting for pages to be fetched */
+ u64 mi_reads_delayed_pending_us;
+
+ /*
+ * Number of reads delayed because of per-uid min_time_us or
+ * min_pending_time_us settings
+ */
+ u32 mi_reads_delayed_min;
+
+ /* Total time waiting because of per-uid min_time_us or
+ * min_pending_time_us settings.
+ *
+ * Note that if a read is initially delayed because we have to wait for
+ * the page, then further delayed because of min_pending_time_us
+ * setting, this counter gets incremented by only the further delay
+ * time.
+ */
+ u64 mi_reads_delayed_min_us;
+};
+
+struct data_file_block {
+ loff_t db_backing_file_data_offset;
+
+ size_t db_stored_size;
+
+ enum incfs_compression_alg db_comp_alg;
+};
+
+struct pending_read {
+ incfs_uuid_t file_id;
+
+ s64 timestamp_us;
+
+ atomic_t done;
+
+ int block_index;
+
+ int serial_number;
+
+ uid_t uid;
+
+ struct list_head mi_reads_list;
+
+ struct list_head segment_reads_list;
+
+ struct rcu_head rcu;
+};
+
+struct data_file_segment {
+ wait_queue_head_t new_data_arrival_wq;
+
+ /* Protects reads and writes from the blockmap */
+ struct rw_semaphore rwsem;
+
+ /* List of active pending_read objects belonging to this segment */
+ /* Protected by mount_info.pending_reads_mutex */
+ struct list_head reads_list_head;
+};
+
+/*
+ * Extra info associated with a file. Just a few bytes set by a user.
+ */
+struct file_attr {
+ loff_t fa_value_offset;
+
+ size_t fa_value_size;
+
+ u32 fa_crc;
+};
+
+
+struct data_file {
+ struct backing_file_context *df_backing_file_context;
+
+ struct mount_info *df_mount_info;
+
+ incfs_uuid_t df_id;
+
+ /*
+ * Array of segments used to reduce lock contention for the file.
+ * Segment is chosen for a block depends on the block's index.
+ */
+ struct data_file_segment df_segments[SEGMENTS_PER_FILE];
+
+ /* Base offset of the first metadata record. */
+ loff_t df_metadata_off;
+
+ /* Base offset of the block map. */
+ loff_t df_blockmap_off;
+
+ /* File size in bytes */
+ loff_t df_size;
+
+ /* File header flags */
+ u32 df_header_flags;
+
+ /* File size in DATA_FILE_BLOCK_SIZE blocks */
+ int df_data_block_count;
+
+ /* Total number of blocks, data + hash */
+ int df_total_block_count;
+
+ /* For mapped files, the offset into the actual file */
+ loff_t df_mapped_offset;
+
+ /* Number of data blocks written to file */
+ atomic_t df_data_blocks_written;
+
+ /* Number of data blocks in the status block */
+ u32 df_initial_data_blocks_written;
+
+ /* Number of hash blocks written to file */
+ atomic_t df_hash_blocks_written;
+
+ /* Number of hash blocks in the status block */
+ u32 df_initial_hash_blocks_written;
+
+ /* Offset to status metadata header */
+ loff_t df_status_offset;
+
+ /*
+ * Mutex acquired while enabling verity. Note that df_hash_tree is set
+ * by enable verity.
+ *
+ * The backing file mutex bc_mutex may be taken while this mutex is
+ * held.
+ */
+ struct mutex df_enable_verity;
+
+ /*
+ * Set either at construction time or during enabling verity. In the
+ * latter case, set via smp_store_release, so use smp_load_acquire to
+ * read it.
+ */
+ struct mtree *df_hash_tree;
+
+ /* Guaranteed set if df_hash_tree is set. */
+ struct incfs_df_signature *df_signature;
+
+ /*
+ * The verity file digest, set when verity is enabled and the file has
+ * been opened
+ */
+ struct mem_range df_verity_file_digest;
+
+ struct incfs_df_verity_signature *df_verity_signature;
+};
+
+struct dir_file {
+ struct mount_info *mount_info;
+
+ struct file *backing_dir;
+};
+
+struct inode_info {
+ struct mount_info *n_mount_info; /* A mount, this file belongs to */
+
+ struct inode *n_backing_inode;
+
+ struct data_file *n_file;
+
+ struct inode n_vfs_inode;
+};
+
+struct dentry_info {
+ struct path backing_path;
+};
+
+enum FILL_PERMISSION {
+ CANT_FILL = 0,
+ CAN_FILL = 1,
+};
+
+struct incfs_file_data {
+ /* Does this file handle have INCFS_IOC_FILL_BLOCKS permission */
+ enum FILL_PERMISSION fd_fill_permission;
+
+ /* If INCFS_IOC_GET_FILLED_BLOCKS has been called, where are we */
+ int fd_get_block_pos;
+
+ /* And how many filled blocks are there up to that point */
+ int fd_filled_data_blocks;
+ int fd_filled_hash_blocks;
+};
+
+struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
+ struct mount_options *options,
+ struct path *backing_dir_path);
+
+int incfs_realloc_mount_info(struct mount_info *mi,
+ struct mount_options *options);
+
+void incfs_free_mount_info(struct mount_info *mi);
+
+char *file_id_to_str(incfs_uuid_t id);
+struct dentry *incfs_lookup_dentry(struct dentry *parent, const char *name);
+struct data_file *incfs_open_data_file(struct mount_info *mi, struct file *bf);
+void incfs_free_data_file(struct data_file *df);
+
+struct dir_file *incfs_open_dir_file(struct mount_info *mi, struct file *bf);
+void incfs_free_dir_file(struct dir_file *dir);
+
+struct incfs_read_data_file_timeouts {
+ u32 min_time_us;
+ u32 min_pending_time_us;
+ u32 max_pending_time_us;
+};
+
+ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
+ int index, struct mem_range tmp,
+ struct incfs_read_data_file_timeouts *timeouts);
+
+ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
+ struct data_file *df, size_t offset);
+
+int incfs_get_filled_blocks(struct data_file *df,
+ struct incfs_file_data *fd,
+ struct incfs_get_filled_blocks_args *arg);
+
+int incfs_read_file_signature(struct data_file *df, struct mem_range dst);
+
+int incfs_process_new_data_block(struct data_file *df,
+ struct incfs_fill_block *block, u8 *data);
+
+int incfs_process_new_hash_block(struct data_file *df,
+ struct incfs_fill_block *block, u8 *data);
+
+bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number);
+
+/*
+ * Collects pending reads and saves them into the array (reads/reads_size).
+ * Only reads with serial_number > sn_lowerbound are reported.
+ * Returns how many reads were saved into the array.
+ */
+int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
+ struct incfs_pending_read_info *reads,
+ struct incfs_pending_read_info2 *reads2,
+ int reads_size, int *new_max_sn);
+
+int incfs_collect_logged_reads(struct mount_info *mi,
+ struct read_log_state *start_state,
+ struct incfs_pending_read_info *reads,
+ struct incfs_pending_read_info2 *reads2,
+ int reads_size);
+struct read_log_state incfs_get_log_state(struct mount_info *mi);
+int incfs_get_uncollected_logs_count(struct mount_info *mi,
+ const struct read_log_state *state);
+
+static inline struct inode_info *get_incfs_node(struct inode *inode)
+{
+ if (!inode)
+ return NULL;
+
+ if (inode->i_sb->s_magic != INCFS_MAGIC_NUMBER) {
+ /* This inode doesn't belong to us. */
+ pr_warn_once("incfs: %s on an alien inode.", __func__);
+ return NULL;
+ }
+
+ return container_of(inode, struct inode_info, n_vfs_inode);
+}
+
+static inline struct data_file *get_incfs_data_file(struct file *f)
+{
+ struct inode_info *node = NULL;
+
+ if (!f)
+ return NULL;
+
+ if (!S_ISREG(f->f_inode->i_mode))
+ return NULL;
+
+ node = get_incfs_node(f->f_inode);
+ if (!node)
+ return NULL;
+
+ return node->n_file;
+}
+
+static inline struct dir_file *get_incfs_dir_file(struct file *f)
+{
+ if (!f)
+ return NULL;
+
+ if (!S_ISDIR(f->f_inode->i_mode))
+ return NULL;
+
+ return (struct dir_file *)f->private_data;
+}
+
+/*
+ * Make sure that inode_info.n_file is initialized and inode can be used
+ * for reading and writing data from/to the backing file.
+ */
+int make_inode_ready_for_data_ops(struct mount_info *mi,
+ struct inode *inode,
+ struct file *backing_file);
+
+static inline struct dentry_info *get_incfs_dentry(const struct dentry *d)
+{
+ if (!d)
+ return NULL;
+
+ return (struct dentry_info *)d->d_fsdata;
+}
+
+static inline void get_incfs_backing_path(const struct dentry *d,
+ struct path *path)
+{
+ struct dentry_info *di = get_incfs_dentry(d);
+
+ if (!di) {
+ *path = (struct path) {};
+ return;
+ }
+
+ *path = di->backing_path;
+ path_get(path);
+}
+
+static inline int get_blocks_count_for_size(u64 size)
+{
+ if (size == 0)
+ return 0;
+ return 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+}
+
+#endif /* _INCFS_DATA_MGMT_H */
diff --git a/fs/incfs/format.c b/fs/incfs/format.c
new file mode 100644
index 0000000..6ffcf33
--- /dev/null
+++ b/fs/incfs/format.c
@@ -0,0 +1,752 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/mm.h>
+#include <linux/falloc.h>
+#include <linux/slab.h>
+#include <linux/crc32.h>
+#include <linux/kernel.h>
+
+#include "format.h"
+#include "data_mgmt.h"
+
+struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
+ struct file *backing_file)
+{
+ struct backing_file_context *result = NULL;
+
+ result = kzalloc(sizeof(*result), GFP_NOFS);
+ if (!result)
+ return ERR_PTR(-ENOMEM);
+
+ result->bc_file = get_file(backing_file);
+ result->bc_cred = mi->mi_owner;
+ mutex_init(&result->bc_mutex);
+ return result;
+}
+
+void incfs_free_bfc(struct backing_file_context *bfc)
+{
+ if (!bfc)
+ return;
+
+ if (bfc->bc_file)
+ fput(bfc->bc_file);
+
+ mutex_destroy(&bfc->bc_mutex);
+ kfree(bfc);
+}
+
+static loff_t incfs_get_end_offset(struct file *f)
+{
+ /*
+ * This function assumes that file size and the end-offset
+ * are the same. This is not always true.
+ */
+ return i_size_read(file_inode(f));
+}
+
+/*
+ * Truncate the tail of the file to the given length.
+ * Used to rollback partially successful multistep writes.
+ */
+static int truncate_backing_file(struct backing_file_context *bfc,
+ loff_t new_end)
+{
+ struct inode *inode = NULL;
+ struct dentry *dentry = NULL;
+ loff_t old_end = 0;
+ struct iattr attr;
+ int result = 0;
+
+ if (!bfc)
+ return -EFAULT;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ if (!bfc->bc_file)
+ return -EFAULT;
+
+ old_end = incfs_get_end_offset(bfc->bc_file);
+ if (old_end == new_end)
+ return 0;
+ if (old_end < new_end)
+ return -EINVAL;
+
+ inode = bfc->bc_file->f_inode;
+ dentry = bfc->bc_file->f_path.dentry;
+
+ attr.ia_size = new_end;
+ attr.ia_valid = ATTR_SIZE;
+
+ inode_lock(inode);
+ result = notify_change(&init_user_ns, dentry, &attr, NULL);
+ inode_unlock(inode);
+
+ return result;
+}
+
+static int write_to_bf(struct backing_file_context *bfc, const void *buf,
+ size_t count, loff_t pos)
+{
+ ssize_t res = incfs_kwrite(bfc, buf, count, pos);
+
+ if (res < 0)
+ return res;
+ if (res != count)
+ return -EIO;
+ return 0;
+}
+
+static int append_zeros_no_fallocate(struct backing_file_context *bfc,
+ size_t file_size, size_t len)
+{
+ u8 buffer[256] = {};
+ size_t i;
+
+ for (i = 0; i < len; i += sizeof(buffer)) {
+ int to_write = len - i > sizeof(buffer)
+ ? sizeof(buffer) : len - i;
+ int err = write_to_bf(bfc, buffer, to_write, file_size + i);
+
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+/* Append a given number of zero bytes to the end of the backing file. */
+static int append_zeros(struct backing_file_context *bfc, size_t len)
+{
+ loff_t file_size = 0;
+ loff_t new_last_byte_offset = 0;
+ int result;
+
+ if (!bfc)
+ return -EFAULT;
+
+ if (len == 0)
+ return 0;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ /*
+ * Allocate only one byte at the new desired end of the file.
+ * It will increase file size and create a zeroed area of
+ * a given size.
+ */
+ file_size = incfs_get_end_offset(bfc->bc_file);
+ new_last_byte_offset = file_size + len - 1;
+ result = vfs_fallocate(bfc->bc_file, 0, new_last_byte_offset, 1);
+ if (result != -EOPNOTSUPP)
+ return result;
+
+ return append_zeros_no_fallocate(bfc, file_size, len);
+}
+
+/*
+ * Append a given metadata record to the backing file and update a previous
+ * record to add the new record the the metadata list.
+ */
+static int append_md_to_backing_file(struct backing_file_context *bfc,
+ struct incfs_md_header *record)
+{
+ int result = 0;
+ loff_t record_offset;
+ loff_t file_pos;
+ __le64 new_md_offset;
+ size_t record_size;
+
+ if (!bfc || !record)
+ return -EFAULT;
+
+ if (bfc->bc_last_md_record_offset < 0)
+ return -EINVAL;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ record_size = le16_to_cpu(record->h_record_size);
+ file_pos = incfs_get_end_offset(bfc->bc_file);
+ record->h_next_md_offset = 0;
+
+ /* Write the metadata record to the end of the backing file */
+ record_offset = file_pos;
+ new_md_offset = cpu_to_le64(record_offset);
+ result = write_to_bf(bfc, record, record_size, file_pos);
+ if (result)
+ return result;
+
+ /* Update next metadata offset in a previous record or a superblock. */
+ if (bfc->bc_last_md_record_offset) {
+ /*
+ * Find a place in the previous md record where new record's
+ * offset needs to be saved.
+ */
+ file_pos = bfc->bc_last_md_record_offset +
+ offsetof(struct incfs_md_header, h_next_md_offset);
+ } else {
+ /*
+ * No metadata yet, file a place to update in the
+ * file_header.
+ */
+ file_pos = offsetof(struct incfs_file_header,
+ fh_first_md_offset);
+ }
+ result = write_to_bf(bfc, &new_md_offset, sizeof(new_md_offset),
+ file_pos);
+ if (result)
+ return result;
+
+ bfc->bc_last_md_record_offset = record_offset;
+ return result;
+}
+
+/*
+ * Reserve 0-filled space for the blockmap body, and append
+ * incfs_blockmap metadata record pointing to it.
+ */
+int incfs_write_blockmap_to_backing_file(struct backing_file_context *bfc,
+ u32 block_count)
+{
+ struct incfs_blockmap blockmap = {};
+ int result = 0;
+ loff_t file_end = 0;
+ size_t map_size = block_count * sizeof(struct incfs_blockmap_entry);
+
+ if (!bfc)
+ return -EFAULT;
+
+ blockmap.m_header.h_md_entry_type = INCFS_MD_BLOCK_MAP;
+ blockmap.m_header.h_record_size = cpu_to_le16(sizeof(blockmap));
+ blockmap.m_header.h_next_md_offset = cpu_to_le64(0);
+ blockmap.m_block_count = cpu_to_le32(block_count);
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ /* Reserve 0-filled space for the blockmap body in the backing file. */
+ file_end = incfs_get_end_offset(bfc->bc_file);
+ result = append_zeros(bfc, map_size);
+ if (result)
+ return result;
+
+ /* Write blockmap metadata record pointing to the body written above. */
+ blockmap.m_base_offset = cpu_to_le64(file_end);
+ result = append_md_to_backing_file(bfc, &blockmap.m_header);
+ if (result)
+ /* Error, rollback file changes */
+ truncate_backing_file(bfc, file_end);
+
+ return result;
+}
+
+int incfs_write_signature_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range sig, u32 tree_size,
+ loff_t *tree_offset, loff_t *sig_offset)
+{
+ struct incfs_file_signature sg = {};
+ int result = 0;
+ loff_t rollback_pos = 0;
+ loff_t tree_area_pos = 0;
+ size_t alignment = 0;
+
+ if (!bfc)
+ return -EFAULT;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ rollback_pos = incfs_get_end_offset(bfc->bc_file);
+
+ sg.sg_header.h_md_entry_type = INCFS_MD_SIGNATURE;
+ sg.sg_header.h_record_size = cpu_to_le16(sizeof(sg));
+ sg.sg_header.h_next_md_offset = cpu_to_le64(0);
+ if (sig.data != NULL && sig.len > 0) {
+ sg.sg_sig_size = cpu_to_le32(sig.len);
+ sg.sg_sig_offset = cpu_to_le64(rollback_pos);
+
+ result = write_to_bf(bfc, sig.data, sig.len, rollback_pos);
+ if (result)
+ goto err;
+ }
+
+ tree_area_pos = incfs_get_end_offset(bfc->bc_file);
+ if (tree_size > 0) {
+ if (tree_size > 5 * INCFS_DATA_FILE_BLOCK_SIZE) {
+ /*
+ * If hash tree is big enough, it makes sense to
+ * align in the backing file for faster access.
+ */
+ loff_t offset = round_up(tree_area_pos, PAGE_SIZE);
+
+ alignment = offset - tree_area_pos;
+ tree_area_pos = offset;
+ }
+
+ /*
+ * If root hash is not the only hash in the tree.
+ * reserve 0-filled space for the tree.
+ */
+ result = append_zeros(bfc, tree_size + alignment);
+ if (result)
+ goto err;
+
+ sg.sg_hash_tree_size = cpu_to_le32(tree_size);
+ sg.sg_hash_tree_offset = cpu_to_le64(tree_area_pos);
+ }
+
+ /* Write a hash tree metadata record pointing to the hash tree above. */
+ result = append_md_to_backing_file(bfc, &sg.sg_header);
+err:
+ if (result)
+ /* Error, rollback file changes */
+ truncate_backing_file(bfc, rollback_pos);
+ else {
+ if (tree_offset)
+ *tree_offset = tree_area_pos;
+ if (sig_offset)
+ *sig_offset = rollback_pos;
+ }
+
+ return result;
+}
+
+static int write_new_status_to_backing_file(struct backing_file_context *bfc,
+ u32 data_blocks_written,
+ u32 hash_blocks_written)
+{
+ int result;
+ loff_t rollback_pos;
+ struct incfs_status is = {
+ .is_header = {
+ .h_md_entry_type = INCFS_MD_STATUS,
+ .h_record_size = cpu_to_le16(sizeof(is)),
+ },
+ .is_data_blocks_written = cpu_to_le32(data_blocks_written),
+ .is_hash_blocks_written = cpu_to_le32(hash_blocks_written),
+ };
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+ rollback_pos = incfs_get_end_offset(bfc->bc_file);
+ result = append_md_to_backing_file(bfc, &is.is_header);
+ if (result)
+ truncate_backing_file(bfc, rollback_pos);
+
+ return result;
+}
+
+int incfs_write_status_to_backing_file(struct backing_file_context *bfc,
+ loff_t status_offset,
+ u32 data_blocks_written,
+ u32 hash_blocks_written)
+{
+ struct incfs_status is;
+ int result;
+
+ if (!bfc)
+ return -EFAULT;
+
+ if (status_offset == 0)
+ return write_new_status_to_backing_file(bfc,
+ data_blocks_written, hash_blocks_written);
+
+ result = incfs_kread(bfc, &is, sizeof(is), status_offset);
+ if (result != sizeof(is))
+ return -EIO;
+
+ is.is_data_blocks_written = cpu_to_le32(data_blocks_written);
+ is.is_hash_blocks_written = cpu_to_le32(hash_blocks_written);
+ result = incfs_kwrite(bfc, &is, sizeof(is), status_offset);
+ if (result != sizeof(is))
+ return -EIO;
+
+ return 0;
+}
+
+int incfs_write_verity_signature_to_backing_file(
+ struct backing_file_context *bfc, struct mem_range signature,
+ loff_t *offset)
+{
+ struct incfs_file_verity_signature vs = {};
+ int result;
+ loff_t pos;
+
+ /* No verity signature section is equivalent to an empty section */
+ if (signature.data == NULL || signature.len == 0)
+ return 0;
+
+ pos = incfs_get_end_offset(bfc->bc_file);
+
+ vs = (struct incfs_file_verity_signature) {
+ .vs_header = (struct incfs_md_header) {
+ .h_md_entry_type = INCFS_MD_VERITY_SIGNATURE,
+ .h_record_size = cpu_to_le16(sizeof(vs)),
+ .h_next_md_offset = cpu_to_le64(0),
+ },
+ .vs_size = cpu_to_le32(signature.len),
+ .vs_offset = cpu_to_le64(pos),
+ };
+
+ result = write_to_bf(bfc, signature.data, signature.len, pos);
+ if (result)
+ goto err;
+
+ result = append_md_to_backing_file(bfc, &vs.vs_header);
+ if (result)
+ goto err;
+
+ *offset = pos;
+err:
+ if (result)
+ /* Error, rollback file changes */
+ truncate_backing_file(bfc, pos);
+ return result;
+}
+
+/*
+ * Write a backing file header
+ * It should always be called only on empty file.
+ * fh.fh_first_md_offset is 0 for now, but will be updated
+ * once first metadata record is added.
+ */
+int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size)
+{
+ struct incfs_file_header fh = {};
+ loff_t file_pos = 0;
+
+ if (!bfc)
+ return -EFAULT;
+
+ fh.fh_magic = cpu_to_le64(INCFS_MAGIC_NUMBER);
+ fh.fh_version = cpu_to_le64(INCFS_FORMAT_CURRENT_VER);
+ fh.fh_header_size = cpu_to_le16(sizeof(fh));
+ fh.fh_first_md_offset = cpu_to_le64(0);
+ fh.fh_data_block_size = cpu_to_le16(INCFS_DATA_FILE_BLOCK_SIZE);
+
+ fh.fh_file_size = cpu_to_le64(file_size);
+ fh.fh_uuid = *uuid;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ file_pos = incfs_get_end_offset(bfc->bc_file);
+ if (file_pos != 0)
+ return -EEXIST;
+
+ return write_to_bf(bfc, &fh, sizeof(fh), file_pos);
+}
+
+/*
+ * Write a backing file header for a mapping file
+ * It should always be called only on empty file.
+ */
+int incfs_write_mapping_fh_to_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size, u64 offset)
+{
+ struct incfs_file_header fh = {};
+ loff_t file_pos = 0;
+
+ if (!bfc)
+ return -EFAULT;
+
+ fh.fh_magic = cpu_to_le64(INCFS_MAGIC_NUMBER);
+ fh.fh_version = cpu_to_le64(INCFS_FORMAT_CURRENT_VER);
+ fh.fh_header_size = cpu_to_le16(sizeof(fh));
+ fh.fh_original_offset = cpu_to_le64(offset);
+ fh.fh_data_block_size = cpu_to_le16(INCFS_DATA_FILE_BLOCK_SIZE);
+
+ fh.fh_mapped_file_size = cpu_to_le64(file_size);
+ fh.fh_original_uuid = *uuid;
+ fh.fh_flags = cpu_to_le32(INCFS_FILE_MAPPED);
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ file_pos = incfs_get_end_offset(bfc->bc_file);
+ if (file_pos != 0)
+ return -EEXIST;
+
+ return write_to_bf(bfc, &fh, sizeof(fh), file_pos);
+}
+
+/* Write a given data block and update file's blockmap to point it. */
+int incfs_write_data_block_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range block, int block_index,
+ loff_t bm_base_off, u16 flags)
+{
+ struct incfs_blockmap_entry bm_entry = {};
+ int result = 0;
+ loff_t data_offset = 0;
+ loff_t bm_entry_off =
+ bm_base_off + sizeof(struct incfs_blockmap_entry) * block_index;
+
+ if (!bfc)
+ return -EFAULT;
+
+ if (block.len >= (1 << 16) || block_index < 0)
+ return -EINVAL;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ data_offset = incfs_get_end_offset(bfc->bc_file);
+ if (data_offset <= bm_entry_off) {
+ /* Blockmap entry is beyond the file's end. It is not normal. */
+ return -EINVAL;
+ }
+
+ /* Write the block data at the end of the backing file. */
+ result = write_to_bf(bfc, block.data, block.len, data_offset);
+ if (result)
+ return result;
+
+ /* Update the blockmap to point to the newly written data. */
+ bm_entry.me_data_offset_lo = cpu_to_le32((u32)data_offset);
+ bm_entry.me_data_offset_hi = cpu_to_le16((u16)(data_offset >> 32));
+ bm_entry.me_data_size = cpu_to_le16((u16)block.len);
+ bm_entry.me_flags = cpu_to_le16(flags);
+
+ return write_to_bf(bfc, &bm_entry, sizeof(bm_entry),
+ bm_entry_off);
+}
+
+int incfs_write_hash_block_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range block,
+ int block_index,
+ loff_t hash_area_off,
+ loff_t bm_base_off,
+ loff_t file_size)
+{
+ struct incfs_blockmap_entry bm_entry = {};
+ int result;
+ loff_t data_offset = 0;
+ loff_t file_end = 0;
+ loff_t bm_entry_off =
+ bm_base_off +
+ sizeof(struct incfs_blockmap_entry) *
+ (block_index + get_blocks_count_for_size(file_size));
+
+ if (!bfc)
+ return -EFAULT;
+
+ LOCK_REQUIRED(bfc->bc_mutex);
+
+ data_offset = hash_area_off + block_index * INCFS_DATA_FILE_BLOCK_SIZE;
+ file_end = incfs_get_end_offset(bfc->bc_file);
+ if (data_offset + block.len > file_end) {
+ /* Block is located beyond the file's end. It is not normal. */
+ return -EINVAL;
+ }
+
+ result = write_to_bf(bfc, block.data, block.len, data_offset);
+ if (result)
+ return result;
+
+ bm_entry.me_data_offset_lo = cpu_to_le32((u32)data_offset);
+ bm_entry.me_data_offset_hi = cpu_to_le16((u16)(data_offset >> 32));
+ bm_entry.me_data_size = cpu_to_le16(INCFS_DATA_FILE_BLOCK_SIZE);
+
+ return write_to_bf(bfc, &bm_entry, sizeof(bm_entry), bm_entry_off);
+}
+
+int incfs_read_blockmap_entry(struct backing_file_context *bfc, int block_index,
+ loff_t bm_base_off,
+ struct incfs_blockmap_entry *bm_entry)
+{
+ int error = incfs_read_blockmap_entries(bfc, bm_entry, block_index, 1,
+ bm_base_off);
+
+ if (error < 0)
+ return error;
+
+ if (error == 0)
+ return -EIO;
+
+ if (error != 1)
+ return -EFAULT;
+
+ return 0;
+}
+
+int incfs_read_blockmap_entries(struct backing_file_context *bfc,
+ struct incfs_blockmap_entry *entries,
+ int start_index, int blocks_number,
+ loff_t bm_base_off)
+{
+ loff_t bm_entry_off =
+ bm_base_off + sizeof(struct incfs_blockmap_entry) * start_index;
+ const size_t bytes_to_read = sizeof(struct incfs_blockmap_entry)
+ * blocks_number;
+ int result = 0;
+
+ if (!bfc || !entries)
+ return -EFAULT;
+
+ if (start_index < 0 || bm_base_off <= 0)
+ return -ENODATA;
+
+ result = incfs_kread(bfc, entries, bytes_to_read, bm_entry_off);
+ if (result < 0)
+ return result;
+ return result / sizeof(*entries);
+}
+
+int incfs_read_file_header(struct backing_file_context *bfc,
+ loff_t *first_md_off, incfs_uuid_t *uuid,
+ u64 *file_size, u32 *flags)
+{
+ ssize_t bytes_read = 0;
+ struct incfs_file_header fh = {};
+
+ if (!bfc || !first_md_off)
+ return -EFAULT;
+
+ bytes_read = incfs_kread(bfc, &fh, sizeof(fh), 0);
+ if (bytes_read < 0)
+ return bytes_read;
+
+ if (bytes_read < sizeof(fh))
+ return -EBADMSG;
+
+ if (le64_to_cpu(fh.fh_magic) != INCFS_MAGIC_NUMBER)
+ return -EILSEQ;
+
+ if (le64_to_cpu(fh.fh_version) > INCFS_FORMAT_CURRENT_VER)
+ return -EILSEQ;
+
+ if (le16_to_cpu(fh.fh_data_block_size) != INCFS_DATA_FILE_BLOCK_SIZE)
+ return -EILSEQ;
+
+ if (le16_to_cpu(fh.fh_header_size) != sizeof(fh))
+ return -EILSEQ;
+
+ if (first_md_off)
+ *first_md_off = le64_to_cpu(fh.fh_first_md_offset);
+ if (uuid)
+ *uuid = fh.fh_uuid;
+ if (file_size)
+ *file_size = le64_to_cpu(fh.fh_file_size);
+ if (flags)
+ *flags = le32_to_cpu(fh.fh_flags);
+ return 0;
+}
+
+/*
+ * Read through metadata records from the backing file one by one
+ * and call provided metadata handlers.
+ */
+int incfs_read_next_metadata_record(struct backing_file_context *bfc,
+ struct metadata_handler *handler)
+{
+ const ssize_t max_md_size = INCFS_MAX_METADATA_RECORD_SIZE;
+ ssize_t bytes_read = 0;
+ size_t md_record_size = 0;
+ loff_t next_record = 0;
+ int res = 0;
+ struct incfs_md_header *md_hdr = NULL;
+
+ if (!bfc || !handler)
+ return -EFAULT;
+
+ if (handler->md_record_offset == 0)
+ return -EPERM;
+
+ memset(&handler->md_buffer, 0, max_md_size);
+ bytes_read = incfs_kread(bfc, &handler->md_buffer, max_md_size,
+ handler->md_record_offset);
+ if (bytes_read < 0)
+ return bytes_read;
+ if (bytes_read < sizeof(*md_hdr))
+ return -EBADMSG;
+
+ md_hdr = &handler->md_buffer.md_header;
+ next_record = le64_to_cpu(md_hdr->h_next_md_offset);
+ md_record_size = le16_to_cpu(md_hdr->h_record_size);
+
+ if (md_record_size > max_md_size) {
+ pr_warn("incfs: The record is too large. Size: %zu",
+ md_record_size);
+ return -EBADMSG;
+ }
+
+ if (bytes_read < md_record_size) {
+ pr_warn("incfs: The record hasn't been fully read.");
+ return -EBADMSG;
+ }
+
+ if (next_record <= handler->md_record_offset && next_record != 0) {
+ pr_warn("incfs: Next record (%lld) points back in file.",
+ next_record);
+ return -EBADMSG;
+ }
+
+ switch (md_hdr->h_md_entry_type) {
+ case INCFS_MD_NONE:
+ break;
+ case INCFS_MD_BLOCK_MAP:
+ if (handler->handle_blockmap)
+ res = handler->handle_blockmap(
+ &handler->md_buffer.blockmap, handler);
+ break;
+ case INCFS_MD_FILE_ATTR:
+ /*
+ * File attrs no longer supported, ignore section for
+ * compatibility
+ */
+ break;
+ case INCFS_MD_SIGNATURE:
+ if (handler->handle_signature)
+ res = handler->handle_signature(
+ &handler->md_buffer.signature, handler);
+ break;
+ case INCFS_MD_STATUS:
+ if (handler->handle_status)
+ res = handler->handle_status(
+ &handler->md_buffer.status, handler);
+ break;
+ case INCFS_MD_VERITY_SIGNATURE:
+ if (handler->handle_verity_signature)
+ res = handler->handle_verity_signature(
+ &handler->md_buffer.verity_signature, handler);
+ break;
+ default:
+ res = -ENOTSUPP;
+ break;
+ }
+
+ if (!res) {
+ if (next_record == 0) {
+ /*
+ * Zero offset for the next record means that the last
+ * metadata record has just been processed.
+ */
+ bfc->bc_last_md_record_offset =
+ handler->md_record_offset;
+ }
+ handler->md_prev_record_offset = handler->md_record_offset;
+ handler->md_record_offset = next_record;
+ }
+ return res;
+}
+
+ssize_t incfs_kread(struct backing_file_context *bfc, void *buf, size_t size,
+ loff_t pos)
+{
+ const struct cred *old_cred = override_creds(bfc->bc_cred);
+ int ret = kernel_read(bfc->bc_file, buf, size, &pos);
+
+ revert_creds(old_cred);
+ return ret;
+}
+
+ssize_t incfs_kwrite(struct backing_file_context *bfc, const void *buf,
+ size_t size, loff_t pos)
+{
+ const struct cred *old_cred = override_creds(bfc->bc_cred);
+ int ret = kernel_write(bfc->bc_file, buf, size, &pos);
+
+ revert_creds(old_cred);
+ return ret;
+}
diff --git a/fs/incfs/format.h b/fs/incfs/format.h
new file mode 100644
index 0000000..14e475b
--- /dev/null
+++ b/fs/incfs/format.h
@@ -0,0 +1,407 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018 Google LLC
+ */
+
+/*
+ * Overview
+ * --------
+ * The backbone of the incremental-fs ondisk format is an append only linked
+ * list of metadata blocks. Each metadata block contains an offset of the next
+ * one. These blocks describe files and directories on the
+ * file system. They also represent actions of adding and removing file names
+ * (hard links).
+ *
+ * Every time incremental-fs instance is mounted, it reads through this list
+ * to recreate filesystem's state in memory. An offset of the first record in
+ * the metadata list is stored in the superblock at the beginning of the backing
+ * file.
+ *
+ * Most of the backing file is taken by data areas and blockmaps.
+ * Since data blocks can be compressed and have different sizes,
+ * single per-file data area can't be pre-allocated. That's why blockmaps are
+ * needed in order to find a location and size of each data block in
+ * the backing file. Each time a file is created, a corresponding block map is
+ * allocated to store future offsets of data blocks.
+ *
+ * Whenever a data block is given by data loader to incremental-fs:
+ * - A data area with the given block is appended to the end of
+ * the backing file.
+ * - A record in the blockmap for the given block index is updated to reflect
+ * its location, size, and compression algorithm.
+
+ * Metadata records
+ * ----------------
+ * incfs_blockmap - metadata record that specifies size and location
+ * of a blockmap area for a given file. This area
+ * contains an array of incfs_blockmap_entry-s.
+ * incfs_file_signature - metadata record that specifies where file signature
+ * and its hash tree can be found in the backing file.
+ *
+ * incfs_file_attr - metadata record that specifies where additional file
+ * attributes blob can be found.
+ *
+ * Metadata header
+ * ---------------
+ * incfs_md_header - header of a metadata record. It's always a part
+ * of other structures and served purpose of metadata
+ * bookkeeping.
+ *
+ * +-----------------------------------------------+ ^
+ * | incfs_md_header | |
+ * | 1. type of body(BLOCKMAP, FILE_ATTR..) | |
+ * | 2. size of the whole record header + body | |
+ * | 3. CRC the whole record header + body | |
+ * | 4. offset of the previous md record |]------+
+ * | 5. offset of the next md record (md link) |]---+
+ * +-----------------------------------------------+ |
+ * | Metadata record body with useful data | |
+ * +-----------------------------------------------+ |
+ * +--->
+ *
+ * Other ondisk structures
+ * -----------------------
+ * incfs_super_block - backing file header
+ * incfs_blockmap_entry - a record in a blockmap area that describes size
+ * and location of a data block.
+ * Data blocks dont have any particular structure, they are written to the
+ * backing file in a raw form as they come from a data loader.
+ *
+ * Backing file layout
+ * -------------------
+ *
+ *
+ * +-------------------------------------------+
+ * | incfs_file_header |]---+
+ * +-------------------------------------------+ |
+ * | metadata |<---+
+ * | incfs_file_signature |]---+
+ * +-------------------------------------------+ |
+ * ......................... |
+ * +-------------------------------------------+ | metadata
+ * +------->| blockmap area | | list links
+ * | | [incfs_blockmap_entry] | |
+ * | | [incfs_blockmap_entry] | |
+ * | | [incfs_blockmap_entry] | |
+ * | +--[| [incfs_blockmap_entry] | |
+ * | | | [incfs_blockmap_entry] | |
+ * | | | [incfs_blockmap_entry] | |
+ * | | +-------------------------------------------+ |
+ * | | ......................... |
+ * | | +-------------------------------------------+ |
+ * | | | metadata |<---+
+ * +----|--[| incfs_blockmap |]---+
+ * | +-------------------------------------------+ |
+ * | ......................... |
+ * | +-------------------------------------------+ |
+ * +-->| data block | |
+ * +-------------------------------------------+ |
+ * ......................... |
+ * +-------------------------------------------+ |
+ * | metadata |<---+
+ * | incfs_file_attr |
+ * +-------------------------------------------+
+ */
+#ifndef _INCFS_FORMAT_H
+#define _INCFS_FORMAT_H
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <uapi/linux/incrementalfs.h>
+
+#include "internal.h"
+
+#define INCFS_MAX_NAME_LEN 255
+#define INCFS_FORMAT_V1 1
+#define INCFS_FORMAT_CURRENT_VER INCFS_FORMAT_V1
+
+enum incfs_metadata_type {
+ INCFS_MD_NONE = 0,
+ INCFS_MD_BLOCK_MAP = 1,
+ INCFS_MD_FILE_ATTR = 2,
+ INCFS_MD_SIGNATURE = 3,
+ INCFS_MD_STATUS = 4,
+ INCFS_MD_VERITY_SIGNATURE = 5,
+};
+
+enum incfs_file_header_flags {
+ INCFS_FILE_MAPPED = 1 << 1,
+};
+
+/* Header included at the beginning of all metadata records on the disk. */
+struct incfs_md_header {
+ __u8 h_md_entry_type;
+
+ /*
+ * Size of the metadata record.
+ * (e.g. inode, dir entry etc) not just this struct.
+ */
+ __le16 h_record_size;
+
+ /*
+ * Was: CRC32 of the metadata record.
+ * (e.g. inode, dir entry etc) not just this struct.
+ */
+ __le32 h_unused1;
+
+ /* Offset of the next metadata entry if any */
+ __le64 h_next_md_offset;
+
+ /* Was: Offset of the previous metadata entry if any */
+ __le64 h_unused2;
+
+} __packed;
+
+/* Backing file header */
+struct incfs_file_header {
+ /* Magic number: INCFS_MAGIC_NUMBER */
+ __le64 fh_magic;
+
+ /* Format version: INCFS_FORMAT_CURRENT_VER */
+ __le64 fh_version;
+
+ /* sizeof(incfs_file_header) */
+ __le16 fh_header_size;
+
+ /* INCFS_DATA_FILE_BLOCK_SIZE */
+ __le16 fh_data_block_size;
+
+ /* File flags, from incfs_file_header_flags */
+ __le32 fh_flags;
+
+ union {
+ /* Standard incfs file */
+ struct {
+ /* Offset of the first metadata record */
+ __le64 fh_first_md_offset;
+
+ /* Full size of the file's content */
+ __le64 fh_file_size;
+
+ /* File uuid */
+ incfs_uuid_t fh_uuid;
+ };
+
+ /* Mapped file - INCFS_FILE_MAPPED set in fh_flags */
+ struct {
+ /* Offset in original file */
+ __le64 fh_original_offset;
+
+ /* Full size of the file's content */
+ __le64 fh_mapped_file_size;
+
+ /* Original file's uuid */
+ incfs_uuid_t fh_original_uuid;
+ };
+ };
+} __packed;
+
+enum incfs_block_map_entry_flags {
+ INCFS_BLOCK_COMPRESSED_LZ4 = 1,
+ INCFS_BLOCK_COMPRESSED_ZSTD = 2,
+
+ /* Reserve 3 bits for compression alg */
+ INCFS_BLOCK_COMPRESSED_MASK = 7,
+};
+
+/* Block map entry pointing to an actual location of the data block. */
+struct incfs_blockmap_entry {
+ /* Offset of the actual data block. Lower 32 bits */
+ __le32 me_data_offset_lo;
+
+ /* Offset of the actual data block. Higher 16 bits */
+ __le16 me_data_offset_hi;
+
+ /* How many bytes the data actually occupies in the backing file */
+ __le16 me_data_size;
+
+ /* Block flags from incfs_block_map_entry_flags */
+ __le16 me_flags;
+} __packed;
+
+/* Metadata record for locations of file blocks. Type = INCFS_MD_BLOCK_MAP */
+struct incfs_blockmap {
+ struct incfs_md_header m_header;
+
+ /* Base offset of the array of incfs_blockmap_entry */
+ __le64 m_base_offset;
+
+ /* Size of the map entry array in blocks */
+ __le32 m_block_count;
+} __packed;
+
+/*
+ * Metadata record for file signature. Type = INCFS_MD_SIGNATURE
+ *
+ * The signature stored here is the APK V4 signature data blob. See the
+ * definition of incfs_new_file_args::signature_info for an explanation of this
+ * blob. Specifically, it contains the root hash, but it does *not* contain
+ * anything that the kernel treats as a signature.
+ *
+ * When FS_IOC_ENABLE_VERITY is called on a file without this record, an APK V4
+ * signature blob and a hash tree are added to the file, and then this metadata
+ * record is created to record their locations.
+ */
+struct incfs_file_signature {
+ struct incfs_md_header sg_header;
+
+ __le32 sg_sig_size; /* The size of the signature. */
+
+ __le64 sg_sig_offset; /* Signature's offset in the backing file */
+
+ __le32 sg_hash_tree_size; /* The size of the hash tree. */
+
+ __le64 sg_hash_tree_offset; /* Hash tree offset in the backing file */
+} __packed;
+
+/* In memory version of above */
+struct incfs_df_signature {
+ u32 sig_size;
+ u64 sig_offset;
+ u32 hash_size;
+ u64 hash_offset;
+};
+
+struct incfs_status {
+ struct incfs_md_header is_header;
+
+ __le32 is_data_blocks_written; /* Number of data blocks written */
+
+ __le32 is_hash_blocks_written; /* Number of hash blocks written */
+
+ __le32 is_dummy[6]; /* Spare fields */
+} __packed;
+
+/*
+ * Metadata record for verity signature. Type = INCFS_MD_VERITY_SIGNATURE
+ *
+ * This record will only exist for verity-enabled files with signatures. Verity
+ * enabled files without signatures do not have this record. This signature is
+ * checked by fs-verity identically to any other fs-verity signature.
+ */
+struct incfs_file_verity_signature {
+ struct incfs_md_header vs_header;
+
+ /* The size of the signature */
+ __le32 vs_size;
+
+ /* Signature's offset in the backing file */
+ __le64 vs_offset;
+} __packed;
+
+/* In memory version of above */
+struct incfs_df_verity_signature {
+ u32 size;
+ u64 offset;
+};
+
+/* State of the backing file. */
+struct backing_file_context {
+ /* Protects writes to bc_file */
+ struct mutex bc_mutex;
+
+ /* File object to read data from */
+ struct file *bc_file;
+
+ /*
+ * Offset of the last known metadata record in the backing file.
+ * 0 means there are no metadata records.
+ */
+ loff_t bc_last_md_record_offset;
+
+ /*
+ * Credentials to set before reads/writes
+ * Note that this is a pointer to the mount_info mi_owner field so
+ * there is no need to get/put the creds
+ */
+ const struct cred *bc_cred;
+};
+
+struct metadata_handler {
+ loff_t md_record_offset;
+ loff_t md_prev_record_offset;
+ void *context;
+
+ union {
+ struct incfs_md_header md_header;
+ struct incfs_blockmap blockmap;
+ struct incfs_file_signature signature;
+ struct incfs_status status;
+ struct incfs_file_verity_signature verity_signature;
+ } md_buffer;
+
+ int (*handle_blockmap)(struct incfs_blockmap *bm,
+ struct metadata_handler *handler);
+ int (*handle_signature)(struct incfs_file_signature *sig,
+ struct metadata_handler *handler);
+ int (*handle_status)(struct incfs_status *sig,
+ struct metadata_handler *handler);
+ int (*handle_verity_signature)(struct incfs_file_verity_signature *s,
+ struct metadata_handler *handler);
+};
+#define INCFS_MAX_METADATA_RECORD_SIZE \
+ sizeof_field(struct metadata_handler, md_buffer)
+
+/* Backing file context management */
+struct mount_info;
+struct backing_file_context *incfs_alloc_bfc(struct mount_info *mi,
+ struct file *backing_file);
+
+void incfs_free_bfc(struct backing_file_context *bfc);
+
+/* Writing stuff */
+int incfs_write_blockmap_to_backing_file(struct backing_file_context *bfc,
+ u32 block_count);
+
+int incfs_write_fh_to_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size);
+
+int incfs_write_mapping_fh_to_backing_file(struct backing_file_context *bfc,
+ incfs_uuid_t *uuid, u64 file_size, u64 offset);
+
+int incfs_write_data_block_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range block,
+ int block_index, loff_t bm_base_off,
+ u16 flags);
+
+int incfs_write_hash_block_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range block,
+ int block_index,
+ loff_t hash_area_off,
+ loff_t bm_base_off,
+ loff_t file_size);
+
+int incfs_write_signature_to_backing_file(struct backing_file_context *bfc,
+ struct mem_range sig, u32 tree_size,
+ loff_t *tree_offset, loff_t *sig_offset);
+
+int incfs_write_status_to_backing_file(struct backing_file_context *bfc,
+ loff_t status_offset,
+ u32 data_blocks_written,
+ u32 hash_blocks_written);
+int incfs_write_verity_signature_to_backing_file(
+ struct backing_file_context *bfc, struct mem_range signature,
+ loff_t *offset);
+
+/* Reading stuff */
+int incfs_read_file_header(struct backing_file_context *bfc,
+ loff_t *first_md_off, incfs_uuid_t *uuid,
+ u64 *file_size, u32 *flags);
+
+int incfs_read_blockmap_entry(struct backing_file_context *bfc, int block_index,
+ loff_t bm_base_off,
+ struct incfs_blockmap_entry *bm_entry);
+
+int incfs_read_blockmap_entries(struct backing_file_context *bfc,
+ struct incfs_blockmap_entry *entries,
+ int start_index, int blocks_number,
+ loff_t bm_base_off);
+
+int incfs_read_next_metadata_record(struct backing_file_context *bfc,
+ struct metadata_handler *handler);
+
+ssize_t incfs_kread(struct backing_file_context *bfc, void *buf, size_t size,
+ loff_t pos);
+ssize_t incfs_kwrite(struct backing_file_context *bfc, const void *buf,
+ size_t size, loff_t pos);
+
+#endif /* _INCFS_FORMAT_H */
diff --git a/fs/incfs/integrity.c b/fs/incfs/integrity.c
new file mode 100644
index 0000000..93e7643
--- /dev/null
+++ b/fs/incfs/integrity.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 Google LLC
+ */
+#include <crypto/sha2.h>
+#include <crypto/hash.h>
+#include <linux/err.h>
+#include <linux/version.h>
+
+#include "integrity.h"
+
+struct incfs_hash_alg *incfs_get_hash_alg(enum incfs_hash_tree_algorithm id)
+{
+ static struct incfs_hash_alg sha256 = {
+ .name = "sha256",
+ .digest_size = SHA256_DIGEST_SIZE,
+ .id = INCFS_HASH_TREE_SHA256
+ };
+ struct incfs_hash_alg *result = NULL;
+ struct crypto_shash *shash;
+
+ if (id == INCFS_HASH_TREE_SHA256) {
+ BUILD_BUG_ON(INCFS_MAX_HASH_SIZE < SHA256_DIGEST_SIZE);
+ result = &sha256;
+ }
+
+ if (result == NULL)
+ return ERR_PTR(-ENOENT);
+
+ /* pairs with cmpxchg_release() below */
+ shash = smp_load_acquire(&result->shash);
+ if (shash)
+ return result;
+
+ shash = crypto_alloc_shash(result->name, 0, 0);
+ if (IS_ERR(shash)) {
+ int err = PTR_ERR(shash);
+
+ pr_err("Can't allocate hash alg %s, error code:%d",
+ result->name, err);
+ return ERR_PTR(err);
+ }
+
+ /* pairs with smp_load_acquire() above */
+ if (cmpxchg_release(&result->shash, NULL, shash) != NULL)
+ crypto_free_shash(shash);
+
+ return result;
+}
+
+struct signature_info {
+ u32 version;
+ enum incfs_hash_tree_algorithm hash_algorithm;
+ u8 log2_blocksize;
+ struct mem_range salt;
+ struct mem_range root_hash;
+};
+
+static bool read_u32(u8 **p, u8 *top, u32 *result)
+{
+ if (*p + sizeof(u32) > top)
+ return false;
+
+ *result = le32_to_cpu(*(__le32 *)*p);
+ *p += sizeof(u32);
+ return true;
+}
+
+static bool read_u8(u8 **p, u8 *top, u8 *result)
+{
+ if (*p + sizeof(u8) > top)
+ return false;
+
+ *result = *(u8 *)*p;
+ *p += sizeof(u8);
+ return true;
+}
+
+static bool read_mem_range(u8 **p, u8 *top, struct mem_range *range)
+{
+ u32 len;
+
+ if (!read_u32(p, top, &len) || *p + len > top)
+ return false;
+
+ range->len = len;
+ range->data = *p;
+ *p += len;
+ return true;
+}
+
+static int incfs_parse_signature(struct mem_range signature,
+ struct signature_info *si)
+{
+ u8 *p = signature.data;
+ u8 *top = signature.data + signature.len;
+ u32 hash_section_size;
+
+ if (signature.len > INCFS_MAX_SIGNATURE_SIZE)
+ return -EINVAL;
+
+ if (!read_u32(&p, top, &si->version) ||
+ si->version != INCFS_SIGNATURE_VERSION)
+ return -EINVAL;
+
+ if (!read_u32(&p, top, &hash_section_size) ||
+ p + hash_section_size > top)
+ return -EINVAL;
+ top = p + hash_section_size;
+
+ if (!read_u32(&p, top, &si->hash_algorithm) ||
+ si->hash_algorithm != INCFS_HASH_TREE_SHA256)
+ return -EINVAL;
+
+ if (!read_u8(&p, top, &si->log2_blocksize) || si->log2_blocksize != 12)
+ return -EINVAL;
+
+ if (!read_mem_range(&p, top, &si->salt))
+ return -EINVAL;
+
+ if (!read_mem_range(&p, top, &si->root_hash))
+ return -EINVAL;
+
+ if (p != top)
+ return -EINVAL;
+
+ return 0;
+}
+
+struct mtree *incfs_alloc_mtree(struct mem_range signature,
+ int data_block_count)
+{
+ int error;
+ struct signature_info si;
+ struct mtree *result = NULL;
+ struct incfs_hash_alg *hash_alg = NULL;
+ int hash_per_block;
+ int lvl;
+ int total_blocks = 0;
+ int blocks_in_level[INCFS_MAX_MTREE_LEVELS];
+ int blocks = data_block_count;
+
+ if (data_block_count <= 0)
+ return ERR_PTR(-EINVAL);
+
+ error = incfs_parse_signature(signature, &si);
+ if (error)
+ return ERR_PTR(error);
+
+ hash_alg = incfs_get_hash_alg(si.hash_algorithm);
+ if (IS_ERR(hash_alg))
+ return ERR_PTR(PTR_ERR(hash_alg));
+
+ if (si.root_hash.len < hash_alg->digest_size)
+ return ERR_PTR(-EINVAL);
+
+ result = kzalloc(sizeof(*result), GFP_NOFS);
+ if (!result)
+ return ERR_PTR(-ENOMEM);
+
+ result->alg = hash_alg;
+ hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / result->alg->digest_size;
+
+ /* Calculating tree geometry. */
+ /* First pass: calculate how many blocks in each tree level. */
+ for (lvl = 0; blocks > 1; lvl++) {
+ if (lvl >= INCFS_MAX_MTREE_LEVELS) {
+ pr_err("incfs: too much data in mtree");
+ goto err;
+ }
+
+ blocks = (blocks + hash_per_block - 1) / hash_per_block;
+ blocks_in_level[lvl] = blocks;
+ total_blocks += blocks;
+ }
+ result->depth = lvl;
+ result->hash_tree_area_size = total_blocks * INCFS_DATA_FILE_BLOCK_SIZE;
+ if (result->hash_tree_area_size > INCFS_MAX_HASH_AREA_SIZE)
+ goto err;
+
+ blocks = 0;
+ /* Second pass: calculate offset of each level. 0th level goes last. */
+ for (lvl = 0; lvl < result->depth; lvl++) {
+ u32 suboffset;
+
+ blocks += blocks_in_level[lvl];
+ suboffset = (total_blocks - blocks)
+ * INCFS_DATA_FILE_BLOCK_SIZE;
+
+ result->hash_level_suboffset[lvl] = suboffset;
+ }
+
+ /* Root hash is stored separately from the rest of the tree. */
+ memcpy(result->root_hash, si.root_hash.data, hash_alg->digest_size);
+ return result;
+
+err:
+ kfree(result);
+ return ERR_PTR(-E2BIG);
+}
+
+void incfs_free_mtree(struct mtree *tree)
+{
+ kfree(tree);
+}
+
+int incfs_calc_digest(struct incfs_hash_alg *alg, struct mem_range data,
+ struct mem_range digest)
+{
+ SHASH_DESC_ON_STACK(desc, alg->shash);
+
+ if (!alg || !alg->shash || !data.data || !digest.data)
+ return -EFAULT;
+
+ if (alg->digest_size > digest.len)
+ return -EINVAL;
+
+ desc->tfm = alg->shash;
+
+ if (data.len < INCFS_DATA_FILE_BLOCK_SIZE) {
+ int err;
+ void *buf = kzalloc(INCFS_DATA_FILE_BLOCK_SIZE, GFP_NOFS);
+
+ if (!buf)
+ return -ENOMEM;
+
+ memcpy(buf, data.data, data.len);
+ err = crypto_shash_digest(desc, buf, INCFS_DATA_FILE_BLOCK_SIZE,
+ digest.data);
+ kfree(buf);
+ return err;
+ }
+ return crypto_shash_digest(desc, data.data, data.len, digest.data);
+}
+
diff --git a/fs/incfs/integrity.h b/fs/incfs/integrity.h
new file mode 100644
index 0000000..cf79b64
--- /dev/null
+++ b/fs/incfs/integrity.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2019 Google LLC
+ */
+#ifndef _INCFS_INTEGRITY_H
+#define _INCFS_INTEGRITY_H
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <crypto/hash.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "internal.h"
+
+#define INCFS_MAX_MTREE_LEVELS 8
+#define INCFS_MAX_HASH_AREA_SIZE (1280 * 1024 * 1024)
+
+struct incfs_hash_alg {
+ const char *name;
+ int digest_size;
+ enum incfs_hash_tree_algorithm id;
+
+ struct crypto_shash *shash;
+};
+
+/* Merkle tree structure. */
+struct mtree {
+ struct incfs_hash_alg *alg;
+
+ u8 root_hash[INCFS_MAX_HASH_SIZE];
+
+ /* Offset of each hash level in the hash area. */
+ u32 hash_level_suboffset[INCFS_MAX_MTREE_LEVELS];
+
+ u32 hash_tree_area_size;
+
+ /* Number of levels in hash_level_suboffset */
+ int depth;
+};
+
+struct incfs_hash_alg *incfs_get_hash_alg(enum incfs_hash_tree_algorithm id);
+
+struct mtree *incfs_alloc_mtree(struct mem_range signature,
+ int data_block_count);
+
+void incfs_free_mtree(struct mtree *tree);
+
+size_t incfs_get_mtree_depth(enum incfs_hash_tree_algorithm alg, loff_t size);
+
+size_t incfs_get_mtree_hash_count(enum incfs_hash_tree_algorithm alg,
+ loff_t size);
+
+int incfs_calc_digest(struct incfs_hash_alg *alg, struct mem_range data,
+ struct mem_range digest);
+
+#endif /* _INCFS_INTEGRITY_H */
diff --git a/fs/incfs/internal.h b/fs/incfs/internal.h
new file mode 100644
index 0000000..c2df8bf
--- /dev/null
+++ b/fs/incfs/internal.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018 Google LLC
+ */
+#ifndef _INCFS_INTERNAL_H
+#define _INCFS_INTERNAL_H
+#include <linux/types.h>
+
+struct mem_range {
+ u8 *data;
+ size_t len;
+};
+
+static inline struct mem_range range(u8 *data, size_t len)
+{
+ return (struct mem_range){ .data = data, .len = len };
+}
+
+#define LOCK_REQUIRED(lock) WARN_ON_ONCE(!mutex_is_locked(&lock))
+
+#define EFSCORRUPTED EUCLEAN
+
+#endif /* _INCFS_INTERNAL_H */
diff --git a/fs/incfs/main.c b/fs/incfs/main.c
new file mode 100644
index 0000000..23347ac
--- /dev/null
+++ b/fs/incfs/main.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/module.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "sysfs.h"
+#include "vfs.h"
+
+static struct file_system_type incfs_fs_type = {
+ .owner = THIS_MODULE,
+ .name = INCFS_NAME,
+ .mount = incfs_mount_fs,
+ .kill_sb = incfs_kill_sb,
+ .fs_flags = 0
+};
+
+static int __init init_incfs_module(void)
+{
+ int err = 0;
+
+ err = incfs_init_sysfs();
+ if (err)
+ return err;
+
+ err = register_filesystem(&incfs_fs_type);
+ if (err)
+ incfs_cleanup_sysfs();
+
+ return err;
+}
+
+static void __exit cleanup_incfs_module(void)
+{
+ incfs_cleanup_sysfs();
+ unregister_filesystem(&incfs_fs_type);
+}
+
+module_init(init_incfs_module);
+module_exit(cleanup_incfs_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Eugene Zemtsov <ezemtsov@google.com>");
+MODULE_DESCRIPTION("Incremental File System");
diff --git a/fs/incfs/pseudo_files.c b/fs/incfs/pseudo_files.c
new file mode 100644
index 0000000..57c3353
--- /dev/null
+++ b/fs/incfs/pseudo_files.c
@@ -0,0 +1,1393 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Google LLC
+ */
+
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/fsnotify.h>
+#include <linux/namei.h>
+#include <linux/poll.h>
+#include <linux/syscalls.h>
+#include <linux/fdtable.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "pseudo_files.h"
+
+#include "data_mgmt.h"
+#include "format.h"
+#include "integrity.h"
+#include "vfs.h"
+
+#define READ_WRITE_FILE_MODE 0666
+
+static bool is_pseudo_filename(struct mem_range name);
+
+/*******************************************************************************
+ * .pending_reads pseudo file definition
+ ******************************************************************************/
+#define INCFS_PENDING_READS_INODE 2
+static const char pending_reads_file_name[] = INCFS_PENDING_READS_FILENAME;
+
+/* State of an open .pending_reads file, unique for each file descriptor. */
+struct pending_reads_state {
+ /* A serial number of the last pending read obtained from this file. */
+ int last_pending_read_sn;
+};
+
+static ssize_t pending_reads_read(struct file *f, char __user *buf, size_t len,
+ loff_t *ppos)
+{
+ struct pending_reads_state *pr_state = f->private_data;
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+ bool report_uid;
+ unsigned long page = 0;
+ struct incfs_pending_read_info *reads_buf = NULL;
+ struct incfs_pending_read_info2 *reads_buf2 = NULL;
+ size_t record_size;
+ size_t reads_to_collect;
+ int last_known_read_sn = READ_ONCE(pr_state->last_pending_read_sn);
+ int new_max_sn = last_known_read_sn;
+ int reads_collected = 0;
+ ssize_t result = 0;
+
+ if (!mi)
+ return -EFAULT;
+
+ report_uid = mi->mi_options.report_uid;
+ record_size = report_uid ? sizeof(*reads_buf2) : sizeof(*reads_buf);
+ reads_to_collect = len / record_size;
+
+ if (!incfs_fresh_pending_reads_exist(mi, last_known_read_sn))
+ return 0;
+
+ page = get_zeroed_page(GFP_NOFS);
+ if (!page)
+ return -ENOMEM;
+
+ if (report_uid)
+ reads_buf2 = (struct incfs_pending_read_info2 *) page;
+ else
+ reads_buf = (struct incfs_pending_read_info *) page;
+
+ reads_to_collect =
+ min_t(size_t, PAGE_SIZE / record_size, reads_to_collect);
+
+ reads_collected = incfs_collect_pending_reads(mi, last_known_read_sn,
+ reads_buf, reads_buf2, reads_to_collect,
+ &new_max_sn);
+
+ if (reads_collected < 0) {
+ result = reads_collected;
+ goto out;
+ }
+
+ /*
+ * Just to make sure that we don't accidentally copy more data
+ * to reads buffer than userspace can handle.
+ */
+ reads_collected = min_t(size_t, reads_collected, reads_to_collect);
+ result = reads_collected * record_size;
+
+ /* Copy reads info to the userspace buffer */
+ if (copy_to_user(buf, (void *)page, result)) {
+ result = -EFAULT;
+ goto out;
+ }
+
+ WRITE_ONCE(pr_state->last_pending_read_sn, new_max_sn);
+ *ppos = 0;
+
+out:
+ free_page(page);
+ return result;
+}
+
+static __poll_t pending_reads_poll(struct file *file, poll_table *wait)
+{
+ struct pending_reads_state *state = file->private_data;
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+ __poll_t ret = 0;
+
+ poll_wait(file, &mi->mi_pending_reads_notif_wq, wait);
+ if (incfs_fresh_pending_reads_exist(mi,
+ state->last_pending_read_sn))
+ ret = EPOLLIN | EPOLLRDNORM;
+
+ return ret;
+}
+
+static int pending_reads_open(struct inode *inode, struct file *file)
+{
+ struct pending_reads_state *state = NULL;
+
+ state = kzalloc(sizeof(*state), GFP_NOFS);
+ if (!state)
+ return -ENOMEM;
+
+ file->private_data = state;
+ return 0;
+}
+
+static int pending_reads_release(struct inode *inode, struct file *file)
+{
+ kfree(file->private_data);
+ return 0;
+}
+
+static long ioctl_permit_fill(struct file *f, void __user *arg)
+{
+ struct incfs_permit_fill __user *usr_permit_fill = arg;
+ struct incfs_permit_fill permit_fill;
+ long error = 0;
+ struct file *file = NULL;
+ struct incfs_file_data *fd;
+
+ if (copy_from_user(&permit_fill, usr_permit_fill, sizeof(permit_fill)))
+ return -EFAULT;
+
+ file = fget(permit_fill.file_descriptor);
+ if (IS_ERR_OR_NULL(file)) {
+ if (!file)
+ return -ENOENT;
+
+ return PTR_ERR(file);
+ }
+
+ if (file->f_op != &incfs_file_ops) {
+ error = -EPERM;
+ goto out;
+ }
+
+ if (file->f_inode->i_sb != f->f_inode->i_sb) {
+ error = -EPERM;
+ goto out;
+ }
+
+ fd = file->private_data;
+
+ switch (fd->fd_fill_permission) {
+ case CANT_FILL:
+ fd->fd_fill_permission = CAN_FILL;
+ break;
+
+ case CAN_FILL:
+ pr_debug("CAN_FILL already set");
+ break;
+
+ default:
+ pr_warn("Invalid file private data");
+ error = -EFAULT;
+ goto out;
+ }
+
+out:
+ fput(file);
+ return error;
+}
+
+static int chmod(struct dentry *dentry, umode_t mode)
+{
+ struct inode *inode = dentry->d_inode;
+ struct inode *delegated_inode = NULL;
+ struct iattr newattrs;
+ int error;
+
+retry_deleg:
+ inode_lock(inode);
+ newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO);
+ newattrs.ia_valid = ATTR_MODE | ATTR_CTIME;
+ error = notify_change(&init_user_ns, dentry, &newattrs, &delegated_inode);
+ inode_unlock(inode);
+ if (delegated_inode) {
+ error = break_deleg_wait(&delegated_inode);
+ if (!error)
+ goto retry_deleg;
+ }
+ return error;
+}
+
+static bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs)
+{
+ if (lhs.len != rhs.len)
+ return false;
+ return memcmp(lhs.data, rhs.data, lhs.len) == 0;
+}
+
+static int validate_name(char *file_name)
+{
+ struct mem_range name = range(file_name, strlen(file_name));
+ int i = 0;
+
+ if (name.len > INCFS_MAX_NAME_LEN)
+ return -ENAMETOOLONG;
+
+ if (is_pseudo_filename(name))
+ return -EINVAL;
+
+ for (i = 0; i < name.len; i++)
+ if (name.data[i] == '/')
+ return -EINVAL;
+
+ return 0;
+}
+
+static int dir_relative_path_resolve(
+ struct mount_info *mi,
+ const char __user *relative_path,
+ struct path *result_path,
+ struct path *base_path)
+{
+ int dir_fd = get_unused_fd_flags(0);
+ struct file *dir_f = NULL;
+ int error = 0;
+
+ if (!base_path)
+ base_path = &mi->mi_backing_dir_path;
+
+ if (dir_fd < 0)
+ return dir_fd;
+
+ dir_f = dentry_open(base_path, O_RDONLY | O_NOATIME, current_cred());
+
+ if (IS_ERR(dir_f)) {
+ error = PTR_ERR(dir_f);
+ goto out;
+ }
+ fd_install(dir_fd, dir_f);
+
+ if (!relative_path) {
+ /* No relative path given, just return the base dir. */
+ *result_path = *base_path;
+ path_get(result_path);
+ goto out;
+ }
+
+ error = user_path_at_empty(dir_fd, relative_path,
+ LOOKUP_FOLLOW | LOOKUP_DIRECTORY, result_path, NULL);
+
+out:
+ close_fd(dir_fd);
+ if (error)
+ pr_debug("Error: %d\n", error);
+ return error;
+}
+
+static struct mem_range incfs_copy_signature_info_from_user(u8 __user *original,
+ u64 size)
+{
+ u8 *result;
+
+ if (!original)
+ return range(NULL, 0);
+
+ if (size > INCFS_MAX_SIGNATURE_SIZE)
+ return range(ERR_PTR(-EFAULT), 0);
+
+ result = kzalloc(size, GFP_NOFS | __GFP_COMP);
+ if (!result)
+ return range(ERR_PTR(-ENOMEM), 0);
+
+ if (copy_from_user(result, original, size)) {
+ kfree(result);
+ return range(ERR_PTR(-EFAULT), 0);
+ }
+
+ return range(result, size);
+}
+
+static int init_new_file(struct mount_info *mi, struct dentry *dentry,
+ incfs_uuid_t *uuid, u64 size, struct mem_range attr,
+ u8 __user *user_signature_info, u64 signature_size)
+{
+ struct path path = {};
+ struct file *new_file;
+ int error = 0;
+ struct backing_file_context *bfc = NULL;
+ u32 block_count;
+ struct mem_range raw_signature = { NULL };
+ struct mtree *hash_tree = NULL;
+
+ if (!mi || !dentry || !uuid)
+ return -EFAULT;
+
+ /* Resize newly created file to its true size. */
+ path = (struct path) {
+ .mnt = mi->mi_backing_dir_path.mnt,
+ .dentry = dentry
+ };
+
+ new_file = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
+ current_cred());
+
+ if (IS_ERR(new_file)) {
+ error = PTR_ERR(new_file);
+ goto out;
+ }
+
+ bfc = incfs_alloc_bfc(mi, new_file);
+ fput(new_file);
+ if (IS_ERR(bfc)) {
+ error = PTR_ERR(bfc);
+ bfc = NULL;
+ goto out;
+ }
+
+ mutex_lock(&bfc->bc_mutex);
+ error = incfs_write_fh_to_backing_file(bfc, uuid, size);
+ if (error)
+ goto out;
+
+ block_count = (u32)get_blocks_count_for_size(size);
+
+ if (user_signature_info) {
+ raw_signature = incfs_copy_signature_info_from_user(
+ user_signature_info, signature_size);
+
+ if (IS_ERR(raw_signature.data)) {
+ error = PTR_ERR(raw_signature.data);
+ raw_signature.data = NULL;
+ goto out;
+ }
+
+ hash_tree = incfs_alloc_mtree(raw_signature, block_count);
+ if (IS_ERR(hash_tree)) {
+ error = PTR_ERR(hash_tree);
+ hash_tree = NULL;
+ goto out;
+ }
+
+ error = incfs_write_signature_to_backing_file(bfc,
+ raw_signature, hash_tree->hash_tree_area_size,
+ NULL, NULL);
+ if (error)
+ goto out;
+
+ block_count += get_blocks_count_for_size(
+ hash_tree->hash_tree_area_size);
+ }
+
+ if (block_count)
+ error = incfs_write_blockmap_to_backing_file(bfc, block_count);
+
+ if (error)
+ goto out;
+
+out:
+ if (bfc) {
+ mutex_unlock(&bfc->bc_mutex);
+ incfs_free_bfc(bfc);
+ }
+ incfs_free_mtree(hash_tree);
+ kfree(raw_signature.data);
+
+ if (error)
+ pr_debug("incfs: %s error: %d\n", __func__, error);
+ return error;
+}
+
+static void notify_create(struct file *pending_reads_file,
+ const char __user *dir_name, const char *file_name,
+ const char *file_id_str, bool incomplete_file)
+{
+ struct mount_info *mi =
+ get_mount_info(file_superblock(pending_reads_file));
+ struct path base_path = {
+ .mnt = pending_reads_file->f_path.mnt,
+ .dentry = pending_reads_file->f_path.dentry->d_parent,
+ };
+ struct path dir_path = {};
+ struct dentry *file = NULL;
+ struct dentry *dir = NULL;
+ int error;
+
+ error = dir_relative_path_resolve(mi, dir_name, &dir_path, &base_path);
+ if (error)
+ goto out;
+
+ file = incfs_lookup_dentry(dir_path.dentry, file_name);
+ if (IS_ERR(file)) {
+ error = PTR_ERR(file);
+ file = NULL;
+ goto out;
+ }
+
+ fsnotify_create(d_inode(dir_path.dentry), file);
+
+ if (file_id_str) {
+ dir = incfs_lookup_dentry(base_path.dentry, INCFS_INDEX_NAME);
+ if (IS_ERR(dir)) {
+ error = PTR_ERR(dir);
+ dir = NULL;
+ goto out;
+ }
+
+ dput(file);
+ file = incfs_lookup_dentry(dir, file_id_str);
+ if (IS_ERR(file)) {
+ error = PTR_ERR(file);
+ file = NULL;
+ goto out;
+ }
+
+ fsnotify_create(d_inode(dir), file);
+
+ if (incomplete_file) {
+ dput(dir);
+ dir = incfs_lookup_dentry(base_path.dentry,
+ INCFS_INCOMPLETE_NAME);
+ if (IS_ERR(dir)) {
+ error = PTR_ERR(dir);
+ dir = NULL;
+ goto out;
+ }
+
+ dput(file);
+ file = incfs_lookup_dentry(dir, file_id_str);
+ if (IS_ERR(file)) {
+ error = PTR_ERR(file);
+ file = NULL;
+ goto out;
+ }
+
+ fsnotify_create(d_inode(dir), file);
+ }
+ }
+out:
+ if (error)
+ pr_warn("%s failed with error %d\n", __func__, error);
+
+ dput(dir);
+ dput(file);
+ path_put(&dir_path);
+}
+
+static long ioctl_create_file(struct file *file,
+ struct incfs_new_file_args __user *usr_args)
+{
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+ struct incfs_new_file_args args;
+ char *file_id_str = NULL;
+ struct dentry *index_file_dentry = NULL;
+ struct dentry *named_file_dentry = NULL;
+ struct dentry *incomplete_file_dentry = NULL;
+ struct path parent_dir_path = {};
+ struct inode *index_dir_inode = NULL;
+ __le64 size_attr_value = 0;
+ char *file_name = NULL;
+ char *attr_value = NULL;
+ int error = 0;
+ bool locked = false;
+ bool index_linked = false;
+ bool name_linked = false;
+ bool incomplete_linked = false;
+
+ if (!mi || !mi->mi_index_dir || !mi->mi_incomplete_dir) {
+ error = -EFAULT;
+ goto out;
+ }
+
+ if (copy_from_user(&args, usr_args, sizeof(args)) > 0) {
+ error = -EFAULT;
+ goto out;
+ }
+
+ file_name = strndup_user(u64_to_user_ptr(args.file_name), PATH_MAX);
+ if (IS_ERR(file_name)) {
+ error = PTR_ERR(file_name);
+ file_name = NULL;
+ goto out;
+ }
+
+ error = validate_name(file_name);
+ if (error)
+ goto out;
+
+ file_id_str = file_id_to_str(args.file_id);
+ if (!file_id_str) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (error)
+ goto out;
+ locked = true;
+
+ /* Find a directory to put the file into. */
+ error = dir_relative_path_resolve(mi,
+ u64_to_user_ptr(args.directory_path),
+ &parent_dir_path, NULL);
+ if (error)
+ goto out;
+
+ if (parent_dir_path.dentry == mi->mi_index_dir) {
+ /* Can't create a file directly inside .index */
+ error = -EBUSY;
+ goto out;
+ }
+
+ if (parent_dir_path.dentry == mi->mi_incomplete_dir) {
+ /* Can't create a file directly inside .incomplete */
+ error = -EBUSY;
+ goto out;
+ }
+
+ /* Look up a dentry in the parent dir. It should be negative. */
+ named_file_dentry = incfs_lookup_dentry(parent_dir_path.dentry,
+ file_name);
+ if (!named_file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(named_file_dentry)) {
+ error = PTR_ERR(named_file_dentry);
+ named_file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(named_file_dentry)) {
+ /* File with this path already exists. */
+ error = -EEXIST;
+ goto out;
+ }
+
+ /* Look up a dentry in the incomplete dir. It should be negative. */
+ incomplete_file_dentry = incfs_lookup_dentry(mi->mi_incomplete_dir,
+ file_id_str);
+ if (!incomplete_file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(incomplete_file_dentry)) {
+ error = PTR_ERR(incomplete_file_dentry);
+ incomplete_file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(incomplete_file_dentry)) {
+ /* File with this path already exists. */
+ error = -EEXIST;
+ goto out;
+ }
+
+ /* Look up a dentry in the .index dir. It should be negative. */
+ index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir, file_id_str);
+ if (!index_file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(index_file_dentry)) {
+ error = PTR_ERR(index_file_dentry);
+ index_file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(index_file_dentry)) {
+ /* File with this ID already exists in index. */
+ error = -EEXIST;
+ goto out;
+ }
+
+ /* Creating a file in the .index dir. */
+ index_dir_inode = d_inode(mi->mi_index_dir);
+ inode_lock_nested(index_dir_inode, I_MUTEX_PARENT);
+ error = vfs_create(&init_user_ns, index_dir_inode, index_file_dentry,
+ args.mode | 0222, true);
+ inode_unlock(index_dir_inode);
+
+ if (error)
+ goto out;
+ if (!d_really_is_positive(index_file_dentry)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ error = chmod(index_file_dentry, args.mode | 0222);
+ if (error) {
+ pr_debug("incfs: chmod err: %d\n", error);
+ goto out;
+ }
+
+ /* Save the file's ID as an xattr for easy fetching in future. */
+ error = vfs_setxattr(&init_user_ns, index_file_dentry, INCFS_XATTR_ID_NAME,
+ file_id_str, strlen(file_id_str), XATTR_CREATE);
+ if (error) {
+ pr_debug("incfs: vfs_setxattr err:%d\n", error);
+ goto out;
+ }
+
+ /* Save the file's size as an xattr for easy fetching in future. */
+ size_attr_value = cpu_to_le64(args.size);
+ error = vfs_setxattr(&init_user_ns, index_file_dentry, INCFS_XATTR_SIZE_NAME,
+ (char *)&size_attr_value, sizeof(size_attr_value),
+ XATTR_CREATE);
+ if (error) {
+ pr_debug("incfs: vfs_setxattr err:%d\n", error);
+ goto out;
+ }
+
+ /* Save the file's attribute as an xattr */
+ if (args.file_attr_len && args.file_attr) {
+ if (args.file_attr_len > INCFS_MAX_FILE_ATTR_SIZE) {
+ error = -E2BIG;
+ goto out;
+ }
+
+ attr_value = kmalloc(args.file_attr_len, GFP_NOFS);
+ if (!attr_value) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ if (copy_from_user(attr_value,
+ u64_to_user_ptr(args.file_attr),
+ args.file_attr_len) > 0) {
+ error = -EFAULT;
+ goto out;
+ }
+
+ error = vfs_setxattr(&init_user_ns, index_file_dentry,
+ INCFS_XATTR_METADATA_NAME,
+ attr_value, args.file_attr_len,
+ XATTR_CREATE);
+
+ if (error)
+ goto out;
+ }
+
+ /* Initializing a newly created file. */
+ error = init_new_file(mi, index_file_dentry, &args.file_id, args.size,
+ range(attr_value, args.file_attr_len),
+ u64_to_user_ptr(args.signature_info),
+ args.signature_size);
+ if (error)
+ goto out;
+ index_linked = true;
+
+ /* Linking a file with its real name from the requested dir. */
+ error = incfs_link(index_file_dentry, named_file_dentry);
+ if (error)
+ goto out;
+ name_linked = true;
+
+ if (args.size) {
+ /* Linking a file with its incomplete entry */
+ error = incfs_link(index_file_dentry, incomplete_file_dentry);
+ if (error)
+ goto out;
+ incomplete_linked = true;
+ }
+
+ notify_create(file, u64_to_user_ptr(args.directory_path), file_name,
+ file_id_str, args.size != 0);
+
+out:
+ if (error) {
+ pr_debug("incfs: %s err:%d\n", __func__, error);
+ if (index_linked)
+ incfs_unlink(index_file_dentry);
+ if (name_linked)
+ incfs_unlink(named_file_dentry);
+ if (incomplete_linked)
+ incfs_unlink(incomplete_file_dentry);
+ }
+
+ kfree(file_id_str);
+ kfree(file_name);
+ kfree(attr_value);
+ dput(named_file_dentry);
+ dput(index_file_dentry);
+ dput(incomplete_file_dentry);
+ path_put(&parent_dir_path);
+ if (locked)
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+
+ return error;
+}
+
+static int init_new_mapped_file(struct mount_info *mi, struct dentry *dentry,
+ incfs_uuid_t *uuid, u64 size, u64 offset)
+{
+ struct path path = {};
+ struct file *new_file;
+ int error = 0;
+ struct backing_file_context *bfc = NULL;
+
+ if (!mi || !dentry || !uuid)
+ return -EFAULT;
+
+ /* Resize newly created file to its true size. */
+ path = (struct path) {
+ .mnt = mi->mi_backing_dir_path.mnt,
+ .dentry = dentry
+ };
+ new_file = dentry_open(&path, O_RDWR | O_NOATIME | O_LARGEFILE,
+ current_cred());
+
+ if (IS_ERR(new_file)) {
+ error = PTR_ERR(new_file);
+ goto out;
+ }
+
+ bfc = incfs_alloc_bfc(mi, new_file);
+ fput(new_file);
+ if (IS_ERR(bfc)) {
+ error = PTR_ERR(bfc);
+ bfc = NULL;
+ goto out;
+ }
+
+ mutex_lock(&bfc->bc_mutex);
+ error = incfs_write_mapping_fh_to_backing_file(bfc, uuid, size, offset);
+ if (error)
+ goto out;
+
+out:
+ if (bfc) {
+ mutex_unlock(&bfc->bc_mutex);
+ incfs_free_bfc(bfc);
+ }
+
+ if (error)
+ pr_debug("incfs: %s error: %d\n", __func__, error);
+ return error;
+}
+
+static long ioctl_create_mapped_file(struct file *file, void __user *arg)
+{
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+ struct incfs_create_mapped_file_args __user *args_usr_ptr = arg;
+ struct incfs_create_mapped_file_args args = {};
+ char *file_name;
+ int error = 0;
+ struct path parent_dir_path = {};
+ char *source_file_name = NULL;
+ struct dentry *source_file_dentry = NULL;
+ u64 source_file_size;
+ struct dentry *file_dentry = NULL;
+ struct inode *parent_inode;
+ __le64 size_attr_value;
+
+ if (copy_from_user(&args, args_usr_ptr, sizeof(args)) > 0)
+ return -EINVAL;
+
+ file_name = strndup_user(u64_to_user_ptr(args.file_name), PATH_MAX);
+ if (IS_ERR(file_name)) {
+ error = PTR_ERR(file_name);
+ file_name = NULL;
+ goto out;
+ }
+
+ error = validate_name(file_name);
+ if (error)
+ goto out;
+
+ if (args.source_offset % INCFS_DATA_FILE_BLOCK_SIZE) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ /* Validate file mapping is in range */
+ source_file_name = file_id_to_str(args.source_file_id);
+ if (!source_file_name) {
+ pr_warn("Failed to alloc source_file_name\n");
+ error = -ENOMEM;
+ goto out;
+ }
+
+ source_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
+ source_file_name);
+ if (!source_file_dentry) {
+ pr_warn("Source file does not exist\n");
+ error = -EINVAL;
+ goto out;
+ }
+ if (IS_ERR(source_file_dentry)) {
+ pr_warn("Error opening source file\n");
+ error = PTR_ERR(source_file_dentry);
+ source_file_dentry = NULL;
+ goto out;
+ }
+ if (!d_really_is_positive(source_file_dentry)) {
+ pr_warn("Source file dentry negative\n");
+ error = -EINVAL;
+ goto out;
+ }
+
+ error = vfs_getxattr(&init_user_ns, source_file_dentry, INCFS_XATTR_SIZE_NAME,
+ (char *)&size_attr_value, sizeof(size_attr_value));
+ if (error < 0)
+ goto out;
+
+ if (error != sizeof(size_attr_value)) {
+ pr_warn("Mapped file has no size attr\n");
+ error = -EINVAL;
+ goto out;
+ }
+
+ source_file_size = le64_to_cpu(size_attr_value);
+ if (args.source_offset + args.size > source_file_size) {
+ pr_warn("Mapped file out of range\n");
+ error = -EINVAL;
+ goto out;
+ }
+
+ /* Find a directory to put the file into. */
+ error = dir_relative_path_resolve(mi,
+ u64_to_user_ptr(args.directory_path),
+ &parent_dir_path, NULL);
+ if (error)
+ goto out;
+
+ if (parent_dir_path.dentry == mi->mi_index_dir) {
+ /* Can't create a file directly inside .index */
+ error = -EBUSY;
+ goto out;
+ }
+
+ /* Look up a dentry in the parent dir. It should be negative. */
+ file_dentry = incfs_lookup_dentry(parent_dir_path.dentry,
+ file_name);
+ if (!file_dentry) {
+ error = -EFAULT;
+ goto out;
+ }
+ if (IS_ERR(file_dentry)) {
+ error = PTR_ERR(file_dentry);
+ file_dentry = NULL;
+ goto out;
+ }
+ if (d_really_is_positive(file_dentry)) {
+ error = -EEXIST;
+ goto out;
+ }
+
+ parent_inode = d_inode(parent_dir_path.dentry);
+ inode_lock_nested(parent_inode, I_MUTEX_PARENT);
+ error = vfs_create(&init_user_ns, parent_inode, file_dentry,
+ args.mode | 0222, true);
+ inode_unlock(parent_inode);
+ if (error)
+ goto out;
+
+ error = chmod(file_dentry, args.mode | 0222);
+ if (error) {
+ pr_debug("incfs: chmod err: %d\n", error);
+ goto delete_file;
+ }
+
+ /* Save the file's size as an xattr for easy fetching in future. */
+ size_attr_value = cpu_to_le64(args.size);
+ error = vfs_setxattr(&init_user_ns, file_dentry, INCFS_XATTR_SIZE_NAME,
+ (char *)&size_attr_value, sizeof(size_attr_value),
+ XATTR_CREATE);
+ if (error) {
+ pr_debug("incfs: vfs_setxattr err:%d\n", error);
+ goto delete_file;
+ }
+
+ error = init_new_mapped_file(mi, file_dentry, &args.source_file_id,
+ args.size, args.source_offset);
+ if (error)
+ goto delete_file;
+
+ notify_create(file, u64_to_user_ptr(args.directory_path), file_name,
+ NULL, false);
+
+ goto out;
+
+delete_file:
+ incfs_unlink(file_dentry);
+
+out:
+ dput(file_dentry);
+ dput(source_file_dentry);
+ path_put(&parent_dir_path);
+ kfree(file_name);
+ kfree(source_file_name);
+ return error;
+}
+
+static long ioctl_get_read_timeouts(struct mount_info *mi, void __user *arg)
+{
+ struct incfs_get_read_timeouts_args __user *args_usr_ptr = arg;
+ struct incfs_get_read_timeouts_args args = {};
+ int error = 0;
+ struct incfs_per_uid_read_timeouts *buffer;
+ int size;
+
+ if (copy_from_user(&args, args_usr_ptr, sizeof(args)))
+ return -EINVAL;
+
+ if (args.timeouts_array_size_out > INCFS_DATA_FILE_BLOCK_SIZE)
+ return -EINVAL;
+
+ buffer = kzalloc(args.timeouts_array_size_out, GFP_NOFS);
+ if (!buffer)
+ return -ENOMEM;
+
+ spin_lock(&mi->mi_per_uid_read_timeouts_lock);
+ size = mi->mi_per_uid_read_timeouts_size;
+ if (args.timeouts_array_size < size)
+ error = -E2BIG;
+ else if (size)
+ memcpy(buffer, mi->mi_per_uid_read_timeouts, size);
+ spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
+
+ args.timeouts_array_size_out = size;
+ if (!error && size)
+ if (copy_to_user(u64_to_user_ptr(args.timeouts_array), buffer,
+ size))
+ error = -EFAULT;
+
+ if (!error || error == -E2BIG)
+ if (copy_to_user(args_usr_ptr, &args, sizeof(args)) > 0)
+ error = -EFAULT;
+
+ kfree(buffer);
+ return error;
+}
+
+static long ioctl_set_read_timeouts(struct mount_info *mi, void __user *arg)
+{
+ struct incfs_set_read_timeouts_args __user *args_usr_ptr = arg;
+ struct incfs_set_read_timeouts_args args = {};
+ int error = 0;
+ int size;
+ struct incfs_per_uid_read_timeouts *buffer = NULL, *tmp;
+ int i;
+
+ if (copy_from_user(&args, args_usr_ptr, sizeof(args)))
+ return -EINVAL;
+
+ size = args.timeouts_array_size;
+ if (size) {
+ if (size > INCFS_DATA_FILE_BLOCK_SIZE ||
+ size % sizeof(*buffer) != 0)
+ return -EINVAL;
+
+ buffer = kzalloc(size, GFP_NOFS);
+ if (!buffer)
+ return -ENOMEM;
+
+ if (copy_from_user(buffer, u64_to_user_ptr(args.timeouts_array),
+ size)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ for (i = 0; i < size / sizeof(*buffer); ++i) {
+ struct incfs_per_uid_read_timeouts *t = &buffer[i];
+
+ if (t->min_pending_time_us > t->max_pending_time_us) {
+ error = -EINVAL;
+ goto out;
+ }
+ }
+ }
+
+ spin_lock(&mi->mi_per_uid_read_timeouts_lock);
+ mi->mi_per_uid_read_timeouts_size = size;
+ tmp = mi->mi_per_uid_read_timeouts;
+ mi->mi_per_uid_read_timeouts = buffer;
+ buffer = tmp;
+ spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
+
+out:
+ kfree(buffer);
+ return error;
+}
+
+static long ioctl_get_last_read_error(struct mount_info *mi, void __user *arg)
+{
+ struct incfs_get_last_read_error_args __user *args_usr_ptr = arg;
+ struct incfs_get_last_read_error_args args = {};
+ int error;
+
+ error = mutex_lock_interruptible(&mi->mi_le_mutex);
+ if (error)
+ return error;
+
+ args.file_id_out = mi->mi_le_file_id;
+ args.time_us_out = mi->mi_le_time_us;
+ args.page_out = mi->mi_le_page;
+ args.errno_out = mi->mi_le_errno;
+ args.uid_out = mi->mi_le_uid;
+
+ mutex_unlock(&mi->mi_le_mutex);
+ if (copy_to_user(args_usr_ptr, &args, sizeof(args)) > 0)
+ error = -EFAULT;
+
+ return error;
+}
+
+static long pending_reads_dispatch_ioctl(struct file *f, unsigned int req,
+ unsigned long arg)
+{
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+
+ switch (req) {
+ case INCFS_IOC_CREATE_FILE:
+ return ioctl_create_file(f, (void __user *)arg);
+ case INCFS_IOC_PERMIT_FILL:
+ return ioctl_permit_fill(f, (void __user *)arg);
+ case INCFS_IOC_CREATE_MAPPED_FILE:
+ return ioctl_create_mapped_file(f, (void __user *)arg);
+ case INCFS_IOC_GET_READ_TIMEOUTS:
+ return ioctl_get_read_timeouts(mi, (void __user *)arg);
+ case INCFS_IOC_SET_READ_TIMEOUTS:
+ return ioctl_set_read_timeouts(mi, (void __user *)arg);
+ case INCFS_IOC_GET_LAST_READ_ERROR:
+ return ioctl_get_last_read_error(mi, (void __user *)arg);
+ default:
+ return -EINVAL;
+ }
+}
+
+static const struct file_operations incfs_pending_reads_file_ops = {
+ .read = pending_reads_read,
+ .poll = pending_reads_poll,
+ .open = pending_reads_open,
+ .release = pending_reads_release,
+ .llseek = noop_llseek,
+ .unlocked_ioctl = pending_reads_dispatch_ioctl,
+ .compat_ioctl = pending_reads_dispatch_ioctl
+};
+
+/*******************************************************************************
+ * .log pseudo file definition
+ ******************************************************************************/
+#define INCFS_LOG_INODE 3
+static const char log_file_name[] = INCFS_LOG_FILENAME;
+
+/* State of an open .log file, unique for each file descriptor. */
+struct log_file_state {
+ struct read_log_state state;
+};
+
+static ssize_t log_read(struct file *f, char __user *buf, size_t len,
+ loff_t *ppos)
+{
+ struct log_file_state *log_state = f->private_data;
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+ int total_reads_collected = 0;
+ int rl_size;
+ ssize_t result = 0;
+ bool report_uid;
+ unsigned long page = 0;
+ struct incfs_pending_read_info *reads_buf = NULL;
+ struct incfs_pending_read_info2 *reads_buf2 = NULL;
+ size_t record_size;
+ ssize_t reads_to_collect;
+ ssize_t reads_per_page;
+
+ if (!mi)
+ return -EFAULT;
+
+ report_uid = mi->mi_options.report_uid;
+ record_size = report_uid ? sizeof(*reads_buf2) : sizeof(*reads_buf);
+ reads_to_collect = len / record_size;
+ reads_per_page = PAGE_SIZE / record_size;
+
+ rl_size = READ_ONCE(mi->mi_log.rl_size);
+ if (rl_size == 0)
+ return 0;
+
+ page = __get_free_page(GFP_NOFS);
+ if (!page)
+ return -ENOMEM;
+
+ if (report_uid)
+ reads_buf2 = (struct incfs_pending_read_info2 *)page;
+ else
+ reads_buf = (struct incfs_pending_read_info *)page;
+
+ reads_to_collect = min_t(ssize_t, rl_size, reads_to_collect);
+ while (reads_to_collect > 0) {
+ struct read_log_state next_state;
+ int reads_collected;
+
+ memcpy(&next_state, &log_state->state, sizeof(next_state));
+ reads_collected = incfs_collect_logged_reads(
+ mi, &next_state, reads_buf, reads_buf2,
+ min_t(ssize_t, reads_to_collect, reads_per_page));
+ if (reads_collected <= 0) {
+ result = total_reads_collected ?
+ total_reads_collected * record_size :
+ reads_collected;
+ goto out;
+ }
+ if (copy_to_user(buf, (void *)page,
+ reads_collected * record_size)) {
+ result = total_reads_collected ?
+ total_reads_collected * record_size :
+ -EFAULT;
+ goto out;
+ }
+
+ memcpy(&log_state->state, &next_state, sizeof(next_state));
+ total_reads_collected += reads_collected;
+ buf += reads_collected * record_size;
+ reads_to_collect -= reads_collected;
+ }
+
+ result = total_reads_collected * record_size;
+ *ppos = 0;
+out:
+ free_page(page);
+ return result;
+}
+
+static __poll_t log_poll(struct file *file, poll_table *wait)
+{
+ struct log_file_state *log_state = file->private_data;
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+ int count;
+ __poll_t ret = 0;
+
+ poll_wait(file, &mi->mi_log.ml_notif_wq, wait);
+ count = incfs_get_uncollected_logs_count(mi, &log_state->state);
+ if (count >= mi->mi_options.read_log_wakeup_count)
+ ret = EPOLLIN | EPOLLRDNORM;
+
+ return ret;
+}
+
+static int log_open(struct inode *inode, struct file *file)
+{
+ struct log_file_state *log_state = NULL;
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+
+ log_state = kzalloc(sizeof(*log_state), GFP_NOFS);
+ if (!log_state)
+ return -ENOMEM;
+
+ log_state->state = incfs_get_log_state(mi);
+ file->private_data = log_state;
+ return 0;
+}
+
+static int log_release(struct inode *inode, struct file *file)
+{
+ kfree(file->private_data);
+ return 0;
+}
+
+static const struct file_operations incfs_log_file_ops = {
+ .read = log_read,
+ .poll = log_poll,
+ .open = log_open,
+ .release = log_release,
+ .llseek = noop_llseek,
+};
+
+/*******************************************************************************
+ * .blocks_written pseudo file definition
+ ******************************************************************************/
+#define INCFS_BLOCKS_WRITTEN_INODE 4
+static const char blocks_written_file_name[] = INCFS_BLOCKS_WRITTEN_FILENAME;
+
+/* State of an open .blocks_written file, unique for each file descriptor. */
+struct blocks_written_file_state {
+ unsigned long blocks_written;
+};
+
+static ssize_t blocks_written_read(struct file *f, char __user *buf, size_t len,
+ loff_t *ppos)
+{
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+ struct blocks_written_file_state *state = f->private_data;
+ unsigned long blocks_written;
+ char string[21];
+ int result = 0;
+
+ if (!mi)
+ return -EFAULT;
+
+ blocks_written = atomic_read(&mi->mi_blocks_written);
+ if (state->blocks_written == blocks_written)
+ return 0;
+
+ result = snprintf(string, sizeof(string), "%lu", blocks_written);
+ if (result > len)
+ result = len;
+ if (copy_to_user(buf, string, result))
+ return -EFAULT;
+
+ state->blocks_written = blocks_written;
+ return result;
+}
+
+static __poll_t blocks_written_poll(struct file *f, poll_table *wait)
+{
+ struct mount_info *mi = get_mount_info(file_superblock(f));
+ struct blocks_written_file_state *state = f->private_data;
+ unsigned long blocks_written;
+
+ if (!mi)
+ return 0;
+
+ poll_wait(f, &mi->mi_blocks_written_notif_wq, wait);
+ blocks_written = atomic_read(&mi->mi_blocks_written);
+ if (state->blocks_written == blocks_written)
+ return 0;
+
+ return EPOLLIN | EPOLLRDNORM;
+}
+
+static int blocks_written_open(struct inode *inode, struct file *file)
+{
+ struct blocks_written_file_state *state =
+ kzalloc(sizeof(*state), GFP_NOFS);
+
+ if (!state)
+ return -ENOMEM;
+
+ state->blocks_written = -1;
+ file->private_data = state;
+ return 0;
+}
+
+static int blocks_written_release(struct inode *inode, struct file *file)
+{
+ kfree(file->private_data);
+ return 0;
+}
+
+static const struct file_operations incfs_blocks_written_file_ops = {
+ .read = blocks_written_read,
+ .poll = blocks_written_poll,
+ .open = blocks_written_open,
+ .release = blocks_written_release,
+ .llseek = noop_llseek,
+};
+
+/*******************************************************************************
+ * Generic inode lookup functionality
+ ******************************************************************************/
+
+const struct mem_range incfs_pseudo_file_names[] = {
+ { .data = (u8 *)pending_reads_file_name,
+ .len = ARRAY_SIZE(pending_reads_file_name) - 1 },
+ { .data = (u8 *)log_file_name, .len = ARRAY_SIZE(log_file_name) - 1 },
+ { .data = (u8 *)blocks_written_file_name,
+ .len = ARRAY_SIZE(blocks_written_file_name) - 1 }
+};
+
+const unsigned long incfs_pseudo_file_inodes[] = { INCFS_PENDING_READS_INODE,
+ INCFS_LOG_INODE,
+ INCFS_BLOCKS_WRITTEN_INODE };
+
+static const struct file_operations *const pseudo_file_operations[] = {
+ &incfs_pending_reads_file_ops, &incfs_log_file_ops,
+ &incfs_blocks_written_file_ops
+};
+
+static bool is_pseudo_filename(struct mem_range name)
+{
+ int i = 0;
+
+ for (; i < ARRAY_SIZE(incfs_pseudo_file_names); ++i)
+ if (incfs_equal_ranges(incfs_pseudo_file_names[i], name))
+ return true;
+ return false;
+}
+
+static bool get_pseudo_inode(int ino, struct inode *inode)
+{
+ int i = 0;
+
+ for (; i < ARRAY_SIZE(incfs_pseudo_file_inodes); ++i)
+ if (ino == incfs_pseudo_file_inodes[i])
+ break;
+ if (i == ARRAY_SIZE(incfs_pseudo_file_inodes))
+ return false;
+
+ inode->i_ctime = (struct timespec64){};
+ inode->i_mtime = inode->i_ctime;
+ inode->i_atime = inode->i_ctime;
+ inode->i_size = 0;
+ inode->i_ino = ino;
+ inode->i_private = NULL;
+ inode_init_owner(&init_user_ns, inode, NULL, S_IFREG | READ_WRITE_FILE_MODE);
+ inode->i_op = &incfs_file_inode_ops;
+ inode->i_fop = pseudo_file_operations[i];
+ return true;
+}
+
+struct inode_search {
+ unsigned long ino;
+};
+
+static int inode_test(struct inode *inode, void *opaque)
+{
+ struct inode_search *search = opaque;
+
+ return inode->i_ino == search->ino;
+}
+
+static int inode_set(struct inode *inode, void *opaque)
+{
+ struct inode_search *search = opaque;
+
+ if (get_pseudo_inode(search->ino, inode))
+ return 0;
+
+ /* Unknown inode requested. */
+ return -EINVAL;
+}
+
+static struct inode *fetch_inode(struct super_block *sb, unsigned long ino)
+{
+ struct inode_search search = {
+ .ino = ino
+ };
+ struct inode *inode = iget5_locked(sb, search.ino, inode_test,
+ inode_set, &search);
+
+ if (!inode)
+ return ERR_PTR(-ENOMEM);
+
+ if (inode->i_state & I_NEW)
+ unlock_new_inode(inode);
+
+ return inode;
+}
+
+int dir_lookup_pseudo_files(struct super_block *sb, struct dentry *dentry)
+{
+ struct mem_range name_range =
+ range((u8 *)dentry->d_name.name, dentry->d_name.len);
+ unsigned long ino;
+ struct inode *inode;
+ int i = 0;
+
+ for (; i < ARRAY_SIZE(incfs_pseudo_file_names); ++i)
+ if (incfs_equal_ranges(incfs_pseudo_file_names[i], name_range))
+ break;
+ if (i == ARRAY_SIZE(incfs_pseudo_file_names))
+ return -ENOENT;
+
+ ino = incfs_pseudo_file_inodes[i];
+
+ inode = fetch_inode(sb, ino);
+ if (IS_ERR(inode))
+ return PTR_ERR(inode);
+
+ d_add(dentry, inode);
+ return 0;
+}
+
+int emit_pseudo_files(struct dir_context *ctx)
+{
+ loff_t i = ctx->pos;
+
+ for (; i < ARRAY_SIZE(incfs_pseudo_file_names); ++i) {
+ if (!dir_emit(ctx, incfs_pseudo_file_names[i].data,
+ incfs_pseudo_file_names[i].len,
+ incfs_pseudo_file_inodes[i], DT_REG))
+ return -EINVAL;
+
+ ctx->pos++;
+ }
+ return 0;
+}
diff --git a/fs/incfs/pseudo_files.h b/fs/incfs/pseudo_files.h
new file mode 100644
index 0000000..1887218
--- /dev/null
+++ b/fs/incfs/pseudo_files.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2020 Google LLC
+ */
+
+#ifndef _INCFS_PSEUDO_FILES_H
+#define _INCFS_PSEUDO_FILES_H
+
+#include "internal.h"
+
+#define PSEUDO_FILE_COUNT 3
+#define INCFS_START_INO_RANGE 10
+
+extern const struct mem_range incfs_pseudo_file_names[PSEUDO_FILE_COUNT];
+extern const unsigned long incfs_pseudo_file_inodes[PSEUDO_FILE_COUNT];
+
+int dir_lookup_pseudo_files(struct super_block *sb, struct dentry *dentry);
+int emit_pseudo_files(struct dir_context *ctx);
+
+#endif
diff --git a/fs/incfs/sysfs.c b/fs/incfs/sysfs.c
new file mode 100644
index 0000000..360f03c
--- /dev/null
+++ b/fs/incfs/sysfs.c
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2021 Google LLC
+ */
+#include <linux/fs.h>
+#include <linux/kobject.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "sysfs.h"
+#include "data_mgmt.h"
+#include "vfs.h"
+
+/******************************************************************************
+ * Define sys/fs/incrementalfs & sys/fs/incrementalfs/features
+ *****************************************************************************/
+#define INCFS_NODE_FEATURES "features"
+#define INCFS_NODE_INSTANCES "instances"
+
+static struct kobject *sysfs_root;
+static struct kobject *features_node;
+static struct kobject *instances_node;
+
+#define DECLARE_FEATURE_FLAG(name) \
+ static ssize_t name##_show(struct kobject *kobj, \
+ struct kobj_attribute *attr, char *buff) \
+{ \
+ return sysfs_emit(buff, "supported\n"); \
+} \
+ \
+static struct kobj_attribute name##_attr = __ATTR_RO(name)
+
+DECLARE_FEATURE_FLAG(corefs);
+DECLARE_FEATURE_FLAG(zstd);
+DECLARE_FEATURE_FLAG(v2);
+
+static struct attribute *attributes[] = {
+ &corefs_attr.attr,
+ &zstd_attr.attr,
+ &v2_attr.attr,
+ NULL,
+};
+
+static const struct attribute_group attr_group = {
+ .attrs = attributes,
+};
+
+int __init incfs_init_sysfs(void)
+{
+ int res = -ENOMEM;
+
+ sysfs_root = kobject_create_and_add(INCFS_NAME, fs_kobj);
+ if (!sysfs_root)
+ return -ENOMEM;
+
+ instances_node = kobject_create_and_add(INCFS_NODE_INSTANCES,
+ sysfs_root);
+ if (!instances_node)
+ goto err_put_root;
+
+ features_node = kobject_create_and_add(INCFS_NODE_FEATURES,
+ sysfs_root);
+ if (!features_node)
+ goto err_put_instances;
+
+ res = sysfs_create_group(features_node, &attr_group);
+ if (res)
+ goto err_put_features;
+
+ return 0;
+
+err_put_features:
+ kobject_put(features_node);
+err_put_instances:
+ kobject_put(instances_node);
+err_put_root:
+ kobject_put(sysfs_root);
+
+ return res;
+}
+
+void incfs_cleanup_sysfs(void)
+{
+ if (features_node) {
+ sysfs_remove_group(features_node, &attr_group);
+ kobject_put(features_node);
+ }
+
+ kobject_put(instances_node);
+ kobject_put(sysfs_root);
+}
+
+/******************************************************************************
+ * Define sys/fs/incrementalfs/instances/<name>/
+ *****************************************************************************/
+#define __DECLARE_STATUS_FLAG(name) \
+static ssize_t name##_show(struct kobject *kobj, \
+ struct kobj_attribute *attr, char *buff) \
+{ \
+ struct incfs_sysfs_node *node = container_of(kobj, \
+ struct incfs_sysfs_node, isn_sysfs_node); \
+ \
+ return sysfs_emit(buff, "%d\n", node->isn_mi->mi_##name); \
+} \
+ \
+static struct kobj_attribute name##_attr = __ATTR_RO(name)
+
+#define __DECLARE_STATUS_FLAG64(name) \
+static ssize_t name##_show(struct kobject *kobj, \
+ struct kobj_attribute *attr, char *buff) \
+{ \
+ struct incfs_sysfs_node *node = container_of(kobj, \
+ struct incfs_sysfs_node, isn_sysfs_node); \
+ \
+ return sysfs_emit(buff, "%lld\n", node->isn_mi->mi_##name); \
+} \
+ \
+static struct kobj_attribute name##_attr = __ATTR_RO(name)
+
+__DECLARE_STATUS_FLAG(reads_failed_timed_out);
+__DECLARE_STATUS_FLAG(reads_failed_hash_verification);
+__DECLARE_STATUS_FLAG(reads_failed_other);
+__DECLARE_STATUS_FLAG(reads_delayed_pending);
+__DECLARE_STATUS_FLAG64(reads_delayed_pending_us);
+__DECLARE_STATUS_FLAG(reads_delayed_min);
+__DECLARE_STATUS_FLAG64(reads_delayed_min_us);
+
+static struct attribute *mount_attributes[] = {
+ &reads_failed_timed_out_attr.attr,
+ &reads_failed_hash_verification_attr.attr,
+ &reads_failed_other_attr.attr,
+ &reads_delayed_pending_attr.attr,
+ &reads_delayed_pending_us_attr.attr,
+ &reads_delayed_min_attr.attr,
+ &reads_delayed_min_us_attr.attr,
+ NULL,
+};
+
+static void incfs_sysfs_release(struct kobject *kobj)
+{
+ struct incfs_sysfs_node *node = container_of(kobj,
+ struct incfs_sysfs_node, isn_sysfs_node);
+
+ complete(&node->isn_completion);
+}
+
+static const struct attribute_group mount_attr_group = {
+ .attrs = mount_attributes,
+};
+
+static struct kobj_type incfs_kobj_node_ktype = {
+ .sysfs_ops = &kobj_sysfs_ops,
+ .release = &incfs_sysfs_release,
+};
+
+struct incfs_sysfs_node *incfs_add_sysfs_node(const char *name,
+ struct mount_info *mi)
+{
+ struct incfs_sysfs_node *node = NULL;
+ int error;
+
+ if (!name)
+ return NULL;
+
+ node = kzalloc(sizeof(*node), GFP_NOFS);
+ if (!node)
+ return ERR_PTR(-ENOMEM);
+
+ node->isn_mi = mi;
+
+ init_completion(&node->isn_completion);
+ kobject_init(&node->isn_sysfs_node, &incfs_kobj_node_ktype);
+ error = kobject_add(&node->isn_sysfs_node, instances_node, "%s", name);
+ if (error)
+ goto err;
+
+ error = sysfs_create_group(&node->isn_sysfs_node, &mount_attr_group);
+ if (error)
+ goto err;
+
+ return node;
+
+err:
+ /*
+ * Note kobject_put always calls release, so incfs_sysfs_release will
+ * free node
+ */
+ kobject_put(&node->isn_sysfs_node);
+ return ERR_PTR(error);
+}
+
+void incfs_free_sysfs_node(struct incfs_sysfs_node *node)
+{
+ if (!node)
+ return;
+
+ sysfs_remove_group(&node->isn_sysfs_node, &mount_attr_group);
+ kobject_put(&node->isn_sysfs_node);
+ wait_for_completion_interruptible(&node->isn_completion);
+ kfree(node);
+}
diff --git a/fs/incfs/sysfs.h b/fs/incfs/sysfs.h
new file mode 100644
index 0000000..65bf554
--- /dev/null
+++ b/fs/incfs/sysfs.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2021 Google LLC
+ */
+#ifndef _INCFS_SYSFS_H
+#define _INCFS_SYSFS_H
+
+struct incfs_sysfs_node {
+ struct kobject isn_sysfs_node;
+
+ struct completion isn_completion;
+
+ struct mount_info *isn_mi;
+};
+
+int incfs_init_sysfs(void);
+void incfs_cleanup_sysfs(void);
+struct incfs_sysfs_node *incfs_add_sysfs_node(const char *name,
+ struct mount_info *mi);
+void incfs_free_sysfs_node(struct incfs_sysfs_node *node);
+
+#endif
diff --git a/fs/incfs/verity.c b/fs/incfs/verity.c
new file mode 100644
index 0000000..b69656b
--- /dev/null
+++ b/fs/incfs/verity.c
@@ -0,0 +1,851 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Google LLC
+ */
+
+/*
+ * fs-verity integration into incfs
+ *
+ * Since incfs has its own merkle tree implementation, most of fs-verity code
+ * is not needed. The key part that is needed is the signature check, since
+ * that is based on the private /proc/sys/fs/verity/require_signatures value
+ * and a private keyring. Thus the first change is to modify verity code to
+ * export a version of fsverity_verify_signature.
+ *
+ * fs-verity integration then consists of the following modifications:
+ *
+ * 1. Add the (optional) verity signature to the incfs file format
+ * 2. Add a pointer to the digest of the fs-verity descriptor struct to the
+ * data_file struct that incfs attaches to each file inode.
+ * 3. Add the following ioclts:
+ * - FS_IOC_ENABLE_VERITY
+ * - FS_IOC_GETFLAGS
+ * - FS_IOC_MEASURE_VERITY
+ * 4. When FS_IOC_ENABLE_VERITY is called on a non-verity file, the
+ * fs-verity descriptor struct is populated and digested. If it passes the
+ * signature check or the signature is NULL and
+ * fs.verity.require_signatures=0, then the S_VERITY flag is set and the
+ * xattr incfs.verity is set. If the signature is non-NULL, an
+ * INCFS_MD_VERITY_SIGNATURE is added to the backing file containing the
+ * signature.
+ * 5. When a file with an incfs.verity xattr's inode is initialized, the
+ * inode’s S_VERITY flag is set.
+ * 6. When a file with the S_VERITY flag set on its inode is opened, the
+ * data_file is checked for its verity digest. If the file doesn’t have a
+ * digest, the file’s digest is calculated as above, checked, and set, or the
+ * open is denied if it is not valid.
+ * 7. FS_IOC_GETFLAGS simply returns the value of the S_VERITY flag
+ * 8. FS_IOC_MEASURE_VERITY simply returns the cached digest
+ * 9. The final complication is that if FS_IOC_ENABLE_VERITY is called on a file
+ * which doesn’t have a merkle tree, the merkle tree is calculated before the
+ * rest of the process is completed.
+ */
+
+#include <crypto/hash.h>
+#include <crypto/sha2.h>
+#include <linux/fsverity.h>
+#include <linux/mount.h>
+
+#include "verity.h"
+
+#include "data_mgmt.h"
+#include "format.h"
+#include "integrity.h"
+#include "vfs.h"
+
+#define FS_VERITY_MAX_SIGNATURE_SIZE 16128
+
+static int incfs_get_root_hash(struct file *filp, u8 *root_hash)
+{
+ struct data_file *df = get_incfs_data_file(filp);
+
+ if (!df)
+ return -EINVAL;
+
+ memcpy(root_hash, df->df_hash_tree->root_hash,
+ df->df_hash_tree->alg->digest_size);
+
+ return 0;
+}
+
+static int incfs_end_enable_verity(struct file *filp, u8 *sig, size_t sig_size)
+{
+ struct inode *inode = file_inode(filp);
+ struct mem_range signature = {
+ .data = sig,
+ .len = sig_size,
+ };
+ struct data_file *df = get_incfs_data_file(filp);
+ struct backing_file_context *bfc;
+ int error;
+ struct incfs_df_verity_signature *vs = NULL;
+ loff_t offset;
+
+ if (!df || !df->df_backing_file_context)
+ return -EFSCORRUPTED;
+
+ if (sig) {
+ vs = kzalloc(sizeof(*vs), GFP_NOFS);
+ if (!vs)
+ return -ENOMEM;
+ }
+
+ bfc = df->df_backing_file_context;
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (error)
+ goto out;
+
+ error = incfs_write_verity_signature_to_backing_file(bfc, signature,
+ &offset);
+ mutex_unlock(&bfc->bc_mutex);
+ if (error)
+ goto out;
+
+ /*
+ * Set verity xattr so we can set S_VERITY without opening backing file
+ */
+ error = vfs_setxattr(&init_user_ns, bfc->bc_file->f_path.dentry,
+ INCFS_XATTR_VERITY_NAME, NULL, 0, XATTR_CREATE);
+ if (error) {
+ pr_warn("incfs: error setting verity xattr: %d\n", error);
+ goto out;
+ }
+
+ if (sig) {
+ *vs = (struct incfs_df_verity_signature) {
+ .size = signature.len,
+ .offset = offset,
+ };
+
+ df->df_verity_signature = vs;
+ vs = NULL;
+ }
+
+ inode_set_flags(inode, S_VERITY, S_VERITY);
+
+out:
+ kfree(vs);
+ return error;
+}
+
+static int incfs_compute_file_digest(struct incfs_hash_alg *alg,
+ struct fsverity_descriptor *desc,
+ u8 *digest)
+{
+ SHASH_DESC_ON_STACK(d, alg->shash);
+
+ d->tfm = alg->shash;
+ return crypto_shash_digest(d, (u8 *)desc, sizeof(*desc), digest);
+}
+
+static enum incfs_hash_tree_algorithm incfs_convert_fsverity_hash_alg(
+ int hash_alg)
+{
+ switch (hash_alg) {
+ case FS_VERITY_HASH_ALG_SHA256:
+ return INCFS_HASH_TREE_SHA256;
+ default:
+ return -EINVAL;
+ }
+}
+
+static struct mem_range incfs_get_verity_digest(struct inode *inode)
+{
+ struct inode_info *node = get_incfs_node(inode);
+ struct data_file *df;
+ struct mem_range verity_file_digest;
+
+ if (!node) {
+ pr_warn("Invalid inode\n");
+ return range(NULL, 0);
+ }
+
+ df = node->n_file;
+
+ /*
+ * Pairs with the cmpxchg_release() in incfs_set_verity_digest().
+ * I.e., another task may publish ->df_verity_file_digest concurrently,
+ * executing a RELEASE barrier. We need to use smp_load_acquire() here
+ * to safely ACQUIRE the memory the other task published.
+ */
+ verity_file_digest.data = smp_load_acquire(
+ &df->df_verity_file_digest.data);
+ verity_file_digest.len = df->df_verity_file_digest.len;
+ return verity_file_digest;
+}
+
+static void incfs_set_verity_digest(struct inode *inode,
+ struct mem_range verity_file_digest)
+{
+ struct inode_info *node = get_incfs_node(inode);
+ struct data_file *df;
+
+ if (!node) {
+ pr_warn("Invalid inode\n");
+ kfree(verity_file_digest.data);
+ return;
+ }
+
+ df = node->n_file;
+ df->df_verity_file_digest.len = verity_file_digest.len;
+
+ /*
+ * Multiple tasks may race to set ->df_verity_file_digest.data, so use
+ * cmpxchg_release(). This pairs with the smp_load_acquire() in
+ * incfs_get_verity_digest(). I.e., here we publish
+ * ->df_verity_file_digest.data, with a RELEASE barrier so that other
+ * tasks can ACQUIRE it.
+ */
+ if (cmpxchg_release(&df->df_verity_file_digest.data, NULL,
+ verity_file_digest.data) != NULL)
+ /* Lost the race, so free the file_digest we allocated. */
+ kfree(verity_file_digest.data);
+}
+
+/*
+ * Calculate the digest of the fsverity_descriptor. The signature (if present)
+ * is also checked.
+ */
+static struct mem_range incfs_calc_verity_digest_from_desc(
+ const struct inode *inode,
+ struct fsverity_descriptor *desc,
+ u8 *signature, size_t sig_size)
+{
+ enum incfs_hash_tree_algorithm incfs_hash_alg;
+ struct mem_range verity_file_digest;
+ int err;
+ struct incfs_hash_alg *hash_alg;
+
+ incfs_hash_alg = incfs_convert_fsverity_hash_alg(desc->hash_algorithm);
+ if (incfs_hash_alg < 0)
+ return range(ERR_PTR(incfs_hash_alg), 0);
+
+ hash_alg = incfs_get_hash_alg(incfs_hash_alg);
+ if (IS_ERR(hash_alg))
+ return range((u8 *)hash_alg, 0);
+
+ verity_file_digest = range(kzalloc(hash_alg->digest_size, GFP_KERNEL),
+ hash_alg->digest_size);
+ if (!verity_file_digest.data)
+ return range(ERR_PTR(-ENOMEM), 0);
+
+ err = incfs_compute_file_digest(hash_alg, desc,
+ verity_file_digest.data);
+ if (err) {
+ pr_err("Error %d computing file digest", err);
+ goto out;
+ }
+ pr_debug("Computed file digest: %s:%*phN\n",
+ hash_alg->name, (int) verity_file_digest.len,
+ verity_file_digest.data);
+
+ err = __fsverity_verify_signature(inode, signature, sig_size,
+ verity_file_digest.data,
+ desc->hash_algorithm);
+out:
+ if (err) {
+ kfree(verity_file_digest.data);
+ verity_file_digest = range(ERR_PTR(err), 0);
+ }
+ return verity_file_digest;
+}
+
+static struct fsverity_descriptor *incfs_get_fsverity_descriptor(
+ struct file *filp, int hash_algorithm)
+{
+ struct inode *inode = file_inode(filp);
+ struct fsverity_descriptor *desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+ int err;
+
+ if (!desc)
+ return ERR_PTR(-ENOMEM);
+
+ *desc = (struct fsverity_descriptor) {
+ .version = 1,
+ .hash_algorithm = hash_algorithm,
+ .log_blocksize = ilog2(INCFS_DATA_FILE_BLOCK_SIZE),
+ .data_size = cpu_to_le64(inode->i_size),
+ };
+
+ err = incfs_get_root_hash(filp, desc->root_hash);
+ if (err) {
+ kfree(desc);
+ return ERR_PTR(err);
+ }
+
+ return desc;
+}
+
+static struct mem_range incfs_calc_verity_digest(
+ struct inode *inode, struct file *filp,
+ u8 *signature, size_t signature_size,
+ int hash_algorithm)
+{
+ struct fsverity_descriptor *desc = incfs_get_fsverity_descriptor(filp,
+ hash_algorithm);
+ struct mem_range verity_file_digest;
+
+ if (IS_ERR(desc))
+ return range((u8 *)desc, 0);
+ verity_file_digest = incfs_calc_verity_digest_from_desc(inode, desc,
+ signature, signature_size);
+ kfree(desc);
+ return verity_file_digest;
+}
+
+static int incfs_build_merkle_tree(struct file *f, struct data_file *df,
+ struct backing_file_context *bfc,
+ struct mtree *hash_tree, loff_t hash_offset,
+ struct incfs_hash_alg *alg, struct mem_range hash)
+{
+ int error = 0;
+ int limit, lvl, i, result;
+ struct mem_range buf = {.len = INCFS_DATA_FILE_BLOCK_SIZE};
+ struct mem_range tmp = {.len = 2 * INCFS_DATA_FILE_BLOCK_SIZE};
+
+ buf.data = (u8 *)__get_free_pages(GFP_NOFS, get_order(buf.len));
+ tmp.data = (u8 *)__get_free_pages(GFP_NOFS, get_order(tmp.len));
+ if (!buf.data || !tmp.data) {
+ error = -ENOMEM;
+ goto out;
+ }
+
+ /*
+ * lvl - 1 is the level we are reading, lvl the level we are writing
+ * lvl == -1 means actual blocks
+ * lvl == hash_tree->depth means root hash
+ */
+ limit = df->df_data_block_count;
+ for (lvl = 0; lvl <= hash_tree->depth; lvl++) {
+ for (i = 0; i < limit; ++i) {
+ loff_t hash_level_offset;
+ struct mem_range partial_buf = buf;
+
+ if (lvl == 0)
+ result = incfs_read_data_file_block(partial_buf,
+ f, i, tmp, NULL);
+ else {
+ hash_level_offset = hash_offset +
+ hash_tree->hash_level_suboffset[lvl - 1];
+
+ result = incfs_kread(bfc, partial_buf.data,
+ partial_buf.len,
+ hash_level_offset + i *
+ INCFS_DATA_FILE_BLOCK_SIZE);
+ }
+
+ if (result < 0) {
+ error = result;
+ goto out;
+ }
+
+ partial_buf.len = result;
+ error = incfs_calc_digest(alg, partial_buf, hash);
+ if (error)
+ goto out;
+
+ /*
+ * last level - only one hash to take and it is stored
+ * in the incfs signature record
+ */
+ if (lvl == hash_tree->depth)
+ break;
+
+ hash_level_offset = hash_offset +
+ hash_tree->hash_level_suboffset[lvl];
+
+ result = incfs_kwrite(bfc, hash.data, hash.len,
+ hash_level_offset + hash.len * i);
+
+ if (result < 0) {
+ error = result;
+ goto out;
+ }
+
+ if (result != hash.len) {
+ error = -EIO;
+ goto out;
+ }
+ }
+ limit = DIV_ROUND_UP(limit,
+ INCFS_DATA_FILE_BLOCK_SIZE / hash.len);
+ }
+
+out:
+ free_pages((unsigned long)tmp.data, get_order(tmp.len));
+ free_pages((unsigned long)buf.data, get_order(buf.len));
+ return error;
+}
+
+/*
+ * incfs files have a signature record that is separate from the
+ * verity_signature record. The signature record does not actually contain a
+ * signature, rather it contains the size/offset of the hash tree, and a binary
+ * blob which contains the root hash and potentially a signature.
+ *
+ * If the file was created with a signature record, then this function simply
+ * returns.
+ *
+ * Otherwise it will create a signature record with a minimal binary blob as
+ * defined by the structure below, create space for the hash tree and then
+ * populate it using incfs_build_merkle_tree
+ */
+static int incfs_add_signature_record(struct file *f)
+{
+ /* See incfs_parse_signature */
+ struct {
+ __le32 version;
+ __le32 size_of_hash_info_section;
+ struct {
+ __le32 hash_algorithm;
+ u8 log2_blocksize;
+ __le32 salt_size;
+ u8 salt[0];
+ __le32 hash_size;
+ u8 root_hash[32];
+ } __packed hash_section;
+ __le32 size_of_signing_info_section;
+ u8 signing_info_section[0];
+ } __packed sig = {
+ .version = cpu_to_le32(INCFS_SIGNATURE_VERSION),
+ .size_of_hash_info_section =
+ cpu_to_le32(sizeof(sig.hash_section)),
+ .hash_section = {
+ .hash_algorithm = cpu_to_le32(INCFS_HASH_TREE_SHA256),
+ .log2_blocksize = ilog2(INCFS_DATA_FILE_BLOCK_SIZE),
+ .hash_size = cpu_to_le32(SHA256_DIGEST_SIZE),
+ },
+ };
+
+ struct data_file *df = get_incfs_data_file(f);
+ struct mtree *hash_tree = NULL;
+ struct backing_file_context *bfc;
+ int error;
+ loff_t hash_offset, sig_offset;
+ struct incfs_hash_alg *alg = incfs_get_hash_alg(INCFS_HASH_TREE_SHA256);
+ u8 hash_buf[INCFS_MAX_HASH_SIZE];
+ int hash_size = alg->digest_size;
+ struct mem_range hash = range(hash_buf, hash_size);
+ int result;
+ struct incfs_df_signature *signature = NULL;
+
+ if (!df)
+ return -EINVAL;
+
+ if (df->df_header_flags & INCFS_FILE_MAPPED)
+ return -EINVAL;
+
+ /* Already signed? */
+ if (df->df_signature && df->df_hash_tree)
+ return 0;
+
+ if (df->df_signature || df->df_hash_tree)
+ return -EFSCORRUPTED;
+
+ /* Add signature metadata record to file */
+ hash_tree = incfs_alloc_mtree(range((u8 *)&sig, sizeof(sig)),
+ df->df_data_block_count);
+ if (IS_ERR(hash_tree))
+ return PTR_ERR(hash_tree);
+
+ bfc = df->df_backing_file_context;
+ if (!bfc) {
+ error = -EFSCORRUPTED;
+ goto out;
+ }
+
+ error = mutex_lock_interruptible(&bfc->bc_mutex);
+ if (error)
+ goto out;
+
+ error = incfs_write_signature_to_backing_file(bfc,
+ range((u8 *)&sig, sizeof(sig)),
+ hash_tree->hash_tree_area_size,
+ &hash_offset, &sig_offset);
+ mutex_unlock(&bfc->bc_mutex);
+ if (error)
+ goto out;
+
+ /* Populate merkle tree */
+ error = incfs_build_merkle_tree(f, df, bfc, hash_tree, hash_offset, alg,
+ hash);
+ if (error)
+ goto out;
+
+ /* Update signature metadata record */
+ memcpy(sig.hash_section.root_hash, hash.data, alg->digest_size);
+ result = incfs_kwrite(bfc, &sig, sizeof(sig), sig_offset);
+ if (result < 0) {
+ error = result;
+ goto out;
+ }
+
+ if (result != sizeof(sig)) {
+ error = -EIO;
+ goto out;
+ }
+
+ /* Update in-memory records */
+ memcpy(hash_tree->root_hash, hash.data, alg->digest_size);
+ signature = kzalloc(sizeof(*signature), GFP_NOFS);
+ if (!signature) {
+ error = -ENOMEM;
+ goto out;
+ }
+ *signature = (struct incfs_df_signature) {
+ .hash_offset = hash_offset,
+ .hash_size = hash_tree->hash_tree_area_size,
+ .sig_offset = sig_offset,
+ .sig_size = sizeof(sig),
+ };
+ df->df_signature = signature;
+ signature = NULL;
+
+ /*
+ * Use memory barrier to prevent readpage seeing the hash tree until
+ * it's fully there
+ */
+ smp_store_release(&df->df_hash_tree, hash_tree);
+ hash_tree = NULL;
+
+out:
+ kfree(signature);
+ kfree(hash_tree);
+ return error;
+}
+
+static int incfs_enable_verity(struct file *filp,
+ const struct fsverity_enable_arg *arg)
+{
+ struct inode *inode = file_inode(filp);
+ struct data_file *df = get_incfs_data_file(filp);
+ u8 *signature = NULL;
+ struct mem_range verity_file_digest = range(NULL, 0);
+ int err;
+
+ if (!df)
+ return -EFSCORRUPTED;
+
+ err = mutex_lock_interruptible(&df->df_enable_verity);
+ if (err)
+ return err;
+
+ if (IS_VERITY(inode)) {
+ err = -EEXIST;
+ goto out;
+ }
+
+ err = incfs_add_signature_record(filp);
+ if (err)
+ goto out;
+
+ /* Get the signature if the user provided one */
+ if (arg->sig_size) {
+ signature = memdup_user(u64_to_user_ptr(arg->sig_ptr),
+ arg->sig_size);
+ if (IS_ERR(signature)) {
+ err = PTR_ERR(signature);
+ signature = NULL;
+ goto out;
+ }
+ }
+
+ verity_file_digest = incfs_calc_verity_digest(inode, filp, signature,
+ arg->sig_size, arg->hash_algorithm);
+ if (IS_ERR(verity_file_digest.data)) {
+ err = PTR_ERR(verity_file_digest.data);
+ verity_file_digest.data = NULL;
+ goto out;
+ }
+
+ err = incfs_end_enable_verity(filp, signature, arg->sig_size);
+ if (err)
+ goto out;
+
+ /* Successfully enabled verity */
+ incfs_set_verity_digest(inode, verity_file_digest);
+ verity_file_digest.data = NULL;
+out:
+ mutex_unlock(&df->df_enable_verity);
+ kfree(signature);
+ kfree(verity_file_digest.data);
+ if (err)
+ pr_err("%s failed with err %d\n", __func__, err);
+ return err;
+}
+
+int incfs_ioctl_enable_verity(struct file *filp, const void __user *uarg)
+{
+ struct inode *inode = file_inode(filp);
+ struct fsverity_enable_arg arg;
+
+ if (copy_from_user(&arg, uarg, sizeof(arg)))
+ return -EFAULT;
+
+ if (arg.version != 1)
+ return -EINVAL;
+
+ if (arg.__reserved1 ||
+ memchr_inv(arg.__reserved2, 0, sizeof(arg.__reserved2)))
+ return -EINVAL;
+
+ if (arg.hash_algorithm != FS_VERITY_HASH_ALG_SHA256)
+ return -EINVAL;
+
+ if (arg.block_size != PAGE_SIZE)
+ return -EINVAL;
+
+ if (arg.salt_size)
+ return -EINVAL;
+
+ if (arg.sig_size > FS_VERITY_MAX_SIGNATURE_SIZE)
+ return -EMSGSIZE;
+
+ if (S_ISDIR(inode->i_mode))
+ return -EISDIR;
+
+ if (!S_ISREG(inode->i_mode))
+ return -EINVAL;
+
+ return incfs_enable_verity(filp, &arg);
+}
+
+static u8 *incfs_get_verity_signature(struct file *filp, size_t *sig_size)
+{
+ struct data_file *df = get_incfs_data_file(filp);
+ struct incfs_df_verity_signature *vs;
+ u8 *signature;
+ int res;
+
+ if (!df || !df->df_backing_file_context)
+ return ERR_PTR(-EFSCORRUPTED);
+
+ vs = df->df_verity_signature;
+ if (!vs) {
+ *sig_size = 0;
+ return NULL;
+ }
+
+ if (!vs->size) {
+ *sig_size = 0;
+ return ERR_PTR(-EFSCORRUPTED);
+ }
+
+ signature = kzalloc(vs->size, GFP_KERNEL);
+ if (!signature)
+ return ERR_PTR(-ENOMEM);
+
+ res = incfs_kread(df->df_backing_file_context,
+ signature, vs->size, vs->offset);
+
+ if (res < 0)
+ goto err_out;
+
+ if (res != vs->size) {
+ res = -EINVAL;
+ goto err_out;
+ }
+
+ *sig_size = vs->size;
+ return signature;
+
+err_out:
+ kfree(signature);
+ return ERR_PTR(res);
+}
+
+/* Ensure data_file->df_verity_file_digest is populated */
+static int ensure_verity_info(struct inode *inode, struct file *filp)
+{
+ struct mem_range verity_file_digest;
+ u8 *signature = NULL;
+ size_t sig_size;
+ int err = 0;
+
+ /* See if this file's verity file digest is already cached */
+ verity_file_digest = incfs_get_verity_digest(inode);
+ if (verity_file_digest.data)
+ return 0;
+
+ signature = incfs_get_verity_signature(filp, &sig_size);
+ if (IS_ERR(signature))
+ return PTR_ERR(signature);
+
+ verity_file_digest = incfs_calc_verity_digest(inode, filp, signature,
+ sig_size,
+ FS_VERITY_HASH_ALG_SHA256);
+ if (IS_ERR(verity_file_digest.data)) {
+ err = PTR_ERR(verity_file_digest.data);
+ goto out;
+ }
+
+ incfs_set_verity_digest(inode, verity_file_digest);
+
+out:
+ kfree(signature);
+ return err;
+}
+
+/**
+ * incfs_fsverity_file_open() - prepare to open a file that may be
+ * verity-enabled
+ * @inode: the inode being opened
+ * @filp: the struct file being set up
+ *
+ * When opening a verity file, set up data_file->df_verity_file_digest if not
+ * already done. Note that incfs does not allow opening for writing, so there is
+ * no need for that check.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+int incfs_fsverity_file_open(struct inode *inode, struct file *filp)
+{
+ if (IS_VERITY(inode))
+ return ensure_verity_info(inode, filp);
+
+ return 0;
+}
+
+int incfs_ioctl_measure_verity(struct file *filp, void __user *_uarg)
+{
+ struct inode *inode = file_inode(filp);
+ struct mem_range verity_file_digest = incfs_get_verity_digest(inode);
+ struct fsverity_digest __user *uarg = _uarg;
+ struct fsverity_digest arg;
+
+ if (!verity_file_digest.data || !verity_file_digest.len)
+ return -ENODATA; /* not a verity file */
+
+ /*
+ * The user specifies the digest_size their buffer has space for; we can
+ * return the digest if it fits in the available space. We write back
+ * the actual size, which may be shorter than the user-specified size.
+ */
+
+ if (get_user(arg.digest_size, &uarg->digest_size))
+ return -EFAULT;
+ if (arg.digest_size < verity_file_digest.len)
+ return -EOVERFLOW;
+
+ memset(&arg, 0, sizeof(arg));
+ arg.digest_algorithm = FS_VERITY_HASH_ALG_SHA256;
+ arg.digest_size = verity_file_digest.len;
+
+ if (copy_to_user(uarg, &arg, sizeof(arg)))
+ return -EFAULT;
+
+ if (copy_to_user(uarg->digest, verity_file_digest.data,
+ verity_file_digest.len))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int incfs_read_merkle_tree(struct file *filp, void __user *buf,
+ u64 start_offset, int length)
+{
+ struct mem_range tmp_buf;
+ size_t offset;
+ int retval = 0;
+ int err = 0;
+ struct data_file *df = get_incfs_data_file(filp);
+
+ if (!df)
+ return -EINVAL;
+
+ tmp_buf = (struct mem_range) {
+ .data = kzalloc(INCFS_DATA_FILE_BLOCK_SIZE, GFP_NOFS),
+ .len = INCFS_DATA_FILE_BLOCK_SIZE,
+ };
+ if (!tmp_buf.data)
+ return -ENOMEM;
+
+ for (offset = start_offset; offset < start_offset + length;
+ offset += tmp_buf.len) {
+ err = incfs_read_merkle_tree_blocks(tmp_buf, df, offset);
+
+ if (err < 0)
+ break;
+
+ if (err != tmp_buf.len)
+ break;
+
+ if (copy_to_user(buf, tmp_buf.data, tmp_buf.len))
+ break;
+
+ buf += tmp_buf.len;
+ retval += tmp_buf.len;
+ }
+
+ kfree(tmp_buf.data);
+ return retval ? retval : err;
+}
+
+static int incfs_read_descriptor(struct file *filp,
+ void __user *buf, u64 offset, int length)
+{
+ int err;
+ struct fsverity_descriptor *desc = incfs_get_fsverity_descriptor(filp,
+ FS_VERITY_HASH_ALG_SHA256);
+
+ if (IS_ERR(desc))
+ return PTR_ERR(desc);
+ length = min_t(u64, length, sizeof(*desc));
+ err = copy_to_user(buf, desc, length);
+ kfree(desc);
+ return err ? err : length;
+}
+
+static int incfs_read_signature(struct file *filp,
+ void __user *buf, u64 offset, int length)
+{
+ size_t sig_size;
+ static u8 *signature;
+ int err;
+
+ signature = incfs_get_verity_signature(filp, &sig_size);
+ if (IS_ERR(signature))
+ return PTR_ERR(signature);
+
+ if (!signature)
+ return -ENODATA;
+
+ length = min_t(u64, length, sig_size);
+ err = copy_to_user(buf, signature, length);
+ kfree(signature);
+ return err ? err : length;
+}
+
+int incfs_ioctl_read_verity_metadata(struct file *filp,
+ const void __user *uarg)
+{
+ struct fsverity_read_metadata_arg arg;
+ int length;
+ void __user *buf;
+
+ if (copy_from_user(&arg, uarg, sizeof(arg)))
+ return -EFAULT;
+
+ if (arg.__reserved)
+ return -EINVAL;
+
+ /* offset + length must not overflow. */
+ if (arg.offset + arg.length < arg.offset)
+ return -EINVAL;
+
+ /* Ensure that the return value will fit in INT_MAX. */
+ length = min_t(u64, arg.length, INT_MAX);
+
+ buf = u64_to_user_ptr(arg.buf_ptr);
+
+ switch (arg.metadata_type) {
+ case FS_VERITY_METADATA_TYPE_MERKLE_TREE:
+ return incfs_read_merkle_tree(filp, buf, arg.offset, length);
+ case FS_VERITY_METADATA_TYPE_DESCRIPTOR:
+ return incfs_read_descriptor(filp, buf, arg.offset, length);
+ case FS_VERITY_METADATA_TYPE_SIGNATURE:
+ return incfs_read_signature(filp, buf, arg.offset, length);
+ default:
+ return -EINVAL;
+ }
+}
diff --git a/fs/incfs/verity.h b/fs/incfs/verity.h
new file mode 100644
index 0000000..8fcdbc8
--- /dev/null
+++ b/fs/incfs/verity.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2020 Google LLC
+ */
+
+#ifndef _INCFS_VERITY_H
+#define _INCFS_VERITY_H
+
+/* Arbitrary limit to bound the kmalloc() size. Can be changed. */
+#define FS_VERITY_MAX_SIGNATURE_SIZE 16128
+
+#ifdef CONFIG_FS_VERITY
+
+int incfs_ioctl_enable_verity(struct file *filp, const void __user *uarg);
+int incfs_ioctl_measure_verity(struct file *filp, void __user *_uarg);
+
+int incfs_fsverity_file_open(struct inode *inode, struct file *filp);
+int incfs_ioctl_read_verity_metadata(struct file *filp,
+ const void __user *uarg);
+
+#else /* !CONFIG_FS_VERITY */
+
+static inline int incfs_ioctl_enable_verity(struct file *filp,
+ const void __user *uarg)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int incfs_ioctl_measure_verity(struct file *filp,
+ void __user *_uarg)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int incfs_fsverity_file_open(struct inode *inode,
+ struct file *filp)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int incfs_ioctl_read_verity_metadata(struct file *filp,
+ const void __user *uarg)
+{
+ return -EOPNOTSUPP;
+}
+
+#endif /* !CONFIG_FS_VERITY */
+
+#endif
diff --git a/fs/incfs/vfs.c b/fs/incfs/vfs.c
new file mode 100644
index 0000000..8f784e0
--- /dev/null
+++ b/fs/incfs/vfs.c
@@ -0,0 +1,1967 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+
+#include <linux/blkdev.h>
+#include <linux/compat.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/fs_stack.h>
+#include <linux/fsnotify.h>
+#include <linux/fsverity.h>
+#include <linux/mmap_lock.h>
+#include <linux/namei.h>
+#include <linux/pagemap.h>
+#include <linux/parser.h>
+#include <linux/seq_file.h>
+#include <linux/backing-dev-defs.h>
+
+#include <uapi/linux/incrementalfs.h>
+
+#include "vfs.h"
+
+#include "data_mgmt.h"
+#include "format.h"
+#include "internal.h"
+#include "pseudo_files.h"
+#include "sysfs.h"
+#include "verity.h"
+
+static int incfs_remount_fs(struct super_block *sb, int *flags, char *data);
+
+static int dentry_revalidate(struct dentry *dentry, unsigned int flags);
+static void dentry_release(struct dentry *d);
+
+static int iterate_incfs_dir(struct file *file, struct dir_context *ctx);
+static struct dentry *dir_lookup(struct inode *dir_inode,
+ struct dentry *dentry, unsigned int flags);
+static int dir_mkdir(struct user_namespace *ns, struct inode *dir,
+ struct dentry *dentry, umode_t mode);
+static int dir_unlink(struct inode *dir, struct dentry *dentry);
+static int dir_link(struct dentry *old_dentry, struct inode *dir,
+ struct dentry *new_dentry);
+static int dir_rmdir(struct inode *dir, struct dentry *dentry);
+static int dir_rename(struct inode *old_dir, struct dentry *old_dentry,
+ struct inode *new_dir, struct dentry *new_dentry,
+ unsigned int flags);
+
+static int file_open(struct inode *inode, struct file *file);
+static int file_release(struct inode *inode, struct file *file);
+static int read_folio(struct file *f, struct folio *folio);
+static long dispatch_ioctl(struct file *f, unsigned int req, unsigned long arg);
+
+#ifdef CONFIG_COMPAT
+static long incfs_compat_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg);
+#endif
+
+static struct inode *alloc_inode(struct super_block *sb);
+static void free_inode(struct inode *inode);
+static void evict_inode(struct inode *inode);
+
+static int incfs_setattr(struct user_namespace *ns, struct dentry *dentry,
+ struct iattr *ia);
+static int incfs_getattr(struct user_namespace *ns, const struct path *path,
+ struct kstat *stat, u32 request_mask,
+ unsigned int query_flags);
+static ssize_t incfs_getxattr(struct dentry *d, const char *name,
+ void *value, size_t size);
+static ssize_t incfs_setxattr(struct user_namespace *ns, struct dentry *d,
+ const char *name, void *value, size_t size,
+ int flags);
+static ssize_t incfs_listxattr(struct dentry *d, char *list, size_t size);
+
+static int show_options(struct seq_file *, struct dentry *);
+
+static const struct super_operations incfs_super_ops = {
+ .statfs = simple_statfs,
+ .remount_fs = incfs_remount_fs,
+ .alloc_inode = alloc_inode,
+ .destroy_inode = free_inode,
+ .evict_inode = evict_inode,
+ .show_options = show_options
+};
+
+static int dir_rename_wrap(struct user_namespace *ns, struct inode *old_dir,
+ struct dentry *old_dentry, struct inode *new_dir,
+ struct dentry *new_dentry, unsigned int flags)
+{
+ return dir_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
+}
+
+static const struct inode_operations incfs_dir_inode_ops = {
+ .lookup = dir_lookup,
+ .mkdir = dir_mkdir,
+ .rename = dir_rename_wrap,
+ .unlink = dir_unlink,
+ .link = dir_link,
+ .rmdir = dir_rmdir,
+ .setattr = incfs_setattr,
+};
+
+static const struct file_operations incfs_dir_fops = {
+ .llseek = generic_file_llseek,
+ .read = generic_read_dir,
+ .iterate = iterate_incfs_dir,
+ .open = file_open,
+ .release = file_release,
+};
+
+static const struct dentry_operations incfs_dentry_ops = {
+ .d_revalidate = dentry_revalidate,
+ .d_release = dentry_release
+};
+
+static const struct address_space_operations incfs_address_space_ops = {
+ .read_folio = read_folio,
+ /* .readpages = readpages */
+};
+
+static vm_fault_t incfs_fault(struct vm_fault *vmf)
+{
+ vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
+ return filemap_fault(vmf);
+}
+
+static const struct vm_operations_struct incfs_file_vm_ops = {
+ .fault = incfs_fault,
+ .map_pages = filemap_map_pages,
+ .page_mkwrite = filemap_page_mkwrite,
+};
+
+/* This is used for a general mmap of a disk file */
+
+static int incfs_file_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct address_space *mapping = file->f_mapping;
+
+ if (!mapping->a_ops->read_folio)
+ return -ENOEXEC;
+ file_accessed(file);
+ vma->vm_ops = &incfs_file_vm_ops;
+ return 0;
+}
+
+const struct file_operations incfs_file_ops = {
+ .open = file_open,
+ .release = file_release,
+ .read_iter = generic_file_read_iter,
+ .mmap = incfs_file_mmap,
+ .splice_read = generic_file_splice_read,
+ .llseek = generic_file_llseek,
+ .unlocked_ioctl = dispatch_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = incfs_compat_ioctl,
+#endif
+};
+
+const struct inode_operations incfs_file_inode_ops = {
+ .setattr = incfs_setattr,
+ .getattr = incfs_getattr,
+ .listxattr = incfs_listxattr
+};
+
+static int incfs_handler_getxattr(const struct xattr_handler *xh,
+ struct dentry *d, struct inode *inode,
+ const char *name, void *buffer, size_t size)
+{
+ return incfs_getxattr(d, name, buffer, size);
+}
+
+static int incfs_handler_setxattr(const struct xattr_handler *xh,
+ struct user_namespace *ns,
+ struct dentry *d, struct inode *inode,
+ const char *name, const void *buffer,
+ size_t size, int flags)
+{
+ return incfs_setxattr(ns, d, name, (void *)buffer, size, flags);
+}
+
+static const struct xattr_handler incfs_xattr_handler = {
+ .prefix = "", /* AKA all attributes */
+ .get = incfs_handler_getxattr,
+ .set = incfs_handler_setxattr,
+};
+
+static const struct xattr_handler *incfs_xattr_ops[] = {
+ &incfs_xattr_handler,
+ NULL,
+};
+
+struct inode_search {
+ unsigned long ino;
+
+ struct dentry *backing_dentry;
+
+ size_t size;
+
+ bool verity;
+};
+
+enum parse_parameter {
+ Opt_read_timeout,
+ Opt_readahead_pages,
+ Opt_rlog_pages,
+ Opt_rlog_wakeup_cnt,
+ Opt_report_uid,
+ Opt_sysfs_name,
+ Opt_err
+};
+
+static const match_table_t option_tokens = {
+ { Opt_read_timeout, "read_timeout_ms=%u" },
+ { Opt_readahead_pages, "readahead=%u" },
+ { Opt_rlog_pages, "rlog_pages=%u" },
+ { Opt_rlog_wakeup_cnt, "rlog_wakeup_cnt=%u" },
+ { Opt_report_uid, "report_uid" },
+ { Opt_sysfs_name, "sysfs_name=%s" },
+ { Opt_err, NULL }
+};
+
+static void free_options(struct mount_options *opts)
+{
+ kfree(opts->sysfs_name);
+ opts->sysfs_name = NULL;
+}
+
+static int parse_options(struct mount_options *opts, char *str)
+{
+ substring_t args[MAX_OPT_ARGS];
+ int value;
+ char *position;
+
+ if (opts == NULL)
+ return -EFAULT;
+
+ *opts = (struct mount_options) {
+ .read_timeout_ms = 1000, /* Default: 1s */
+ .readahead_pages = 10,
+ .read_log_pages = 2,
+ .read_log_wakeup_count = 10,
+ };
+
+ if (str == NULL || *str == 0)
+ return 0;
+
+ while ((position = strsep(&str, ",")) != NULL) {
+ int token;
+
+ if (!*position)
+ continue;
+
+ token = match_token(position, option_tokens, args);
+
+ switch (token) {
+ case Opt_read_timeout:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ if (value > 3600000)
+ return -EINVAL;
+ opts->read_timeout_ms = value;
+ break;
+ case Opt_readahead_pages:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ opts->readahead_pages = value;
+ break;
+ case Opt_rlog_pages:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ opts->read_log_pages = value;
+ break;
+ case Opt_rlog_wakeup_cnt:
+ if (match_int(&args[0], &value))
+ return -EINVAL;
+ opts->read_log_wakeup_count = value;
+ break;
+ case Opt_report_uid:
+ opts->report_uid = true;
+ break;
+ case Opt_sysfs_name:
+ opts->sysfs_name = match_strdup(&args[0]);
+ break;
+ default:
+ free_options(opts);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+/* Read file size from the attribute. Quicker than reading the header */
+static u64 read_size_attr(struct dentry *backing_dentry)
+{
+ __le64 attr_value;
+ ssize_t bytes_read;
+
+ bytes_read = vfs_getxattr(&init_user_ns, backing_dentry, INCFS_XATTR_SIZE_NAME,
+ (char *)&attr_value, sizeof(attr_value));
+
+ if (bytes_read != sizeof(attr_value))
+ return 0;
+
+ return le64_to_cpu(attr_value);
+}
+
+/* Read verity flag from the attribute. Quicker than reading the header */
+static bool read_verity_attr(struct dentry *backing_dentry)
+{
+ return vfs_getxattr(&init_user_ns, backing_dentry, INCFS_XATTR_VERITY_NAME, NULL, 0)
+ >= 0;
+}
+
+static int inode_test(struct inode *inode, void *opaque)
+{
+ struct inode_search *search = opaque;
+ struct inode_info *node = get_incfs_node(inode);
+ struct inode *backing_inode = d_inode(search->backing_dentry);
+
+ if (!node)
+ return 0;
+
+ return node->n_backing_inode == backing_inode &&
+ inode->i_ino == search->ino;
+}
+
+static int inode_set(struct inode *inode, void *opaque)
+{
+ struct inode_search *search = opaque;
+ struct inode_info *node = get_incfs_node(inode);
+ struct dentry *backing_dentry = search->backing_dentry;
+ struct inode *backing_inode = d_inode(backing_dentry);
+
+ fsstack_copy_attr_all(inode, backing_inode);
+ if (S_ISREG(inode->i_mode)) {
+ u64 size = search->size;
+
+ inode->i_size = size;
+ inode->i_blocks = get_blocks_count_for_size(size);
+ inode->i_mapping->a_ops = &incfs_address_space_ops;
+ inode->i_op = &incfs_file_inode_ops;
+ inode->i_fop = &incfs_file_ops;
+ inode->i_mode &= ~0222;
+ if (search->verity)
+ inode_set_flags(inode, S_VERITY, S_VERITY);
+ } else if (S_ISDIR(inode->i_mode)) {
+ inode->i_size = 0;
+ inode->i_blocks = 1;
+ inode->i_mapping->a_ops = &incfs_address_space_ops;
+ inode->i_op = &incfs_dir_inode_ops;
+ inode->i_fop = &incfs_dir_fops;
+ } else {
+ pr_warn_once("incfs: Unexpected inode type\n");
+ return -EBADF;
+ }
+
+ ihold(backing_inode);
+ node->n_backing_inode = backing_inode;
+ node->n_mount_info = get_mount_info(inode->i_sb);
+ inode->i_ctime = backing_inode->i_ctime;
+ inode->i_mtime = backing_inode->i_mtime;
+ inode->i_atime = backing_inode->i_atime;
+ inode->i_ino = backing_inode->i_ino;
+ if (backing_inode->i_ino < INCFS_START_INO_RANGE) {
+ pr_warn("incfs: ino conflict with backing FS %ld\n",
+ backing_inode->i_ino);
+ }
+
+ return 0;
+}
+
+static struct inode *fetch_regular_inode(struct super_block *sb,
+ struct dentry *backing_dentry)
+{
+ struct inode *backing_inode = d_inode(backing_dentry);
+ struct inode_search search = {
+ .ino = backing_inode->i_ino,
+ .backing_dentry = backing_dentry,
+ .size = read_size_attr(backing_dentry),
+ .verity = read_verity_attr(backing_dentry),
+ };
+ struct inode *inode = iget5_locked(sb, search.ino, inode_test,
+ inode_set, &search);
+
+ if (!inode)
+ return ERR_PTR(-ENOMEM);
+
+ if (inode->i_state & I_NEW)
+ unlock_new_inode(inode);
+
+ return inode;
+}
+
+static int iterate_incfs_dir(struct file *file, struct dir_context *ctx)
+{
+ struct dir_file *dir = get_incfs_dir_file(file);
+ int error = 0;
+ struct mount_info *mi = get_mount_info(file_superblock(file));
+ bool root;
+
+ if (!dir) {
+ error = -EBADF;
+ goto out;
+ }
+
+ root = dir->backing_dir->f_inode
+ == d_inode(mi->mi_backing_dir_path.dentry);
+
+ if (root) {
+ error = emit_pseudo_files(ctx);
+ if (error)
+ goto out;
+ }
+
+ ctx->pos -= PSEUDO_FILE_COUNT;
+ error = iterate_dir(dir->backing_dir, ctx);
+ ctx->pos += PSEUDO_FILE_COUNT;
+ file->f_pos = dir->backing_dir->f_pos;
+out:
+ if (error)
+ pr_warn("incfs: %s %s %d\n", __func__,
+ file->f_path.dentry->d_name.name, error);
+ return error;
+}
+
+static int incfs_init_dentry(struct dentry *dentry, struct path *path)
+{
+ struct dentry_info *d_info = NULL;
+
+ if (!dentry || !path)
+ return -EFAULT;
+
+ d_info = kzalloc(sizeof(*d_info), GFP_NOFS);
+ if (!d_info)
+ return -ENOMEM;
+
+ d_info->backing_path = *path;
+ path_get(path);
+
+ dentry->d_fsdata = d_info;
+ return 0;
+}
+
+static struct dentry *open_or_create_special_dir(struct dentry *backing_dir,
+ const char *name,
+ bool *created)
+{
+ struct dentry *index_dentry;
+ struct inode *backing_inode = d_inode(backing_dir);
+ int err = 0;
+
+ index_dentry = incfs_lookup_dentry(backing_dir, name);
+ if (!index_dentry) {
+ return ERR_PTR(-EINVAL);
+ } else if (IS_ERR(index_dentry)) {
+ return index_dentry;
+ } else if (d_really_is_positive(index_dentry)) {
+ /* Index already exists. */
+ *created = false;
+ return index_dentry;
+ }
+
+ /* Index needs to be created. */
+ inode_lock_nested(backing_inode, I_MUTEX_PARENT);
+ err = vfs_mkdir(&init_user_ns, backing_inode, index_dentry, 0777);
+ inode_unlock(backing_inode);
+
+ if (err) {
+ dput(index_dentry);
+ return ERR_PTR(err);
+ }
+
+ if (!d_really_is_positive(index_dentry) ||
+ unlikely(d_unhashed(index_dentry))) {
+ dput(index_dentry);
+ return ERR_PTR(-EINVAL);
+ }
+
+ *created = true;
+ return index_dentry;
+}
+
+static int read_single_page_timeouts(struct data_file *df, struct file *f,
+ int block_index, struct mem_range range,
+ struct mem_range tmp)
+{
+ struct mount_info *mi = df->df_mount_info;
+ struct incfs_read_data_file_timeouts timeouts = {
+ .max_pending_time_us = U32_MAX,
+ };
+ int uid = current_uid().val;
+ int i;
+
+ spin_lock(&mi->mi_per_uid_read_timeouts_lock);
+ for (i = 0; i < mi->mi_per_uid_read_timeouts_size /
+ sizeof(*mi->mi_per_uid_read_timeouts); ++i) {
+ struct incfs_per_uid_read_timeouts *t =
+ &mi->mi_per_uid_read_timeouts[i];
+
+ if(t->uid == uid) {
+ timeouts.min_time_us = t->min_time_us;
+ timeouts.min_pending_time_us = t->min_pending_time_us;
+ timeouts.max_pending_time_us = t->max_pending_time_us;
+ break;
+ }
+ }
+ spin_unlock(&mi->mi_per_uid_read_timeouts_lock);
+ if (timeouts.max_pending_time_us == U32_MAX) {
+ u64 read_timeout_us = (u64)mi->mi_options.read_timeout_ms *
+ 1000;
+
+ timeouts.max_pending_time_us = read_timeout_us <= U32_MAX ?
+ read_timeout_us : U32_MAX;
+ }
+
+ return incfs_read_data_file_block(range, f, block_index, tmp,
+ &timeouts);
+}
+
+static int read_folio(struct file *f, struct folio *folio)
+{
+ struct page *page = &folio->page;
+ loff_t offset = 0;
+ loff_t size = 0;
+ ssize_t bytes_to_read = 0;
+ ssize_t read_result = 0;
+ struct data_file *df = get_incfs_data_file(f);
+ int result = 0;
+ void *page_start;
+ int block_index;
+
+ if (!df) {
+ SetPageError(page);
+ unlock_page(page);
+ return -EBADF;
+ }
+
+ page_start = kmap(page);
+ offset = page_offset(page);
+ block_index = (offset + df->df_mapped_offset) /
+ INCFS_DATA_FILE_BLOCK_SIZE;
+ size = df->df_size;
+
+ if (offset < size) {
+ struct mem_range tmp = {
+ .len = 2 * INCFS_DATA_FILE_BLOCK_SIZE
+ };
+ tmp.data = (u8 *)__get_free_pages(GFP_NOFS, get_order(tmp.len));
+ if (!tmp.data) {
+ read_result = -ENOMEM;
+ goto err;
+ }
+ bytes_to_read = min_t(loff_t, size - offset, PAGE_SIZE);
+
+ read_result = read_single_page_timeouts(df, f, block_index,
+ range(page_start, bytes_to_read), tmp);
+
+ free_pages((unsigned long)tmp.data, get_order(tmp.len));
+ } else {
+ bytes_to_read = 0;
+ read_result = 0;
+ }
+
+err:
+ if (read_result < 0)
+ result = read_result;
+ else if (read_result < PAGE_SIZE)
+ zero_user(page, read_result, PAGE_SIZE - read_result);
+
+ if (result == 0)
+ SetPageUptodate(page);
+ else
+ SetPageError(page);
+
+ flush_dcache_page(page);
+ kunmap(page);
+ unlock_page(page);
+ return result;
+}
+
+int incfs_link(struct dentry *what, struct dentry *where)
+{
+ struct dentry *parent_dentry = dget_parent(where);
+ struct inode *pinode = d_inode(parent_dentry);
+ int error = 0;
+
+ inode_lock_nested(pinode, I_MUTEX_PARENT);
+ error = vfs_link(what, &init_user_ns, pinode, where, NULL);
+ inode_unlock(pinode);
+
+ dput(parent_dentry);
+ return error;
+}
+
+int incfs_unlink(struct dentry *dentry)
+{
+ struct dentry *parent_dentry = dget_parent(dentry);
+ struct inode *pinode = d_inode(parent_dentry);
+ int error = 0;
+
+ inode_lock_nested(pinode, I_MUTEX_PARENT);
+ error = vfs_unlink(&init_user_ns, pinode, dentry, NULL);
+ inode_unlock(pinode);
+
+ dput(parent_dentry);
+ return error;
+}
+
+static int incfs_rmdir(struct dentry *dentry)
+{
+ struct dentry *parent_dentry = dget_parent(dentry);
+ struct inode *pinode = d_inode(parent_dentry);
+ int error = 0;
+
+ inode_lock_nested(pinode, I_MUTEX_PARENT);
+ error = vfs_rmdir(&init_user_ns, pinode, dentry);
+ inode_unlock(pinode);
+
+ dput(parent_dentry);
+ return error;
+}
+
+static void notify_unlink(struct dentry *dentry, const char *file_id_str,
+ const char *special_directory)
+{
+ struct dentry *root = dentry;
+ struct dentry *file = NULL;
+ struct dentry *dir = NULL;
+ int error = 0;
+ bool take_lock = root->d_parent != root->d_parent->d_parent;
+
+ while (root != root->d_parent)
+ root = root->d_parent;
+
+ if (take_lock)
+ dir = incfs_lookup_dentry(root, special_directory);
+ else
+ dir = lookup_one_len(special_directory, root,
+ strlen(special_directory));
+
+ if (IS_ERR(dir)) {
+ error = PTR_ERR(dir);
+ goto out;
+ }
+ if (d_is_negative(dir)) {
+ error = -ENOENT;
+ goto out;
+ }
+
+ file = incfs_lookup_dentry(dir, file_id_str);
+ if (IS_ERR(file)) {
+ error = PTR_ERR(file);
+ goto out;
+ }
+ if (d_is_negative(file)) {
+ error = -ENOENT;
+ goto out;
+ }
+
+ fsnotify_unlink(d_inode(dir), file);
+ d_delete(file);
+
+out:
+ if (error)
+ pr_warn("%s failed with error %d\n", __func__, error);
+
+ dput(dir);
+ dput(file);
+}
+
+static void maybe_delete_incomplete_file(struct file *f,
+ struct data_file *df)
+{
+ struct backing_file_context *bfc;
+ struct mount_info *mi = df->df_mount_info;
+ char *file_id_str = NULL;
+ struct dentry *incomplete_file_dentry = NULL;
+ const struct cred *old_cred = override_creds(mi->mi_owner);
+ int error;
+
+ if (atomic_read(&df->df_data_blocks_written) < df->df_data_block_count)
+ goto out;
+
+ /* Truncate file to remove any preallocated space */
+ bfc = df->df_backing_file_context;
+ if (bfc) {
+ struct file *f = bfc->bc_file;
+
+ if (f) {
+ loff_t size = i_size_read(file_inode(f));
+
+ error = vfs_truncate(&f->f_path, size);
+ if (error)
+ /* No useful action on failure */
+ pr_warn("incfs: Failed to truncate complete file: %d\n",
+ error);
+ }
+ }
+
+ /* This is best effort - there is no useful action to take on failure */
+ file_id_str = file_id_to_str(df->df_id);
+ if (!file_id_str)
+ goto out;
+
+ incomplete_file_dentry = incfs_lookup_dentry(
+ df->df_mount_info->mi_incomplete_dir,
+ file_id_str);
+ if (!incomplete_file_dentry || IS_ERR(incomplete_file_dentry)) {
+ incomplete_file_dentry = NULL;
+ goto out;
+ }
+
+ if (!d_really_is_positive(incomplete_file_dentry))
+ goto out;
+
+ vfs_fsync(df->df_backing_file_context->bc_file, 0);
+ error = incfs_unlink(incomplete_file_dentry);
+ if (error) {
+ pr_warn("incfs: Deleting incomplete file failed: %d\n", error);
+ goto out;
+ }
+
+ notify_unlink(f->f_path.dentry, file_id_str, INCFS_INCOMPLETE_NAME);
+
+out:
+ dput(incomplete_file_dentry);
+ kfree(file_id_str);
+ revert_creds(old_cred);
+}
+
+static long ioctl_fill_blocks(struct file *f, void __user *arg)
+{
+ struct incfs_fill_blocks __user *usr_fill_blocks = arg;
+ struct incfs_fill_blocks fill_blocks;
+ struct incfs_fill_block __user *usr_fill_block_array;
+ struct data_file *df = get_incfs_data_file(f);
+ struct incfs_file_data *fd = f->private_data;
+ const ssize_t data_buf_size = 2 * INCFS_DATA_FILE_BLOCK_SIZE;
+ u8 *data_buf = NULL;
+ ssize_t error = 0;
+ int i = 0;
+
+ if (!df)
+ return -EBADF;
+
+ if (!fd || fd->fd_fill_permission != CAN_FILL)
+ return -EPERM;
+
+ if (copy_from_user(&fill_blocks, usr_fill_blocks, sizeof(fill_blocks)))
+ return -EFAULT;
+
+ usr_fill_block_array = u64_to_user_ptr(fill_blocks.fill_blocks);
+ data_buf = (u8 *)__get_free_pages(GFP_NOFS | __GFP_COMP,
+ get_order(data_buf_size));
+ if (!data_buf)
+ return -ENOMEM;
+
+ for (i = 0; i < fill_blocks.count; i++) {
+ struct incfs_fill_block fill_block = {};
+
+ if (copy_from_user(&fill_block, &usr_fill_block_array[i],
+ sizeof(fill_block)) > 0) {
+ error = -EFAULT;
+ break;
+ }
+
+ if (fill_block.data_len > data_buf_size) {
+ error = -E2BIG;
+ break;
+ }
+
+ if (copy_from_user(data_buf, u64_to_user_ptr(fill_block.data),
+ fill_block.data_len) > 0) {
+ error = -EFAULT;
+ break;
+ }
+ fill_block.data = 0; /* To make sure nobody uses it. */
+ if (fill_block.flags & INCFS_BLOCK_FLAGS_HASH) {
+ error = incfs_process_new_hash_block(df, &fill_block,
+ data_buf);
+ } else {
+ error = incfs_process_new_data_block(df, &fill_block,
+ data_buf);
+ }
+ if (error)
+ break;
+ }
+
+ if (data_buf)
+ free_pages((unsigned long)data_buf, get_order(data_buf_size));
+
+ maybe_delete_incomplete_file(f, df);
+
+ /*
+ * Only report the error if no records were processed, otherwise
+ * just return how many were processed successfully.
+ */
+ if (i == 0)
+ return error;
+
+ return i;
+}
+
+static long ioctl_read_file_signature(struct file *f, void __user *arg)
+{
+ struct incfs_get_file_sig_args __user *args_usr_ptr = arg;
+ struct incfs_get_file_sig_args args = {};
+ u8 *sig_buffer = NULL;
+ size_t sig_buf_size = 0;
+ int error = 0;
+ int read_result = 0;
+ struct data_file *df = get_incfs_data_file(f);
+
+ if (!df)
+ return -EINVAL;
+
+ if (copy_from_user(&args, args_usr_ptr, sizeof(args)) > 0)
+ return -EINVAL;
+
+ sig_buf_size = args.file_signature_buf_size;
+ if (sig_buf_size > INCFS_MAX_SIGNATURE_SIZE)
+ return -E2BIG;
+
+ sig_buffer = kzalloc(sig_buf_size, GFP_NOFS | __GFP_COMP);
+ if (!sig_buffer)
+ return -ENOMEM;
+
+ read_result = incfs_read_file_signature(df,
+ range(sig_buffer, sig_buf_size));
+
+ if (read_result < 0) {
+ error = read_result;
+ goto out;
+ }
+
+ if (copy_to_user(u64_to_user_ptr(args.file_signature), sig_buffer,
+ read_result)) {
+ error = -EFAULT;
+ goto out;
+ }
+
+ args.file_signature_len_out = read_result;
+ if (copy_to_user(args_usr_ptr, &args, sizeof(args)))
+ error = -EFAULT;
+
+out:
+ kfree(sig_buffer);
+
+ return error;
+}
+
+static long ioctl_get_filled_blocks(struct file *f, void __user *arg)
+{
+ struct incfs_get_filled_blocks_args __user *args_usr_ptr = arg;
+ struct incfs_get_filled_blocks_args args = {};
+ struct data_file *df = get_incfs_data_file(f);
+ struct incfs_file_data *fd = f->private_data;
+ int error;
+
+ if (!df || !fd)
+ return -EINVAL;
+
+ if (fd->fd_fill_permission != CAN_FILL)
+ return -EPERM;
+
+ if (copy_from_user(&args, args_usr_ptr, sizeof(args)) > 0)
+ return -EINVAL;
+
+ error = incfs_get_filled_blocks(df, fd, &args);
+
+ if (copy_to_user(args_usr_ptr, &args, sizeof(args)))
+ return -EFAULT;
+
+ return error;
+}
+
+static long ioctl_get_block_count(struct file *f, void __user *arg)
+{
+ struct incfs_get_block_count_args __user *args_usr_ptr = arg;
+ struct incfs_get_block_count_args args = {};
+ struct data_file *df = get_incfs_data_file(f);
+
+ if (!df)
+ return -EINVAL;
+
+ args.total_data_blocks_out = df->df_data_block_count;
+ args.filled_data_blocks_out = atomic_read(&df->df_data_blocks_written);
+ args.total_hash_blocks_out = df->df_total_block_count -
+ df->df_data_block_count;
+ args.filled_hash_blocks_out = atomic_read(&df->df_hash_blocks_written);
+
+ if (copy_to_user(args_usr_ptr, &args, sizeof(args)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int incfs_ioctl_get_flags(struct file *f, void __user *arg)
+{
+ u32 flags = IS_VERITY(file_inode(f)) ? FS_VERITY_FL : 0;
+
+ return put_user(flags, (int __user *) arg);
+}
+
+static long dispatch_ioctl(struct file *f, unsigned int req, unsigned long arg)
+{
+ switch (req) {
+ case INCFS_IOC_FILL_BLOCKS:
+ return ioctl_fill_blocks(f, (void __user *)arg);
+ case INCFS_IOC_READ_FILE_SIGNATURE:
+ return ioctl_read_file_signature(f, (void __user *)arg);
+ case INCFS_IOC_GET_FILLED_BLOCKS:
+ return ioctl_get_filled_blocks(f, (void __user *)arg);
+ case INCFS_IOC_GET_BLOCK_COUNT:
+ return ioctl_get_block_count(f, (void __user *)arg);
+ case FS_IOC_ENABLE_VERITY:
+ return incfs_ioctl_enable_verity(f, (const void __user *)arg);
+ case FS_IOC_GETFLAGS:
+ return incfs_ioctl_get_flags(f, (void __user *) arg);
+ case FS_IOC_MEASURE_VERITY:
+ return incfs_ioctl_measure_verity(f, (void __user *)arg);
+ case FS_IOC_READ_VERITY_METADATA:
+ return incfs_ioctl_read_verity_metadata(f, (void __user *)arg);
+ default:
+ return -EINVAL;
+ }
+}
+
+#ifdef CONFIG_COMPAT
+static long incfs_compat_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ switch (cmd) {
+ case FS_IOC32_GETFLAGS:
+ cmd = FS_IOC_GETFLAGS;
+ break;
+ case INCFS_IOC_FILL_BLOCKS:
+ case INCFS_IOC_READ_FILE_SIGNATURE:
+ case INCFS_IOC_GET_FILLED_BLOCKS:
+ case INCFS_IOC_GET_BLOCK_COUNT:
+ case FS_IOC_ENABLE_VERITY:
+ case FS_IOC_MEASURE_VERITY:
+ case FS_IOC_READ_VERITY_METADATA:
+ break;
+ default:
+ return -ENOIOCTLCMD;
+ }
+ return dispatch_ioctl(file, cmd, (unsigned long) compat_ptr(arg));
+}
+#endif
+
+static struct dentry *dir_lookup(struct inode *dir_inode, struct dentry *dentry,
+ unsigned int flags)
+{
+ struct mount_info *mi = get_mount_info(dir_inode->i_sb);
+ struct dentry *dir_dentry = NULL;
+ struct dentry *backing_dentry = NULL;
+ struct path dir_backing_path = {};
+ struct inode_info *dir_info = get_incfs_node(dir_inode);
+ int err = 0;
+
+ if (!mi || !dir_info || !dir_info->n_backing_inode)
+ return ERR_PTR(-EBADF);
+
+ if (d_inode(mi->mi_backing_dir_path.dentry) ==
+ dir_info->n_backing_inode) {
+ /* We do lookup in the FS root. Show pseudo files. */
+ err = dir_lookup_pseudo_files(dir_inode->i_sb, dentry);
+ if (err != -ENOENT)
+ goto out;
+ err = 0;
+ }
+
+ dir_dentry = dget_parent(dentry);
+ get_incfs_backing_path(dir_dentry, &dir_backing_path);
+ backing_dentry = incfs_lookup_dentry(dir_backing_path.dentry,
+ dentry->d_name.name);
+
+ if (!backing_dentry || IS_ERR(backing_dentry)) {
+ err = IS_ERR(backing_dentry)
+ ? PTR_ERR(backing_dentry)
+ : -EFAULT;
+ backing_dentry = NULL;
+ goto out;
+ } else {
+ struct inode *inode = NULL;
+ struct path backing_path = {
+ .mnt = dir_backing_path.mnt,
+ .dentry = backing_dentry
+ };
+
+ err = incfs_init_dentry(dentry, &backing_path);
+ if (err)
+ goto out;
+
+ if (!d_really_is_positive(backing_dentry)) {
+ /*
+ * No such entry found in the backing dir.
+ * Create a negative entry.
+ */
+ d_add(dentry, NULL);
+ err = 0;
+ goto out;
+ }
+
+ if (d_inode(backing_dentry)->i_sb !=
+ dir_info->n_backing_inode->i_sb) {
+ /*
+ * Somehow after the path lookup we ended up in a
+ * different fs mount. If we keep going it's going
+ * to end badly.
+ */
+ err = -EXDEV;
+ goto out;
+ }
+
+ inode = fetch_regular_inode(dir_inode->i_sb, backing_dentry);
+ if (IS_ERR(inode)) {
+ err = PTR_ERR(inode);
+ goto out;
+ }
+
+ d_add(dentry, inode);
+ }
+
+out:
+ dput(dir_dentry);
+ dput(backing_dentry);
+ path_put(&dir_backing_path);
+ if (err)
+ pr_debug("incfs: %s %s %d\n", __func__,
+ dentry->d_name.name, err);
+ return ERR_PTR(err);
+}
+
+static int dir_mkdir(struct user_namespace *ns, struct inode *dir, struct dentry *dentry, umode_t mode)
+{
+ struct mount_info *mi = get_mount_info(dir->i_sb);
+ struct inode_info *dir_node = get_incfs_node(dir);
+ struct dentry *backing_dentry = NULL;
+ struct path backing_path = {};
+ int err = 0;
+
+
+ if (!mi || !dir_node || !dir_node->n_backing_inode)
+ return -EBADF;
+
+ err = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (err)
+ return err;
+
+ get_incfs_backing_path(dentry, &backing_path);
+ backing_dentry = backing_path.dentry;
+
+ if (!backing_dentry) {
+ err = -EBADF;
+ goto path_err;
+ }
+
+ if (backing_dentry->d_parent == mi->mi_index_dir) {
+ /* Can't create a subdir inside .index */
+ err = -EBUSY;
+ goto out;
+ }
+
+ if (backing_dentry->d_parent == mi->mi_incomplete_dir) {
+ /* Can't create a subdir inside .incomplete */
+ err = -EBUSY;
+ goto out;
+ }
+ inode_lock_nested(dir_node->n_backing_inode, I_MUTEX_PARENT);
+ err = vfs_mkdir(ns, dir_node->n_backing_inode, backing_dentry, mode | 0222);
+ inode_unlock(dir_node->n_backing_inode);
+ if (!err) {
+ struct inode *inode = NULL;
+
+ if (d_really_is_negative(backing_dentry) ||
+ unlikely(d_unhashed(backing_dentry))) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ inode = fetch_regular_inode(dir->i_sb, backing_dentry);
+ if (IS_ERR(inode)) {
+ err = PTR_ERR(inode);
+ goto out;
+ }
+ d_instantiate(dentry, inode);
+ }
+
+out:
+ if (d_really_is_negative(dentry))
+ d_drop(dentry);
+ path_put(&backing_path);
+
+path_err:
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ if (err)
+ pr_debug("incfs: %s err:%d\n", __func__, err);
+ return err;
+}
+
+/*
+ * Delete file referenced by backing_dentry and if appropriate its hardlink
+ * from .index and .incomplete
+ */
+static int file_delete(struct mount_info *mi, struct dentry *dentry,
+ struct dentry *backing_dentry, int nlink)
+{
+ struct dentry *index_file_dentry = NULL;
+ struct dentry *incomplete_file_dentry = NULL;
+ /* 2 chars per byte of file ID + 1 char for \0 */
+ char file_id_str[2 * sizeof(incfs_uuid_t) + 1] = {0};
+ ssize_t uuid_size = 0;
+ int error = 0;
+
+ WARN_ON(!mutex_is_locked(&mi->mi_dir_struct_mutex));
+
+ if (nlink > 3)
+ goto just_unlink;
+
+ uuid_size = vfs_getxattr(&init_user_ns, backing_dentry, INCFS_XATTR_ID_NAME,
+ file_id_str, 2 * sizeof(incfs_uuid_t));
+ if (uuid_size < 0) {
+ error = uuid_size;
+ goto out;
+ }
+
+ if (uuid_size != 2 * sizeof(incfs_uuid_t)) {
+ error = -EBADMSG;
+ goto out;
+ }
+
+ index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir, file_id_str);
+ if (IS_ERR(index_file_dentry)) {
+ error = PTR_ERR(index_file_dentry);
+ index_file_dentry = NULL;
+ goto out;
+ }
+
+ if (d_really_is_positive(index_file_dentry) && nlink > 0)
+ nlink--;
+
+ if (nlink > 2)
+ goto just_unlink;
+
+ incomplete_file_dentry = incfs_lookup_dentry(mi->mi_incomplete_dir,
+ file_id_str);
+ if (IS_ERR(incomplete_file_dentry)) {
+ error = PTR_ERR(incomplete_file_dentry);
+ incomplete_file_dentry = NULL;
+ goto out;
+ }
+
+ if (d_really_is_positive(incomplete_file_dentry) && nlink > 0)
+ nlink--;
+
+ if (nlink > 1)
+ goto just_unlink;
+
+ if (d_really_is_positive(index_file_dentry)) {
+ error = incfs_unlink(index_file_dentry);
+ if (error)
+ goto out;
+ notify_unlink(dentry, file_id_str, INCFS_INDEX_NAME);
+ }
+
+ if (d_really_is_positive(incomplete_file_dentry)) {
+ error = incfs_unlink(incomplete_file_dentry);
+ if (error)
+ goto out;
+ notify_unlink(dentry, file_id_str, INCFS_INCOMPLETE_NAME);
+ }
+
+just_unlink:
+ error = incfs_unlink(backing_dentry);
+
+out:
+ dput(index_file_dentry);
+ dput(incomplete_file_dentry);
+ if (error)
+ pr_debug("incfs: delete_file_from_index err:%d\n", error);
+ return error;
+}
+
+static int dir_unlink(struct inode *dir, struct dentry *dentry)
+{
+ struct mount_info *mi = get_mount_info(dir->i_sb);
+ struct path backing_path = {};
+ struct kstat stat;
+ int err = 0;
+
+ if (!mi)
+ return -EBADF;
+
+ err = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (err)
+ return err;
+
+ get_incfs_backing_path(dentry, &backing_path);
+ if (!backing_path.dentry) {
+ err = -EBADF;
+ goto path_err;
+ }
+
+ if (backing_path.dentry->d_parent == mi->mi_index_dir) {
+ /* Direct unlink from .index are not allowed. */
+ err = -EBUSY;
+ goto out;
+ }
+
+ if (backing_path.dentry->d_parent == mi->mi_incomplete_dir) {
+ /* Direct unlink from .incomplete are not allowed. */
+ err = -EBUSY;
+ goto out;
+ }
+
+ err = vfs_getattr(&backing_path, &stat, STATX_NLINK,
+ AT_STATX_SYNC_AS_STAT);
+ if (err)
+ goto out;
+
+ err = file_delete(mi, dentry, backing_path.dentry, stat.nlink);
+
+ d_drop(dentry);
+out:
+ path_put(&backing_path);
+path_err:
+ if (err)
+ pr_debug("incfs: %s err:%d\n", __func__, err);
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ return err;
+}
+
+static int dir_link(struct dentry *old_dentry, struct inode *dir,
+ struct dentry *new_dentry)
+{
+ struct mount_info *mi = get_mount_info(dir->i_sb);
+ struct path backing_old_path = {};
+ struct path backing_new_path = {};
+ int error = 0;
+
+ if (!mi)
+ return -EBADF;
+
+ error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (error)
+ return error;
+
+ get_incfs_backing_path(old_dentry, &backing_old_path);
+ get_incfs_backing_path(new_dentry, &backing_new_path);
+
+ if (backing_new_path.dentry->d_parent == mi->mi_index_dir) {
+ /* Can't link to .index */
+ error = -EBUSY;
+ goto out;
+ }
+
+ if (backing_new_path.dentry->d_parent == mi->mi_incomplete_dir) {
+ /* Can't link to .incomplete */
+ error = -EBUSY;
+ goto out;
+ }
+
+ error = incfs_link(backing_old_path.dentry, backing_new_path.dentry);
+ if (!error) {
+ struct inode *inode = NULL;
+ struct dentry *bdentry = backing_new_path.dentry;
+
+ if (d_really_is_negative(bdentry)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ inode = fetch_regular_inode(dir->i_sb, bdentry);
+ if (IS_ERR(inode)) {
+ error = PTR_ERR(inode);
+ goto out;
+ }
+ d_instantiate(new_dentry, inode);
+ }
+
+out:
+ path_put(&backing_old_path);
+ path_put(&backing_new_path);
+ if (error)
+ pr_debug("incfs: %s err:%d\n", __func__, error);
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ return error;
+}
+
+static int dir_rmdir(struct inode *dir, struct dentry *dentry)
+{
+ struct mount_info *mi = get_mount_info(dir->i_sb);
+ struct path backing_path = {};
+ int err = 0;
+
+ if (!mi)
+ return -EBADF;
+
+ err = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (err)
+ return err;
+
+ get_incfs_backing_path(dentry, &backing_path);
+ if (!backing_path.dentry) {
+ err = -EBADF;
+ goto path_err;
+ }
+
+ if (backing_path.dentry == mi->mi_index_dir) {
+ /* Can't delete .index */
+ err = -EBUSY;
+ goto out;
+ }
+
+ if (backing_path.dentry == mi->mi_incomplete_dir) {
+ /* Can't delete .incomplete */
+ err = -EBUSY;
+ goto out;
+ }
+
+ err = incfs_rmdir(backing_path.dentry);
+ if (!err)
+ d_drop(dentry);
+out:
+ path_put(&backing_path);
+
+path_err:
+ if (err)
+ pr_debug("incfs: %s err:%d\n", __func__, err);
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ return err;
+}
+
+static int dir_rename(struct inode *old_dir, struct dentry *old_dentry,
+ struct inode *new_dir, struct dentry *new_dentry,
+ unsigned int flags)
+{
+ struct mount_info *mi = get_mount_info(old_dir->i_sb);
+ struct dentry *backing_old_dentry;
+ struct dentry *backing_new_dentry;
+ struct dentry *backing_old_dir_dentry;
+ struct dentry *backing_new_dir_dentry;
+ struct inode *target_inode;
+ struct dentry *trap;
+ struct renamedata rd = {};
+ int error = 0;
+
+ error = mutex_lock_interruptible(&mi->mi_dir_struct_mutex);
+ if (error)
+ return error;
+
+ backing_old_dentry = get_incfs_dentry(old_dentry)->backing_path.dentry;
+
+ if (!backing_old_dentry || backing_old_dentry == mi->mi_index_dir ||
+ backing_old_dentry == mi->mi_incomplete_dir) {
+ /* Renaming .index or .incomplete not allowed */
+ error = -EBUSY;
+ goto exit;
+ }
+
+ backing_new_dentry = get_incfs_dentry(new_dentry)->backing_path.dentry;
+ dget(backing_old_dentry);
+ dget(backing_new_dentry);
+
+ backing_old_dir_dentry = dget_parent(backing_old_dentry);
+ backing_new_dir_dentry = dget_parent(backing_new_dentry);
+ target_inode = d_inode(new_dentry);
+
+ if (backing_old_dir_dentry == mi->mi_index_dir ||
+ backing_old_dir_dentry == mi->mi_incomplete_dir) {
+ /* Direct moves from .index or .incomplete are not allowed. */
+ error = -EBUSY;
+ goto out;
+ }
+
+ trap = lock_rename(backing_old_dir_dentry, backing_new_dir_dentry);
+
+ if (trap == backing_old_dentry) {
+ error = -EINVAL;
+ goto unlock_out;
+ }
+ if (trap == backing_new_dentry) {
+ error = -ENOTEMPTY;
+ goto unlock_out;
+ }
+
+ rd.old_dir = d_inode(backing_old_dir_dentry);
+ rd.old_dentry = backing_old_dentry;
+ rd.new_dir = d_inode(backing_new_dir_dentry);
+ rd.new_dentry = backing_new_dentry;
+ rd.flags = flags;
+ rd.old_mnt_userns = &init_user_ns;
+ rd.new_mnt_userns = &init_user_ns;
+ rd.delegated_inode = NULL;
+
+ error = vfs_rename(&rd);
+ if (error)
+ goto unlock_out;
+ if (target_inode)
+ fsstack_copy_attr_all(target_inode,
+ get_incfs_node(target_inode)->n_backing_inode);
+ fsstack_copy_attr_all(new_dir, d_inode(backing_new_dir_dentry));
+ if (new_dir != old_dir)
+ fsstack_copy_attr_all(old_dir, d_inode(backing_old_dir_dentry));
+
+unlock_out:
+ unlock_rename(backing_old_dir_dentry, backing_new_dir_dentry);
+
+out:
+ dput(backing_new_dir_dentry);
+ dput(backing_old_dir_dentry);
+ dput(backing_new_dentry);
+ dput(backing_old_dentry);
+
+exit:
+ mutex_unlock(&mi->mi_dir_struct_mutex);
+ if (error)
+ pr_debug("incfs: %s err:%d\n", __func__, error);
+ return error;
+}
+
+
+static int file_open(struct inode *inode, struct file *file)
+{
+ struct mount_info *mi = get_mount_info(inode->i_sb);
+ struct file *backing_file = NULL;
+ struct path backing_path = {};
+ int err = 0;
+ int flags = O_NOATIME | O_LARGEFILE |
+ (S_ISDIR(inode->i_mode) ? O_RDONLY : O_RDWR);
+ const struct cred *old_cred;
+
+ WARN_ON(file->private_data);
+
+ if (!mi)
+ return -EBADF;
+
+ get_incfs_backing_path(file->f_path.dentry, &backing_path);
+ if (!backing_path.dentry)
+ return -EBADF;
+
+ old_cred = override_creds(mi->mi_owner);
+ backing_file = dentry_open(&backing_path, flags, current_cred());
+ revert_creds(old_cred);
+ path_put(&backing_path);
+
+ if (IS_ERR(backing_file)) {
+ err = PTR_ERR(backing_file);
+ backing_file = NULL;
+ goto out;
+ }
+
+ if (S_ISREG(inode->i_mode)) {
+ struct incfs_file_data *fd = kzalloc(sizeof(*fd), GFP_NOFS);
+
+ if (!fd) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ *fd = (struct incfs_file_data) {
+ .fd_fill_permission = CANT_FILL,
+ };
+ file->private_data = fd;
+
+ err = make_inode_ready_for_data_ops(mi, inode, backing_file);
+ if (err)
+ goto out;
+
+ err = incfs_fsverity_file_open(inode, file);
+ if (err)
+ goto out;
+ } else if (S_ISDIR(inode->i_mode)) {
+ struct dir_file *dir = NULL;
+
+ dir = incfs_open_dir_file(mi, backing_file);
+ if (IS_ERR(dir))
+ err = PTR_ERR(dir);
+ else
+ file->private_data = dir;
+ } else
+ err = -EBADF;
+
+out:
+ if (err) {
+ pr_debug("name:%s err: %d\n",
+ file->f_path.dentry->d_name.name, err);
+ if (S_ISREG(inode->i_mode))
+ kfree(file->private_data);
+ else if (S_ISDIR(inode->i_mode))
+ incfs_free_dir_file(file->private_data);
+
+ file->private_data = NULL;
+ }
+
+ if (backing_file)
+ fput(backing_file);
+ return err;
+}
+
+static int file_release(struct inode *inode, struct file *file)
+{
+ if (S_ISREG(inode->i_mode)) {
+ kfree(file->private_data);
+ file->private_data = NULL;
+ } else if (S_ISDIR(inode->i_mode)) {
+ struct dir_file *dir = get_incfs_dir_file(file);
+
+ incfs_free_dir_file(dir);
+ }
+
+ return 0;
+}
+
+static int dentry_revalidate(struct dentry *d, unsigned int flags)
+{
+ struct path backing_path = {};
+ struct inode_info *info = get_incfs_node(d_inode(d));
+ struct inode *binode = (info == NULL) ? NULL : info->n_backing_inode;
+ struct dentry *backing_dentry = NULL;
+ int result = 0;
+
+ if (flags & LOOKUP_RCU)
+ return -ECHILD;
+
+ get_incfs_backing_path(d, &backing_path);
+ backing_dentry = backing_path.dentry;
+ if (!backing_dentry)
+ goto out;
+
+ if (d_inode(backing_dentry) != binode) {
+ /*
+ * Backing inodes obtained via dentry and inode don't match.
+ * It indicates that most likely backing dir has changed
+ * directly bypassing Incremental FS interface.
+ */
+ goto out;
+ }
+
+ if (backing_dentry->d_flags & DCACHE_OP_REVALIDATE) {
+ result = backing_dentry->d_op->d_revalidate(backing_dentry,
+ flags);
+ } else
+ result = 1;
+
+out:
+ path_put(&backing_path);
+ return result;
+}
+
+static void dentry_release(struct dentry *d)
+{
+ struct dentry_info *di = get_incfs_dentry(d);
+
+ if (di)
+ path_put(&di->backing_path);
+ kfree(d->d_fsdata);
+ d->d_fsdata = NULL;
+}
+
+static struct inode *alloc_inode(struct super_block *sb)
+{
+ struct inode_info *node = kzalloc(sizeof(*node), GFP_NOFS);
+
+ /* TODO: add a slab-based cache here. */
+ if (!node)
+ return NULL;
+ inode_init_once(&node->n_vfs_inode);
+ return &node->n_vfs_inode;
+}
+
+static void free_inode(struct inode *inode)
+{
+ struct inode_info *node = get_incfs_node(inode);
+
+ kfree(node);
+}
+
+static void evict_inode(struct inode *inode)
+{
+ struct inode_info *node = get_incfs_node(inode);
+
+ if (node) {
+ if (node->n_backing_inode) {
+ iput(node->n_backing_inode);
+ node->n_backing_inode = NULL;
+ }
+ if (node->n_file) {
+ incfs_free_data_file(node->n_file);
+ node->n_file = NULL;
+ }
+ }
+
+ truncate_inode_pages(&inode->i_data, 0);
+ clear_inode(inode);
+}
+
+static int incfs_setattr(struct user_namespace *ns, struct dentry *dentry,
+ struct iattr *ia)
+{
+ struct dentry_info *di = get_incfs_dentry(dentry);
+ struct dentry *backing_dentry;
+ struct inode *backing_inode;
+ int error;
+
+ if (ia->ia_valid & ATTR_SIZE)
+ return -EINVAL;
+
+ if ((ia->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) &&
+ (ia->ia_valid & ATTR_MODE))
+ return -EINVAL;
+
+ if (!di)
+ return -EINVAL;
+ backing_dentry = di->backing_path.dentry;
+ if (!backing_dentry)
+ return -EINVAL;
+
+ backing_inode = d_inode(backing_dentry);
+
+ /* incfs files are readonly, but the backing files must be writeable */
+ if (S_ISREG(backing_inode->i_mode)) {
+ if ((ia->ia_valid & ATTR_MODE) && (ia->ia_mode & 0222))
+ return -EINVAL;
+
+ ia->ia_mode |= 0222;
+ }
+
+ inode_lock(d_inode(backing_dentry));
+ error = notify_change(ns, backing_dentry, ia, NULL);
+ inode_unlock(d_inode(backing_dentry));
+
+ if (error)
+ return error;
+
+ if (S_ISREG(backing_inode->i_mode))
+ ia->ia_mode &= ~0222;
+
+ return simple_setattr(ns, dentry, ia);
+}
+
+
+static int incfs_getattr(struct user_namespace *ns, const struct path *path,
+ struct kstat *stat, u32 request_mask,
+ unsigned int query_flags)
+{
+ struct inode *inode = d_inode(path->dentry);
+
+ generic_fillattr(ns, inode, stat);
+
+ if (inode->i_ino < INCFS_START_INO_RANGE)
+ return 0;
+
+ stat->attributes &= ~STATX_ATTR_VERITY;
+ if (IS_VERITY(inode))
+ stat->attributes |= STATX_ATTR_VERITY;
+ stat->attributes_mask |= STATX_ATTR_VERITY;
+
+ if (request_mask & STATX_BLOCKS) {
+ struct kstat backing_kstat;
+ struct dentry_info *di = get_incfs_dentry(path->dentry);
+ int error = 0;
+ struct path *backing_path;
+
+ if (!di)
+ return -EFSCORRUPTED;
+ backing_path = &di->backing_path;
+ error = vfs_getattr(backing_path, &backing_kstat, STATX_BLOCKS,
+ AT_STATX_SYNC_AS_STAT);
+ if (error)
+ return error;
+
+ stat->blocks = backing_kstat.blocks;
+ }
+
+ return 0;
+}
+
+static ssize_t incfs_getxattr(struct dentry *d, const char *name,
+ void *value, size_t size)
+{
+ struct dentry_info *di = get_incfs_dentry(d);
+ struct mount_info *mi = get_mount_info(d->d_sb);
+ char *stored_value;
+ size_t stored_size;
+ int i;
+
+ if (di && di->backing_path.dentry)
+ return vfs_getxattr(&init_user_ns, di->backing_path.dentry, name, value, size);
+
+ if (strcmp(name, "security.selinux"))
+ return -ENODATA;
+
+ for (i = 0; i < PSEUDO_FILE_COUNT; ++i)
+ if (!strcmp(d->d_iname, incfs_pseudo_file_names[i].data))
+ break;
+ if (i == PSEUDO_FILE_COUNT)
+ return -ENODATA;
+
+ stored_value = mi->pseudo_file_xattr[i].data;
+ stored_size = mi->pseudo_file_xattr[i].len;
+ if (!stored_value)
+ return -ENODATA;
+
+ if (stored_size > size)
+ return -E2BIG;
+
+ memcpy(value, stored_value, stored_size);
+ return stored_size;
+}
+
+
+static ssize_t incfs_setxattr(struct user_namespace *ns, struct dentry *d,
+ const char *name, void *value, size_t size,
+ int flags)
+{
+ struct dentry_info *di = get_incfs_dentry(d);
+ struct mount_info *mi = get_mount_info(d->d_sb);
+ u8 **stored_value;
+ size_t *stored_size;
+ int i;
+
+ if (di && di->backing_path.dentry)
+ return vfs_setxattr(ns, di->backing_path.dentry, name, value,
+ size, flags);
+
+ if (strcmp(name, "security.selinux"))
+ return -ENODATA;
+
+ if (size > INCFS_MAX_FILE_ATTR_SIZE)
+ return -E2BIG;
+
+ for (i = 0; i < PSEUDO_FILE_COUNT; ++i)
+ if (!strcmp(d->d_iname, incfs_pseudo_file_names[i].data))
+ break;
+ if (i == PSEUDO_FILE_COUNT)
+ return -ENODATA;
+
+ stored_value = &mi->pseudo_file_xattr[i].data;
+ stored_size = &mi->pseudo_file_xattr[i].len;
+ kfree (*stored_value);
+ *stored_value = kzalloc(size, GFP_NOFS);
+ if (!*stored_value)
+ return -ENOMEM;
+
+ memcpy(*stored_value, value, size);
+ *stored_size = size;
+ return 0;
+}
+
+static ssize_t incfs_listxattr(struct dentry *d, char *list, size_t size)
+{
+ struct dentry_info *di = get_incfs_dentry(d);
+
+ if (!di || !di->backing_path.dentry)
+ return -ENODATA;
+
+ return vfs_listxattr(di->backing_path.dentry, list, size);
+}
+
+struct dentry *incfs_mount_fs(struct file_system_type *type, int flags,
+ const char *dev_name, void *data)
+{
+ struct mount_options options = {};
+ struct mount_info *mi = NULL;
+ struct path backing_dir_path = {};
+ struct dentry *index_dir = NULL;
+ struct dentry *incomplete_dir = NULL;
+ struct super_block *src_fs_sb = NULL;
+ struct inode *root_inode = NULL;
+ struct super_block *sb = sget(type, NULL, set_anon_super, flags, NULL);
+ bool dir_created = false;
+ int error = 0;
+
+ if (IS_ERR(sb))
+ return ERR_CAST(sb);
+
+ sb->s_op = &incfs_super_ops;
+ sb->s_d_op = &incfs_dentry_ops;
+ sb->s_flags |= S_NOATIME;
+ sb->s_magic = INCFS_MAGIC_NUMBER;
+ sb->s_time_gran = 1;
+ sb->s_blocksize = INCFS_DATA_FILE_BLOCK_SIZE;
+ sb->s_blocksize_bits = blksize_bits(sb->s_blocksize);
+ sb->s_xattr = incfs_xattr_ops;
+
+ BUILD_BUG_ON(PAGE_SIZE != INCFS_DATA_FILE_BLOCK_SIZE);
+
+ if (!dev_name) {
+ pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n");
+ error = -ENOENT;
+ goto err_deactivate;
+ }
+
+ error = parse_options(&options, (char *)data);
+ if (error != 0) {
+ pr_err("incfs: Options parsing error. %d\n", error);
+ goto err_deactivate;
+ }
+
+ sb->s_bdi->ra_pages = options.readahead_pages;
+ if (!dev_name) {
+ pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n");
+ error = -ENOENT;
+ goto err_free_opts;
+ }
+
+ error = kern_path(dev_name, LOOKUP_FOLLOW | LOOKUP_DIRECTORY,
+ &backing_dir_path);
+ if (error || backing_dir_path.dentry == NULL ||
+ !d_really_is_positive(backing_dir_path.dentry)) {
+ pr_err("incfs: Error accessing: %s.\n",
+ dev_name);
+ goto err_free_opts;
+ }
+ src_fs_sb = backing_dir_path.dentry->d_sb;
+ sb->s_maxbytes = src_fs_sb->s_maxbytes;
+ sb->s_stack_depth = src_fs_sb->s_stack_depth + 1;
+
+ if (sb->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) {
+ error = -EINVAL;
+ goto err_put_path;
+ }
+
+ mi = incfs_alloc_mount_info(sb, &options, &backing_dir_path);
+ if (IS_ERR_OR_NULL(mi)) {
+ error = PTR_ERR(mi);
+ pr_err("incfs: Error allocating mount info. %d\n", error);
+ goto err_put_path;
+ }
+
+ sb->s_fs_info = mi;
+ mi->mi_backing_dir_path = backing_dir_path;
+ index_dir = open_or_create_special_dir(backing_dir_path.dentry,
+ INCFS_INDEX_NAME, &dir_created);
+ if (IS_ERR_OR_NULL(index_dir)) {
+ error = PTR_ERR(index_dir);
+ pr_err("incfs: Can't find or create .index dir in %s\n",
+ dev_name);
+ /* No need to null index_dir since we don't put it */
+ goto err_put_path;
+ }
+
+ mi->mi_index_dir = index_dir;
+ mi->mi_index_free = dir_created;
+
+ incomplete_dir = open_or_create_special_dir(backing_dir_path.dentry,
+ INCFS_INCOMPLETE_NAME,
+ &dir_created);
+ if (IS_ERR_OR_NULL(incomplete_dir)) {
+ error = PTR_ERR(incomplete_dir);
+ pr_err("incfs: Can't find or create .incomplete dir in %s\n",
+ dev_name);
+ /* No need to null incomplete_dir since we don't put it */
+ goto err_put_path;
+ }
+ mi->mi_incomplete_dir = incomplete_dir;
+ mi->mi_incomplete_free = dir_created;
+
+ root_inode = fetch_regular_inode(sb, backing_dir_path.dentry);
+ if (IS_ERR(root_inode)) {
+ error = PTR_ERR(root_inode);
+ goto err_put_path;
+ }
+
+ sb->s_root = d_make_root(root_inode);
+ if (!sb->s_root) {
+ error = -ENOMEM;
+ goto err_put_path;
+ }
+ error = incfs_init_dentry(sb->s_root, &backing_dir_path);
+ if (error)
+ goto err_put_path;
+
+ path_put(&backing_dir_path);
+ sb->s_flags |= SB_ACTIVE;
+
+ pr_debug("incfs: mount\n");
+ return dget(sb->s_root);
+
+err_put_path:
+ path_put(&backing_dir_path);
+err_free_opts:
+ free_options(&options);
+err_deactivate:
+ deactivate_locked_super(sb);
+ pr_err("incfs: mount failed %d\n", error);
+ return ERR_PTR(error);
+}
+
+static int incfs_remount_fs(struct super_block *sb, int *flags, char *data)
+{
+ struct mount_options options;
+ struct mount_info *mi = get_mount_info(sb);
+ int err = 0;
+
+ sync_filesystem(sb);
+ err = parse_options(&options, (char *)data);
+ if (err)
+ return err;
+
+ if (options.report_uid != mi->mi_options.report_uid) {
+ pr_err("incfs: Can't change report_uid mount option on remount\n");
+ err = -EOPNOTSUPP;
+ goto out;
+ }
+
+ err = incfs_realloc_mount_info(mi, &options);
+ if (err)
+ goto out;
+
+ pr_debug("incfs: remount\n");
+
+out:
+ free_options(&options);
+ return err;
+}
+
+void incfs_kill_sb(struct super_block *sb)
+{
+ struct mount_info *mi = sb->s_fs_info;
+ struct inode *dinode = NULL;
+
+ pr_debug("incfs: unmount\n");
+
+ if (mi) {
+ if (mi->mi_backing_dir_path.dentry)
+ dinode = d_inode(mi->mi_backing_dir_path.dentry);
+
+ if (dinode) {
+ if (mi->mi_index_dir && mi->mi_index_free)
+ vfs_rmdir(&init_user_ns, dinode,
+ mi->mi_index_dir);
+
+ if (mi->mi_incomplete_dir && mi->mi_incomplete_free)
+ vfs_rmdir(&init_user_ns, dinode,
+ mi->mi_incomplete_dir);
+ }
+
+ incfs_free_mount_info(mi);
+ sb->s_fs_info = NULL;
+ }
+ kill_anon_super(sb);
+}
+
+static int show_options(struct seq_file *m, struct dentry *root)
+{
+ struct mount_info *mi = get_mount_info(root->d_sb);
+
+ seq_printf(m, ",read_timeout_ms=%u", mi->mi_options.read_timeout_ms);
+ seq_printf(m, ",readahead=%u", mi->mi_options.readahead_pages);
+ if (mi->mi_options.read_log_pages != 0) {
+ seq_printf(m, ",rlog_pages=%u", mi->mi_options.read_log_pages);
+ seq_printf(m, ",rlog_wakeup_cnt=%u",
+ mi->mi_options.read_log_wakeup_count);
+ }
+ if (mi->mi_options.report_uid)
+ seq_puts(m, ",report_uid");
+
+ if (mi->mi_sysfs_node)
+ seq_printf(m, ",sysfs_name=%s",
+ kobject_name(&mi->mi_sysfs_node->isn_sysfs_node));
+ return 0;
+}
diff --git a/fs/incfs/vfs.h b/fs/incfs/vfs.h
new file mode 100644
index 0000000..79fdf24
--- /dev/null
+++ b/fs/incfs/vfs.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018 Google LLC
+ */
+
+#ifndef _INCFS_VFS_H
+#define _INCFS_VFS_H
+
+extern const struct file_operations incfs_file_ops;
+extern const struct inode_operations incfs_file_inode_ops;
+
+void incfs_kill_sb(struct super_block *sb);
+struct dentry *incfs_mount_fs(struct file_system_type *type, int flags,
+ const char *dev_name, void *data);
+int incfs_link(struct dentry *what, struct dentry *where);
+int incfs_unlink(struct dentry *dentry);
+
+static inline struct mount_info *get_mount_info(struct super_block *sb)
+{
+ struct mount_info *result = sb->s_fs_info;
+
+ WARN_ON(!result);
+ return result;
+}
+
+static inline struct super_block *file_superblock(struct file *f)
+{
+ struct inode *inode = file_inode(f);
+
+ return inode->i_sb;
+}
+
+#endif
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c
index 1c4bfda..05299fb 100644
--- a/fs/notify/inotify/inotify_user.c
+++ b/fs/notify/inotify/inotify_user.c
@@ -733,6 +733,8 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname,
struct fsnotify_group *group;
struct inode *inode;
struct path path;
+ struct path alteredpath;
+ struct path *canonical_path = &path;
struct fd f;
int ret;
unsigned flags = 0;
@@ -779,13 +781,23 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname,
if (ret)
goto fput_and_out;
+ /* support stacked filesystems */
+ if (path.dentry && path.dentry->d_op) {
+ if (path.dentry->d_op->d_canonical_path) {
+ path.dentry->d_op->d_canonical_path(&path,
+ &alteredpath);
+ canonical_path = &alteredpath;
+ path_put(&path);
+ }
+ }
+
/* inode held in place by reference to path; group by fget on fd */
- inode = path.dentry->d_inode;
+ inode = canonical_path->dentry->d_inode;
group = f.file->private_data;
/* create/update an inode mark */
ret = inotify_update_watch(group, inode, mask);
- path_put(&path);
+ path_put(canonical_path);
fput_and_out:
fdput(f);
return ret;
diff --git a/fs/open.c b/fs/open.c
index 82c1a28..e9da9bd 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -35,6 +35,7 @@
#include <linux/mnt_idmapping.h>
#include "internal.h"
+#include <trace/hooks/syscall_check.h>
int do_truncate(struct user_namespace *mnt_userns, struct dentry *dentry,
loff_t length, unsigned int time_attrs, struct file *filp)
@@ -865,6 +866,7 @@ static int do_dentry_open(struct file *f,
error = -ENODEV;
goto cleanup_all;
}
+ trace_android_vh_check_file_open(f);
error = security_file_open(f);
if (error)
diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c
index 6e4e65e..f1f9a12 100644
--- a/fs/overlayfs/copy_up.c
+++ b/fs/overlayfs/copy_up.c
@@ -1093,7 +1093,7 @@ static int ovl_copy_up_flags(struct dentry *dentry, int flags)
dput(parent);
dput(next);
}
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return err;
}
diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
index f61e37f..dc40974 100644
--- a/fs/overlayfs/dir.c
+++ b/fs/overlayfs/dir.c
@@ -556,7 +556,7 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
struct ovl_cattr *attr, bool origin)
{
int err;
- const struct cred *old_cred;
+ const struct cred *old_cred, *hold_cred = NULL;
struct cred *override_cred;
struct dentry *parent = dentry->d_parent;
@@ -597,13 +597,14 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
override_cred->fsuid = inode->i_uid;
override_cred->fsgid = inode->i_gid;
err = security_dentry_create_files_as(dentry,
- attr->mode, &dentry->d_name, old_cred,
+ attr->mode, &dentry->d_name,
+ old_cred ? old_cred : current_cred(),
override_cred);
if (err) {
put_cred(override_cred);
goto out_revert_creds;
}
- put_cred(override_creds(override_cred));
+ hold_cred = override_creds(override_cred);
put_cred(override_cred);
}
@@ -613,7 +614,9 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
err = ovl_create_over_whiteout(dentry, inode, attr);
out_revert_creds:
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred ?: hold_cred);
+ if (old_cred && hold_cred)
+ put_cred(hold_cred);
return err;
}
@@ -690,7 +693,7 @@ static int ovl_set_link_redirect(struct dentry *dentry)
old_cred = ovl_override_creds(dentry->d_sb);
err = ovl_set_redirect(dentry, false);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return err;
}
@@ -909,7 +912,7 @@ static int ovl_do_remove(struct dentry *dentry, bool is_dir)
err = ovl_remove_upper(dentry, is_dir, &list);
else
err = ovl_remove_and_whiteout(dentry, &list);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (!err) {
if (is_dir)
clear_nlink(dentry->d_inode);
@@ -1284,7 +1287,7 @@ static int ovl_rename(struct user_namespace *mnt_userns, struct inode *olddir,
out_unlock:
unlock_rename(new_upperdir, old_upperdir);
out_revert_creds:
- revert_creds(old_cred);
+ ovl_revert_creds(old->d_sb, old_cred);
if (update_nlink)
ovl_nlink_end(new);
out_drop_write:
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index c9d0c36..2654b1d 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -15,6 +15,8 @@
#include <linux/fs.h>
#include "overlayfs.h"
+#define OVL_IOCB_MASK (IOCB_DSYNC | IOCB_HIPRI | IOCB_NOWAIT | IOCB_SYNC)
+
struct ovl_aio_req {
struct kiocb iocb;
refcount_t ref;
@@ -58,13 +60,14 @@ static struct file *ovl_open_realfile(const struct file *file,
if (err) {
realfile = ERR_PTR(err);
} else {
- if (!inode_owner_or_capable(real_mnt_userns, realinode))
+ if (old_cred && !inode_owner_or_capable(real_mnt_userns,
+ realinode))
flags &= ~O_NOATIME;
realfile = open_with_fake_path(&file->f_path, flags, realinode,
current_cred());
}
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
pr_debug("open(%p[%pD2/%c], 0%o) -> (%p, 0%o)\n",
file, file, ovl_whatisit(inode, realinode), file->f_flags,
@@ -209,7 +212,7 @@ static loff_t ovl_llseek(struct file *file, loff_t offset, int whence)
old_cred = ovl_override_creds(inode->i_sb);
ret = vfs_llseek(real.file, offset, whence);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
file->f_pos = real.file->f_pos;
ovl_inode_unlock(inode);
@@ -241,22 +244,6 @@ static void ovl_file_accessed(struct file *file)
touch_atime(&file->f_path);
}
-static rwf_t ovl_iocb_to_rwf(int ifl)
-{
- rwf_t flags = 0;
-
- if (ifl & IOCB_NOWAIT)
- flags |= RWF_NOWAIT;
- if (ifl & IOCB_HIPRI)
- flags |= RWF_HIPRI;
- if (ifl & IOCB_DSYNC)
- flags |= RWF_DSYNC;
- if (ifl & IOCB_SYNC)
- flags |= RWF_SYNC;
-
- return flags;
-}
-
static inline void ovl_aio_put(struct ovl_aio_req *aio_req)
{
if (refcount_dec_and_test(&aio_req->ref)) {
@@ -316,7 +303,8 @@ static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter)
old_cred = ovl_override_creds(file_inode(file)->i_sb);
if (is_sync_kiocb(iocb)) {
ret = vfs_iter_read(real.file, iter, &iocb->ki_pos,
- ovl_iocb_to_rwf(iocb->ki_flags));
+ iocb_to_rw_flags(iocb->ki_flags,
+ OVL_IOCB_MASK));
} else {
struct ovl_aio_req *aio_req;
@@ -337,7 +325,7 @@ static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter)
ovl_aio_cleanup_handler(aio_req);
}
out:
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
ovl_file_accessed(file);
out_fdput:
fdput(real);
@@ -380,7 +368,7 @@ static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter)
if (is_sync_kiocb(iocb)) {
file_start_write(real.file);
ret = vfs_iter_write(real.file, iter, &iocb->ki_pos,
- ovl_iocb_to_rwf(ifl));
+ iocb_to_rw_flags(ifl, OVL_IOCB_MASK));
file_end_write(real.file);
/* Update size */
ovl_copyattr(inode);
@@ -409,7 +397,7 @@ static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter)
ovl_aio_cleanup_handler(aio_req);
}
out:
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
out_fdput:
fdput(real);
@@ -454,7 +442,7 @@ static ssize_t ovl_splice_write(struct pipe_inode_info *pipe, struct file *out,
file_end_write(real.file);
/* Update size */
ovl_copyattr(inode);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
fdput(real);
out_unlock:
@@ -481,7 +469,7 @@ static int ovl_fsync(struct file *file, loff_t start, loff_t end, int datasync)
if (file_inode(real.file) == ovl_inode_upper(file_inode(file))) {
old_cred = ovl_override_creds(file_inode(file)->i_sb);
ret = vfs_fsync_range(real.file, start, end, datasync);
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
}
fdput(real);
@@ -505,7 +493,7 @@ static int ovl_mmap(struct file *file, struct vm_area_struct *vma)
old_cred = ovl_override_creds(file_inode(file)->i_sb);
ret = call_mmap(vma->vm_file, vma);
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
ovl_file_accessed(file);
return ret;
@@ -531,7 +519,7 @@ static long ovl_fallocate(struct file *file, int mode, loff_t offset, loff_t len
old_cred = ovl_override_creds(file_inode(file)->i_sb);
ret = vfs_fallocate(real.file, mode, offset, len);
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
/* Update size */
ovl_copyattr(inode);
@@ -556,7 +544,7 @@ static int ovl_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
old_cred = ovl_override_creds(file_inode(file)->i_sb);
ret = vfs_fadvise(real.file, offset, len, advice);
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
fdput(real);
@@ -615,7 +603,7 @@ static loff_t ovl_copyfile(struct file *file_in, loff_t pos_in,
flags);
break;
}
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file_out)->i_sb, old_cred);
/* Update size */
ovl_copyattr(inode_out);
@@ -677,7 +665,7 @@ static int ovl_flush(struct file *file, fl_owner_t id)
if (real.file->f_op->flush) {
old_cred = ovl_override_creds(file_inode(file)->i_sb);
err = real.file->f_op->flush(real.file, id);
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
}
fdput(real);
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index ee6dfa5..ecbba82 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -81,7 +81,7 @@ int ovl_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
inode_lock(upperdentry->d_inode);
old_cred = ovl_override_creds(dentry->d_sb);
err = ovl_do_notify_change(ofs, upperdentry, attr);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (!err)
ovl_copyattr(dentry->d_inode);
inode_unlock(upperdentry->d_inode);
@@ -273,7 +273,7 @@ int ovl_getattr(struct user_namespace *mnt_userns, const struct path *path,
stat->nlink = dentry->d_inode->i_nlink;
out:
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return err;
}
@@ -311,7 +311,7 @@ int ovl_permission(struct user_namespace *mnt_userns,
mask |= MAY_READ;
}
err = inode_permission(mnt_user_ns(realpath.mnt), realinode, mask);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
return err;
}
@@ -328,7 +328,7 @@ static const char *ovl_get_link(struct dentry *dentry,
old_cred = ovl_override_creds(dentry->d_sb);
p = vfs_get_link(ovl_dentry_real(dentry), done);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return p;
}
@@ -362,7 +362,7 @@ int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name,
ovl_path_lower(dentry, &realpath);
old_cred = ovl_override_creds(dentry->d_sb);
err = vfs_getxattr(mnt_user_ns(realpath.mnt), realdentry, name, NULL, 0);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (err < 0)
goto out_drop_write;
}
@@ -383,7 +383,7 @@ int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name,
WARN_ON(flags != XATTR_REPLACE);
err = ovl_do_removexattr(ofs, realdentry, name);
}
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
/* copy c/mtime */
ovl_copyattr(inode);
@@ -404,7 +404,7 @@ int ovl_xattr_get(struct dentry *dentry, struct inode *inode, const char *name,
ovl_i_path_real(inode, &realpath);
old_cred = ovl_override_creds(dentry->d_sb);
res = vfs_getxattr(mnt_user_ns(realpath.mnt), realpath.dentry, name, value, size);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return res;
}
@@ -432,7 +432,7 @@ ssize_t ovl_listxattr(struct dentry *dentry, char *list, size_t size)
old_cred = ovl_override_creds(dentry->d_sb);
res = vfs_listxattr(realdentry, list, size);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (res <= 0 || size == 0)
return res;
@@ -587,7 +587,7 @@ struct posix_acl *do_ovl_get_acl(struct user_namespace *mnt_userns,
old_cred = ovl_override_creds(inode->i_sb);
acl = ovl_get_acl_path(&realpath, posix_acl_xattr_name(type), noperm);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
}
return acl;
@@ -715,7 +715,7 @@ static int ovl_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
old_cred = ovl_override_creds(inode->i_sb);
err = realinode->i_op->fiemap(realinode, fieinfo, start, len);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
return err;
}
@@ -786,7 +786,7 @@ int ovl_fileattr_set(struct user_namespace *mnt_userns,
err = ovl_set_protattr(inode, upperpath.dentry, fa);
if (!err)
err = ovl_real_fileattr_set(&upperpath, fa);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
/*
* Merge real inode flags with inode flags read from
@@ -848,7 +848,7 @@ int ovl_fileattr_get(struct dentry *dentry, struct fileattr *fa)
old_cred = ovl_override_creds(inode->i_sb);
err = ovl_real_fileattr_get(&realpath, fa);
ovl_fileattr_prot_flags(inode, fa);
- revert_creds(old_cred);
+ ovl_revert_creds(inode->i_sb, old_cred);
return err;
}
diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c
index 4675313..9249bd2 100644
--- a/fs/overlayfs/namei.c
+++ b/fs/overlayfs/namei.c
@@ -1125,7 +1125,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry,
ovl_dentry_update_reval(dentry, upperdentry,
DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (origin_path) {
dput(origin_path->dentry);
kfree(origin_path);
@@ -1152,7 +1152,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry,
kfree(upperredirect);
out:
kfree(d.redirect);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return ERR_PTR(err);
}
@@ -1204,7 +1204,7 @@ bool ovl_lower_positive(struct dentry *dentry)
dput(this);
}
}
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return positive;
}
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index 1df7f85..8da6bf5 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -368,6 +368,7 @@ int ovl_want_write(struct dentry *dentry);
void ovl_drop_write(struct dentry *dentry);
struct dentry *ovl_workdir(struct dentry *dentry);
const struct cred *ovl_override_creds(struct super_block *sb);
+void ovl_revert_creds(struct super_block *sb, const struct cred *oldcred);
int ovl_can_decode_fh(struct super_block *sb);
struct dentry *ovl_indexdir(struct super_block *sb);
bool ovl_index_all(struct super_block *sb);
diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h
index e1af8f6..17c13f7 100644
--- a/fs/overlayfs/ovl_entry.h
+++ b/fs/overlayfs/ovl_entry.h
@@ -20,6 +20,7 @@ struct ovl_config {
bool metacopy;
bool userxattr;
bool ovl_volatile;
+ bool override_creds;
};
struct ovl_sb {
diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c
index 8cd2b99..d072c30 100644
--- a/fs/overlayfs/readdir.c
+++ b/fs/overlayfs/readdir.c
@@ -286,7 +286,7 @@ static int ovl_check_whiteouts(const struct path *path, struct ovl_readdir_data
}
inode_unlock(dir->d_inode);
}
- revert_creds(old_cred);
+ ovl_revert_creds(rdd->dentry->d_sb, old_cred);
return err;
}
@@ -791,7 +791,7 @@ static int ovl_iterate(struct file *file, struct dir_context *ctx)
}
err = 0;
out:
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
return err;
}
@@ -843,7 +843,7 @@ static struct file *ovl_dir_open_realfile(const struct file *file,
old_cred = ovl_override_creds(file_inode(file)->i_sb);
res = ovl_path_open(realpath, O_RDONLY | (file->f_flags & O_LARGEFILE));
- revert_creds(old_cred);
+ ovl_revert_creds(file_inode(file)->i_sb, old_cred);
return res;
}
@@ -969,7 +969,7 @@ int ovl_check_empty_dir(struct dentry *dentry, struct list_head *list)
old_cred = ovl_override_creds(dentry->d_sb);
err = ovl_dir_read_merged(dentry, list, &root);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
if (err)
return err;
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index 85b8911..4c800136 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -54,6 +54,11 @@ module_param_named(xino_auto, ovl_xino_auto_def, bool, 0644);
MODULE_PARM_DESC(xino_auto,
"Auto enable xino feature");
+static bool __read_mostly ovl_override_creds_def = true;
+module_param_named(override_creds, ovl_override_creds_def, bool, 0644);
+MODULE_PARM_DESC(ovl_override_creds_def,
+ "Use mounter's credentials for accesses");
+
static void ovl_entry_stack_free(struct ovl_entry *oe)
{
unsigned int i;
@@ -391,6 +396,9 @@ static int ovl_show_options(struct seq_file *m, struct dentry *dentry)
seq_puts(m, ",volatile");
if (ofs->config.userxattr)
seq_puts(m, ",userxattr");
+ if (ofs->config.override_creds != ovl_override_creds_def)
+ seq_show_option(m, "override_creds",
+ ofs->config.override_creds ? "on" : "off");
return 0;
}
@@ -446,6 +454,8 @@ enum {
OPT_METACOPY_ON,
OPT_METACOPY_OFF,
OPT_VOLATILE,
+ OPT_OVERRIDE_CREDS_ON,
+ OPT_OVERRIDE_CREDS_OFF,
OPT_ERR,
};
@@ -468,6 +478,8 @@ static const match_table_t ovl_tokens = {
{OPT_METACOPY_ON, "metacopy=on"},
{OPT_METACOPY_OFF, "metacopy=off"},
{OPT_VOLATILE, "volatile"},
+ {OPT_OVERRIDE_CREDS_ON, "override_creds=on"},
+ {OPT_OVERRIDE_CREDS_OFF, "override_creds=off"},
{OPT_ERR, NULL}
};
@@ -527,6 +539,7 @@ static int ovl_parse_opt(char *opt, struct ovl_config *config)
config->redirect_mode = kstrdup(ovl_redirect_mode_def(), GFP_KERNEL);
if (!config->redirect_mode)
return -ENOMEM;
+ config->override_creds = ovl_override_creds_def;
while ((p = ovl_next_opt(&opt)) != NULL) {
int token;
@@ -628,6 +641,14 @@ static int ovl_parse_opt(char *opt, struct ovl_config *config)
config->userxattr = true;
break;
+ case OPT_OVERRIDE_CREDS_ON:
+ config->override_creds = true;
+ break;
+
+ case OPT_OVERRIDE_CREDS_OFF:
+ config->override_creds = false;
+ break;
+
default:
pr_err("unrecognized mount option \"%s\" or missing value\n",
p);
@@ -2071,7 +2092,6 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent)
kfree(splitlower);
sb->s_root = root_dentry;
-
return 0;
out_free_oe:
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index bde2916..7921de4 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -38,9 +38,18 @@ const struct cred *ovl_override_creds(struct super_block *sb)
{
struct ovl_fs *ofs = sb->s_fs_info;
+ if (!ofs->config.override_creds)
+ return NULL;
return override_creds(ofs->creator_cred);
}
+void ovl_revert_creds(struct super_block *sb, const struct cred *old_cred)
+{
+ if (old_cred)
+ revert_creds(old_cred);
+}
+
+
/*
* Check if underlying fs supports file handles and try to determine encoding
* type, in order to deduce maximum inode number used by fs.
@@ -925,7 +934,7 @@ int ovl_nlink_start(struct dentry *dentry)
* value relative to the upper inode nlink in an upper inode xattr.
*/
err = ovl_set_nlink_upper(dentry);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
out:
if (err)
@@ -943,7 +952,7 @@ void ovl_nlink_end(struct dentry *dentry)
old_cred = ovl_override_creds(dentry->d_sb);
ovl_cleanup_index(dentry);
- revert_creds(old_cred);
+ ovl_revert_creds(dentry->d_sb, old_cred);
}
ovl_inode_unlock(inode);
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 9e479d7d..f7c6474c 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -96,6 +96,7 @@
#include <linux/time_namespace.h>
#include <linux/resctrl.h>
#include <linux/cn_proc.h>
+#include <linux/cpufreq_times.h>
#include <trace/events/oom.h>
#include "internal.h"
#include "fd.h"
@@ -3336,6 +3337,9 @@ static const struct pid_entry tgid_base_stuff[] = {
#ifdef CONFIG_LIVEPATCH
ONE("patch_state", S_IRUSR, proc_pid_patch_state),
#endif
+#ifdef CONFIG_CPU_FREQ_TIMES
+ ONE("time_in_state", 0444, proc_time_in_state_show),
+#endif
#ifdef CONFIG_STACKLEAK_METRICS
ONE("stack_depth", S_IRUGO, proc_stack_depth),
#endif
@@ -3687,6 +3691,9 @@ static const struct pid_entry tid_base_stuff[] = {
ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages),
ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat),
#endif
+#ifdef CONFIG_CPU_FREQ_TIMES
+ ONE("time_in_state", 0444, proc_time_in_state_show),
+#endif
};
static int proc_tid_base_readdir(struct file *file, struct dir_context *ctx)
diff --git a/fs/select.c b/fs/select.c
index 0ee55af..794e2a9 100644
--- a/fs/select.c
+++ b/fs/select.c
@@ -476,6 +476,7 @@ static inline void wait_key_set(poll_table *wait, unsigned long in,
wait->_key |= POLLOUT_SET;
}
+noinline_for_stack
static int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time)
{
ktime_t expire, *to = NULL;
diff --git a/fs/sync.c b/fs/sync.c
index dc725914..df95543 100644
--- a/fs/sync.c
+++ b/fs/sync.c
@@ -10,7 +10,7 @@
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/namei.h>
-#include <linux/sched.h>
+#include <linux/sched/xacct.h>
#include <linux/writeback.h>
#include <linux/syscalls.h>
#include <linux/linkage.h>
@@ -211,6 +211,7 @@ static int do_fsync(unsigned int fd, int datasync)
if (f.file) {
ret = vfs_fsync(f.file, datasync);
fdput(f);
+ inc_syscfs(current);
}
return ret;
}
diff --git a/fs/verity/signature.c b/fs/verity/signature.c
index 143a530..e33f6c4 100644
--- a/fs/verity/signature.c
+++ b/fs/verity/signature.c
@@ -40,11 +40,38 @@ static struct key *fsverity_keyring;
int fsverity_verify_signature(const struct fsverity_info *vi,
const u8 *signature, size_t sig_size)
{
- const struct inode *inode = vi->inode;
- const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg;
+ unsigned int digest_algorithm =
+ vi->tree_params.hash_alg - fsverity_hash_algs;
+
+ return __fsverity_verify_signature(vi->inode, signature, sig_size,
+ vi->file_digest, digest_algorithm);
+}
+
+/**
+ * __fsverity_verify_signature() - check a verity file's signature
+ * @inode: the file's inode
+ * @signature: the file's signature
+ * @sig_size: size of @signature. Can be 0 if there is no signature
+ * @file_digest: the file's digest
+ * @digest_algorithm: the digest algorithm used
+ *
+ * Takes the file's digest and optional signature and verifies the signature
+ * against the digest and the fs-verity keyring if appropriate
+ *
+ * Return: 0 on success (signature valid or not required); -errno on failure
+ */
+int __fsverity_verify_signature(const struct inode *inode, const u8 *signature,
+ size_t sig_size, const u8 *file_digest,
+ unsigned int digest_algorithm)
+{
struct fsverity_formatted_digest *d;
+ struct fsverity_hash_alg *hash_alg = fsverity_get_hash_alg(inode,
+ digest_algorithm);
int err;
+ if (IS_ERR(hash_alg))
+ return PTR_ERR(hash_alg);
+
if (sig_size == 0) {
if (fsverity_require_signatures) {
fsverity_err(inode,
@@ -60,7 +87,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi,
memcpy(d->magic, "FSVerity", 8);
d->digest_algorithm = cpu_to_le16(hash_alg - fsverity_hash_algs);
d->digest_size = cpu_to_le16(hash_alg->digest_size);
- memcpy(d->digest, vi->file_digest, hash_alg->digest_size);
+ memcpy(d->digest, file_digest, hash_alg->digest_size);
err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size,
signature, sig_size, fsverity_keyring,
@@ -83,9 +110,10 @@ int fsverity_verify_signature(const struct fsverity_info *vi,
}
pr_debug("Valid signature for file digest %s:%*phN\n",
- hash_alg->name, hash_alg->digest_size, vi->file_digest);
+ hash_alg->name, hash_alg->digest_size, file_digest);
return 0;
}
+EXPORT_SYMBOL_GPL(__fsverity_verify_signature);
#ifdef CONFIG_SYSCTL
static struct ctl_table_header *fsverity_sysctl_header;
diff --git a/include/OWNERS b/include/OWNERS
new file mode 100644
index 0000000..4b815c2
--- /dev/null
+++ b/include/OWNERS
@@ -0,0 +1 @@
+per-file net/**=file:/net/OWNERS
diff --git a/include/drm/drm_mode_object.h b/include/drm/drm_mode_object.h
index 912f1e4..08d7a7f 100644
--- a/include/drm/drm_mode_object.h
+++ b/include/drm/drm_mode_object.h
@@ -60,7 +60,7 @@ struct drm_mode_object {
void (*free_cb)(struct kref *kref);
};
-#define DRM_OBJECT_MAX_PROPERTY 24
+#define DRM_OBJECT_MAX_PROPERTY 64
/**
* struct drm_object_properties - property tracking for &drm_mode_object
*/
diff --git a/include/linux/OWNERS b/include/linux/OWNERS
new file mode 100644
index 0000000..68b6ded
--- /dev/null
+++ b/include/linux/OWNERS
@@ -0,0 +1,4 @@
+per-file bio.h=file:/block/OWNERS
+per-file blk*.h=file:/block/OWNERS
+per-file f2fs**=file:/fs/f2fs/OWNERS
+per-file net**=file:/net/OWNERS
diff --git a/include/linux/android_kabi.h b/include/linux/android_kabi.h
new file mode 100644
index 0000000..f6dd7f0
--- /dev/null
+++ b/include/linux/android_kabi.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * android_kabi.h - Android kernel abi abstraction header
+ *
+ * Copyright (C) 2020 Google, Inc.
+ *
+ * Heavily influenced by rh_kabi.h which came from the RHEL/CENTOS kernel and
+ * was:
+ * Copyright (c) 2014 Don Zickus
+ * Copyright (c) 2015-2018 Jiri Benc
+ * Copyright (c) 2015 Sabrina Dubroca, Hannes Frederic Sowa
+ * Copyright (c) 2016-2018 Prarit Bhargava
+ * Copyright (c) 2017 Paolo Abeni, Larry Woodman
+ *
+ * These macros are to be used to try to help alleviate future kernel abi
+ * changes that will occur as LTS and other kernel patches are merged into the
+ * tree during a period in which the kernel abi is wishing to not be disturbed.
+ *
+ * There are two times these macros should be used:
+ * - Before the kernel abi is "frozen"
+ * Padding can be added to various kernel structures that have in the past
+ * been known to change over time. That will give "room" in the structure
+ * that can then be used when fields are added so that the structure size
+ * will not change.
+ *
+ * - After the kernel abi is "frozen"
+ * If a structure's field is changed to a type that is identical in size to
+ * the previous type, it can be changed with a union macro
+ * If a field is added to a structure, the padding fields can be used to add
+ * the new field in a "safe" way.
+ */
+#ifndef _ANDROID_KABI_H
+#define _ANDROID_KABI_H
+
+#include <linux/compiler.h>
+
+/*
+ * Worker macros, don't use these, use the ones without a leading '_'
+ */
+
+#define __ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new) \
+ union { \
+ _Static_assert(sizeof(struct{_new;}) <= sizeof(struct{_orig;}), \
+ __FILE__ ":" __stringify(__LINE__) ": " \
+ __stringify(_new) \
+ " is larger than " \
+ __stringify(_orig) ); \
+ _Static_assert(__alignof__(struct{_new;}) <= __alignof__(struct{_orig;}), \
+ __FILE__ ":" __stringify(__LINE__) ": " \
+ __stringify(_orig) \
+ " is not aligned the same as " \
+ __stringify(_new) ); \
+ }
+
+#ifdef __GENKSYMS__
+
+#define _ANDROID_KABI_REPLACE(_orig, _new) _orig
+
+#else
+
+#define _ANDROID_KABI_REPLACE(_orig, _new) \
+ union { \
+ _new; \
+ struct { \
+ _orig; \
+ }; \
+ __ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new); \
+ }
+
+#endif /* __GENKSYMS__ */
+
+#define _ANDROID_KABI_RESERVE(n) u64 android_kabi_reserved##n
+
+
+/*
+ * Macros to use _before_ the ABI is frozen
+ */
+
+/*
+ * ANDROID_KABI_RESERVE
+ * Reserve some "padding" in a structure for potential future use.
+ * This normally placed at the end of a structure.
+ * number: the "number" of the padding variable in the structure. Start with
+ * 1 and go up.
+ */
+#ifdef CONFIG_ANDROID_KABI_RESERVE
+#define ANDROID_KABI_RESERVE(number) _ANDROID_KABI_RESERVE(number)
+#else
+#define ANDROID_KABI_RESERVE(number)
+#endif
+
+
+/*
+ * Macros to use _after_ the ABI is frozen
+ */
+
+/*
+ * ANDROID_KABI_USE(number, _new)
+ * Use a previous padding entry that was defined with ANDROID_KABI_RESERVE
+ * number: the previous "number" of the padding variable
+ * _new: the variable to use now instead of the padding variable
+ */
+#define ANDROID_KABI_USE(number, _new) \
+ _ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), _new)
+
+/*
+ * ANDROID_KABI_USE2(number, _new1, _new2)
+ * Use a previous padding entry that was defined with ANDROID_KABI_RESERVE for
+ * two new variables that fit into 64 bits. This is good for when you do not
+ * want to "burn" a 64bit padding variable for a smaller variable size if not
+ * needed.
+ */
+#define ANDROID_KABI_USE2(number, _new1, _new2) \
+ _ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), struct{ _new1; _new2; })
+
+
+#endif /* _ANDROID_KABI_H */
diff --git a/include/linux/android_vendor.h b/include/linux/android_vendor.h
new file mode 100644
index 0000000..af3014c
--- /dev/null
+++ b/include/linux/android_vendor.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * android_vendor.h - Android vendor data
+ *
+ * Copyright 2020 Google LLC
+ *
+ * These macros are to be used to reserve space in kernel data structures
+ * for use by vendor modules.
+ *
+ * These macros should be used before the kernel abi is "frozen".
+ * Fields can be added to various kernel structures that need space
+ * for functionality implemented in vendor modules. The use of
+ * these fields is vendor specific.
+ */
+#ifndef _ANDROID_VENDOR_H
+#define _ANDROID_VENDOR_H
+
+/*
+ * ANDROID_VENDOR_DATA
+ * Reserve some "padding" in a structure for potential future use.
+ * This normally placed at the end of a structure.
+ * number: the "number" of the padding variable in the structure. Start with
+ * 1 and go up.
+ *
+ * ANDROID_VENDOR_DATA_ARRAY
+ * Same as ANDROID_VENDOR_DATA but allocates an array of u64 with
+ * the specified size
+ */
+#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA
+#define ANDROID_VENDOR_DATA(n) u64 android_vendor_data##n
+#define ANDROID_VENDOR_DATA_ARRAY(n, s) u64 android_vendor_data##n[s]
+
+#define ANDROID_OEM_DATA(n) u64 android_oem_data##n
+#define ANDROID_OEM_DATA_ARRAY(n, s) u64 android_oem_data##n[s]
+
+#define android_init_vendor_data(p, n) \
+ memset(&p->android_vendor_data##n, 0, sizeof(p->android_vendor_data##n))
+#define android_init_oem_data(p, n) \
+ memset(&p->android_oem_data##n, 0, sizeof(p->android_oem_data##n))
+#else
+#define ANDROID_VENDOR_DATA(n)
+#define ANDROID_VENDOR_DATA_ARRAY(n, s)
+#define ANDROID_OEM_DATA(n)
+#define ANDROID_OEM_DATA_ARRAY(n, s)
+
+#define android_init_vendor_data(p, n)
+#define android_init_oem_data(p, n)
+#endif
+
+#endif /* _ANDROID_VENDOR_H */
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index a07b510..589f2a2 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -93,5 +93,6 @@ void remove_cpu_topology(unsigned int cpuid);
void reset_cpu_topology(void);
int parse_acpi_topology(void);
#endif
+extern bool topology_update_done;
#endif /* _LINUX_ARCH_TOPOLOGY_H_ */
diff --git a/include/linux/blk-crypto-profile.h b/include/linux/blk-crypto-profile.h
index e6802b6..8b30d04 100644
--- a/include/linux/blk-crypto-profile.h
+++ b/include/linux/blk-crypto-profile.h
@@ -57,6 +57,20 @@ struct blk_crypto_ll_ops {
int (*keyslot_evict)(struct blk_crypto_profile *profile,
const struct blk_crypto_key *key,
unsigned int slot);
+
+ /**
+ * @derive_sw_secret: Derive the software secret from a hardware-wrapped
+ * key in ephemerally-wrapped form.
+ *
+ * This only needs to be implemented if BLK_CRYPTO_KEY_TYPE_HW_WRAPPED
+ * is supported.
+ *
+ * Must return 0 on success, -EBADMSG if the key is invalid, or another
+ * -errno code on other errors.
+ */
+ int (*derive_sw_secret)(struct blk_crypto_profile *profile,
+ const u8 *eph_key, size_t eph_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);
};
/**
@@ -85,6 +99,12 @@ struct blk_crypto_profile {
unsigned int max_dun_bytes_supported;
/**
+ * @key_types_supported: A bitmask of the supported key types:
+ * BLK_CRYPTO_KEY_TYPE_STANDARD and/or BLK_CRYPTO_KEY_TYPE_HW_WRAPPED.
+ */
+ unsigned int key_types_supported;
+
+ /**
* @modes_supported: Array of bitmasks that specifies whether each
* combination of crypto mode and data unit size is supported.
* Specifically, the i'th bit of modes_supported[crypto_mode] is set if
diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h
index 1e3e5d0..ef771c9 100644
--- a/include/linux/blk-crypto.h
+++ b/include/linux/blk-crypto.h
@@ -17,7 +17,58 @@ enum blk_crypto_mode_num {
BLK_ENCRYPTION_MODE_MAX,
};
-#define BLK_CRYPTO_MAX_KEY_SIZE 64
+/*
+ * Supported types of keys. Must be bitflags due to their use in
+ * blk_crypto_profile::key_types_supported.
+ */
+enum blk_crypto_key_type {
+ /*
+ * Standard keys (i.e. "software keys"). These keys are simply kept in
+ * raw, plaintext form in kernel memory.
+ */
+ BLK_CRYPTO_KEY_TYPE_STANDARD = 1 << 0,
+
+ /*
+ * Hardware-wrapped keys. These keys are only present in kernel memory
+ * in ephemerally-wrapped form, and they can only be unwrapped by
+ * dedicated hardware. For details, see the "Hardware-wrapped keys"
+ * section of Documentation/block/inline-encryption.rst.
+ */
+ BLK_CRYPTO_KEY_TYPE_HW_WRAPPED = 1 << 1,
+};
+
+/*
+ * Currently the maximum standard key size is 64 bytes, as that is the key size
+ * of BLK_ENCRYPTION_MODE_AES_256_XTS which takes the longest key.
+ *
+ * The maximum hardware-wrapped key size depends on the hardware's key wrapping
+ * algorithm, which is a hardware implementation detail, so it isn't precisely
+ * specified. But currently 128 bytes is plenty in practice. Implementations
+ * are recommended to wrap a 32-byte key for the hardware KDF with AES-256-GCM,
+ * which should result in a size closer to 64 bytes than 128.
+ *
+ * Both of these values can trivially be increased if ever needed.
+ */
+#define BLK_CRYPTO_MAX_STANDARD_KEY_SIZE 64
+#define BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE 128
+
+/* This should use max(), but max() doesn't work in a struct definition. */
+#define BLK_CRYPTO_MAX_ANY_KEY_SIZE \
+ (BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE > \
+ BLK_CRYPTO_MAX_STANDARD_KEY_SIZE ? \
+ BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE : BLK_CRYPTO_MAX_STANDARD_KEY_SIZE)
+
+/*
+ * Size of the "software secret" which can be derived from a hardware-wrapped
+ * key. This is currently always 32 bytes. Note, the choice of 32 bytes
+ * assumes that the software secret is only used directly for algorithms that
+ * don't require more than a 256-bit key to get the desired security strength.
+ * If it were to be used e.g. directly as an AES-256-XTS key, then this would
+ * need to be increased (which is possible if hardware supports it, but care
+ * would need to be taken to avoid breaking users who need exactly 32 bytes).
+ */
+#define BLK_CRYPTO_SW_SECRET_SIZE 32
+
/**
* struct blk_crypto_config - an inline encryption key's crypto configuration
* @crypto_mode: encryption algorithm this key is for
@@ -26,20 +77,23 @@ enum blk_crypto_mode_num {
* ciphertext. This is always a power of 2. It might be e.g. the
* filesystem block size or the disk sector size.
* @dun_bytes: the maximum number of bytes of DUN used when using this key
+ * @key_type: the type of this key -- either standard or hardware-wrapped
*/
struct blk_crypto_config {
enum blk_crypto_mode_num crypto_mode;
unsigned int data_unit_size;
unsigned int dun_bytes;
+ enum blk_crypto_key_type key_type;
};
/**
* struct blk_crypto_key - an inline encryption key
- * @crypto_cfg: the crypto configuration (like crypto_mode, key size) for this
- * key
+ * @crypto_cfg: the crypto mode, data unit size, key type, and other
+ * characteristics of this key and how it will be used
* @data_unit_size_bits: log2 of data_unit_size
- * @size: size of this key in bytes (determined by @crypto_cfg.crypto_mode)
- * @raw: the raw bytes of this key. Only the first @size bytes are used.
+ * @size: size of this key in bytes. The size of a standard key is fixed for a
+ * given crypto mode, but the size of a hardware-wrapped key can vary.
+ * @raw: the bytes of this key. Only the first @size bytes are significant.
*
* A blk_crypto_key is immutable once created, and many bios can reference it at
* the same time. It must not be freed until all bios using it have completed
@@ -49,7 +103,7 @@ struct blk_crypto_key {
struct blk_crypto_config crypto_cfg;
unsigned int data_unit_size_bits;
unsigned int size;
- u8 raw[BLK_CRYPTO_MAX_KEY_SIZE];
+ u8 raw[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
};
#define BLK_CRYPTO_MAX_IV_SIZE 32
@@ -87,7 +141,9 @@ bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc,
unsigned int bytes,
const u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]);
-int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
+int blk_crypto_init_key(struct blk_crypto_key *blk_key,
+ const u8 *raw_key, size_t raw_key_size,
+ enum blk_crypto_key_type key_type,
enum blk_crypto_mode_num crypto_mode,
unsigned int dun_bytes,
unsigned int data_unit_size);
@@ -103,6 +159,10 @@ bool blk_crypto_config_supported_natively(struct block_device *bdev,
bool blk_crypto_config_supported(struct block_device *bdev,
const struct blk_crypto_config *cfg);
+int blk_crypto_derive_sw_secret(struct block_device *bdev,
+ const u8 *eph_key, size_t eph_key_size,
+ u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);
+
#else /* CONFIG_BLK_INLINE_ENCRYPTION */
static inline bool bio_has_crypt_ctx(struct bio *bio)
@@ -112,6 +172,9 @@ static inline bool bio_has_crypt_ctx(struct bio *bio)
#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
+static inline void bio_clone_skip_dm_default_key(struct bio *dst,
+ const struct bio *src);
+
int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask);
/**
* bio_crypt_clone - clone bio encryption context
@@ -127,9 +190,42 @@ int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask);
static inline int bio_crypt_clone(struct bio *dst, struct bio *src,
gfp_t gfp_mask)
{
+ bio_clone_skip_dm_default_key(dst, src);
if (bio_has_crypt_ctx(src))
return __bio_crypt_clone(dst, src, gfp_mask);
return 0;
}
+#if IS_ENABLED(CONFIG_DM_DEFAULT_KEY)
+static inline void bio_set_skip_dm_default_key(struct bio *bio)
+{
+ bio->bi_skip_dm_default_key = true;
+}
+
+static inline bool bio_should_skip_dm_default_key(const struct bio *bio)
+{
+ return bio->bi_skip_dm_default_key;
+}
+
+static inline void bio_clone_skip_dm_default_key(struct bio *dst,
+ const struct bio *src)
+{
+ dst->bi_skip_dm_default_key = src->bi_skip_dm_default_key;
+}
+#else /* CONFIG_DM_DEFAULT_KEY */
+static inline void bio_set_skip_dm_default_key(struct bio *bio)
+{
+}
+
+static inline bool bio_should_skip_dm_default_key(const struct bio *bio)
+{
+ return false;
+}
+
+static inline void bio_clone_skip_dm_default_key(struct bio *dst,
+ const struct bio *src)
+{
+}
+#endif /* !CONFIG_DM_DEFAULT_KEY */
+
#endif /* __LINUX_BLK_CRYPTO_H */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 99be590..4f7eca4 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -281,6 +281,9 @@ struct bio {
#ifdef CONFIG_BLK_INLINE_ENCRYPTION
struct bio_crypt_ctx *bi_crypt_context;
+#if IS_ENABLED(CONFIG_DM_DEFAULT_KEY)
+ bool bi_skip_dm_default_key;
+#endif
#endif
union {
diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h
index 267cd06..e989ab74 100644
--- a/include/linux/clk-provider.h
+++ b/include/linux/clk-provider.h
@@ -32,6 +32,7 @@
#define CLK_OPS_PARENT_ENABLE BIT(12)
/* duty cycle call may be forwarded to the parent clock */
#define CLK_DUTY_CYCLE_PARENT BIT(13)
+#define CLK_DONT_HOLD_STATE BIT(14) /* Don't hold state */
struct clk;
struct clk_hw;
@@ -217,6 +218,13 @@ struct clk_duty {
* directory is provided as an argument. Called with
* prepare_lock held. Returns 0 on success, -EERROR otherwise.
*
+ * @pre_rate_change: Optional callback for a clock to fulfill its rate
+ * change requirements before any rate change has occurred in
+ * its clock tree. Returns 0 on success, -EERROR otherwise.
+ *
+ * @post_rate_change: Optional callback for a clock to clean up any
+ * requirements that were needed while the clock and its tree
+ * was changing states. Returns 0 on success, -EERROR otherwise.
*
* The clk_enable/clk_disable and clk_prepare/clk_unprepare pairs allow
* implementations to split any work between atomic (enable) and sleepable
@@ -264,6 +272,12 @@ struct clk_ops {
int (*init)(struct clk_hw *hw);
void (*terminate)(struct clk_hw *hw);
void (*debug_init)(struct clk_hw *hw, struct dentry *dentry);
+ int (*pre_rate_change)(struct clk_hw *hw,
+ unsigned long rate,
+ unsigned long new_rate);
+ int (*post_rate_change)(struct clk_hw *hw,
+ unsigned long old_rate,
+ unsigned long rate);
};
/**
@@ -1272,6 +1286,7 @@ int __must_check of_clk_hw_register(struct device_node *node, struct clk_hw *hw)
void clk_unregister(struct clk *clk);
void clk_hw_unregister(struct clk_hw *hw);
+void clk_sync_state(struct device *dev);
/* helper functions */
const char *__clk_get_name(const struct clk *clk);
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 6a94a6e..3b28b47 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -1195,14 +1195,6 @@ static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_
}
#endif
-#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
-void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
- struct cpufreq_governor *old_gov);
-#else
-static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
- struct cpufreq_governor *old_gov) { }
-#endif
-
extern unsigned int arch_freq_get_on_cpu(int cpu);
#ifndef arch_set_freq_scale
diff --git a/include/linux/cpufreq_times.h b/include/linux/cpufreq_times.h
new file mode 100644
index 0000000..38272a5
--- /dev/null
+++ b/include/linux/cpufreq_times.h
@@ -0,0 +1,42 @@
+/* drivers/cpufreq/cpufreq_times.c
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _LINUX_CPUFREQ_TIMES_H
+#define _LINUX_CPUFREQ_TIMES_H
+
+#include <linux/cpufreq.h>
+#include <linux/pid.h>
+
+#ifdef CONFIG_CPU_FREQ_TIMES
+void cpufreq_task_times_init(struct task_struct *p);
+void cpufreq_task_times_alloc(struct task_struct *p);
+void cpufreq_task_times_exit(struct task_struct *p);
+int proc_time_in_state_show(struct seq_file *m, struct pid_namespace *ns,
+ struct pid *pid, struct task_struct *p);
+void cpufreq_acct_update_power(struct task_struct *p, u64 cputime);
+void cpufreq_times_create_policy(struct cpufreq_policy *policy);
+void cpufreq_times_record_transition(struct cpufreq_policy *policy,
+ unsigned int new_freq);
+#else
+static inline void cpufreq_task_times_init(struct task_struct *p) {}
+static inline void cpufreq_task_times_alloc(struct task_struct *p) {}
+static inline void cpufreq_task_times_exit(struct task_struct *p) {}
+static inline void cpufreq_acct_update_power(struct task_struct *p,
+ u64 cputime) {}
+static inline void cpufreq_times_create_policy(struct cpufreq_policy *policy) {}
+static inline void cpufreq_times_record_transition(
+ struct cpufreq_policy *policy, unsigned int new_freq) {}
+#endif /* CONFIG_CPU_FREQ_TIMES */
+#endif /* _LINUX_CPUFREQ_TIMES_H */
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index d58e047..92b0a92 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -71,8 +71,6 @@ extern void cpuset_init_smp(void);
extern void cpuset_force_rebuild(void);
extern void cpuset_update_active_cpus(void);
extern void cpuset_wait_for_hotplug(void);
-extern void cpuset_read_lock(void);
-extern void cpuset_read_unlock(void);
extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
@@ -196,9 +194,6 @@ static inline void cpuset_update_active_cpus(void)
static inline void cpuset_wait_for_hotplug(void) { }
-static inline void cpuset_read_lock(void) { }
-static inline void cpuset_read_unlock(void) { }
-
static inline void cpuset_cpus_allowed(struct task_struct *p,
struct cpumask *mask)
{
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 6b351e0..ccdb1c2 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -140,6 +140,7 @@ struct dentry_operations {
struct vfsmount *(*d_automount)(struct path *);
int (*d_manage)(const struct path *, bool);
struct dentry *(*d_real)(struct dentry *, const struct inode *);
+ void (*d_canonical_path)(const struct path *, struct path *);
} ____cacheline_aligned;
/*
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 6fa8d4e..c873145 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -230,6 +230,41 @@ struct dma_buf_ops {
int (*begin_cpu_access)(struct dma_buf *, enum dma_data_direction);
/**
+ * @begin_cpu_access_partial:
+ *
+ * This is called from dma_buf_begin_cpu_access_partial() and allows the
+ * exporter to ensure that the memory specified in the range is
+ * available for cpu access - the exporter might need to allocate or
+ * swap-in and pin the backing storage.
+ * The exporter also needs to ensure that cpu access is
+ * coherent for the access direction. The direction can be used by the
+ * exporter to optimize the cache flushing, i.e. access with a different
+ * direction (read instead of write) might return stale or even bogus
+ * data (e.g. when the exporter needs to copy the data to temporary
+ * storage).
+ *
+ * This callback is optional.
+ *
+ * FIXME: This is both called through the DMA_BUF_IOCTL_SYNC command
+ * from userspace (where storage shouldn't be pinned to avoid handing
+ * de-factor mlock rights to userspace) and for the kernel-internal
+ * users of the various kmap interfaces, where the backing storage must
+ * be pinned to guarantee that the atomic kmap calls can succeed. Since
+ * there's no in-kernel users of the kmap interfaces yet this isn't a
+ * real problem.
+ *
+ * Returns:
+ *
+ * 0 on success or a negative error code on failure. This can for
+ * example fail when the backing storage can't be allocated. Can also
+ * return -ERESTARTSYS or -EINTR when the call has been interrupted and
+ * needs to be restarted.
+ */
+ int (*begin_cpu_access_partial)(struct dma_buf *dmabuf,
+ enum dma_data_direction,
+ unsigned int offset, unsigned int len);
+
+ /**
* @end_cpu_access:
*
* This is called from dma_buf_end_cpu_access() when the importer is
@@ -247,6 +282,28 @@ struct dma_buf_ops {
int (*end_cpu_access)(struct dma_buf *, enum dma_data_direction);
/**
+ * @end_cpu_access_partial:
+ *
+ * This is called from dma_buf_end_cpu_access_partial() when the
+ * importer is done accessing the CPU. The exporter can use to limit
+ * cache flushing to only the range specefied and to unpin any
+ * resources pinned in @begin_cpu_access_umapped.
+ * The result of any dma_buf kmap calls after end_cpu_access_partial is
+ * undefined.
+ *
+ * This callback is optional.
+ *
+ * Returns:
+ *
+ * 0 on success or a negative error code on failure. Can return
+ * -ERESTARTSYS or -EINTR when the call has been interrupted and needs
+ * to be restarted.
+ */
+ int (*end_cpu_access_partial)(struct dma_buf *dmabuf,
+ enum dma_data_direction,
+ unsigned int offset, unsigned int len);
+
+ /**
* @mmap:
*
* This callback is used by the dma_buf_mmap() function
@@ -285,6 +342,20 @@ struct dma_buf_ops {
int (*vmap)(struct dma_buf *dmabuf, struct iosys_map *map);
void (*vunmap)(struct dma_buf *dmabuf, struct iosys_map *map);
+
+ /**
+ * @get_flags:
+ *
+ * This is called by dma_buf_get_flags and is used to get the buffer's
+ * flags.
+ * This callback is optional.
+ *
+ * Returns:
+ *
+ * 0 on success or a negative error code on failure. On success flags
+ * will be populated with the buffer's flags.
+ */
+ int (*get_flags)(struct dma_buf *dmabuf, unsigned long *flags);
};
/**
@@ -495,6 +566,8 @@ struct dma_buf_attach_ops {
* @importer_ops: importer operations for this attachment, if provided
* dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
* @importer_priv: importer specific attachment data.
+ * @dma_map_attrs: DMA attributes to be used when the exporter maps the buffer
+ * through dma_buf_map_attachment.
*
* This structure holds the attachment information between the dma_buf buffer
* and its user device(s). The list contains one attachment struct per device
@@ -515,6 +588,7 @@ struct dma_buf_attachment {
const struct dma_buf_attach_ops *importer_ops;
void *importer_priv;
void *priv;
+ unsigned long dma_map_attrs;
};
/**
@@ -616,8 +690,14 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *, struct sg_table *,
void dma_buf_move_notify(struct dma_buf *dma_buf);
int dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
enum dma_data_direction dir);
+int dma_buf_begin_cpu_access_partial(struct dma_buf *dma_buf,
+ enum dma_data_direction dir,
+ unsigned int offset, unsigned int len);
int dma_buf_end_cpu_access(struct dma_buf *dma_buf,
enum dma_data_direction dir);
+int dma_buf_end_cpu_access_partial(struct dma_buf *dma_buf,
+ enum dma_data_direction dir,
+ unsigned int offset, unsigned int len);
struct sg_table *
dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach,
enum dma_data_direction direction);
@@ -631,4 +711,5 @@ int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map);
void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map);
int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map);
+int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags);
#endif /* __DMA_BUF_H__ */
diff --git a/include/linux/dma-heap.h b/include/linux/dma-heap.h
index 0c05561..44f1050 100644
--- a/include/linux/dma-heap.h
+++ b/include/linux/dma-heap.h
@@ -51,6 +51,15 @@ struct dma_heap_export_info {
void *dma_heap_get_drvdata(struct dma_heap *heap);
/**
+ * dma_heap_get_dev() - get device struct for the heap
+ * @heap: DMA-Heap to retrieve device struct from
+ *
+ * Returns:
+ * The device struct for the heap.
+ */
+struct device *dma_heap_get_dev(struct dma_heap *heap);
+
+/**
* dma_heap_get_name() - get heap name
* @heap: DMA-Heap to retrieve private data for
*
@@ -65,4 +74,49 @@ const char *dma_heap_get_name(struct dma_heap *heap);
*/
struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info);
+/**
+ * dma_heap_put - drops a reference to a dmabuf heaps, potentially freeing it
+ * @heap: heap pointer
+ */
+void dma_heap_put(struct dma_heap *heap);
+
+/**
+ * dma_heap_find - Returns the registered dma_heap with the specified name
+ * @name: Name of the heap to find
+ *
+ * NOTE: dma_heaps returned from this function MUST be released
+ * using dma_heap_put() when the user is done.
+ */
+struct dma_heap *dma_heap_find(const char *name);
+
+/**
+ * dma_heap_buffer_alloc - Allocate dma-buf from a dma_heap
+ * @heap: dma_heap to allocate from
+ * @len: size to allocate
+ * @fd_flags: flags to set on returned dma-buf fd
+ * @heap_flags: flags to pass to the dma heap
+ *
+ * This is for internal dma-buf allocations only.
+ */
+struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
+ unsigned int fd_flags,
+ unsigned int heap_flags);
+
+/** dma_heap_buffer_free - Free dma_buf allocated by dma_heap_buffer_alloc
+ * @dma_buf: dma_buf to free
+ *
+ * This is really only a simple wrapper to dma_buf_put()
+ */
+void dma_heap_buffer_free(struct dma_buf *);
+
+/**
+ * dma_heap_bufferfd_alloc - Allocate dma-buf fd from a dma_heap
+ * @heap: dma_heap to allocate from
+ * @len: size to allocate
+ * @fd_flags: flags to set on returned dma-buf fd
+ * @heap_flags: flags to pass to the dma heap
+ */
+int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len,
+ unsigned int fd_flags,
+ unsigned int heap_flags);
#endif /* _DMA_HEAPS_H */
diff --git a/include/linux/export.h b/include/linux/export.h
index 3f31ced..b674ca6 100644
--- a/include/linux/export.h
+++ b/include/linux/export.h
@@ -106,7 +106,7 @@ struct kernel_symbol {
*/
#define __EXPORT_SYMBOL(sym, sec, ns)
-#elif defined(CONFIG_TRIM_UNUSED_KSYMS)
+#elif defined(CONFIG_TRIM_UNUSED_KSYMS) && !defined(MODULE)
#include <generated/autoksyms.h>
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 066555a..1e7b5ca 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -3433,6 +3433,11 @@ static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
return 0;
}
+static inline rwf_t iocb_to_rw_flags(int ifl, int iocb_mask)
+{
+ return ifl & iocb_mask;
+}
+
static inline ino_t parent_ino(struct dentry *dentry)
{
ino_t res;
diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
index 4f5f8a6..c431ad6 100644
--- a/include/linux/fscrypt.h
+++ b/include/linux/fscrypt.h
@@ -809,6 +809,20 @@ static inline u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk,
}
#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
+#if IS_ENABLED(CONFIG_FS_ENCRYPTION) && IS_ENABLED(CONFIG_DM_DEFAULT_KEY)
+static inline bool
+fscrypt_inode_should_skip_dm_default_key(const struct inode *inode)
+{
+ return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
+}
+#else
+static inline bool
+fscrypt_inode_should_skip_dm_default_key(const struct inode *inode)
+{
+ return false;
+}
+#endif
+
/**
* fscrypt_inode_uses_inline_crypto() - test whether an inode uses inline
* encryption
diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h
index 40f14e5..0beb972 100644
--- a/include/linux/fsverity.h
+++ b/include/linux/fsverity.h
@@ -254,4 +254,18 @@ static inline bool fsverity_active(const struct inode *inode)
return fsverity_get_info(inode) != NULL;
}
+#ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES
+int __fsverity_verify_signature(const struct inode *inode, const u8 *signature,
+ size_t sig_size, const u8 *file_digest,
+ unsigned int digest_algorithm);
+#else /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */
+static inline int __fsverity_verify_signature(const struct inode *inode,
+ const u8 *signature, size_t sig_size,
+ const u8 *file_digest,
+ unsigned int digest_algorithm)
+{
+ return 0;
+}
+#endif /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */
+
#endif /* _LINUX_FSVERITY_H */
diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
index 37dfdcf..03cead7 100644
--- a/include/linux/ipv6.h
+++ b/include/linux/ipv6.h
@@ -43,6 +43,7 @@ struct ipv6_devconf {
__s32 accept_ra_rt_info_max_plen;
#endif
#endif
+ __s32 accept_ra_rt_table;
__s32 proxy_ndp;
__s32 accept_source_route;
__s32 accept_ra_from_local;
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9c49ec5..8c89da0 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -148,9 +148,7 @@ enum zone_stat_item {
NR_MLOCK, /* mlock()ed pages found and moved off LRU */
/* Second 128 byte cacheline */
NR_BOUNCE,
-#if IS_ENABLED(CONFIG_ZSMALLOC)
NR_ZSPAGES, /* allocated in zsmalloc */
-#endif
NR_FREE_CMA_PAGES,
NR_VM_ZONE_STAT_ITEMS };
diff --git a/include/linux/module.h b/include/linux/module.h
index ec61fb5..6aa2bc8 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -404,10 +404,12 @@ struct module {
const s32 *gpl_crcs;
bool using_gplonly_symbols;
-#ifdef CONFIG_MODULE_SIG
- /* Signature was verified. */
+ /*
+ * Signature was verified. Unconditionally compiled in Android to
+ * preserve ABI compatibility between kernels without module
+ * signing enabled and signed modules.
+ */
bool sig_ok;
-#endif
bool async_probe_requested;
diff --git a/include/linux/netfilter/xt_quota2.h b/include/linux/netfilter/xt_quota2.h
new file mode 100644
index 0000000..a391871
--- /dev/null
+++ b/include/linux/netfilter/xt_quota2.h
@@ -0,0 +1,26 @@
+#ifndef _XT_QUOTA_H
+#define _XT_QUOTA_H
+#include <linux/types.h>
+
+enum xt_quota_flags {
+ XT_QUOTA_INVERT = 1 << 0,
+ XT_QUOTA_GROW = 1 << 1,
+ XT_QUOTA_PACKET = 1 << 2,
+ XT_QUOTA_NO_CHANGE = 1 << 3,
+ XT_QUOTA_MASK = 0x0F,
+};
+
+struct xt_quota_counter;
+
+struct xt_quota_mtinfo2 {
+ char name[15];
+ u_int8_t flags;
+
+ /* Comparison-invariant */
+ aligned_u64 quota;
+
+ /* Used internally by the kernel */
+ struct xt_quota_counter *master __attribute__((aligned(8)));
+};
+
+#endif /* _XT_QUOTA_H */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index c6a3bac..dc21c6a 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1361,22 +1361,40 @@ extern void perf_event_bpf_event(struct bpf_prog *prog,
#ifdef CONFIG_GUEST_PERF_EVENTS
extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
-
-DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);
-DECLARE_STATIC_CALL(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
-DECLARE_STATIC_CALL(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
-
+static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void)
+{
+ /*
+ * Callbacks are RCU-protected and must be READ_ONCE to avoid reloading
+ * the callbacks between a !NULL check and dereferences, to ensure
+ * pending stores/changes to the callback pointers are visible before a
+ * non-NULL perf_guest_cbs is visible to readers, and to prevent a
+ * module from unloading callbacks while readers are active.
+ */
+ return rcu_dereference(perf_guest_cbs);
+}
static inline unsigned int perf_guest_state(void)
{
- return static_call(__perf_guest_state)();
+ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+
+ return guest_cbs ? guest_cbs->state() : 0;
}
static inline unsigned long perf_guest_get_ip(void)
{
- return static_call(__perf_guest_get_ip)();
+ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+
+ /*
+ * Arbitrarily return '0' in the unlikely scenario that the callbacks
+ * are unregistered between checking guest state and getting the IP.
+ */
+ return guest_cbs ? guest_cbs->get_ip() : 0;
}
static inline unsigned int perf_guest_handle_intel_pt_intr(void)
{
- return static_call(__perf_guest_handle_intel_pt_intr)();
+ struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
+
+ if (guest_cbs && guest_cbs->handle_intel_pt_intr)
+ return guest_cbs->handle_intel_pt_intr();
+ return 0;
}
extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
diff --git a/include/linux/poll.h b/include/linux/poll.h
index a9e0e1c..d1ea4f3 100644
--- a/include/linux/poll.h
+++ b/include/linux/poll.h
@@ -14,11 +14,7 @@
/* ~832 bytes of stack space used max in sys_select/sys_poll before allocating
additional memory. */
-#ifdef __clang__
-#define MAX_STACK_ALLOC 768
-#else
#define MAX_STACK_ALLOC 832
-#endif
#define FRONTEND_STACK_ALLOC 256
#define SELECT_STACK_ALLOC FRONTEND_STACK_ALLOC
#define POLL_STACK_ALLOC FRONTEND_STACK_ALLOC
diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h
index aa2c4a7..b0cc4b8 100644
--- a/include/linux/power_supply.h
+++ b/include/linux/power_supply.h
@@ -50,6 +50,12 @@ enum {
POWER_SUPPLY_CHARGE_TYPE_CUSTOM, /* use CHARGE_CONTROL_* props */
POWER_SUPPLY_CHARGE_TYPE_LONGLIFE, /* slow speed, longer life */
POWER_SUPPLY_CHARGE_TYPE_BYPASS, /* bypassing the charger */
+
+ /*
+ * force to 50 to minimize the chances of userspace binary
+ * incompatibility on newer upstream kernels
+ */
+ POWER_SUPPLY_CHARGE_TYPE_TAPER_EXT = 50, /* charging in CV phase */
};
enum {
@@ -780,12 +786,22 @@ static inline struct power_supply *power_supply_get_by_name(const char *name)
#ifdef CONFIG_OF
extern struct power_supply *power_supply_get_by_phandle(struct device_node *np,
const char *property);
+extern int power_supply_get_by_phandle_array(struct device_node *np,
+ const char *property,
+ struct power_supply **psy,
+ ssize_t size);
extern struct power_supply *devm_power_supply_get_by_phandle(
struct device *dev, const char *property);
#else /* !CONFIG_OF */
static inline struct power_supply *
power_supply_get_by_phandle(struct device_node *np, const char *property)
{ return NULL; }
+static inline int
+power_supply_get_by_phandle_array(struct device_node *np,
+ const char *property,
+ struct power_supply **psy,
+ int size)
+{ return 0; }
static inline struct power_supply *
devm_power_supply_get_by_phandle(struct device *dev, const char *property)
{ return NULL; }
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5affff1..77d24cf 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -36,6 +36,7 @@
#include <linux/seqlock.h>
#include <linux/kcsan.h>
#include <linux/rv.h>
+#include <linux/android_vendor.h>
#include <asm/kmap_size.h>
/* task_struct member predeclarations (sorted alphabetically): */
@@ -311,6 +312,8 @@ extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
extern long io_schedule_timeout(long timeout);
extern void io_schedule(void);
+extern struct task_struct *pick_migrate_task(struct rq *rq);
+extern int select_fallback_rq(int cpu, struct task_struct *p);
/**
* struct prev_cputime - snapshot of system and user cputime
@@ -1020,6 +1023,10 @@ struct task_struct {
u64 stimescaled;
#endif
u64 gtime;
+#ifdef CONFIG_CPU_FREQ_TIMES
+ u64 *time_in_state;
+ unsigned int max_state;
+#endif
struct prev_cputime prev_cputime;
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
struct vtime vtime;
@@ -1132,6 +1139,7 @@ struct task_struct {
raw_spinlock_t pi_lock;
struct wake_q_node wake_q;
+ int wake_q_count;
#ifdef CONFIG_RT_MUTEXES
/* PI waiters blocked on a rt_mutex held by this task: */
@@ -1497,6 +1505,8 @@ struct task_struct {
struct callback_head mce_kill_me;
int mce_count;
#endif
+ ANDROID_VENDOR_DATA_ARRAY(1, 64);
+ ANDROID_OEM_DATA_ARRAY(1, 6);
#ifdef CONFIG_KRETPROBES
struct llist_head kretprobe_instances;
diff --git a/include/linux/sched/wake_q.h b/include/linux/sched/wake_q.h
index 06cd8fb..9a1394f 100644
--- a/include/linux/sched/wake_q.h
+++ b/include/linux/sched/wake_q.h
@@ -38,6 +38,7 @@
struct wake_q_head {
struct wake_q_node *first;
struct wake_q_node **lastp;
+ int count;
};
#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)
@@ -52,6 +53,7 @@ static inline void wake_q_init(struct wake_q_head *head)
{
head->first = WAKE_Q_TAIL;
head->lastp = &head->first;
+ head->count = 0;
}
static inline bool wake_q_empty(struct wake_q_head *head)
diff --git a/include/linux/sched/xacct.h b/include/linux/sched/xacct.h
index c078f0a..9544c9d 100644
--- a/include/linux/sched/xacct.h
+++ b/include/linux/sched/xacct.h
@@ -28,6 +28,11 @@ static inline void inc_syscw(struct task_struct *tsk)
{
tsk->ioac.syscw++;
}
+
+static inline void inc_syscfs(struct task_struct *tsk)
+{
+ tsk->ioac.syscfs++;
+}
#else
static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
{
@@ -44,6 +49,10 @@ static inline void inc_syscr(struct task_struct *tsk)
static inline void inc_syscw(struct task_struct *tsk)
{
}
+
+static inline void inc_syscfs(struct task_struct *tsk)
+{
+}
#endif
#endif /* _LINUX_SCHED_XACCT_H */
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 45af703..c44f34b 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -668,7 +668,7 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag
}
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
- unsigned long caller) __alloc_size(1);
+ unsigned long caller);
#define kmalloc_node_track_caller(size, flags, node) \
__kmalloc_node_track_caller(size, flags, node, \
_RET_IP_)
diff --git a/include/linux/suspend.h b/include/linux/suspend.h
index cfe19a0..d1ab3ff 100644
--- a/include/linux/suspend.h
+++ b/include/linux/suspend.h
@@ -510,6 +510,7 @@ extern bool pm_get_wakeup_count(unsigned int *count, bool block);
extern bool pm_save_wakeup_count(unsigned int count);
extern void pm_wakep_autosleep_enabled(bool set);
extern void pm_print_active_wakeup_sources(void);
+extern void pm_get_active_wakeup_sources(char *pending_sources, size_t max);
extern unsigned int lock_system_sleep(void);
extern void unlock_system_sleep(unsigned int);
diff --git a/include/linux/task_io_accounting.h b/include/linux/task_io_accounting.h
index 6f6acce..bb26108 100644
--- a/include/linux/task_io_accounting.h
+++ b/include/linux/task_io_accounting.h
@@ -19,6 +19,8 @@ struct task_io_accounting {
u64 syscr;
/* # of write syscalls */
u64 syscw;
+ /* # of fsync syscalls */
+ u64 syscfs;
#endif /* CONFIG_TASK_XACCT */
#ifdef CONFIG_TASK_IO_ACCOUNTING
diff --git a/include/linux/task_io_accounting_ops.h b/include/linux/task_io_accounting_ops.h
index bb5498b..733ab62 100644
--- a/include/linux/task_io_accounting_ops.h
+++ b/include/linux/task_io_accounting_ops.h
@@ -97,6 +97,7 @@ static inline void task_chr_io_accounting_add(struct task_io_accounting *dst,
dst->wchar += src->wchar;
dst->syscr += src->syscr;
dst->syscw += src->syscw;
+ dst->syscfs += src->syscfs;
}
#else
static inline void task_chr_io_accounting_add(struct task_io_accounting *dst,
diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h
index 43ac3fa..814f6579 100644
--- a/include/linux/usb/composite.h
+++ b/include/linux/usb/composite.h
@@ -578,6 +578,7 @@ struct usb_function_instance {
struct config_group group;
struct list_head cfs_list;
struct usb_function_driver *fd;
+ struct usb_function *f;
int (*set_inst_name)(struct usb_function_instance *inst,
const char *name);
void (*free_func_inst)(struct usb_function_instance *inst);
diff --git a/include/linux/usb/f_accessory.h b/include/linux/usb/f_accessory.h
new file mode 100644
index 0000000..ebe3c4d
--- /dev/null
+++ b/include/linux/usb/f_accessory.h
@@ -0,0 +1,23 @@
+/*
+ * Gadget Function Driver for Android USB accessories
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Author: Mike Lockwood <lockwood@android.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __LINUX_USB_F_ACCESSORY_H
+#define __LINUX_USB_F_ACCESSORY_H
+
+#include <uapi/linux/usb/f_accessory.h>
+
+#endif /* __LINUX_USB_F_ACCESSORY_H */
diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h
index 35d7eed..de584a7 100644
--- a/include/linux/virtio_vsock.h
+++ b/include/linux/virtio_vsock.h
@@ -9,7 +9,8 @@
#define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE (1024 * 4)
#define VIRTIO_VSOCK_MAX_BUF_SIZE 0xFFFFFFFFUL
-#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE (1024 * 64)
+#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE virtio_transport_max_vsock_pkt_buf_size
+extern uint virtio_transport_max_vsock_pkt_buf_size;
enum {
VSOCK_VQ_RX = 0, /* for host to guest data */
diff --git a/include/linux/wait.h b/include/linux/wait.h
index a0307b5..40dbc01 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -229,6 +229,7 @@ void __wake_up_pollfree(struct wait_queue_head *wq_head);
#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
+#define wake_up_sync(x) __wake_up_sync((x), TASK_NORMAL)
/*
* Wakeup macros to be used to report events to the targets.
diff --git a/include/linux/wakeup_reason.h b/include/linux/wakeup_reason.h
new file mode 100644
index 0000000..54f5caa
--- /dev/null
+++ b/include/linux/wakeup_reason.h
@@ -0,0 +1,37 @@
+/*
+ * include/linux/wakeup_reason.h
+ *
+ * Logs the reason which caused the kernel to resume
+ * from the suspend mode.
+ *
+ * Copyright (C) 2014 Google, Inc.
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _LINUX_WAKEUP_REASON_H
+#define _LINUX_WAKEUP_REASON_H
+
+#define MAX_SUSPEND_ABORT_LEN 256
+
+#ifdef CONFIG_SUSPEND
+void log_irq_wakeup_reason(int irq);
+void log_threaded_irq_wakeup_reason(int irq, int parent_irq);
+void log_suspend_abort_reason(const char *fmt, ...);
+void log_abnormal_wakeup_reason(const char *fmt, ...);
+void clear_wakeup_reasons(void);
+#else
+static inline void log_irq_wakeup_reason(int irq) { }
+static inline void log_threaded_irq_wakeup_reason(int irq, int parent_irq) { }
+static inline void log_suspend_abort_reason(const char *fmt, ...) { }
+static inline void log_abnormal_wakeup_reason(const char *fmt, ...) { }
+static inline void clear_wakeup_reasons(void) { }
+#endif
+
+#endif /* _LINUX_WAKEUP_REASON_H */
diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
index 4b6a9d2..cacd498 100644
--- a/include/media/videobuf2-core.h
+++ b/include/media/videobuf2-core.h
@@ -20,7 +20,7 @@
#include <media/media-request.h>
#include <media/frame_vector.h>
-#define VB2_MAX_FRAME (32)
+#define VB2_MAX_FRAME (64)
#define VB2_MAX_PLANES (8)
/**
diff --git a/include/net/TEST_MAPPING b/include/net/TEST_MAPPING
new file mode 100644
index 0000000..1eb8d43
--- /dev/null
+++ b/include/net/TEST_MAPPING
@@ -0,0 +1,12 @@
+{
+ "presubmit": [
+ {
+ "name": "CtsNetTestCases",
+ "options": [
+ {
+ "exclude-annotation": "com.android.testutils.SkipPresubmit"
+ }
+ ]
+ }
+ ]
+}
diff --git a/include/net/addrconf.h b/include/net/addrconf.h
index c04f359..7ae7175 100644
--- a/include/net/addrconf.h
+++ b/include/net/addrconf.h
@@ -271,6 +271,18 @@ static inline bool ipv6_is_mld(struct sk_buff *skb, int nexthdr, int offset)
void addrconf_prefix_rcv(struct net_device *dev,
u8 *opt, int len, bool sllao);
+/* Determines into what table to put autoconf PIO/RIO/default routes
+ * learned on this device.
+ *
+ * - If 0, use the same table for every device. This puts routes into
+ * one of RT_TABLE_{PREFIX,INFO,DFLT} depending on the type of route
+ * (but note that these three are currently all equal to
+ * RT6_TABLE_MAIN).
+ * - If > 0, use the specified table.
+ * - If < 0, put routes into table dev->ifindex + (-rt_table).
+ */
+u32 addrconf_rt_table(const struct net_device *dev, u32 default_table);
+
/*
* anycast prototypes (anycast.c)
*/
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index dbc81f5..380eea4 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -1597,9 +1597,7 @@ int xfrm_trans_queue(struct sk_buff *skb,
int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err);
int xfrm_output(struct sock *sk, struct sk_buff *skb);
-#if IS_ENABLED(CONFIG_NET_PKTGEN)
int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb);
-#endif
void xfrm_local_error(struct sk_buff *skb, int mtu);
int xfrm4_extract_input(struct xfrm_state *x, struct sk_buff *skb);
diff --git a/include/sound/soc.h b/include/sound/soc.h
index 37bbfc8..f16861c 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -253,6 +253,14 @@
.get = xhandler_get, .put = xhandler_put, \
.private_value = SOC_DOUBLE_R_VALUE(reg_left, reg_right, xshift, \
xmax, xinvert) }
+#define SOC_SINGLE_MULTI_EXT(xname, xreg, xshift, xmax, xinvert, xcount,\
+ xhandler_get, xhandler_put) \
+{ .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+ .info = snd_soc_info_multi_ext, \
+ .get = xhandler_get, .put = xhandler_put, \
+ .private_value = (unsigned long)&(struct soc_multi_mixer_control) \
+ {.reg = xreg, .shift = xshift, .rshift = xshift, .max = xmax, \
+ .count = xcount, .platform_max = xmax, .invert = xinvert} }
#define SOC_SINGLE_EXT_TLV(xname, xreg, xshift, xmax, xinvert,\
xhandler_get, xhandler_put, tlv_array) \
{ .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
@@ -606,6 +614,8 @@ int snd_soc_get_strobe(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol);
int snd_soc_put_strobe(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol);
+int snd_soc_info_multi_ext(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_info *uinfo);
/* SoC PCM stream information */
struct snd_soc_pcm_stream {
@@ -1165,6 +1175,11 @@ struct soc_mreg_control {
unsigned int regbase, regcount, nbits, invert;
};
+struct soc_multi_mixer_control {
+ int min, max, platform_max, count;
+ unsigned int reg, rreg, shift, rshift, invert;
+};
+
/* enumerated kcontrol */
struct soc_enum {
int reg;
diff --git a/include/trace/events/OWNERS b/include/trace/events/OWNERS
new file mode 100644
index 0000000..a63dbf4
--- /dev/null
+++ b/include/trace/events/OWNERS
@@ -0,0 +1 @@
+per-file f2fs**=file:/fs/f2fs/OWNERS
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index fbb99a6..cd969a6 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -488,6 +488,30 @@ DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_blocked,
TP_ARGS(tsk, delay));
/*
+ * Tracepoint for recording the cause of uninterruptible sleep.
+ */
+TRACE_EVENT(sched_blocked_reason,
+
+ TP_PROTO(struct task_struct *tsk),
+
+ TP_ARGS(tsk),
+
+ TP_STRUCT__entry(
+ __field( pid_t, pid )
+ __field( void*, caller )
+ __field( bool, io_wait )
+ ),
+
+ TP_fast_assign(
+ __entry->pid = tsk->pid;
+ __entry->caller = (void *)__get_wchan(tsk);
+ __entry->io_wait = tsk->in_iowait;
+ ),
+
+ TP_printk("pid=%d iowait=%d caller=%pS", __entry->pid, __entry->io_wait, __entry->caller)
+);
+
+/*
* Tracepoint for accounting runtime (time the task is executing
* on a CPU).
*/
diff --git a/include/trace/hooks/avc.h b/include/trace/hooks/avc.h
new file mode 100644
index 0000000..5100dde7
--- /dev/null
+++ b/include/trace/hooks/avc.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM avc
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_AVC_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_AVC_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct avc_node;
+DECLARE_RESTRICTED_HOOK(android_rvh_selinux_avc_insert,
+ TP_PROTO(const struct avc_node *node),
+ TP_ARGS(node), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_selinux_avc_node_delete,
+ TP_PROTO(const struct avc_node *node),
+ TP_ARGS(node), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_selinux_avc_node_replace,
+ TP_PROTO(const struct avc_node *old, const struct avc_node *new),
+ TP_ARGS(old, new), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_selinux_avc_lookup,
+ TP_PROTO(const struct avc_node *node, u32 ssid, u32 tsid, u16 tclass),
+ TP_ARGS(node, ssid, tsid, tclass), 1);
+
+#endif /* _TRACE_HOOK_AVC_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/binder.h b/include/trace/hooks/binder.h
new file mode 100644
index 0000000..4cb0d78
--- /dev/null
+++ b/include/trace/hooks/binder.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM binder
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_BINDER_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_BINDER_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct binder_transaction;
+struct task_struct;
+DECLARE_HOOK(android_vh_binder_transaction_init,
+ TP_PROTO(struct binder_transaction *t),
+ TP_ARGS(t));
+DECLARE_HOOK(android_vh_binder_set_priority,
+ TP_PROTO(struct binder_transaction *t, struct task_struct *task),
+ TP_ARGS(t, task));
+DECLARE_HOOK(android_vh_binder_restore_priority,
+ TP_PROTO(struct binder_transaction *t, struct task_struct *task),
+ TP_ARGS(t, task));
+struct binder_proc;
+struct binder_thread;
+DECLARE_HOOK(android_vh_binder_wakeup_ilocked,
+ TP_PROTO(struct task_struct *task, bool sync, struct binder_proc *proc),
+ TP_ARGS(task, sync, proc));
+DECLARE_HOOK(android_vh_binder_wait_for_work,
+ TP_PROTO(bool do_proc_work, struct binder_thread *tsk, struct binder_proc *proc),
+ TP_ARGS(do_proc_work, tsk, proc));
+DECLARE_HOOK(android_vh_sync_txn_recvd,
+ TP_PROTO(struct task_struct *tsk, struct task_struct *from),
+ TP_ARGS(tsk, from));
+#endif /* _TRACE_HOOK_BINDER_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/cgroup.h b/include/trace/hooks/cgroup.h
new file mode 100644
index 0000000..b8510d6
--- /dev/null
+++ b/include/trace/hooks/cgroup.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cgroup
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_CGROUP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CGROUP_H
+#include <trace/hooks/vendor_hooks.h>
+
+struct task_struct;
+DECLARE_HOOK(android_vh_cgroup_set_task,
+ TP_PROTO(int ret, struct task_struct *task),
+ TP_ARGS(ret, task));
+
+struct cgroup_subsys;
+struct cgroup_taskset;
+DECLARE_HOOK(android_vh_cgroup_attach,
+ TP_PROTO(struct cgroup_subsys *ss, struct cgroup_taskset *tset),
+ TP_ARGS(ss, tset))
+DECLARE_RESTRICTED_HOOK(android_rvh_cgroup_force_kthread_migration,
+ TP_PROTO(struct task_struct *tsk, struct cgroup *dst_cgrp, bool *force_migration),
+ TP_ARGS(tsk, dst_cgrp, force_migration), 1);
+
+struct cgroup_taskset;
+struct cgroup_subsys;
+
+DECLARE_RESTRICTED_HOOK(android_rvh_cpu_cgroup_attach,
+ TP_PROTO(struct cgroup_taskset *tset),
+ TP_ARGS(tset), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_cpu_cgroup_online,
+ TP_PROTO(struct cgroup_subsys_state *css),
+ TP_ARGS(css), 1);
+#endif
+
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/cpufreq.h b/include/trace/hooks/cpufreq.h
new file mode 100644
index 0000000..3c9fdeb
--- /dev/null
+++ b/include/trace/hooks/cpufreq.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cpufreq
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_CPUFREQ_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CPUFREQ_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct cpufreq_policy;
+
+DECLARE_RESTRICTED_HOOK(android_rvh_show_max_freq,
+ TP_PROTO(struct cpufreq_policy *policy, unsigned int *max_freq),
+ TP_ARGS(policy, max_freq), 1);
+
+DECLARE_HOOK(android_vh_freq_table_limits,
+ TP_PROTO(struct cpufreq_policy *policy, unsigned int min_freq,
+ unsigned int max_freq),
+ TP_ARGS(policy, min_freq, max_freq));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_cpufreq_transition,
+ TP_PROTO(struct cpufreq_policy *policy),
+ TP_ARGS(policy), 1);
+
+#endif /* _TRACE_HOOK_CPUFREQ_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/cpuidle.h b/include/trace/hooks/cpuidle.h
new file mode 100644
index 0000000..b1ee27e
--- /dev/null
+++ b/include/trace/hooks/cpuidle.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cpuidle
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_CPUIDLE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CPUIDLE_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct cpuidle_device;
+
+DECLARE_HOOK(android_vh_cpu_idle_enter,
+ TP_PROTO(int *state, struct cpuidle_device *dev),
+ TP_ARGS(state, dev))
+DECLARE_HOOK(android_vh_cpu_idle_exit,
+ TP_PROTO(int state, struct cpuidle_device *dev),
+ TP_ARGS(state, dev))
+
+#endif /* _TRACE_HOOK_CPUIDLE_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
+
diff --git a/include/trace/hooks/cpuidle_psci.h b/include/trace/hooks/cpuidle_psci.h
new file mode 100644
index 0000000..eef0032
--- /dev/null
+++ b/include/trace/hooks/cpuidle_psci.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cpuidle_psci
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_CPUIDLE_PSCI_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CPUIDLE_PSCI_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+
+struct cpuidle_device;
+DECLARE_HOOK(android_vh_cpuidle_psci_enter,
+ TP_PROTO(struct cpuidle_device *dev, bool s2idle),
+ TP_ARGS(dev, s2idle));
+
+DECLARE_HOOK(android_vh_cpuidle_psci_exit,
+ TP_PROTO(struct cpuidle_device *dev, bool s2idle),
+ TP_ARGS(dev, s2idle));
+
+#endif /* _TRACE_HOOK_CPUIDLE_PSCI_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/creds.h b/include/trace/hooks/creds.h
new file mode 100644
index 0000000..69a6808
--- /dev/null
+++ b/include/trace/hooks/creds.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM creds
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_CREDS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CREDS_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct cred;
+struct task_struct;
+DECLARE_RESTRICTED_HOOK(android_rvh_commit_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *new),
+ TP_ARGS(task, new), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_exit_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *cred),
+ TP_ARGS(task, cred), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_override_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *new),
+ TP_ARGS(task, new), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_revert_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *old),
+ TP_ARGS(task, old), 1);
+
+#endif /* _TRACE_HOOK_CREDS_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/debug.h b/include/trace/hooks/debug.h
new file mode 100644
index 0000000..5a20141
--- /dev/null
+++ b/include/trace/hooks/debug.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM debug
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_DEBUG_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_DEBUG_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct pt_regs;
+
+DECLARE_HOOK(android_vh_ipi_stop,
+ TP_PROTO(struct pt_regs *regs),
+ TP_ARGS(regs))
+
+#endif /* _TRACE_HOOK_DEBUG_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/epoch.h b/include/trace/hooks/epoch.h
new file mode 100644
index 0000000..ccee2d8
--- /dev/null
+++ b/include/trace/hooks/epoch.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM epoch
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_EPOCH_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_EPOCH_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_HOOK(android_vh_show_suspend_epoch_val,
+ TP_PROTO(u64 suspend_ns, u64 suspend_cycles),
+ TP_ARGS(suspend_ns, suspend_cycles));
+
+DECLARE_HOOK(android_vh_show_resume_epoch_val,
+ TP_PROTO(u64 resume_cycles),
+ TP_ARGS(resume_cycles));
+
+#endif /* _TRACE_HOOK_EPOCH_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/ftrace_dump.h b/include/trace/hooks/ftrace_dump.h
new file mode 100644
index 0000000..3d0b980
--- /dev/null
+++ b/include/trace/hooks/ftrace_dump.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ftrace_dump
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_FTRACE_DUMP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_FTRACE_DUMP_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct trace_seq;
+
+DECLARE_HOOK(android_vh_ftrace_oops_enter,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_oops_exit,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_size_check,
+ TP_PROTO(unsigned long size, bool *ftrace_check),
+ TP_ARGS(size, ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_format_check,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_dump_buffer,
+ TP_PROTO(struct trace_seq *trace_buf, bool *dump_printk),
+ TP_ARGS(trace_buf, dump_printk));
+
+#endif /* _TRACE_HOOK_FTRACE_DUMP_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/iommu.h b/include/trace/hooks/iommu.h
new file mode 100644
index 0000000..6357a6b
--- /dev/null
+++ b/include/trace/hooks/iommu.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM iommu
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_IOMMU_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_IOMMU_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_RESTRICTED_HOOK(android_rvh_iommu_setup_dma_ops,
+ TP_PROTO(struct device *dev, u64 dma_base, u64 dma_limit),
+ TP_ARGS(dev, dma_base, dma_limit), 1);
+
+struct iova_domain;
+
+DECLARE_HOOK(android_vh_iommu_iovad_alloc_iova,
+ TP_PROTO(struct device *dev, struct iova_domain *iovad, dma_addr_t iova, size_t size),
+ TP_ARGS(dev, iovad, iova, size));
+
+DECLARE_HOOK(android_vh_iommu_iovad_free_iova,
+ TP_PROTO(struct iova_domain *iovad, dma_addr_t iova, size_t size),
+ TP_ARGS(iovad, iova, size));
+
+#endif /* _TRACE_HOOK_IOMMU_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h
new file mode 100644
index 0000000..b3912a9
--- /dev/null
+++ b/include/trace/hooks/mm.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mm
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_MM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_MM_H
+
+#include <trace/hooks/vendor_hooks.h>
+/*
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_skip_swapcache_flags,
+ TP_PROTO(gfp_t *flags),
+ TP_ARGS(flags), 1);
+DECLARE_RESTRICTED_HOOK(android_rvh_set_gfp_zone_flags,
+ TP_PROTO(gfp_t *flags),
+ TP_ARGS(flags), 1);
+DECLARE_RESTRICTED_HOOK(android_rvh_set_readahead_gfp_mask,
+ TP_PROTO(gfp_t *flags),
+ TP_ARGS(flags), 1);
+
+*/
+#endif /* _TRACE_HOOK_MM_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/module.h b/include/trace/hooks/module.h
new file mode 100644
index 0000000..780e767
--- /dev/null
+++ b/include/trace/hooks/module.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM module
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_MODULE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_MODULE_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct module;
+DECLARE_RESTRICTED_HOOK(android_rvh_set_module_permit_before_init,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_module_permit_after_init,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_module_core_rw_nx,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_module_init_rw_nx,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod), 1);
+
+#endif /* _TRACE_HOOK_MODULE_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/mpam.h b/include/trace/hooks/mpam.h
new file mode 100644
index 0000000..50f5a68
--- /dev/null
+++ b/include/trace/hooks/mpam.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mpam
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_MPAM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_MPAM_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct task_struct;
+DECLARE_HOOK(android_vh_mpam_set,
+ TP_PROTO(struct task_struct *prev, struct task_struct *next),
+ TP_ARGS(prev, next));
+
+#endif /* _TRACE_HOOK_MPAM_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/net.h b/include/trace/hooks/net.h
new file mode 100644
index 0000000..381649b
--- /dev/null
+++ b/include/trace/hooks/net.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM net
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_NET_VH_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_NET_VH_H
+#include <trace/hooks/vendor_hooks.h>
+
+struct packet_type;
+struct list_head;
+DECLARE_HOOK(android_vh_ptype_head,
+ TP_PROTO(const struct packet_type *pt, struct list_head *vendor_pt),
+ TP_ARGS(pt, vendor_pt));
+
+/* macro versions of hooks are no longer required */
+
+#endif /* _TRACE_HOOK_NET_VH_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/pm_domain.h b/include/trace/hooks/pm_domain.h
new file mode 100644
index 0000000..2a530d1
--- /dev/null
+++ b/include/trace/hooks/pm_domain.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM pm_domain
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_PM_DOMAIN_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_PM_DOMAIN_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct generic_pm_domain;
+DECLARE_HOOK(android_vh_allow_domain_state,
+ TP_PROTO(struct generic_pm_domain *genpd, uint32_t idx, bool *allow),
+ TP_ARGS(genpd, idx, allow))
+
+#endif /* _TRACE_HOOK_PM_DOMAIN_H */
+
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/preemptirq.h b/include/trace/hooks/preemptirq.h
new file mode 100644
index 0000000..52308c8
--- /dev/null
+++ b/include/trace/hooks/preemptirq.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM preemptirq
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_PREEMPTIRQ_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_PREEMPTIRQ_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_RESTRICTED_HOOK(android_rvh_preempt_disable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_preempt_enable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_irqs_disable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_irqs_enable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+#endif /* _TRACE_HOOK_PREEMPTIRQ_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/printk.h b/include/trace/hooks/printk.h
new file mode 100644
index 0000000..2598490
--- /dev/null
+++ b/include/trace/hooks/printk.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM printk
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_PRINTK_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_HOOK(android_vh_printk_hotplug,
+ TP_PROTO(int *flag),
+ TP_ARGS(flag));
+
+#endif /* _TRACE_HOOK_PRINTK_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/remoteproc.h b/include/trace/hooks/remoteproc.h
new file mode 100644
index 0000000..55fae70
--- /dev/null
+++ b/include/trace/hooks/remoteproc.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM remoteproc
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_RPROC_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_RPROC_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+struct rproc;
+
+/* When recovery succeeds */
+DECLARE_HOOK(android_vh_rproc_recovery,
+ TP_PROTO(struct rproc *rproc),
+ TP_ARGS(rproc));
+
+/* When recovery mode is enabled or disabled by sysfs */
+DECLARE_HOOK(android_vh_rproc_recovery_set,
+ TP_PROTO(struct rproc *rproc),
+ TP_ARGS(rproc));
+
+#endif /* _TRACE_HOOK_RPROC_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h
new file mode 100644
index 0000000..c6fa88f
--- /dev/null
+++ b/include/trace/hooks/sched.h
@@ -0,0 +1,319 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sched
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_SCHED_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SCHED_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct task_struct;
+DECLARE_RESTRICTED_HOOK(android_rvh_select_task_rq_fair,
+ TP_PROTO(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu),
+ TP_ARGS(p, prev_cpu, sd_flag, wake_flags, new_cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_select_task_rq_rt,
+ TP_PROTO(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu),
+ TP_ARGS(p, prev_cpu, sd_flag, wake_flags, new_cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_select_fallback_rq,
+ TP_PROTO(int cpu, struct task_struct *p, int *new_cpu),
+ TP_ARGS(cpu, p, new_cpu), 1);
+
+struct rq;
+DECLARE_HOOK(android_vh_scheduler_tick,
+ TP_PROTO(struct rq *rq),
+ TP_ARGS(rq));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_enqueue_task,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_dequeue_task,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_can_migrate_task,
+ TP_PROTO(struct task_struct *p, int dst_cpu, int *can_migrate),
+ TP_ARGS(p, dst_cpu, can_migrate), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_find_lowest_rq,
+ TP_PROTO(struct task_struct *p, struct cpumask *local_cpu_mask,
+ int ret, int *lowest_cpu),
+ TP_ARGS(p, local_cpu_mask, ret, lowest_cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_prepare_prio_fork,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_finish_prio_fork,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_rtmutex_prepare_setprio,
+ TP_PROTO(struct task_struct *p, struct task_struct *pi_task),
+ TP_ARGS(p, pi_task), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_rto_next_cpu,
+ TP_PROTO(int rto_cpu, struct cpumask *rto_mask, int *cpu),
+ TP_ARGS(rto_cpu, rto_mask, cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_is_cpu_allowed,
+ TP_PROTO(struct task_struct *p, int cpu, bool *allowed),
+ TP_ARGS(p, cpu, allowed), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_get_nohz_timer_target,
+ TP_PROTO(int *cpu, bool *done),
+ TP_ARGS(cpu, done), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_user_nice,
+ TP_PROTO(struct task_struct *p, long *nice, bool *allowed),
+ TP_ARGS(p, nice, allowed), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_setscheduler,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+struct sched_group;
+DECLARE_RESTRICTED_HOOK(android_rvh_find_busiest_group,
+ TP_PROTO(struct sched_group *busiest, struct rq *dst_rq, int *out_balance),
+ TP_ARGS(busiest, dst_rq, out_balance), 1);
+
+DECLARE_HOOK(android_vh_dump_throttled_rt_tasks,
+ TP_PROTO(int cpu, u64 clock, ktime_t rt_period, u64 rt_runtime,
+ s64 rt_period_timer_expires),
+ TP_ARGS(cpu, clock, rt_period, rt_runtime, rt_period_timer_expires));
+
+DECLARE_HOOK(android_vh_jiffies_update,
+ TP_PROTO(void *unused),
+ TP_ARGS(unused));
+
+struct rq_flags;
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_newidle_balance,
+ TP_PROTO(struct rq *this_rq, struct rq_flags *rf,
+ int *pulled_task, int *done),
+ TP_ARGS(this_rq, rf, pulled_task, done), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_nohz_balancer_kick,
+ TP_PROTO(struct rq *rq, unsigned int *flags, int *done),
+ TP_ARGS(rq, flags, done), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_rebalance_domains,
+ TP_PROTO(struct rq *rq, int *continue_balancing),
+ TP_ARGS(rq, continue_balancing), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_find_busiest_queue,
+ TP_PROTO(int dst_cpu, struct sched_group *group,
+ struct cpumask *env_cpus, struct rq **busiest,
+ int *done),
+ TP_ARGS(dst_cpu, group, env_cpus, busiest, done), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_migrate_queued_task,
+ TP_PROTO(struct rq *rq, struct rq_flags *rf,
+ struct task_struct *p, int new_cpu,
+ int *detached),
+ TP_ARGS(rq, rf, p, new_cpu, detached), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_cpu_overutilized,
+ TP_PROTO(int cpu, int *overutilized),
+ TP_ARGS(cpu, overutilized), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_setaffinity,
+ TP_PROTO(struct task_struct *p, const struct cpumask *in_mask, int *retval),
+ TP_ARGS(p, in_mask, retval), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_getaffinity,
+ TP_PROTO(struct task_struct *p, struct cpumask *in_mask),
+ TP_ARGS(p, in_mask), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu,
+ TP_PROTO(struct task_struct *p, unsigned int new_cpu),
+ TP_ARGS(p, new_cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up_success,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_wake_up_new_task,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_new_task_stats,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_flush_task,
+ TP_PROTO(struct task_struct *prev),
+ TP_ARGS(prev), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_tick_entry,
+ TP_PROTO(struct rq *rq),
+ TP_ARGS(rq), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_schedule,
+ TP_PROTO(struct task_struct *prev, struct task_struct *next, struct rq *rq),
+ TP_ARGS(prev, next, rq), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_starting,
+ TP_PROTO(int cpu),
+ TP_ARGS(cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_dying,
+ TP_PROTO(int cpu),
+ TP_ARGS(cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_account_irq,
+ TP_PROTO(struct task_struct *curr, int cpu, s64 delta, bool start),
+ TP_ARGS(curr, cpu, delta, start), 1);
+
+struct sched_entity;
+DECLARE_RESTRICTED_HOOK(android_rvh_place_entity,
+ TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 *vruntime),
+ TP_ARGS(cfs_rq, se, initial, vruntime), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains,
+ TP_PROTO(bool *eas_check),
+ TP_ARGS(eas_check), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_update_cpu_capacity,
+ TP_PROTO(int cpu, unsigned long *capacity),
+ TP_ARGS(cpu, capacity), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_update_misfit_status,
+ TP_PROTO(struct task_struct *p, struct rq *rq, bool *need_update),
+ TP_ARGS(p, rq, need_update), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork_init,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_ttwu_cond,
+ TP_PROTO(int cpu, bool *cond),
+ TP_ARGS(cpu, cond), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_schedule_bug,
+ TP_PROTO(void *unused),
+ TP_ARGS(unused), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_exec,
+ TP_PROTO(bool *cond),
+ TP_ARGS(cond), 1);
+
+DECLARE_HOOK(android_vh_build_sched_domains,
+ TP_PROTO(bool has_asym),
+ TP_ARGS(has_asym));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_check_preempt_tick,
+ TP_PROTO(struct task_struct *p, unsigned long *ideal_runtime, bool *skip_preempt,
+ unsigned long delta_exec, struct cfs_rq *cfs_rq, struct sched_entity *curr,
+ unsigned int granularity),
+ TP_ARGS(p, ideal_runtime, skip_preempt, delta_exec, cfs_rq, curr, granularity), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_check_preempt_wakeup_ignore,
+ TP_PROTO(struct task_struct *p, bool *ignore),
+ TP_ARGS(p, ignore), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_replace_next_task_fair,
+ TP_PROTO(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick,
+ bool simple, struct task_struct *prev),
+ TP_ARGS(rq, p, se, repick, simple, prev), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_sched_balance_rt,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int *done),
+ TP_ARGS(rq, p, done), 1);
+
+struct cfs_rq;
+DECLARE_RESTRICTED_HOOK(android_rvh_pick_next_entity,
+ TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *curr,
+ struct sched_entity **se),
+ TP_ARGS(cfs_rq, curr, se), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_check_preempt_wakeup,
+ TP_PROTO(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt,
+ int wake_flags, struct sched_entity *se, struct sched_entity *pse,
+ int next_buddy_marked, unsigned int granularity),
+ TP_ARGS(rq, p, preempt, nopreempt, wake_flags, se, pse, next_buddy_marked,
+ granularity), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_cpus_allowed_by_task,
+ TP_PROTO(const struct cpumask *cpu_valid_mask, const struct cpumask *new_mask,
+ struct task_struct *p, unsigned int *dest_cpu),
+ TP_ARGS(cpu_valid_mask, new_mask, p, dest_cpu), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_do_sched_yield,
+ TP_PROTO(struct rq *rq),
+ TP_ARGS(rq), 1);
+
+DECLARE_HOOK(android_vh_free_task,
+ TP_PROTO(struct task_struct *p),
+ TP_ARGS(p));
+
+enum uclamp_id;
+struct uclamp_se;
+DECLARE_RESTRICTED_HOOK(android_rvh_uclamp_eff_get,
+ TP_PROTO(struct task_struct *p, enum uclamp_id clamp_id,
+ struct uclamp_se *uclamp_max, struct uclamp_se *uclamp_eff, int *ret),
+ TP_ARGS(p, clamp_id, uclamp_max, uclamp_eff, ret), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_after_enqueue_task,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_after_dequeue_task,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+struct cfs_rq;
+struct sched_entity;
+struct rq_flags;
+DECLARE_RESTRICTED_HOOK(android_rvh_enqueue_entity,
+ TP_PROTO(struct cfs_rq *cfs, struct sched_entity *se),
+ TP_ARGS(cfs, se), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_dequeue_entity,
+ TP_PROTO(struct cfs_rq *cfs, struct sched_entity *se),
+ TP_ARGS(cfs, se), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_entity_tick,
+ TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se),
+ TP_ARGS(cfs_rq, se), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_enqueue_task_fair,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_dequeue_task_fair,
+ TP_PROTO(struct rq *rq, struct task_struct *p, int flags),
+ TP_ARGS(rq, p, flags), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_util_est_update,
+ TP_PROTO(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep, int *ret),
+ TP_ARGS(cfs_rq, p, task_sleep, ret), 1);
+
+DECLARE_HOOK(android_vh_setscheduler_uclamp,
+ TP_PROTO(struct task_struct *tsk, int clamp_id, unsigned int value),
+ TP_ARGS(tsk, clamp_id, value));
+
+DECLARE_HOOK(android_vh_update_topology_flags_workfn,
+ TP_PROTO(void *unused),
+ TP_ARGS(unused));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_update_thermal_stats,
+ TP_PROTO(int cpu),
+ TP_ARGS(cpu), 1);
+
+/* macro versions of hooks are no longer required */
+
+#endif /* _TRACE_HOOK_SCHED_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/selinux.h b/include/trace/hooks/selinux.h
new file mode 100644
index 0000000..0b65631b
--- /dev/null
+++ b/include/trace/hooks/selinux.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM selinux
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_SELINUX_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SELINUX_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct selinux_state;
+DECLARE_RESTRICTED_HOOK(android_rvh_selinux_is_initialized,
+ TP_PROTO(const struct selinux_state *state),
+ TP_ARGS(state), 1);
+
+#endif /* _TRACE_HOOK_SELINUX_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/sys.h b/include/trace/hooks/sys.h
new file mode 100644
index 0000000..e2d5d6d
--- /dev/null
+++ b/include/trace/hooks/sys.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sys
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_SYS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SYS_H
+#include <trace/hooks/vendor_hooks.h>
+
+struct task_struct;
+DECLARE_HOOK(android_vh_syscall_prctl_finished,
+ TP_PROTO(int option, struct task_struct *task),
+ TP_ARGS(option, task));
+#endif
+
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/syscall_check.h b/include/trace/hooks/syscall_check.h
new file mode 100644
index 0000000..56d8267
--- /dev/null
+++ b/include/trace/hooks/syscall_check.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM syscall_check
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_SYSCALL_CHECK_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SYSCALL_CHECK_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct file;
+union bpf_attr;
+DECLARE_HOOK(android_vh_check_mmap_file,
+ TP_PROTO(const struct file *file, unsigned long prot,
+ unsigned long flag, unsigned long ret),
+ TP_ARGS(file, prot, flag, ret));
+
+DECLARE_HOOK(android_vh_check_file_open,
+ TP_PROTO(const struct file *file),
+ TP_ARGS(file));
+
+DECLARE_HOOK(android_vh_check_bpf_syscall,
+ TP_PROTO(int cmd, const union bpf_attr *attr, unsigned int size),
+ TP_ARGS(cmd, attr, size));
+
+#endif /* _TRACE_HOOK_SYSCALL_CHECK_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/sysrqcrash.h b/include/trace/hooks/sysrqcrash.h
new file mode 100644
index 0000000..92e7bc7
--- /dev/null
+++ b/include/trace/hooks/sysrqcrash.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sysrqcrash
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_SYSRQCRASH_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SYSRQCRASH_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+DECLARE_HOOK(android_vh_sysrq_crash,
+ TP_PROTO(void *data),
+ TP_ARGS(data));
+
+#endif /* _TRACE_HOOK_SYSRQCRASH_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/ufshcd.h b/include/trace/hooks/ufshcd.h
new file mode 100644
index 0000000..7f5c545
--- /dev/null
+++ b/include/trace/hooks/ufshcd.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ufshcd
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_UFSHCD_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_UFSHCD_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct ufs_hba;
+struct request;
+struct ufshcd_lrb;
+
+DECLARE_HOOK(android_vh_ufs_fill_prdt,
+ TP_PROTO(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
+ unsigned int segments, int *err),
+ TP_ARGS(hba, lrbp, segments, err));
+
+DECLARE_HOOK(android_vh_ufs_prepare_command,
+ TP_PROTO(struct ufs_hba *hba, struct request *rq,
+ struct ufshcd_lrb *lrbp, int *err),
+ TP_ARGS(hba, rq, lrbp, err));
+
+DECLARE_HOOK(android_vh_ufs_update_sysfs,
+ TP_PROTO(struct ufs_hba *hba),
+ TP_ARGS(hba));
+
+DECLARE_HOOK(android_vh_ufs_send_command,
+ TP_PROTO(struct ufs_hba *hba, struct ufshcd_lrb *lrbp),
+ TP_ARGS(hba, lrbp));
+
+DECLARE_HOOK(android_vh_ufs_compl_command,
+ TP_PROTO(struct ufs_hba *hba, struct ufshcd_lrb *lrbp),
+ TP_ARGS(hba, lrbp));
+
+struct uic_command;
+DECLARE_HOOK(android_vh_ufs_send_uic_command,
+ TP_PROTO(struct ufs_hba *hba, const struct uic_command *ucmd,
+ int str_t),
+ TP_ARGS(hba, ucmd, str_t));
+
+DECLARE_HOOK(android_vh_ufs_send_tm_command,
+ TP_PROTO(struct ufs_hba *hba, int tag, int str_t),
+ TP_ARGS(hba, tag, str_t));
+
+DECLARE_HOOK(android_vh_ufs_check_int_errors,
+ TP_PROTO(struct ufs_hba *hba, bool queue_eh_work),
+ TP_ARGS(hba, queue_eh_work));
+
+#endif /* _TRACE_HOOK_UFSHCD_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/vendor_hooks.h b/include/trace/hooks/vendor_hooks.h
new file mode 100644
index 0000000..48ee67a
--- /dev/null
+++ b/include/trace/hooks/vendor_hooks.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Note: we intentionally omit include file ifdef protection
+ * This is due to the way trace events work. If a file includes two
+ * trace event headers under one "CREATE_TRACE_POINTS" the first include
+ * will override the DECLARE_RESTRICTED_HOOK and break the second include.
+ */
+
+#ifndef __GENKSYMS__
+#include <linux/tracepoint.h>
+#endif
+
+#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS)
+
+#define DECLARE_HOOK DECLARE_TRACE
+
+int android_rvh_probe_register(struct tracepoint *tp, void *probe, void *data);
+
+#ifdef TRACE_HEADER_MULTI_READ
+
+#define DEFINE_HOOK_FN(_name, _reg, _unreg, proto, args) \
+ static const char __tpstrtab_##_name[] \
+ __section("__tracepoints_strings") = #_name; \
+ extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
+ int __traceiter_##_name(void *__data, proto); \
+ struct tracepoint __tracepoint_##_name __used \
+ __section("__tracepoints") = { \
+ .name = __tpstrtab_##_name, \
+ .key = STATIC_KEY_INIT_FALSE, \
+ .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
+ .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
+ .iterator = &__traceiter_##_name, \
+ .regfunc = _reg, \
+ .unregfunc = _unreg, \
+ .funcs = NULL }; \
+ __TRACEPOINT_ENTRY(_name); \
+ int __traceiter_##_name(void *__data, proto) \
+ { \
+ struct tracepoint_func *it_func_ptr; \
+ void *it_func; \
+ \
+ it_func_ptr = (&__tracepoint_##_name)->funcs; \
+ it_func = (it_func_ptr)->func; \
+ do { \
+ __data = (it_func_ptr)->data; \
+ ((void(*)(void *, proto))(it_func))(__data, args); \
+ it_func = READ_ONCE((++it_func_ptr)->func); \
+ } while (it_func); \
+ return 0; \
+ } \
+ DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
+
+#undef DECLARE_RESTRICTED_HOOK
+#define DECLARE_RESTRICTED_HOOK(name, proto, args, cond) \
+ DEFINE_HOOK_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args))
+
+/* prevent additional recursion */
+#undef TRACE_HEADER_MULTI_READ
+#else /* TRACE_HEADER_MULTI_READ */
+
+#ifdef CONFIG_HAVE_STATIC_CALL
+#define __DO_RESTRICTED_HOOK_CALL(name, args) \
+ do { \
+ struct tracepoint_func *it_func_ptr; \
+ void *__data; \
+ it_func_ptr = (&__tracepoint_##name)->funcs; \
+ if (it_func_ptr) { \
+ __data = (it_func_ptr)->data; \
+ static_call(tp_func_##name)(__data, args); \
+ } \
+ } while (0)
+#else
+#define __DO_RESTRICTED_HOOK_CALL(name, args) __traceiter_##name(NULL, args)
+#endif
+
+#define DO_RESTRICTED_HOOK(name, args, cond) \
+ do { \
+ if (!(cond)) \
+ return; \
+ \
+ __DO_RESTRICTED_HOOK_CALL(name, TP_ARGS(args)); \
+ } while (0)
+
+#define __DECLARE_RESTRICTED_HOOK(name, proto, args, cond, data_proto) \
+ extern int __traceiter_##name(data_proto); \
+ DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \
+ extern struct tracepoint __tracepoint_##name; \
+ static inline void trace_##name(proto) \
+ { \
+ if (static_key_false(&__tracepoint_##name.key)) \
+ DO_RESTRICTED_HOOK(name, \
+ TP_ARGS(args), \
+ TP_CONDITION(cond)); \
+ } \
+ static inline bool \
+ trace_##name##_enabled(void) \
+ { \
+ return static_key_false(&__tracepoint_##name.key); \
+ } \
+ static inline int \
+ register_trace_##name(void (*probe)(data_proto), void *data) \
+ { \
+ return android_rvh_probe_register(&__tracepoint_##name, \
+ (void *)probe, data); \
+ } \
+ /* vendor hooks cannot be unregistered */ \
+
+#undef DECLARE_RESTRICTED_HOOK
+#define DECLARE_RESTRICTED_HOOK(name, proto, args, cond) \
+ __DECLARE_RESTRICTED_HOOK(name, PARAMS(proto), PARAMS(args), \
+ cond, \
+ PARAMS(void *__data, proto))
+
+#endif /* TRACE_HEADER_MULTI_READ */
+
+#else /* !CONFIG_TRACEPOINTS || !CONFIG_ANDROID_VENDOR_HOOKS */
+/* suppress trace hooks */
+#define DECLARE_HOOK DECLARE_EVENT_NOP
+#define DECLARE_RESTRICTED_HOOK(name, proto, args, cond) \
+ DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args))
+#endif
diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h
new file mode 100644
index 0000000..83116f7
--- /dev/null
+++ b/include/trace/hooks/vmscan.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM vmscan
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_VMSCAN_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_VMSCAN_H
+
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim,
+ TP_PROTO(bool *balance_anon_file_reclaim),
+ TP_ARGS(balance_anon_file_reclaim), 1);
+#endif /* _TRACE_HOOK_VMSCAN_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/wqlockup.h b/include/trace/hooks/wqlockup.h
new file mode 100644
index 0000000..92e01ab
--- /dev/null
+++ b/include/trace/hooks/wqlockup.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM wqlockup
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_WQLOCKUP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_WQLOCKUP_H
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+DECLARE_HOOK(android_vh_wq_lockup_pool,
+ TP_PROTO(int cpu, unsigned long pool_ts),
+ TP_ARGS(cpu, pool_ts));
+
+#endif /* _TRACE_HOOK_WQLOCKUP_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/uapi/linux/OWNERS b/include/uapi/linux/OWNERS
new file mode 100644
index 0000000..8aed640
--- /dev/null
+++ b/include/uapi/linux/OWNERS
@@ -0,0 +1,3 @@
+per-file f2fs**=file:/fs/f2fs/OWNERS
+per-file fuse**=file:/fs/fuse/OWNERS
+per-file net**=file:/net/OWNERS
diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h
index e72e4de..7fa36a8 100644
--- a/include/uapi/linux/android/binder.h
+++ b/include/uapi/linux/android/binder.h
@@ -38,11 +38,59 @@ enum {
BINDER_TYPE_PTR = B_PACK_CHARS('p', 't', '*', B_TYPE_LARGE),
};
-enum {
+/**
+ * enum flat_binder_object_shifts: shift values for flat_binder_object_flags
+ * @FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT: shift for getting scheduler policy.
+ *
+ */
+enum flat_binder_object_shifts {
+ FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT = 9,
+};
+
+/**
+ * enum flat_binder_object_flags - flags for use in flat_binder_object.flags
+ */
+enum flat_binder_object_flags {
+ /**
+ * @FLAT_BINDER_FLAG_PRIORITY_MASK: bit-mask for min scheduler priority
+ *
+ * These bits can be used to set the minimum scheduler priority
+ * at which transactions into this node should run. Valid values
+ * in these bits depend on the scheduler policy encoded in
+ * @FLAT_BINDER_FLAG_SCHED_POLICY_MASK.
+ *
+ * For SCHED_NORMAL/SCHED_BATCH, the valid range is between [-20..19]
+ * For SCHED_FIFO/SCHED_RR, the value can run between [1..99]
+ */
FLAT_BINDER_FLAG_PRIORITY_MASK = 0xff,
+ /**
+ * @FLAT_BINDER_FLAG_ACCEPTS_FDS: whether the node accepts fds.
+ */
FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100,
/**
+ * @FLAT_BINDER_FLAG_SCHED_POLICY_MASK: bit-mask for scheduling policy
+ *
+ * These two bits can be used to set the min scheduling policy at which
+ * transactions on this node should run. These match the UAPI
+ * scheduler policy values, eg:
+ * 00b: SCHED_NORMAL
+ * 01b: SCHED_FIFO
+ * 10b: SCHED_RR
+ * 11b: SCHED_BATCH
+ */
+ FLAT_BINDER_FLAG_SCHED_POLICY_MASK =
+ 3U << FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT,
+
+ /**
+ * @FLAT_BINDER_FLAG_INHERIT_RT: whether the node inherits RT policy
+ *
+ * Only when set, calls into this node will inherit a real-time
+ * scheduling policy from the caller (for synchronous transactions).
+ */
+ FLAT_BINDER_FLAG_INHERIT_RT = 0x800,
+
+ /**
* @FLAT_BINDER_FLAG_TXN_SECURITY_CTX: request security contexts
*
* Only when set, causes senders to include their security
diff --git a/include/uapi/linux/dm-user.h b/include/uapi/linux/dm-user.h
new file mode 100644
index 0000000..6d8f535b
--- /dev/null
+++ b/include/uapi/linux/dm-user.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: LGPL-2.0+ WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2020 Google, Inc
+ * Copyright (C) 2020 Palmer Dabbelt <palmerdabbelt@google.com>
+ */
+
+#ifndef _LINUX_DM_USER_H
+#define _LINUX_DM_USER_H
+
+#include <linux/types.h>
+
+/*
+ * dm-user proxies device mapper ops between the kernel and userspace. It's
+ * essentially just an RPC mechanism: all kernel calls create a request,
+ * userspace handles that with a response. Userspace obtains requests via
+ * read() and provides responses via write().
+ *
+ * See Documentation/block/dm-user.rst for more information.
+ */
+
+#define DM_USER_REQ_MAP_READ 0
+#define DM_USER_REQ_MAP_WRITE 1
+#define DM_USER_REQ_MAP_FLUSH 2
+#define DM_USER_REQ_MAP_DISCARD 3
+#define DM_USER_REQ_MAP_SECURE_ERASE 4
+#define DM_USER_REQ_MAP_WRITE_SAME 5
+#define DM_USER_REQ_MAP_WRITE_ZEROES 6
+#define DM_USER_REQ_MAP_ZONE_OPEN 7
+#define DM_USER_REQ_MAP_ZONE_CLOSE 8
+#define DM_USER_REQ_MAP_ZONE_FINISH 9
+#define DM_USER_REQ_MAP_ZONE_APPEND 10
+#define DM_USER_REQ_MAP_ZONE_RESET 11
+#define DM_USER_REQ_MAP_ZONE_RESET_ALL 12
+
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_DEV 0x00001
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_TRANSPORT 0x00002
+#define DM_USER_REQ_MAP_FLAG_FAILFAST_DRIVER 0x00004
+#define DM_USER_REQ_MAP_FLAG_SYNC 0x00008
+#define DM_USER_REQ_MAP_FLAG_META 0x00010
+#define DM_USER_REQ_MAP_FLAG_PRIO 0x00020
+#define DM_USER_REQ_MAP_FLAG_NOMERGE 0x00040
+#define DM_USER_REQ_MAP_FLAG_IDLE 0x00080
+#define DM_USER_REQ_MAP_FLAG_INTEGRITY 0x00100
+#define DM_USER_REQ_MAP_FLAG_FUA 0x00200
+#define DM_USER_REQ_MAP_FLAG_PREFLUSH 0x00400
+#define DM_USER_REQ_MAP_FLAG_RAHEAD 0x00800
+#define DM_USER_REQ_MAP_FLAG_BACKGROUND 0x01000
+#define DM_USER_REQ_MAP_FLAG_NOWAIT 0x02000
+#define DM_USER_REQ_MAP_FLAG_CGROUP_PUNT 0x04000
+#define DM_USER_REQ_MAP_FLAG_NOUNMAP 0x08000
+#define DM_USER_REQ_MAP_FLAG_HIPRI 0x10000
+#define DM_USER_REQ_MAP_FLAG_DRV 0x20000
+#define DM_USER_REQ_MAP_FLAG_SWAP 0x40000
+
+#define DM_USER_RESP_SUCCESS 0
+#define DM_USER_RESP_ERROR 1
+#define DM_USER_RESP_UNSUPPORTED 2
+
+struct dm_user_message {
+ __u64 seq;
+ __u64 type;
+ __u64 flags;
+ __u64 sector;
+ __u64 len;
+ __u8 buf[];
+};
+
+#endif
diff --git a/include/uapi/linux/fscrypt.h b/include/uapi/linux/fscrypt.h
index fd1fb0d..024def3 100644
--- a/include/uapi/linux/fscrypt.h
+++ b/include/uapi/linux/fscrypt.h
@@ -127,7 +127,10 @@ struct fscrypt_add_key_arg {
struct fscrypt_key_specifier key_spec;
__u32 raw_size;
__u32 key_id;
- __u32 __reserved[8];
+ __u32 __reserved[7];
+ /* N.B.: "temporary" flag, not reserved upstream */
+#define __FSCRYPT_ADD_KEY_FLAG_HW_WRAPPED 0x00000001
+ __u32 __flags;
__u8 raw[];
};
diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index e3c5410..eba89ce 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -399,6 +399,18 @@ struct fuse_file_lock {
#define FUSE_SECURITY_CTX (1ULL << 32)
#define FUSE_HAS_INODE_DAX (1ULL << 33)
+/*
+ * For FUSE < 7.36 FUSE_PASSTHROUGH has value (1 << 31).
+ * This condition check is not really required, but would prevent having a
+ * broken commit in the tree.
+ */
+#if FUSE_KERNEL_VERSION > 7 || \
+ (FUSE_KERNEL_VERSION == 7 && FUSE_KERNEL_MINOR_VERSION >= 36)
+#define FUSE_PASSTHROUGH (1ULL << 63)
+#else
+#define FUSE_PASSTHROUGH (1 << 31)
+#endif
+
/**
* CUSE INIT request/reply flags
*
@@ -553,6 +565,7 @@ enum fuse_opcode {
FUSE_REMOVEMAPPING = 49,
FUSE_SYNCFS = 50,
FUSE_TMPFILE = 51,
+ FUSE_CANONICAL_PATH = 2016,
/* CUSE specific operations */
CUSE_INIT = 4096,
@@ -679,7 +692,7 @@ struct fuse_create_in {
struct fuse_open_out {
uint64_t fh;
uint32_t open_flags;
- uint32_t padding;
+ uint32_t passthrough_fh;
};
struct fuse_release_in {
@@ -969,6 +982,9 @@ struct fuse_notify_retrieve_in {
/* Device ioctls: */
#define FUSE_DEV_IOC_MAGIC 229
#define FUSE_DEV_IOC_CLONE _IOR(FUSE_DEV_IOC_MAGIC, 0, uint32_t)
+/* 127 is reserved for the V1 interface implementation in Android (deprecated) */
+/* 126 is reserved for the V2 interface implementation in Android */
+#define FUSE_DEV_IOC_PASSTHROUGH_OPEN _IOW(FUSE_DEV_IOC_MAGIC, 126, uint32_t)
struct fuse_lseek_in {
uint64_t fh;
diff --git a/include/uapi/linux/icmp.h b/include/uapi/linux/icmp.h
index 163c099..d3242d5 100644
--- a/include/uapi/linux/icmp.h
+++ b/include/uapi/linux/icmp.h
@@ -97,7 +97,11 @@ struct icmphdr {
} echo;
__be32 gateway;
struct {
+#ifdef __BIONIC__
+ __be16 __linux_unused;
+#else
__be16 __unused;
+#endif
__be16 mtu;
} frag;
__u8 reserved[4];
diff --git a/include/uapi/linux/incrementalfs.h b/include/uapi/linux/incrementalfs.h
new file mode 100644
index 0000000..f8338af
--- /dev/null
+++ b/include/uapi/linux/incrementalfs.h
@@ -0,0 +1,590 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Userspace interface for Incremental FS.
+ *
+ * Incremental FS is special-purpose Linux virtual file system that allows
+ * execution of a program while its binary and resource files are still being
+ * lazily downloaded over the network, USB etc.
+ *
+ * Copyright 2019 Google LLC
+ */
+#ifndef _UAPI_LINUX_INCREMENTALFS_H
+#define _UAPI_LINUX_INCREMENTALFS_H
+
+#include <linux/limits.h>
+#include <linux/ioctl.h>
+#include <linux/types.h>
+#include <linux/xattr.h>
+
+/* ===== constants ===== */
+#define INCFS_NAME "incremental-fs"
+
+/*
+ * Magic number used in file header and in memory superblock
+ * Note that it is a 5 byte unsigned long. Thus on 32 bit kernels, it is
+ * truncated to a 4 byte number
+ */
+#define INCFS_MAGIC_NUMBER (0x5346434e49ul & ULONG_MAX)
+
+#define INCFS_DATA_FILE_BLOCK_SIZE 4096
+#define INCFS_HEADER_VER 1
+
+/* TODO: This value is assumed in incfs_copy_signature_info_from_user to be the
+ * actual signature length. Set back to 64 when fixed.
+ */
+#define INCFS_MAX_HASH_SIZE 32
+#define INCFS_MAX_FILE_ATTR_SIZE 512
+
+#define INCFS_INDEX_NAME ".index"
+#define INCFS_INCOMPLETE_NAME ".incomplete"
+#define INCFS_PENDING_READS_FILENAME ".pending_reads"
+#define INCFS_LOG_FILENAME ".log"
+#define INCFS_BLOCKS_WRITTEN_FILENAME ".blocks_written"
+#define INCFS_XATTR_ID_NAME (XATTR_USER_PREFIX "incfs.id")
+#define INCFS_XATTR_SIZE_NAME (XATTR_USER_PREFIX "incfs.size")
+#define INCFS_XATTR_METADATA_NAME (XATTR_USER_PREFIX "incfs.metadata")
+#define INCFS_XATTR_VERITY_NAME (XATTR_USER_PREFIX "incfs.verity")
+
+#define INCFS_MAX_SIGNATURE_SIZE 8096
+#define INCFS_SIGNATURE_VERSION 2
+#define INCFS_SIGNATURE_SECTIONS 2
+
+#define INCFS_IOCTL_BASE_CODE 'g'
+
+/* ===== ioctl requests on the command dir ===== */
+
+/*
+ * Create a new file
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_CREATE_FILE \
+ _IOWR(INCFS_IOCTL_BASE_CODE, 30, struct incfs_new_file_args)
+
+/* Read file signature */
+#define INCFS_IOC_READ_FILE_SIGNATURE \
+ _IOR(INCFS_IOCTL_BASE_CODE, 31, struct incfs_get_file_sig_args)
+
+/*
+ * Fill in one or more data block. This may only be called on a handle
+ * passed as a parameter to INCFS_IOC_PERMIT_FILLING
+ *
+ * Returns number of blocks filled in, or error if none were
+ */
+#define INCFS_IOC_FILL_BLOCKS \
+ _IOR(INCFS_IOCTL_BASE_CODE, 32, struct incfs_fill_blocks)
+
+/*
+ * Permit INCFS_IOC_FILL_BLOCKS on the given file descriptor
+ * May only be called on .pending_reads file
+ *
+ * Returns 0 on success or error
+ */
+#define INCFS_IOC_PERMIT_FILL \
+ _IOW(INCFS_IOCTL_BASE_CODE, 33, struct incfs_permit_fill)
+
+/*
+ * Fills buffer with ranges of populated blocks
+ *
+ * Returns 0 if all ranges written
+ * error otherwise
+ *
+ * Either way, range_buffer_size_out is set to the number
+ * of bytes written. Should be set to 0 by caller. The ranges
+ * filled are valid, but if an error was returned there might
+ * be more ranges to come.
+ *
+ * Ranges are ranges of filled blocks:
+ *
+ * 1 2 7 9
+ *
+ * means blocks 1, 2, 7, 8, 9 are filled, 0, 3, 4, 5, 6 and 10 on
+ * are not
+ *
+ * If hashing is enabled for the file, the hash blocks are simply
+ * treated as though they immediately followed the data blocks.
+ */
+#define INCFS_IOC_GET_FILLED_BLOCKS \
+ _IOR(INCFS_IOCTL_BASE_CODE, 34, struct incfs_get_filled_blocks_args)
+
+/*
+ * Creates a new mapped file
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_CREATE_MAPPED_FILE \
+ _IOWR(INCFS_IOCTL_BASE_CODE, 35, struct incfs_create_mapped_file_args)
+
+/*
+ * Get number of blocks, total and filled
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_GET_BLOCK_COUNT \
+ _IOR(INCFS_IOCTL_BASE_CODE, 36, struct incfs_get_block_count_args)
+
+/*
+ * Get per UID read timeouts
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_GET_READ_TIMEOUTS \
+ _IOR(INCFS_IOCTL_BASE_CODE, 37, struct incfs_get_read_timeouts_args)
+
+/*
+ * Set per UID read timeouts
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_SET_READ_TIMEOUTS \
+ _IOW(INCFS_IOCTL_BASE_CODE, 38, struct incfs_set_read_timeouts_args)
+
+/*
+ * Get last read error
+ * May only be called on .pending_reads file
+ */
+#define INCFS_IOC_GET_LAST_READ_ERROR \
+ _IOW(INCFS_IOCTL_BASE_CODE, 39, struct incfs_get_last_read_error_args)
+
+/* ===== sysfs feature flags ===== */
+/*
+ * Each flag is represented by a file in /sys/fs/incremental-fs/features
+ * If the file exists the feature is supported
+ * Also the file contents will be the line "supported"
+ */
+
+/*
+ * Basic flag stating that the core incfs file system is available
+ */
+#define INCFS_FEATURE_FLAG_COREFS "corefs"
+
+/*
+ * zstd compression support
+ */
+#define INCFS_FEATURE_FLAG_ZSTD "zstd"
+
+/*
+ * v2 feature set support. Covers:
+ * INCFS_IOC_CREATE_MAPPED_FILE
+ * INCFS_IOC_GET_BLOCK_COUNT
+ * INCFS_IOC_GET_READ_TIMEOUTS/INCFS_IOC_SET_READ_TIMEOUTS
+ * .blocks_written status file
+ * .incomplete folder
+ * report_uid mount option
+ */
+#define INCFS_FEATURE_FLAG_V2 "v2"
+
+enum incfs_compression_alg {
+ COMPRESSION_NONE = 0,
+ COMPRESSION_LZ4 = 1,
+ COMPRESSION_ZSTD = 2,
+};
+
+enum incfs_block_flags {
+ INCFS_BLOCK_FLAGS_NONE = 0,
+ INCFS_BLOCK_FLAGS_HASH = 1,
+};
+
+typedef struct {
+ __u8 bytes[16];
+} incfs_uuid_t __attribute__((aligned (8)));
+
+/*
+ * Description of a pending read. A pending read - a read call by
+ * a userspace program for which the filesystem currently doesn't have data.
+ *
+ * Reads from .pending_reads and .log return an array of these structure
+ */
+struct incfs_pending_read_info {
+ /* Id of a file that is being read from. */
+ incfs_uuid_t file_id;
+
+ /* A number of microseconds since system boot to the read. */
+ __aligned_u64 timestamp_us;
+
+ /* Index of a file block that is being read. */
+ __u32 block_index;
+
+ /* A serial number of this pending read. */
+ __u32 serial_number;
+};
+
+/*
+ * Description of a pending read. A pending read - a read call by
+ * a userspace program for which the filesystem currently doesn't have data.
+ *
+ * This version of incfs_pending_read_info is used whenever the file system is
+ * mounted with the report_uid flag
+ */
+struct incfs_pending_read_info2 {
+ /* Id of a file that is being read from. */
+ incfs_uuid_t file_id;
+
+ /* A number of microseconds since system boot to the read. */
+ __aligned_u64 timestamp_us;
+
+ /* Index of a file block that is being read. */
+ __u32 block_index;
+
+ /* A serial number of this pending read. */
+ __u32 serial_number;
+
+ /* The UID of the reading process */
+ __u32 uid;
+
+ __u32 reserved;
+};
+
+/*
+ * Description of a data or hash block to add to a data file.
+ */
+struct incfs_fill_block {
+ /* Index of a data block. */
+ __u32 block_index;
+
+ /* Length of data */
+ __u32 data_len;
+
+ /*
+ * A pointer to an actual data for the block.
+ *
+ * Equivalent to: __u8 *data;
+ */
+ __aligned_u64 data;
+
+ /*
+ * Compression algorithm used to compress the data block.
+ * Values from enum incfs_compression_alg.
+ */
+ __u8 compression;
+
+ /* Values from enum incfs_block_flags */
+ __u8 flags;
+
+ __u16 reserved1;
+
+ __u32 reserved2;
+
+ __aligned_u64 reserved3;
+};
+
+/*
+ * Description of a number of blocks to add to a data file
+ *
+ * Argument for INCFS_IOC_FILL_BLOCKS
+ */
+struct incfs_fill_blocks {
+ /* Number of blocks */
+ __u64 count;
+
+ /* A pointer to an array of incfs_fill_block structs */
+ __aligned_u64 fill_blocks;
+};
+
+/*
+ * Permit INCFS_IOC_FILL_BLOCKS on the given file descriptor
+ * May only be called on .pending_reads file
+ *
+ * Argument for INCFS_IOC_PERMIT_FILL
+ */
+struct incfs_permit_fill {
+ /* File to permit fills on */
+ __u32 file_descriptor;
+};
+
+enum incfs_hash_tree_algorithm {
+ INCFS_HASH_TREE_NONE = 0,
+ INCFS_HASH_TREE_SHA256 = 1
+};
+
+/*
+ * Create a new file or directory.
+ */
+struct incfs_new_file_args {
+ /* Id of a file to create. */
+ incfs_uuid_t file_id;
+
+ /*
+ * Total size of the new file. Ignored if S_ISDIR(mode).
+ */
+ __aligned_u64 size;
+
+ /*
+ * File mode. Permissions and dir flag.
+ */
+ __u16 mode;
+
+ __u16 reserved1;
+
+ __u32 reserved2;
+
+ /*
+ * A pointer to a null-terminated relative path to the file's parent
+ * dir.
+ * Max length: PATH_MAX
+ *
+ * Equivalent to: char *directory_path;
+ */
+ __aligned_u64 directory_path;
+
+ /*
+ * A pointer to a null-terminated file's name.
+ * Max length: PATH_MAX
+ *
+ * Equivalent to: char *file_name;
+ */
+ __aligned_u64 file_name;
+
+ /*
+ * A pointer to a file attribute to be set on creation.
+ *
+ * Equivalent to: u8 *file_attr;
+ */
+ __aligned_u64 file_attr;
+
+ /*
+ * Length of the data buffer specfied by file_attr.
+ * Max value: INCFS_MAX_FILE_ATTR_SIZE
+ */
+ __u32 file_attr_len;
+
+ __u32 reserved4;
+
+ /*
+ * Points to an APK V4 Signature data blob
+ * Signature must have two sections
+ * Format is:
+ * u32 version
+ * u32 size_of_hash_info_section
+ * u8 hash_info_section[]
+ * u32 size_of_signing_info_section
+ * u8 signing_info_section[]
+ *
+ * Note that incfs does not care about what is in signing_info_section
+ *
+ * hash_info_section has following format:
+ * u32 hash_algorithm; // Must be SHA256 == 1
+ * u8 log2_blocksize; // Must be 12 for 4096 byte blocks
+ * u32 salt_size;
+ * u8 salt[];
+ * u32 hash_size;
+ * u8 root_hash[];
+ */
+ __aligned_u64 signature_info;
+
+ /* Size of signature_info */
+ __aligned_u64 signature_size;
+
+ __aligned_u64 reserved6;
+};
+
+/*
+ * Request a digital signature blob for a given file.
+ * Argument for INCFS_IOC_READ_FILE_SIGNATURE ioctl
+ */
+struct incfs_get_file_sig_args {
+ /*
+ * A pointer to the data buffer to save an signature blob to.
+ *
+ * Equivalent to: u8 *file_signature;
+ */
+ __aligned_u64 file_signature;
+
+ /* Size of the buffer at file_signature. */
+ __u32 file_signature_buf_size;
+
+ /*
+ * Number of bytes save file_signature buffer.
+ * It is set after ioctl done.
+ */
+ __u32 file_signature_len_out;
+};
+
+struct incfs_filled_range {
+ __u32 begin;
+ __u32 end;
+};
+
+/*
+ * Request ranges of filled blocks
+ * Argument for INCFS_IOC_GET_FILLED_BLOCKS
+ */
+struct incfs_get_filled_blocks_args {
+ /*
+ * A buffer to populate with ranges of filled blocks
+ *
+ * Equivalent to struct incfs_filled_ranges *range_buffer
+ */
+ __aligned_u64 range_buffer;
+
+ /* Size of range_buffer */
+ __u32 range_buffer_size;
+
+ /* Start index to read from */
+ __u32 start_index;
+
+ /*
+ * End index to read to. 0 means read to end. This is a range,
+ * so incfs will read from start_index to end_index - 1
+ */
+ __u32 end_index;
+
+ /* Actual number of blocks in file */
+ __u32 total_blocks_out;
+
+ /* The number of data blocks in file */
+ __u32 data_blocks_out;
+
+ /* Number of bytes written to range buffer */
+ __u32 range_buffer_size_out;
+
+ /* Sector scanned up to, if the call was interrupted */
+ __u32 index_out;
+};
+
+/*
+ * Create a new mapped file
+ * Argument for INCFS_IOC_CREATE_MAPPED_FILE
+ */
+struct incfs_create_mapped_file_args {
+ /*
+ * Total size of the new file.
+ */
+ __aligned_u64 size;
+
+ /*
+ * File mode. Permissions and dir flag.
+ */
+ __u16 mode;
+
+ __u16 reserved1;
+
+ __u32 reserved2;
+
+ /*
+ * A pointer to a null-terminated relative path to the incfs mount
+ * point
+ * Max length: PATH_MAX
+ *
+ * Equivalent to: char *directory_path;
+ */
+ __aligned_u64 directory_path;
+
+ /*
+ * A pointer to a null-terminated file name.
+ * Max length: PATH_MAX
+ *
+ * Equivalent to: char *file_name;
+ */
+ __aligned_u64 file_name;
+
+ /* Id of source file to map. */
+ incfs_uuid_t source_file_id;
+
+ /*
+ * Offset in source file to start mapping. Must be a multiple of
+ * INCFS_DATA_FILE_BLOCK_SIZE
+ */
+ __aligned_u64 source_offset;
+};
+
+/*
+ * Get information about the blocks in this file
+ * Argument for INCFS_IOC_GET_BLOCK_COUNT
+ */
+struct incfs_get_block_count_args {
+ /* Total number of data blocks in the file */
+ __u32 total_data_blocks_out;
+
+ /* Number of filled data blocks in the file */
+ __u32 filled_data_blocks_out;
+
+ /* Total number of hash blocks in the file */
+ __u32 total_hash_blocks_out;
+
+ /* Number of filled hash blocks in the file */
+ __u32 filled_hash_blocks_out;
+};
+
+/* Description of timeouts for one UID */
+struct incfs_per_uid_read_timeouts {
+ /* UID to apply these timeouts to */
+ __u32 uid;
+
+ /*
+ * Min time in microseconds to read any block. Note that this doesn't
+ * apply to reads which are satisfied from the page cache.
+ */
+ __u32 min_time_us;
+
+ /*
+ * Min time in microseconds to satisfy a pending read. Any pending read
+ * which is filled before this time will be delayed so that the total
+ * read time >= this value.
+ */
+ __u32 min_pending_time_us;
+
+ /*
+ * Max time in microseconds to satisfy a pending read before the read
+ * times out. If set to U32_MAX, defaults to mount options
+ * read_timeout_ms * 1000. Must be >= min_pending_time_us
+ */
+ __u32 max_pending_time_us;
+};
+
+/*
+ * Get the read timeouts array
+ * Argument for INCFS_IOC_GET_READ_TIMEOUTS
+ */
+struct incfs_get_read_timeouts_args {
+ /*
+ * A pointer to a buffer to fill with the current timeouts
+ *
+ * Equivalent to struct incfs_per_uid_read_timeouts *
+ */
+ __aligned_u64 timeouts_array;
+
+ /* Size of above buffer in bytes */
+ __u32 timeouts_array_size;
+
+ /* Size used in bytes, or size needed if -ENOMEM returned */
+ __u32 timeouts_array_size_out;
+};
+
+/*
+ * Set the read timeouts array
+ * Arguments for INCFS_IOC_SET_READ_TIMEOUTS
+ */
+struct incfs_set_read_timeouts_args {
+ /*
+ * A pointer to an array containing the new timeouts
+ * This will replace any existing timeouts
+ *
+ * Equivalent to struct incfs_per_uid_read_timeouts *
+ */
+ __aligned_u64 timeouts_array;
+
+ /* Size of above array in bytes. Must be < 256 */
+ __u32 timeouts_array_size;
+};
+
+/*
+ * Get last read error struct
+ * Arguments for INCFS_IOC_GET_LAST_READ_ERROR
+ */
+struct incfs_get_last_read_error_args {
+ /* File id of last file that had a read error */
+ incfs_uuid_t file_id_out;
+
+ /* Time of last read error, in us, from CLOCK_MONOTONIC */
+ __u64 time_us_out;
+
+ /* Index of page that was being read at last read error */
+ __u32 page_out;
+
+ /* errno of last read error */
+ __u32 errno_out;
+
+ /* uid of last read error */
+ __u32 uid_out;
+
+ __u32 reserved1;
+ __u64 reserved2;
+};
+
+#endif /* _UAPI_LINUX_INCREMENTALFS_H */
diff --git a/include/uapi/linux/netfilter/xt_IDLETIMER.h b/include/uapi/linux/netfilter/xt_IDLETIMER.h
index 7bfb31a..104ac32 100644
--- a/include/uapi/linux/netfilter/xt_IDLETIMER.h
+++ b/include/uapi/linux/netfilter/xt_IDLETIMER.h
@@ -33,7 +33,7 @@ struct idletimer_tg_info_v1 {
char label[MAX_IDLETIMER_LABEL_SIZE];
- __u8 send_nl_msg; /* unused: for compatibility with Android */
+ __u8 send_nl_msg;
__u8 timer_type;
/* for kernel module internal use only */
diff --git a/include/uapi/linux/usb/f_accessory.h b/include/uapi/linux/usb/f_accessory.h
new file mode 100644
index 0000000..0baeb7d
--- /dev/null
+++ b/include/uapi/linux/usb/f_accessory.h
@@ -0,0 +1,146 @@
+/*
+ * Gadget Function Driver for Android USB accessories
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Author: Mike Lockwood <lockwood@android.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _UAPI_LINUX_USB_F_ACCESSORY_H
+#define _UAPI_LINUX_USB_F_ACCESSORY_H
+
+/* Use Google Vendor ID when in accessory mode */
+#define USB_ACCESSORY_VENDOR_ID 0x18D1
+
+
+/* Product ID to use when in accessory mode */
+#define USB_ACCESSORY_PRODUCT_ID 0x2D00
+
+/* Product ID to use when in accessory mode and adb is enabled */
+#define USB_ACCESSORY_ADB_PRODUCT_ID 0x2D01
+
+/* Indexes for strings sent by the host via ACCESSORY_SEND_STRING */
+#define ACCESSORY_STRING_MANUFACTURER 0
+#define ACCESSORY_STRING_MODEL 1
+#define ACCESSORY_STRING_DESCRIPTION 2
+#define ACCESSORY_STRING_VERSION 3
+#define ACCESSORY_STRING_URI 4
+#define ACCESSORY_STRING_SERIAL 5
+
+/* Control request for retrieving device's protocol version
+ *
+ * requestType: USB_DIR_IN | USB_TYPE_VENDOR
+ * request: ACCESSORY_GET_PROTOCOL
+ * value: 0
+ * index: 0
+ * data version number (16 bits little endian)
+ * 1 for original accessory support
+ * 2 adds HID and device to host audio support
+ */
+#define ACCESSORY_GET_PROTOCOL 51
+
+/* Control request for host to send a string to the device
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_SEND_STRING
+ * value: 0
+ * index: string ID
+ * data zero terminated UTF8 string
+ *
+ * The device can later retrieve these strings via the
+ * ACCESSORY_GET_STRING_* ioctls
+ */
+#define ACCESSORY_SEND_STRING 52
+
+/* Control request for starting device in accessory mode.
+ * The host sends this after setting all its strings to the device.
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_START
+ * value: 0
+ * index: 0
+ * data none
+ */
+#define ACCESSORY_START 53
+
+/* Control request for registering a HID device.
+ * Upon registering, a unique ID is sent by the accessory in the
+ * value parameter. This ID will be used for future commands for
+ * the device
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_REGISTER_HID_DEVICE
+ * value: Accessory assigned ID for the HID device
+ * index: total length of the HID report descriptor
+ * data none
+ */
+#define ACCESSORY_REGISTER_HID 54
+
+/* Control request for unregistering a HID device.
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_REGISTER_HID
+ * value: Accessory assigned ID for the HID device
+ * index: 0
+ * data none
+ */
+#define ACCESSORY_UNREGISTER_HID 55
+
+/* Control request for sending the HID report descriptor.
+ * If the HID descriptor is longer than the endpoint zero max packet size,
+ * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC
+ * commands. The data for the descriptor must be sent sequentially
+ * if multiple packets are needed.
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_SET_HID_REPORT_DESC
+ * value: Accessory assigned ID for the HID device
+ * index: offset of data in descriptor
+ * (needed when HID descriptor is too big for one packet)
+ * data the HID report descriptor
+ */
+#define ACCESSORY_SET_HID_REPORT_DESC 56
+
+/* Control request for sending HID events.
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_SEND_HID_EVENT
+ * value: Accessory assigned ID for the HID device
+ * index: 0
+ * data the HID report for the event
+ */
+#define ACCESSORY_SEND_HID_EVENT 57
+
+/* Control request for setting the audio mode.
+ *
+ * requestType: USB_DIR_OUT | USB_TYPE_VENDOR
+ * request: ACCESSORY_SET_AUDIO_MODE
+ * value: 0 - no audio
+ * 1 - device to host, 44100 16-bit stereo PCM
+ * index: 0
+ * data none
+ */
+#define ACCESSORY_SET_AUDIO_MODE 58
+
+/* ioctls for retrieving strings set by the host */
+#define ACCESSORY_GET_STRING_MANUFACTURER _IOW('M', 1, char[256])
+#define ACCESSORY_GET_STRING_MODEL _IOW('M', 2, char[256])
+#define ACCESSORY_GET_STRING_DESCRIPTION _IOW('M', 3, char[256])
+#define ACCESSORY_GET_STRING_VERSION _IOW('M', 4, char[256])
+#define ACCESSORY_GET_STRING_URI _IOW('M', 5, char[256])
+#define ACCESSORY_GET_STRING_SERIAL _IOW('M', 6, char[256])
+/* returns 1 if there is a start request pending */
+#define ACCESSORY_IS_START_REQUESTED _IO('M', 7)
+/* returns audio mode (set via the ACCESSORY_SET_AUDIO_MODE control request) */
+#define ACCESSORY_GET_AUDIO_MODE _IO('M', 8)
+
+#endif /* _UAPI_LINUX_USB_F_ACCESSORY_H */
diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h
index 1befd18..5390414 100644
--- a/include/uapi/linux/videodev2.h
+++ b/include/uapi/linux/videodev2.h
@@ -70,7 +70,7 @@
* Common stuff for both V4L1 and V4L2
* Moved from videodev.h
*/
-#define VIDEO_MAX_FRAME 32
+#define VIDEO_MAX_FRAME 64
#define VIDEO_MAX_PLANES 8
/*
diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 9f28349..badb928 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -593,6 +593,29 @@ enum ufshcd_quirks {
* auto-hibernate capability but it's FASTAUTO only.
*/
UFSHCD_QUIRK_HIBERN_FASTAUTO = 1 << 18,
+
+ /*
+ * This quirk needs to be enabled if the host controller supports inline
+ * encryption, but it needs to initialize the crypto capabilities in a
+ * nonstandard way and/or it needs to override blk_crypto_ll_ops. If
+ * enabled, the standard code won't initialize the blk_crypto_profile;
+ * ufs_hba_variant_ops::init() must do it instead.
+ */
+ UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE = 1 << 20,
+
+ /*
+ * This quirk needs to be enabled if the host controller supports inline
+ * encryption, but the CRYPTO_GENERAL_ENABLE bit is not implemented and
+ * breaks the HCE sequence if used.
+ */
+ UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE = 1 << 21,
+
+ /*
+ * This quirk needs to be enabled if the host controller requires that
+ * the PRDT be cleared after each encrypted request because encryption
+ * keys were stored in it.
+ */
+ UFSHCD_QUIRK_KEYS_IN_PRDT = 1 << 22,
};
enum ufshcd_caps {
@@ -754,6 +777,7 @@ struct ufs_hba_monitor {
* @vops: pointer to variant specific operations
* @vps: pointer to variant specific parameters
* @priv: pointer to variant specific private data
+ * @sg_entry_size: size of struct ufshcd_sg_entry (may include variant fields)
* @irq: Irq number of the controller
* @is_irq_enabled: whether or not the UFS controller interrupt is enabled.
* @dev_ref_clk_freq: reference clock frequency
@@ -875,6 +899,7 @@ struct ufs_hba {
const struct ufs_hba_variant_ops *vops;
struct ufs_hba_variant_params *vps;
void *priv;
+ size_t sg_entry_size;
unsigned int irq;
bool is_irq_enabled;
enum ufs_ref_clk_freq dev_ref_clk_freq;
@@ -1175,6 +1200,20 @@ static inline int ufshcd_disable_host_tx_lcc(struct ufs_hba *hba)
return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0);
}
+int ufshcd_read_desc_param(struct ufs_hba *hba,
+ enum desc_idn desc_id,
+ int desc_index,
+ u8 param_offset,
+ u8 *param_read_buf,
+ u8 param_size);
+int ufshcd_query_attr_retry(struct ufs_hba *hba,
+ enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
+ u32 *attr_val);
+int ufshcd_query_flag_retry(struct ufs_hba *hba,
+ enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res);
+
+int ufshcd_bkops_ctrl(struct ufs_hba *hba, enum bkops_status status);
+
void ufshcd_auto_hibern8_enable(struct ufs_hba *hba);
void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h
index f525566a..36961ac 100644
--- a/include/ufs/ufshci.h
+++ b/include/ufs/ufshci.h
@@ -422,20 +422,28 @@ struct ufshcd_sg_entry {
__le64 addr;
__le32 reserved;
__le32 size;
+ /*
+ * followed by variant-specific fields if
+ * hba->sg_entry_size != sizeof(struct ufshcd_sg_entry)
+ */
};
/**
* struct utp_transfer_cmd_desc - UTP Command Descriptor (UCD)
* @command_upiu: Command UPIU Frame address
* @response_upiu: Response UPIU Frame address
- * @prd_table: Physical Region Descriptor
+ * @prd_table: Physical Region Descriptor: an array of SG_ALL struct
+ * ufshcd_sg_entry's. Variant-specific fields may be present after each.
*/
struct utp_transfer_cmd_desc {
u8 command_upiu[ALIGNED_UPIU_SIZE];
u8 response_upiu[ALIGNED_UPIU_SIZE];
- struct ufshcd_sg_entry prd_table[SG_ALL];
+ u8 prd_table[];
};
+#define sizeof_utp_transfer_cmd_desc(hba) \
+ (sizeof(struct utp_transfer_cmd_desc) + SG_ALL * (hba)->sg_entry_size)
+
/**
* struct request_desc_header - Descriptor Header common to both UTRD and UTMRD
* @dword0: Descriptor Header DW0
diff --git a/init/Kconfig b/init/Kconfig
index 94125d3..971df98 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -155,7 +155,7 @@
config WERROR
bool "Compile the kernel with warnings as errors"
- default COMPILE_TEST
+ default y
help
A kernel build should not cause any compiler warnings, and this
enables the '-Werror' (for C) and '-Dwarnings' (for Rust) flags
@@ -2007,3 +2007,5 @@
# <asm/syscall_wrapper.h>.
config ARCH_HAS_SYSCALL_WRAPPER
def_bool n
+
+source "init/Kconfig.gki"
diff --git a/init/Kconfig.gki b/init/Kconfig.gki
new file mode 100644
index 0000000..29eb1ee
--- /dev/null
+++ b/init/Kconfig.gki
@@ -0,0 +1,282 @@
+config GKI_HIDDEN_DRM_CONFIGS
+ bool "Hidden DRM configs needed for GKI"
+ select DRM_KMS_HELPER if (HAS_IOMEM && DRM)
+ select DRM_GEM_SHMEM_HELPER if (DRM)
+ select DRM_MIPI_DSI
+ select DRM_TTM if (HAS_IOMEM && DRM)
+ select VIDEOMODE_HELPERS
+ select WANT_DEV_COREDUMP
+ select INTERVAL_TREE
+ help
+ Dummy config option used to enable hidden DRM configs.
+ These are normally selected implicitly when including a
+ DRM module, but for GKI, the modules are built out-of-tree.
+
+config GKI_HIDDEN_MCP251XFD_CONFIGS
+ bool "Hidden MCP251XFD configs needed for GKI"
+ select CAN_RX_OFFLOAD
+ help
+ Dummy config option used to enable hidden MCP251XFD configs.
+ These are normally selected implicitly when including a
+ MCP251XFD module, but for GKI, the modules are built out-of-tree.
+
+config GKI_HIDDEN_REGMAP_CONFIGS
+ bool "Hidden Regmap configs needed for GKI"
+ select REGMAP_IRQ
+ select REGMAP_MMIO
+ select REGMAP_SPMI
+ select SPMI
+ help
+ Dummy config option used to enable hidden regmap configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_CRYPTO_CONFIGS
+ bool "Hidden CRYPTO configs needed for GKI"
+ select CRYPTO_ENGINE
+ help
+ Dummy config option used to enable hidden CRYPTO configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_SND_CONFIGS
+ bool "Hidden SND configs needed for GKI"
+ select SND_VMASTER
+ select SND_PCM_ELD
+ select SND_JACK
+ select SND_JACK_INPUT_DEV
+ select SND_INTEL_NHLT if (ACPI)
+ help
+ Dummy config option used to enable hidden SND configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_SND_SOC_CONFIGS
+ bool "Hidden SND_SOC configs needed for GKI"
+ select SND_SOC_GENERIC_DMAENGINE_PCM if (SND_SOC && SND)
+ select SND_PCM_IEC958
+ select SND_SOC_COMPRESS if (SND_SOC && SND)
+ select SND_SOC_TOPOLOGY if (SND_SOC && SND)
+ select DMADEVICES
+ select DMA_VIRTUAL_CHANNELS
+ help
+ Dummy config option used to enable hidden SND_SOC configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_MMC_CONFIGS
+ bool "Hidden MMC configs needed for GKI"
+ select MMC_SDHCI_IO_ACCESSORS if (MMC_SDHCI)
+ help
+ Dummy config option used to enable hidden MMC configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_GPIO_CONFIGS
+ bool "Hidden GPIO configs needed for GKI"
+ select PINCTRL_SINGLE if (PINCTRL && OF && HAS_IOMEM)
+ select GPIO_PL061 if (HAS_IOMEM && ARM_AMBA && GPIOLIB)
+ help
+ Dummy config option used to enable hidden GPIO configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_QCOM_CONFIGS
+ bool "Hidden QCOM configs needed for GKI"
+ select QCOM_SMEM_STATE
+ select QCOM_GDSC if (ARCH_QCOM)
+ select IOMMU_IO_PGTABLE_LPAE if (ARCH_QCOM)
+ select INTERCONNECT_QCOM if (ARCH_QCOM)
+
+ help
+ Dummy config option used to enable hidden QCOM configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_MEDIA_CONFIGS
+ bool "Hidden Media configs needed for GKI"
+ select VIDEOBUF2_CORE
+ select V4L2_MEM2MEM_DEV
+ select MEDIA_CONTROLLER
+ select MEDIA_CONTROLLER_REQUEST_API
+ select MEDIA_SUPPORT
+ select FRAME_VECTOR
+ select CEC_CORE
+ select CEC_NOTIFIER
+ select CEC_PIN
+ select VIDEOBUF2_DMA_CONTIG
+ select VIDEOBUF2_DMA_SG
+ select VIDEO_V4L2_SUBDEV_API
+ help
+ Dummy config option used to enable hidden media configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_VIRTUAL_CONFIGS
+ bool "Hidden Virtual configs needed for GKI"
+ select HVC_DRIVER
+ help
+ Dummy config option used to enable hidden virtual device configs.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+# LEGACY_WEXT_ALLCONFIG Discussed upstream, soundly rejected as a unique
+# problem for GKI to solve. It should be noted that these extensions are
+# in-effect deprecated and generally unsupported and we should pressure
+# the SOC vendors to drop any modules that require these extensions.
+config GKI_LEGACY_WEXT_ALLCONFIG
+ bool "Hidden wireless extension configs needed for GKI"
+ select WIRELESS_EXT
+ select WEXT_CORE
+ select WEXT_PROC
+ select WEXT_SPY
+ select WEXT_PRIV
+ help
+ Dummy config option used to enable all the hidden legacy wireless
+ extensions to the core wireless network functionality used by
+ add-in modules.
+
+ If you are not building a kernel to be used for a variety of
+ out-of-kernel built wireless modules, say N here.
+
+config GKI_HIDDEN_USB_CONFIGS
+ bool "Hiddel USB configurations needed for GKI"
+ select USB_PHY
+ help
+ Dummy config option used to enable all USB related hidden configs.
+ These configurations are usually only selected by another config
+ option or a combination of them.
+
+ If you are not building a kernel to be used for a variety of
+ out-of-kernel build USB drivers, say N here.
+
+config GKI_HIDDEN_SOC_BUS_CONFIGS
+ bool "Hidden SoC bus configuration needed for GKI"
+ select SOC_BUS
+ help
+ Dummy config option used to enable SOC_BUS hidden Kconfig.
+ The configuration is required for SoCs to register themselves to the bus.
+
+ If you are not building a kernel to be used for a variety of SoCs and
+ out-of-tree drivers, say N here.
+
+config GKI_HIDDEN_RPMSG_CONFIGS
+ bool "Hidden RPMSG configuration needed for GKI"
+ select RPMSG
+ help
+ Dummy config option used to enable the hidden RPMSG config.
+ This configuration is usually only selected by another config
+ option or a combination of them.
+
+ If you are not building a kernel to be used for a variety of
+ out-of-kernel build RPMSG drivers, say N here.
+
+config GKI_HIDDEN_GPU_CONFIGS
+ bool "Hidden GPU configuration needed for GKI"
+ select TRACE_GPU_MEM
+ help
+ Dummy config option used to enable the hidden GPU config.
+ These are normally selected implicitly when a module
+ that relies on it is configured.
+
+config GKI_HIDDEN_IRQ_CONFIGS
+ bool "Hidden IRQ configuration needed for GKI"
+ select GENERIC_IRQ_CHIP
+ select IRQ_DOMAIN_HIERARCHY
+ select IRQ_FASTEOI_HIERARCHY_HANDLERS
+ help
+ Dummy config option used to enable GENERIC_IRQ_CHIP hidden
+ config, required by various SoC platforms. This is usually
+ selected by ARCH_*.
+
+config GKI_HIDDEN_HYPERVISOR_CONFIGS
+ bool "Hidden hypervisor configuration needed for GKI"
+ select SYS_HYPERVISOR
+ help
+ Dummy config option used to enable the SYS_HYPERVISOR hidden
+ config, required by various SoC platforms. This is usually
+ selected by XEN or S390.
+
+config GKI_HIDDEN_NET_CONFIGS
+ bool "Hidden networking configuration needed for GKI"
+ select PAGE_POOL
+ select NET_PTP_CLASSIFY
+ help
+ Dummy config option used to enable the networking hidden
+ config, required by various SoC platforms.
+
+config GKI_HIDDEN_PHY_CONFIGS
+ bool "Hidden PHY configuration needed for GKI"
+ select GENERIC_PHY_MIPI_DPHY
+ help
+ Dummy config option used to enable the hidden PHY configs,
+ required by various SoC platforms.
+
+config GKI_HIDDEN_MM_CONFIGS
+ bool "Hidden MM configuration needed for GKI"
+ select PAGE_REPORTING
+ select BALLOON_COMPACTION
+ select MEMORY_BALLOON
+ help
+ Dummy config option used to enable hidden MM configs,
+ currently required for VIRTIO_BALLOON
+
+config GKI_HIDDEN_ETHERNET_CONFIGS
+ bool "Hidden Ethernet configuration needed for GKI"
+ select PHYLINK
+ help
+ Dummy config option used to enable the hidden Ethernet PHYLINK
+ configs, required by various ethernet devices.
+
+config GKI_HIDDEN_DMA_CONFIGS
+ bool "Hidden DMA configuration needed for GKI"
+ select ASYNC_TX_ENABLE_CHANNEL_SWITCH
+ help
+ Dummy config option used to enable the hidden DMA configs,
+ required by various SoC platforms.
+
+config GKI_NET_XFRM_HACKS
+ bool "XFRM changes required by Android"
+ help
+ Android Networking tests fail without this.
+
+# Atrocities needed for
+# a) building GKI modules in separate tree, or
+# b) building drivers that are not modularizable
+#
+# All of these should be reworked into an upstream solution
+# if possible.
+#
+config GKI_HACKS_TO_FIX
+ bool "GKI Dummy config options"
+ select GKI_NET_XFRM_HACKS
+ select GKI_HIDDEN_CRYPTO_CONFIGS
+ select GKI_HIDDEN_DRM_CONFIGS
+ select GKI_HIDDEN_MCP251XFD_CONFIGS
+ select GKI_HIDDEN_REGMAP_CONFIGS
+ select GKI_HIDDEN_SND_CONFIGS
+ select GKI_HIDDEN_SND_SOC_CONFIGS
+ select GKI_HIDDEN_MMC_CONFIGS
+ select GKI_HIDDEN_GPIO_CONFIGS
+ select GKI_HIDDEN_QCOM_CONFIGS
+ select GKI_LEGACY_WEXT_ALLCONFIG
+ select GKI_HIDDEN_MEDIA_CONFIGS
+ select GKI_HIDDEN_VIRTUAL_CONFIGS
+ select GKI_HIDDEN_USB_CONFIGS
+ select GKI_HIDDEN_SOC_BUS_CONFIGS
+ select GKI_HIDDEN_RPMSG_CONFIGS
+ select GKI_HIDDEN_GPU_CONFIGS
+ select GKI_HIDDEN_IRQ_CONFIGS
+ select GKI_HIDDEN_HYPERVISOR_CONFIGS
+ select GKI_HIDDEN_NET_CONFIGS
+ select GKI_HIDDEN_PHY_CONFIGS
+ select GKI_HIDDEN_MM_CONFIGS
+ select GKI_HIDDEN_ETHERNET_CONFIGS
+ select GKI_HIDDEN_DMA_CONFIGS
+
+ help
+ Dummy config option used to enable core functionality used by
+ modules that may not be selectable in this config.
+
+ Unless you are building a GKI kernel to be used with modules
+ built from a different config, say N here.
diff --git a/init/init_task.c b/init/init_task.c
index ff6c4b9..31ceb0e 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -210,6 +210,10 @@ struct task_struct init_task
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
+#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA
+ .android_vendor_data1 = {0, },
+ .android_oem_data1 = {0, },
+#endif
};
EXPORT_SYMBOL(init_task);
diff --git a/init/main.c b/init/main.c
index d213371c..9c276095 100644
--- a/init/main.c
+++ b/init/main.c
@@ -112,8 +112,6 @@
#define CREATE_TRACE_POINTS
#include <trace/events/initcall.h>
-#include <kunit/test.h>
-
static int kernel_init(void *);
extern void init_IRQ(void);
@@ -1633,8 +1631,6 @@ static noinline void __init kernel_init_freeable(void)
do_basic_setup();
- kunit_run_all_tests();
-
wait_for_initramfs();
console_on_rootfs();
diff --git a/kernel/.gitignore b/kernel/.gitignore
index c6b299a..43a96fc 100644
--- a/kernel/.gitignore
+++ b/kernel/.gitignore
@@ -1,3 +1,5 @@
# SPDX-License-Identifier: GPL-2.0-only
/config_data
/kheaders.md5
+/gki_module_exported.h
+/gki_module_protected.h
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 7b373a5..0d37692 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -36,6 +36,8 @@
#include <linux/memcontrol.h>
#include <linux/trace_events.h>
+#include <trace/hooks/syscall_check.h>
+
#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
@@ -4938,6 +4940,8 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
if (copy_from_bpfptr(&attr, uattr, size) != 0)
return -EFAULT;
+ trace_android_vh_check_bpf_syscall(cmd, &attr, size);
+
err = security_bpf(cmd, &attr, size);
if (err < 0)
return err;
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index 52bb5a74..c6822bf 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -17,6 +17,7 @@
#include <linux/fs_parser.h>
#include <trace/events/cgroup.h>
+#include <trace/hooks/cgroup.h>
/*
* pidlists linger the following amount before being destroyed. The goal
@@ -514,13 +515,15 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of,
tcred = get_task_cred(task);
if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
!uid_eq(cred->euid, tcred->uid) &&
- !uid_eq(cred->euid, tcred->suid))
+ !uid_eq(cred->euid, tcred->suid) &&
+ !ns_capable(tcred->user_ns, CAP_SYS_NICE))
ret = -EACCES;
put_cred(tcred);
if (ret)
goto out_finish;
ret = cgroup_attach_task(cgrp, task, threadgroup);
+ trace_android_vh_cgroup_set_task(ret, task);
out_finish:
cgroup_procs_write_finish(task, locked);
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 15cc265..5d74fce 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -62,6 +62,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cgroup.h>
+#undef CREATE_TRACE_POINTS
+
+#include <trace/hooks/cgroup.h>
#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
MAX_CFTYPE_NAME + 2)
@@ -2502,6 +2505,7 @@ struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
return cgroup_taskset_next(tset, dst_cssp);
}
+EXPORT_SYMBOL_GPL(cgroup_taskset_first);
/**
* cgroup_taskset_next - iterate to the next task in taskset
@@ -2548,6 +2552,7 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
return NULL;
}
+EXPORT_SYMBOL_GPL(cgroup_taskset_next);
/**
* cgroup_migrate_execute - migrate a taskset
@@ -2618,6 +2623,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
if (ss->attach) {
tset->ssid = ssid;
+ trace_android_vh_cgroup_attach(ss, tset);
ss->attach(tset);
}
} while_each_subsys_mask();
@@ -4561,6 +4567,7 @@ struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
return next;
return NULL;
}
+EXPORT_SYMBOL_GPL(css_next_child);
/**
* css_next_descendant_pre - find the next descendant for pre-order walk
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 589827c..e429df3 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -404,17 +404,6 @@ static struct cpuset top_cpuset = {
*/
DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem);
-
-void cpuset_read_lock(void)
-{
- percpu_down_read(&cpuset_rwsem);
-}
-
-void cpuset_read_unlock(void)
-{
- percpu_up_read(&cpuset_rwsem);
-}
-
static DEFINE_SPINLOCK(callback_lock);
static struct workqueue_struct *cpuset_migrate_mm_wq;
@@ -1201,6 +1190,7 @@ void rebuild_sched_domains(void)
percpu_up_write(&cpuset_rwsem);
cpus_read_unlock();
}
+EXPORT_SYMBOL_GPL(rebuild_sched_domains);
/**
* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
diff --git a/kernel/cred.c b/kernel/cred.c
index e10c15f..d4a529c 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -17,6 +17,8 @@
#include <linux/cn_proc.h>
#include <linux/uidgid.h>
+#include <trace/hooks/creds.h>
+
#if 0
#define kdebug(FMT, ...) \
printk("[%-5.5s%5u] " FMT "\n", \
@@ -181,6 +183,7 @@ void exit_creds(struct task_struct *tsk)
key_put(tsk->cached_requested_key);
tsk->cached_requested_key = NULL;
#endif
+ trace_android_rvh_exit_creds(tsk, cred);
}
/**
@@ -499,6 +502,7 @@ int commit_creds(struct cred *new)
inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1);
rcu_assign_pointer(task->real_cred, new);
rcu_assign_pointer(task->cred, new);
+ trace_android_rvh_commit_creds(task, new);
if (new->user != old->user || new->user_ns != old->user_ns)
dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1);
alter_cred_subscribers(old, -2);
@@ -576,6 +580,7 @@ const struct cred *override_creds(const struct cred *new)
get_new_cred((struct cred *)new);
alter_cred_subscribers(new, 1);
rcu_assign_pointer(current->cred, new);
+ trace_android_rvh_override_creds(current, new);
alter_cred_subscribers(old, -1);
kdebug("override_creds() = %p{%d,%d}", old,
@@ -604,6 +609,7 @@ void revert_creds(const struct cred *old)
validate_creds(override);
alter_cred_subscribers(old, 1);
rcu_assign_pointer(current->cred, old);
+ trace_android_rvh_revert_creds(current, old);
alter_cred_subscribers(override, -1);
put_cred(override);
}
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 6ea80ae..d06c598 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -58,6 +58,7 @@
#endif
struct cma *dma_contiguous_default_area;
+EXPORT_SYMBOL_GPL(dma_contiguous_default_area);
/*
* Default global CMA area size can be defined in kernel's .config.
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 18c93c2..082329b 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -1134,10 +1134,11 @@ static void check_sync(struct device *dev,
dir2name[entry->direction],
dir2name[ref->direction]);
+ /* sg list count can be less than map count when partial cache sync */
if (ref->sg_call_ents && ref->type == dma_debug_sg &&
- ref->sg_call_ents != entry->sg_call_ents) {
+ ref->sg_call_ents > entry->sg_call_ents) {
err_printk(ref->dev, entry, "device driver syncs "
- "DMA sg list with different entry count "
+ "DMA sg list count larger than map count "
"[map count=%d] [sync count=%d]\n",
entry->sg_call_ents, ref->sg_call_ents);
}
diff --git a/kernel/events/core.c b/kernel/events/core.c
index e47914a..77affa7 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -4583,6 +4583,7 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
return ret;
}
+EXPORT_SYMBOL_GPL(perf_event_read_local);
static int perf_event_read(struct perf_event *event, bool group)
{
@@ -6769,23 +6770,12 @@ static void perf_pending_task(struct callback_head *head)
#ifdef CONFIG_GUEST_PERF_EVENTS
struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
-DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
-DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
-DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
-
void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
return;
rcu_assign_pointer(perf_guest_cbs, cbs);
- static_call_update(__perf_guest_state, cbs->state);
- static_call_update(__perf_guest_get_ip, cbs->get_ip);
-
- /* Implementing ->handle_intel_pt_intr is optional. */
- if (cbs->handle_intel_pt_intr)
- static_call_update(__perf_guest_handle_intel_pt_intr,
- cbs->handle_intel_pt_intr);
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
@@ -6795,10 +6785,6 @@ void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
return;
rcu_assign_pointer(perf_guest_cbs, NULL);
- static_call_update(__perf_guest_state, (void *)&__static_call_return0);
- static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
- static_call_update(__perf_guest_handle_intel_pt_intr,
- (void *)&__static_call_return0);
synchronize_rcu();
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
diff --git a/kernel/fork.c b/kernel/fork.c
index 89b8b6c..978142a 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -97,6 +97,7 @@
#include <linux/io_uring.h>
#include <linux/bpf.h>
#include <linux/stackprotector.h>
+#include <linux/cpufreq_times.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
@@ -109,6 +110,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/task.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/sched.h>
/*
* Minimum number of threads to boot the kernel
*/
@@ -119,6 +122,8 @@
*/
#define MAX_THREADS FUTEX_TID_MASK
+EXPORT_TRACEPOINT_SYMBOL_GPL(task_newtask);
+
/*
* Protected counters by write_lock_irq(&tasklist_lock)
*/
@@ -139,6 +144,7 @@ static const char * const resident_page_types[] = {
DEFINE_PER_CPU(unsigned long, process_counts) = 0;
__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
+EXPORT_SYMBOL_GPL(tasklist_lock);
#ifdef CONFIG_PROVE_RCU
int lockdep_tasklist_lock_is_held(void)
@@ -538,9 +544,11 @@ void free_task(struct task_struct *tsk)
#ifdef CONFIG_SECCOMP
WARN_ON_ONCE(tsk->seccomp.filter);
#endif
+ cpufreq_task_times_exit(tsk);
release_user_cpus_ptr(tsk);
scs_release(tsk);
+ trace_android_vh_free_task(tsk);
#ifndef CONFIG_THREAD_INFO_IN_TASK
/*
* The task is finally done with both the stack and thread_info,
@@ -1050,6 +1058,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
tsk->reported_split_lock = 0;
#endif
+ android_init_vendor_data(tsk, 1);
+ android_init_oem_data(tsk, 1);
+
return tsk;
free_stack:
@@ -2090,6 +2101,8 @@ static __latent_entropy struct task_struct *copy_process(
siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP));
}
+ cpufreq_task_times_init(p);
+
p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
/*
* Clear TID on mm_release()?
@@ -2670,6 +2683,8 @@ pid_t kernel_clone(struct kernel_clone_args *args)
if (IS_ERR(p))
return PTR_ERR(p);
+ cpufreq_task_times_alloc(p);
+
/*
* Do this prior waking up the new thread - the thread pointer
* might get invalid after that point, if the thread exits quickly.
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 49e7bc8..53d7914 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -14,6 +14,7 @@
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/irqdomain.h>
+#include <linux/wakeup_reason.h>
#include <trace/events/irq.h>
@@ -509,8 +510,22 @@ static bool irq_may_run(struct irq_desc *desc)
* If the interrupt is not in progress and is not an armed
* wakeup interrupt, proceed.
*/
- if (!irqd_has_set(&desc->irq_data, mask))
+ if (!irqd_has_set(&desc->irq_data, mask)) {
+#ifdef CONFIG_PM_SLEEP
+ if (unlikely(desc->no_suspend_depth &&
+ irqd_is_wakeup_set(&desc->irq_data))) {
+ unsigned int irq = irq_desc_get_irq(desc);
+ const char *name = "(unnamed)";
+
+ if (desc->action && desc->action->name)
+ name = desc->action->name;
+
+ log_abnormal_wakeup_reason("misconfigured IRQ %u %s",
+ irq, name);
+ }
+#endif
return true;
+ }
/*
* If the interrupt is an armed wakeup source, mark it pending
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index fd09962..53b6358 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -355,9 +355,7 @@ struct irq_desc *irq_to_desc(unsigned int irq)
{
return radix_tree_lookup(&irq_desc_tree, irq);
}
-#ifdef CONFIG_KVM_BOOK3S_64_HV_MODULE
EXPORT_SYMBOL_GPL(irq_to_desc);
-#endif
static void delete_irq_desc(unsigned int irq)
{
@@ -936,6 +934,7 @@ unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
return desc && desc->kstat_irqs ?
*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}
+EXPORT_SYMBOL_GPL(kstat_irqs_cpu);
static bool irq_is_nmi(struct irq_desc *desc)
{
@@ -993,3 +992,4 @@ void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class
}
EXPORT_SYMBOL_GPL(__irq_set_lockdep_class);
#endif
+EXPORT_SYMBOL_GPL(kstat_irqs_usr);
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 7afa40f..fa98e18 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -170,6 +170,7 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
return true;
#endif /* CONFIG_SMP */
}
+EXPORT_SYMBOL_GPL(irq_work_queue_on);
bool irq_work_needs_cpu(void)
{
diff --git a/kernel/kthread.c b/kernel/kthread.c
index f97fd01..2610210 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -541,6 +541,7 @@ void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
{
__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
}
+EXPORT_SYMBOL_GPL(kthread_bind_mask);
/**
* kthread_bind - bind a just-created kthread to a cpu.
diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig
index 26ea5d0..4f0ccf6 100644
--- a/kernel/module/Kconfig
+++ b/kernel/module/Kconfig
@@ -117,6 +117,19 @@
Reject unsigned modules or signed modules for which we don't have a
key. Without this, such modules will simply taint the kernel.
+config MODULE_SIG_PROTECT
+ bool "Android GKI module protection"
+ depends on MODULE_SIG && !MODULE_SIG_FORCE
+ help
+ Enables Android GKI symbol and export protection support.
+
+ This modifies the behavior of the MODULE_SIG_FORCE as follows:
+ - Allows Android GKI Modules signed using MODULE_SIG_ALL during build.
+ - Allows other modules to load if they don't violate the access to
+ Android GKI protected symbols and do not export the symbols already
+ exported by the Android GKI modules. Loading will fail and return
+ -EACCES (Permission denied) if symbol access conditions are not met.
+
config MODULE_SIG_ALL
bool "Automatically sign all modules"
default y
diff --git a/kernel/module/Makefile b/kernel/module/Makefile
index 948efea..dbcd507 100644
--- a/kernel/module/Makefile
+++ b/kernel/module/Makefile
@@ -10,6 +10,7 @@
obj-y += main.o strict_rwx.o
obj-$(CONFIG_MODULE_DECOMPRESS) += decompress.o
obj-$(CONFIG_MODULE_SIG) += signing.o
+obj-$(CONFIG_MODULE_SIG_PROTECT) += gki_module.o
obj-$(CONFIG_LIVEPATCH) += livepatch.o
obj-$(CONFIG_MODULES_TREE_LOOKUP) += tree_lookup.o
obj-$(CONFIG_DEBUG_KMEMLEAK) += debug_kmemleak.o
@@ -19,3 +20,28 @@
obj-$(CONFIG_KGDB_KDB) += kdb.o
obj-$(CONFIG_MODVERSIONS) += version.o
obj-$(CONFIG_MODULE_UNLOAD_TAINT_TRACKING) += tracking.o
+
+#
+# ANDROID: GKI: Generate headerfiles required for gki_module.o
+#
+# Dependencies on generated files need to be listed explicitly
+$(obj)/gki_module.o: $(obj)/gki_module_protected_exports.h \
+ $(obj)/gki_module_unprotected.h
+
+ALL_KMI_SYMBOLS := all_kmi_symbols
+
+$(obj)/gki_module_unprotected.h: $(ALL_KMI_SYMBOLS) \
+ $(srctree)/scripts/gen_gki_modules_headers.sh
+ $(Q)$(CONFIG_SHELL) $(srctree)/scripts/gen_gki_modules_headers.sh $@ \
+ "$(srctree)" \
+ $(ALL_KMI_SYMBOLS)
+
+# Generate symbol list with union of all symbol list for arm64; empty for others
+$(ALL_KMI_SYMBOLS): $(if $(filter arm64,$(ARCH)),$(wildcard $(srctree)/android/abi_gki_aarch64*),)
+ $(if $(strip $^),cat $^ > $(ALL_KMI_SYMBOLS), echo "" > $(ALL_KMI_SYMBOLS))
+
+$(obj)/gki_module_protected_exports.h: $(srctree)/android/abi_gki_protected_exports \
+ $(srctree)/scripts/gen_gki_modules_headers.sh
+ $(Q)$(CONFIG_SHELL) $(srctree)/scripts/gen_gki_modules_headers.sh $@ \
+ "$(srctree)" \
+ $<
diff --git a/kernel/module/gki_module.c b/kernel/module/gki_module.c
new file mode 100644
index 0000000..4f124f9
--- /dev/null
+++ b/kernel/module/gki_module.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2022 Google LLC
+ * Author: ramjiyani@google.com (Ramji Jiyani)
+ */
+
+#include <linux/bsearch.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/string.h>
+
+/*
+ * Build time generated header files
+ *
+ * gki_module_protected_exports.h -- Symbols protected from _export_ by unsigned modules
+ * gki_module_unprotected.h -- Symbols allowed to _access_ by unsigned modules
+ */
+#include "gki_module_protected_exports.h"
+#include "gki_module_unprotected.h"
+
+#define MAX_STRCMP_LEN (max(MAX_UNPROTECTED_NAME_LEN, MAX_PROTECTED_EXPORTS_NAME_LEN))
+
+/* bsearch() comparision callback */
+static int cmp_name(const void *sym, const void *protected_sym)
+{
+ return strncmp(sym, protected_sym, MAX_STRCMP_LEN);
+}
+
+/**
+ * gki_is_module_protected_export - Is a symbol exported from a protected GKI module?
+ *
+ * @name: Symbol being checked against exported symbols from protected GKI modules
+ */
+bool gki_is_module_protected_export(const char *name)
+{
+ if (NR_UNPROTECTED_SYMBOLS) {
+ return bsearch(name, gki_protected_exports_symbols, NR_PROTECTED_EXPORTS_SYMBOLS,
+ MAX_PROTECTED_EXPORTS_NAME_LEN, cmp_name) != NULL;
+ } else {
+ /*
+ * If there are no symbols in unprotected list; We don't need to
+ * protect exports as there is no KMI enforcement.
+ * Treat everything exportable in this case.
+ */
+ return false;
+ }
+}
+
+/**
+ * gki_is_module_unprotected_symbol - Is a symbol unprotected for unsigned module?
+ *
+ * @name: Symbol being checked in list of unprotected symbols
+ */
+bool gki_is_module_unprotected_symbol(const char *name)
+{
+ if (NR_UNPROTECTED_SYMBOLS) {
+ return bsearch(name, gki_unprotected_symbols, NR_UNPROTECTED_SYMBOLS,
+ MAX_UNPROTECTED_NAME_LEN, cmp_name) != NULL;
+ } else {
+ /*
+ * If there are no symbols in unprotected list;
+ * there isn't a KMI enforcement for the kernel.
+ * Treat everything accessible in this case.
+ */
+ return true;
+ }
+}
diff --git a/kernel/module/internal.h b/kernel/module/internal.h
index 2e2bf23..d51b047 100644
--- a/kernel/module/internal.h
+++ b/kernel/module/internal.h
@@ -303,3 +303,17 @@ static inline int same_magic(const char *amagic, const char *bmagic, bool has_cr
return strcmp(amagic, bmagic) == 0;
}
#endif /* CONFIG_MODVERSIONS */
+
+#ifdef CONFIG_MODULE_SIG_PROTECT
+extern bool gki_is_module_unprotected_symbol(const char *name);
+extern bool gki_is_module_protected_export(const char *name);
+#else
+static inline bool gki_is_module_unprotected_symbol(const char *name)
+{
+ return true;
+}
+static inline bool gki_is_module_protected_export(const char *name)
+{
+ return false;
+}
+#endif /* CONFIG_MODULE_SIG_PROTECT */
diff --git a/kernel/module/main.c b/kernel/module/main.c
index d02d39c7..3aaa90e 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -60,6 +60,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/module.h>
+
/*
* Mutex protects:
* 1) List of modules (also safely readable with preempt_disable),
@@ -1095,6 +1098,21 @@ static const struct kernel_symbol *resolve_symbol(struct module *mod,
goto getname;
}
+ /*
+ * ANDROID: GKI:
+ * In case of an unsigned module symbol resolves only if:
+ * 1. Symbol is in the list of unprotected symbol list OR
+ * 2. If symbol owner is not NULL i.e. owner is another module;
+ * it has to be an unsigned module and not signed GKI module
+ * to protect symbols exported by signed GKI modules.
+ */
+ if (!mod->sig_ok &&
+ !gki_is_module_unprotected_symbol(name) &&
+ fsa.owner && fsa.owner->sig_ok) {
+ fsa.sym = ERR_PTR(-EACCES);
+ goto getname;
+ }
+
err = ref_module(mod, fsa.owner);
if (err) {
fsa.sym = ERR_PTR(err);
@@ -1191,6 +1209,7 @@ static void free_module(struct module *mod)
/* This may be empty, but that's OK */
module_arch_freeing_init(mod);
+ trace_android_rvh_set_module_init_rw_nx(mod);
module_memfree(mod->init_layout.base);
kfree(mod->args);
percpu_modfree(mod);
@@ -1199,6 +1218,7 @@ static void free_module(struct module *mod)
lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size);
/* Finally, free the core (containing the module structure) */
+ trace_android_rvh_set_module_core_rw_nx(mod);
module_memfree(mod->core_layout.base);
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
vfree(mod->data_layout.base);
@@ -1247,6 +1267,14 @@ static int verify_exported_symbols(struct module *mod)
.name = kernel_symbol_name(s),
.gplok = true,
};
+
+ if (!mod->sig_ok && gki_is_module_protected_export(
+ kernel_symbol_name(s))) {
+ pr_err("%s: exports protected symbol %s\n",
+ mod->name, kernel_symbol_name(s));
+ return -EACCES;
+ }
+
if (find_symbol(&fsa)) {
pr_err("%s: exports duplicate symbol %s"
" (owned by %s)\n",
@@ -1327,9 +1355,15 @@ static int simplify_symbols(struct module *mod, const struct load_info *info)
ignore_undef_symbol(info->hdr->e_machine, name)))
break;
- ret = PTR_ERR(ksym) ?: -ENOENT;
- pr_warn("%s: Unknown symbol %s (err %d)\n",
- mod->name, name, ret);
+ if (PTR_ERR(ksym) == -EACCES) {
+ ret = -EACCES;
+ pr_warn("%s: Protected symbol: %s (err %d)\n",
+ mod->name, name, ret);
+ } else {
+ ret = PTR_ERR(ksym) ?: -ENOENT;
+ pr_warn("%s: Unknown symbol %s (err %d)\n",
+ mod->name, name, ret);
+ }
break;
default:
@@ -2342,7 +2376,9 @@ static void module_deallocate(struct module *mod, struct load_info *info)
{
percpu_modfree(mod);
module_arch_freeing_init(mod);
+ trace_android_rvh_set_module_init_rw_nx(mod);
module_memfree(mod->init_layout.base);
+ trace_android_rvh_set_module_core_rw_nx(mod);
module_memfree(mod->core_layout.base);
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
vfree(mod->data_layout.base);
@@ -2494,8 +2530,10 @@ static noinline int do_init_module(struct module *mod)
rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
#endif
module_enable_ro(mod, true);
+ trace_android_rvh_set_module_permit_after_init(mod);
mod_tree_remove_init(mod);
module_arch_freeing_init(mod);
+ trace_android_rvh_set_module_init_rw_nx(mod);
mod->init_layout.base = NULL;
mod->init_layout.size = 0;
mod->init_layout.ro_size = 0;
@@ -2610,6 +2648,7 @@ static int complete_formation(struct module *mod, struct load_info *info)
module_enable_ro(mod, false);
module_enable_nx(mod);
module_enable_x(mod);
+ trace_android_rvh_set_module_permit_before_init(mod);
/*
* Mark state as coming so strong_try_module_get() ignores us,
@@ -2749,6 +2788,8 @@ static int load_module(struct load_info *info, const char __user *uargs,
"kernel\n", mod->name);
add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
}
+#else
+ mod->sig_ok = 0;
#endif
/* To avoid stressing percpu allocator, do this once we're unique. */
diff --git a/kernel/module/signing.c b/kernel/module/signing.c
index a2ff4242..7ffb2ae 100644
--- a/kernel/module/signing.c
+++ b/kernel/module/signing.c
@@ -19,8 +19,20 @@
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "module."
+/*
+ * ANDROID: GKI:
+ * Only enforce signature if SIG_PROTECT is not set
+ */
+#ifndef CONFIG_MODULE_SIG_PROTECT
static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
module_param(sig_enforce, bool_enable_only, 0644);
+void set_module_sig_enforced(void)
+{
+ sig_enforce = true;
+}
+#else
+#define sig_enforce false
+#endif
/*
* Export sig_enforce kernel cmdline parameter to allow other subsystems rely
@@ -32,11 +44,6 @@ bool is_module_sig_enforced(void)
}
EXPORT_SYMBOL(is_module_sig_enforced);
-void set_module_sig_enforced(void)
-{
- sig_enforce = true;
-}
-
/*
* Verify the signature on a module.
*/
@@ -121,5 +128,13 @@ int module_sig_check(struct load_info *info, int flags)
return -EKEYREJECTED;
}
+/*
+ * ANDROID: GKI: Do not prevent loading of unsigned modules;
+ * as all modules except GKI modules are not signed.
+ */
+#ifndef CONFIG_MODULE_SIG_PROTECT
return security_locked_down(LOCKDOWN_MODULE_SIGNATURE);
+#else
+ return 0;
+#endif
}
diff --git a/kernel/pid.c b/kernel/pid.c
index 3fbc5e4..9efcf71 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -421,6 +421,7 @@ struct task_struct *find_task_by_vpid(pid_t vnr)
{
return find_task_by_pid_ns(vnr, task_active_pid_ns(current));
}
+EXPORT_SYMBOL_GPL(find_task_by_vpid);
struct task_struct *find_get_task_by_vpid(pid_t nr)
{
diff --git a/kernel/power/Makefile b/kernel/power/Makefile
index 874ad83..3e84020 100644
--- a/kernel/power/Makefile
+++ b/kernel/power/Makefile
@@ -21,4 +21,5 @@
obj-$(CONFIG_MAGIC_SYSRQ) += poweroff.o
+obj-$(CONFIG_SUSPEND) += wakeup_reason.o
obj-$(CONFIG_ENERGY_MODEL) += energy_model.o
diff --git a/kernel/power/process.c b/kernel/power/process.c
index 6c1c7e5..961a56b 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -85,17 +85,19 @@ static int try_to_freeze_tasks(bool user_only)
elapsed = ktime_sub(end, start);
elapsed_msecs = ktime_to_ms(elapsed);
- if (todo) {
- pr_err("Freezing %s %s after %d.%03d seconds "
+ if (wakeup) {
+ pr_err("Freezing %s aborted after %d.%03d seconds\n", what,
+ elapsed_msecs / 1000, elapsed_msecs % 1000);
+ } else if (todo) {
+ pr_err("Freezing %s failed after %d.%03d seconds "
"(%d tasks refusing to freeze, wq_busy=%d):\n", what,
- wakeup ? "aborted" : "failed",
elapsed_msecs / 1000, elapsed_msecs % 1000,
todo - wq_busy, wq_busy);
if (wq_busy)
show_all_workqueues();
- if (!wakeup || pm_debug_messages_on) {
+ if (pm_debug_messages_on) {
read_lock(&tasklist_lock);
for_each_process_thread(g, p) {
if (p != current && freezing(p) && !frozen(p))
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index fa3bf16..c61c378 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -30,6 +30,7 @@
#include <trace/events/power.h>
#include <linux/compiler.h>
#include <linux/moduleparam.h>
+#include <linux/wakeup_reason.h>
#include "power.h"
@@ -138,6 +139,8 @@ static void s2idle_loop(void)
break;
}
+ clear_wakeup_reasons();
+
if (s2idle_ops && s2idle_ops->check)
s2idle_ops->check();
@@ -367,6 +370,7 @@ static int suspend_prepare(suspend_state_t state)
if (!error)
return 0;
+ log_suspend_abort_reason("One or more tasks refusing to freeze");
suspend_stats.failed_freeze++;
dpm_save_failed_step(SUSPEND_FREEZE);
pm_notifier_call_chain(PM_POST_SUSPEND);
@@ -396,7 +400,7 @@ void __weak arch_suspend_enable_irqs(void)
*/
static int suspend_enter(suspend_state_t state, bool *wakeup)
{
- int error;
+ int error, last_dev;
error = platform_suspend_prepare(state);
if (error)
@@ -404,7 +408,11 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
error = dpm_suspend_late(PMSG_SUSPEND);
if (error) {
+ last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1;
+ last_dev %= REC_FAILED_NUM;
pr_err("late suspend of devices failed\n");
+ log_suspend_abort_reason("late suspend of %s device failed",
+ suspend_stats.failed_devs[last_dev]);
goto Platform_finish;
}
error = platform_suspend_prepare_late(state);
@@ -413,7 +421,11 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
error = dpm_suspend_noirq(PMSG_SUSPEND);
if (error) {
+ last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1;
+ last_dev %= REC_FAILED_NUM;
pr_err("noirq suspend of devices failed\n");
+ log_suspend_abort_reason("noirq suspend of %s device failed",
+ suspend_stats.failed_devs[last_dev]);
goto Platform_early_resume;
}
error = platform_suspend_prepare_noirq(state);
@@ -429,8 +441,10 @@ static int suspend_enter(suspend_state_t state, bool *wakeup)
}
error = pm_sleep_disable_secondary_cpus();
- if (error || suspend_test(TEST_CPUS))
+ if (error || suspend_test(TEST_CPUS)) {
+ log_suspend_abort_reason("Disabling non-boot cpus failed");
goto Enable_cpus;
+ }
arch_suspend_disable_irqs();
BUG_ON(!irqs_disabled());
@@ -501,6 +515,8 @@ int suspend_devices_and_enter(suspend_state_t state)
error = dpm_suspend_start(PMSG_SUSPEND);
if (error) {
pr_err("Some devices failed to suspend, or early wake event detected\n");
+ log_suspend_abort_reason(
+ "Some devices failed to suspend, or early wake event detected");
goto Recover_platform;
}
suspend_test_finish("suspend devices");
diff --git a/kernel/power/wakeup_reason.c b/kernel/power/wakeup_reason.c
new file mode 100644
index 0000000..8fefaa3
--- /dev/null
+++ b/kernel/power/wakeup_reason.c
@@ -0,0 +1,438 @@
+/*
+ * kernel/power/wakeup_reason.c
+ *
+ * Logs the reasons which caused the kernel to resume from
+ * the suspend mode.
+ *
+ * Copyright (C) 2020 Google, Inc.
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/wakeup_reason.h>
+#include <linux/kernel.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kobject.h>
+#include <linux/sysfs.h>
+#include <linux/init.h>
+#include <linux/spinlock.h>
+#include <linux/notifier.h>
+#include <linux/suspend.h>
+#include <linux/slab.h>
+
+/*
+ * struct wakeup_irq_node - stores data and relationships for IRQs logged as
+ * either base or nested wakeup reasons during suspend/resume flow.
+ * @siblings - for membership on leaf or parent IRQ lists
+ * @irq - the IRQ number
+ * @irq_name - the name associated with the IRQ, or a default if none
+ */
+struct wakeup_irq_node {
+ struct list_head siblings;
+ int irq;
+ const char *irq_name;
+};
+
+enum wakeup_reason_flag {
+ RESUME_NONE = 0,
+ RESUME_IRQ,
+ RESUME_ABORT,
+ RESUME_ABNORMAL,
+};
+
+static DEFINE_SPINLOCK(wakeup_reason_lock);
+
+static LIST_HEAD(leaf_irqs); /* kept in ascending IRQ sorted order */
+static LIST_HEAD(parent_irqs); /* unordered */
+
+static struct kmem_cache *wakeup_irq_nodes_cache;
+
+static const char *default_irq_name = "(unnamed)";
+
+static struct kobject *kobj;
+
+static bool capture_reasons;
+static int wakeup_reason;
+static char non_irq_wake_reason[MAX_SUSPEND_ABORT_LEN];
+
+static ktime_t last_monotime; /* monotonic time before last suspend */
+static ktime_t curr_monotime; /* monotonic time after last suspend */
+static ktime_t last_stime; /* monotonic boottime offset before last suspend */
+static ktime_t curr_stime; /* monotonic boottime offset after last suspend */
+
+static void init_node(struct wakeup_irq_node *p, int irq)
+{
+ struct irq_desc *desc;
+
+ INIT_LIST_HEAD(&p->siblings);
+
+ p->irq = irq;
+ desc = irq_to_desc(irq);
+ if (desc && desc->action && desc->action->name)
+ p->irq_name = desc->action->name;
+ else
+ p->irq_name = default_irq_name;
+}
+
+static struct wakeup_irq_node *create_node(int irq)
+{
+ struct wakeup_irq_node *result;
+
+ result = kmem_cache_alloc(wakeup_irq_nodes_cache, GFP_ATOMIC);
+ if (unlikely(!result))
+ pr_warn("Failed to log wakeup IRQ %d\n", irq);
+ else
+ init_node(result, irq);
+
+ return result;
+}
+
+static void delete_list(struct list_head *head)
+{
+ struct wakeup_irq_node *n;
+
+ while (!list_empty(head)) {
+ n = list_first_entry(head, struct wakeup_irq_node, siblings);
+ list_del(&n->siblings);
+ kmem_cache_free(wakeup_irq_nodes_cache, n);
+ }
+}
+
+static bool add_sibling_node_sorted(struct list_head *head, int irq)
+{
+ struct wakeup_irq_node *n = NULL;
+ struct list_head *predecessor = head;
+
+ if (unlikely(WARN_ON(!head)))
+ return NULL;
+
+ if (!list_empty(head))
+ list_for_each_entry(n, head, siblings) {
+ if (n->irq < irq)
+ predecessor = &n->siblings;
+ else if (n->irq == irq)
+ return true;
+ else
+ break;
+ }
+
+ n = create_node(irq);
+ if (n) {
+ list_add(&n->siblings, predecessor);
+ return true;
+ }
+
+ return false;
+}
+
+static struct wakeup_irq_node *find_node_in_list(struct list_head *head,
+ int irq)
+{
+ struct wakeup_irq_node *n;
+
+ if (unlikely(WARN_ON(!head)))
+ return NULL;
+
+ list_for_each_entry(n, head, siblings)
+ if (n->irq == irq)
+ return n;
+
+ return NULL;
+}
+
+void log_irq_wakeup_reason(int irq)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+ if (wakeup_reason == RESUME_ABNORMAL || wakeup_reason == RESUME_ABORT) {
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ if (!capture_reasons) {
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ if (find_node_in_list(&parent_irqs, irq) == NULL)
+ add_sibling_node_sorted(&leaf_irqs, irq);
+
+ wakeup_reason = RESUME_IRQ;
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+}
+
+void log_threaded_irq_wakeup_reason(int irq, int parent_irq)
+{
+ struct wakeup_irq_node *parent;
+ unsigned long flags;
+
+ /*
+ * Intentionally unsynchronized. Calls that come in after we have
+ * resumed should have a fast exit path since there's no work to be
+ * done, any any coherence issue that could cause a wrong value here is
+ * both highly improbable - given the set/clear timing - and very low
+ * impact (parent IRQ gets logged instead of the specific child).
+ */
+ if (!capture_reasons)
+ return;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+
+ if (wakeup_reason == RESUME_ABNORMAL || wakeup_reason == RESUME_ABORT) {
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ if (!capture_reasons || (find_node_in_list(&leaf_irqs, irq) != NULL)) {
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ parent = find_node_in_list(&parent_irqs, parent_irq);
+ if (parent != NULL)
+ add_sibling_node_sorted(&leaf_irqs, irq);
+ else {
+ parent = find_node_in_list(&leaf_irqs, parent_irq);
+ if (parent != NULL) {
+ list_del_init(&parent->siblings);
+ list_add_tail(&parent->siblings, &parent_irqs);
+ add_sibling_node_sorted(&leaf_irqs, irq);
+ }
+ }
+
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+}
+EXPORT_SYMBOL_GPL(log_threaded_irq_wakeup_reason);
+
+static void __log_abort_or_abnormal_wake(bool abort, const char *fmt,
+ va_list args)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+
+ /* Suspend abort or abnormal wake reason has already been logged. */
+ if (wakeup_reason != RESUME_NONE) {
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ if (abort)
+ wakeup_reason = RESUME_ABORT;
+ else
+ wakeup_reason = RESUME_ABNORMAL;
+
+ vsnprintf(non_irq_wake_reason, MAX_SUSPEND_ABORT_LEN, fmt, args);
+
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+}
+
+void log_suspend_abort_reason(const char *fmt, ...)
+{
+ va_list args;
+
+ va_start(args, fmt);
+ __log_abort_or_abnormal_wake(true, fmt, args);
+ va_end(args);
+}
+EXPORT_SYMBOL_GPL(log_suspend_abort_reason);
+
+void log_abnormal_wakeup_reason(const char *fmt, ...)
+{
+ va_list args;
+
+ va_start(args, fmt);
+ __log_abort_or_abnormal_wake(false, fmt, args);
+ va_end(args);
+}
+EXPORT_SYMBOL_GPL(log_abnormal_wakeup_reason);
+
+void clear_wakeup_reasons(void)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+
+ delete_list(&leaf_irqs);
+ delete_list(&parent_irqs);
+ wakeup_reason = RESUME_NONE;
+ capture_reasons = true;
+
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+}
+
+static void print_wakeup_sources(void)
+{
+ struct wakeup_irq_node *n;
+ unsigned long flags;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+
+ capture_reasons = false;
+
+ if (wakeup_reason == RESUME_ABORT) {
+ pr_info("Abort: %s\n", non_irq_wake_reason);
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return;
+ }
+
+ if (wakeup_reason == RESUME_IRQ && !list_empty(&leaf_irqs))
+ list_for_each_entry(n, &leaf_irqs, siblings)
+ pr_info("Resume caused by IRQ %d, %s\n", n->irq,
+ n->irq_name);
+ else if (wakeup_reason == RESUME_ABNORMAL)
+ pr_info("Resume caused by %s\n", non_irq_wake_reason);
+ else
+ pr_info("Resume cause unknown\n");
+
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+}
+
+static ssize_t last_resume_reason_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ ssize_t buf_offset = 0;
+ struct wakeup_irq_node *n;
+ unsigned long flags;
+
+ spin_lock_irqsave(&wakeup_reason_lock, flags);
+
+ if (wakeup_reason == RESUME_ABORT) {
+ buf_offset = scnprintf(buf, PAGE_SIZE, "Abort: %s",
+ non_irq_wake_reason);
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+ return buf_offset;
+ }
+
+ if (wakeup_reason == RESUME_IRQ && !list_empty(&leaf_irqs))
+ list_for_each_entry(n, &leaf_irqs, siblings)
+ buf_offset += scnprintf(buf + buf_offset,
+ PAGE_SIZE - buf_offset,
+ "%d %s\n", n->irq, n->irq_name);
+ else if (wakeup_reason == RESUME_ABNORMAL)
+ buf_offset = scnprintf(buf, PAGE_SIZE, "-1 %s",
+ non_irq_wake_reason);
+
+ spin_unlock_irqrestore(&wakeup_reason_lock, flags);
+
+ return buf_offset;
+}
+
+static ssize_t last_suspend_time_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct timespec64 sleep_time;
+ struct timespec64 total_time;
+ struct timespec64 suspend_resume_time;
+
+ /*
+ * total_time is calculated from monotonic bootoffsets because
+ * unlike CLOCK_MONOTONIC it include the time spent in suspend state.
+ */
+ total_time = ktime_to_timespec64(ktime_sub(curr_stime, last_stime));
+
+ /*
+ * suspend_resume_time is calculated as monotonic (CLOCK_MONOTONIC)
+ * time interval before entering suspend and post suspend.
+ */
+ suspend_resume_time =
+ ktime_to_timespec64(ktime_sub(curr_monotime, last_monotime));
+
+ /* sleep_time = total_time - suspend_resume_time */
+ sleep_time = timespec64_sub(total_time, suspend_resume_time);
+
+ /* Export suspend_resume_time and sleep_time in pair here. */
+ return sprintf(buf, "%llu.%09lu %llu.%09lu\n",
+ (unsigned long long)suspend_resume_time.tv_sec,
+ suspend_resume_time.tv_nsec,
+ (unsigned long long)sleep_time.tv_sec,
+ sleep_time.tv_nsec);
+}
+
+static struct kobj_attribute resume_reason = __ATTR_RO(last_resume_reason);
+static struct kobj_attribute suspend_time = __ATTR_RO(last_suspend_time);
+
+static struct attribute *attrs[] = {
+ &resume_reason.attr,
+ &suspend_time.attr,
+ NULL,
+};
+static struct attribute_group attr_group = {
+ .attrs = attrs,
+};
+
+/* Detects a suspend and clears all the previous wake up reasons*/
+static int wakeup_reason_pm_event(struct notifier_block *notifier,
+ unsigned long pm_event, void *unused)
+{
+ switch (pm_event) {
+ case PM_SUSPEND_PREPARE:
+ /* monotonic time since boot */
+ last_monotime = ktime_get();
+ /* monotonic time since boot including the time spent in suspend */
+ last_stime = ktime_get_boottime();
+ clear_wakeup_reasons();
+ break;
+ case PM_POST_SUSPEND:
+ /* monotonic time since boot */
+ curr_monotime = ktime_get();
+ /* monotonic time since boot including the time spent in suspend */
+ curr_stime = ktime_get_boottime();
+ print_wakeup_sources();
+ break;
+ default:
+ break;
+ }
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block wakeup_reason_pm_notifier_block = {
+ .notifier_call = wakeup_reason_pm_event,
+};
+
+static int __init wakeup_reason_init(void)
+{
+ if (register_pm_notifier(&wakeup_reason_pm_notifier_block)) {
+ pr_warn("[%s] failed to register PM notifier\n", __func__);
+ goto fail;
+ }
+
+ kobj = kobject_create_and_add("wakeup_reasons", kernel_kobj);
+ if (!kobj) {
+ pr_warn("[%s] failed to create a sysfs kobject\n", __func__);
+ goto fail_unregister_pm_notifier;
+ }
+
+ if (sysfs_create_group(kobj, &attr_group)) {
+ pr_warn("[%s] failed to create a sysfs group\n", __func__);
+ goto fail_kobject_put;
+ }
+
+ wakeup_irq_nodes_cache =
+ kmem_cache_create("wakeup_irq_node_cache",
+ sizeof(struct wakeup_irq_node), 0, 0, NULL);
+ if (!wakeup_irq_nodes_cache)
+ goto fail_remove_group;
+
+ return 0;
+
+fail_remove_group:
+ sysfs_remove_group(kobj, &attr_group);
+fail_kobject_put:
+ kobject_put(kobj);
+fail_unregister_pm_notifier:
+ unregister_pm_notifier(&wakeup_reason_pm_notifier_block);
+fail:
+ return 1;
+}
+
+late_initcall(wakeup_reason_init);
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 7decf1e..e21a07c 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -54,6 +54,8 @@
#include <trace/events/initcall.h>
#define CREATE_TRACE_POINTS
#include <trace/events/printk.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/printk.h>
#include "printk_ringbuffer.h"
#include "console_cmdline.h"
@@ -2633,6 +2635,12 @@ void resume_console(void)
*/
static int console_cpu_notify(unsigned int cpu)
{
+ int flag = 0;
+
+ trace_android_vh_printk_hotplug(&flag);
+ if (flag)
+ return 0;
+
if (!cpuhp_tasks_frozen) {
/* If trylock fails, someone else is doing the printing */
if (console_trylock())
@@ -3795,6 +3803,7 @@ int _printk_deferred(const char *fmt, ...)
return r;
}
+EXPORT_SYMBOL_GPL(_printk_deferred);
/*
* printk rate limiting, lifted from the networking subsystem.
diff --git a/kernel/reboot.c b/kernel/reboot.c
index 3bba88c..a032312 100644
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -35,6 +35,7 @@ EXPORT_SYMBOL(cad_pid);
enum reboot_mode reboot_mode DEFAULT_REBOOT_MODE;
EXPORT_SYMBOL_GPL(reboot_mode);
enum reboot_mode panic_reboot_mode = REBOOT_UNDEFINED;
+EXPORT_SYMBOL_GPL(panic_reboot_mode);
/*
* This variable is used privately to keep track of whether or not
diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
index 976092b..ff1738c 100644
--- a/kernel/sched/Makefile
+++ b/kernel/sched/Makefile
@@ -32,3 +32,4 @@
obj-y += fair.o
obj-y += build_policy.o
obj-y += build_utility.o
+obj-$(CONFIG_ANDROID_VENDOR_HOOKS) += vendor_hooks.o
diff --git a/kernel/sched/OWNERS b/kernel/sched/OWNERS
new file mode 100644
index 0000000..09a9f8e
--- /dev/null
+++ b/kernel/sched/OWNERS
@@ -0,0 +1,4 @@
+connoro@google.com
+elavila@google.com
+qperret@google.com
+tkjos@google.com
diff --git a/kernel/sched/android.h b/kernel/sched/android.h
new file mode 100644
index 0000000..5fe6bb7
--- /dev/null
+++ b/kernel/sched/android.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Android scheduler hooks and modifications
+ *
+ * Put all of the android-specific scheduler hooks and changes
+ * in this .h file to make merges and modifications easier. It's also
+ * simpler to notice what is, and is not, an upstream change this way over time.
+ */
+
+
+/*
+ * task_may_not_preempt - check whether a task may not be preemptible soon
+ */
+static inline bool task_may_not_preempt(struct task_struct *task, int cpu)
+{
+ return false;
+}
+
+static inline bool uclamp_boosted(struct task_struct *p)
+{
+ return false;
+}
+
+static inline bool uclamp_latency_sensitive(struct task_struct *p)
+{
+ return false;
+}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 78b2d5c..617eeea 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -95,6 +95,9 @@
#include "../../io_uring/io-wq.h"
#include "../smpboot.h"
+#include <trace/hooks/sched.h>
+#include <trace/hooks/cgroup.h>
+
/*
* Export tracepoints that act as a bare tracehook (ie: have no trace event
* associated with them) to allow external modules to probe them.
@@ -110,8 +113,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp);
+EXPORT_TRACEPOINT_SYMBOL_GPL(sched_switch);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+EXPORT_SYMBOL_GPL(runqueues);
#ifdef CONFIG_SCHED_DEBUG
/*
@@ -126,6 +131,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
const_debug unsigned int sysctl_sched_features =
#include "features.h"
0;
+EXPORT_SYMBOL_GPL(sysctl_sched_features);
#undef SCHED_FEAT
/*
@@ -551,6 +557,7 @@ void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
raw_spin_unlock(lock);
}
}
+EXPORT_SYMBOL_GPL(raw_spin_rq_lock_nested);
bool raw_spin_rq_trylock(struct rq *rq)
{
@@ -580,6 +587,7 @@ void raw_spin_rq_unlock(struct rq *rq)
{
raw_spin_unlock(rq_lockp(rq));
}
+EXPORT_SYMBOL_GPL(raw_spin_rq_unlock);
#ifdef CONFIG_SMP
/*
@@ -598,6 +606,7 @@ void double_rq_lock(struct rq *rq1, struct rq *rq2)
double_rq_clock_clear_update(rq1, rq2);
}
+EXPORT_SYMBOL_GPL(double_rq_lock);
#endif
/*
@@ -623,6 +632,7 @@ struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
cpu_relax();
}
}
+EXPORT_SYMBOL_GPL(__task_rq_lock);
/*
* task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
@@ -665,6 +675,7 @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
cpu_relax();
}
}
+EXPORT_SYMBOL_GPL(task_rq_lock);
/*
* RQ-clock updating methods:
@@ -746,6 +757,7 @@ void update_rq_clock(struct rq *rq)
rq->clock += delta;
update_rq_clock_task(rq, delta);
}
+EXPORT_SYMBOL_GPL(update_rq_clock);
#ifdef CONFIG_SCHED_HRTICK
/*
@@ -944,6 +956,7 @@ static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
*/
*head->lastp = node;
head->lastp = &node->next;
+ head->count++;
return true;
}
@@ -999,12 +1012,14 @@ void wake_up_q(struct wake_q_head *head)
/* Task can safely be re-inserted now: */
node = node->next;
task->wake_q.next = NULL;
+ task->wake_q_count = head->count;
/*
* wake_up_process() executes a full barrier, which pairs with
* the queueing in wake_q_add() so as not to miss wakeups.
*/
wake_up_process(task);
+ task->wake_q_count = 0;
put_task_struct(task);
}
}
@@ -1039,6 +1054,7 @@ void resched_curr(struct rq *rq)
else
trace_sched_wake_idle_without_ipi(cpu);
}
+EXPORT_SYMBOL_GPL(resched_curr);
void resched_cpu(int cpu)
{
@@ -1066,6 +1082,11 @@ int get_nohz_timer_target(void)
int i, cpu = smp_processor_id(), default_cpu = -1;
struct sched_domain *sd;
const struct cpumask *hk_mask;
+ bool done = false;
+
+ trace_android_rvh_get_nohz_timer_target(&cpu, &done);
+ if (done)
+ return cpu;
if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) {
if (!idle_cpu(cpu))
@@ -1341,6 +1362,7 @@ static struct uclamp_se uclamp_default[UCLAMP_CNT];
* * An admin modifying the cgroup cpu.uclamp.{min, max}
*/
DEFINE_STATIC_KEY_FALSE(sched_uclamp_used);
+EXPORT_SYMBOL_GPL(sched_uclamp_used);
/* Integer rounded range for each bucket */
#define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS)
@@ -1487,6 +1509,12 @@ uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
{
struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
struct uclamp_se uc_max = uclamp_default[clamp_id];
+ struct uclamp_se uc_eff;
+ int ret = 0;
+
+ trace_android_rvh_uclamp_eff_get(p, clamp_id, &uc_max, &uc_eff, &ret);
+ if (ret)
+ return uc_eff;
/* System default restrictions always apply */
if (unlikely(uc_req.value > uc_max.value))
@@ -1507,6 +1535,7 @@ unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
return (unsigned long)uc_eff.value;
}
+EXPORT_SYMBOL_GPL(uclamp_eff_value);
/*
* When a task is enqueued on a rq, the clamp bucket currently defined by the
@@ -1934,12 +1963,14 @@ static void __setscheduler_uclamp(struct task_struct *p,
attr->sched_util_min != -1) {
uclamp_se_set(&p->uclamp_req[UCLAMP_MIN],
attr->sched_util_min, true);
+ trace_android_vh_setscheduler_uclamp(p, UCLAMP_MIN, attr->sched_util_min);
}
if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX &&
attr->sched_util_max != -1) {
uclamp_se_set(&p->uclamp_req[UCLAMP_MAX],
attr->sched_util_max, true);
+ trace_android_vh_setscheduler_uclamp(p, UCLAMP_MAX, attr->sched_util_max);
}
}
@@ -2057,7 +2088,9 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
}
uclamp_rq_inc(rq, p);
+ trace_android_rvh_enqueue_task(rq, p, flags);
p->sched_class->enqueue_task(rq, p, flags);
+ trace_android_rvh_after_enqueue_task(rq, p, flags);
if (sched_core_enabled(rq))
sched_core_enqueue(rq, p);
@@ -2077,7 +2110,9 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
}
uclamp_rq_dec(rq, p);
+ trace_android_rvh_dequeue_task(rq, p, flags);
p->sched_class->dequeue_task(rq, p, flags);
+ trace_android_rvh_after_dequeue_task(rq, p, flags);
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
@@ -2086,6 +2121,7 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
p->on_rq = TASK_ON_RQ_QUEUED;
}
+EXPORT_SYMBOL_GPL(activate_task);
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
@@ -2093,6 +2129,7 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
dequeue_task(rq, p, flags);
}
+EXPORT_SYMBOL_GPL(deactivate_task);
static inline int __normal_prio(int policy, int rt_prio, int nice)
{
@@ -2185,6 +2222,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
rq_clock_skip_update(rq);
}
+EXPORT_SYMBOL_GPL(check_preempt_curr);
#ifdef CONFIG_SMP
@@ -2275,6 +2313,8 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
*/
static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
{
+ bool allowed = true;
+
/* When not in the task's cpumask, no point in looking further. */
if (!cpumask_test_cpu(cpu, p->cpus_ptr))
return false;
@@ -2283,14 +2323,20 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
if (is_migration_disabled(p))
return cpu_online(cpu);
+ /* check for all cases */
+ trace_android_rvh_is_cpu_allowed(p, cpu, &allowed);
+
/* Non kernel threads are not allowed during either online or offline. */
if (!(p->flags & PF_KTHREAD))
- return cpu_active(cpu) && task_cpu_possible(cpu, p);
+ return cpu_active(cpu) && task_cpu_possible(cpu, p) && allowed;
/* KTHREAD_IS_PER_CPU is always allowed. */
if (kthread_is_per_cpu(p))
return cpu_online(cpu);
+ if (!allowed)
+ return false;
+
/* Regular kernel threads don't get to stay during offline. */
if (cpu_dying(cpu))
return false;
@@ -2321,12 +2367,24 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
struct task_struct *p, int new_cpu)
{
+ int detached = 0;
+
lockdep_assert_rq_held(rq);
+ /*
+ * The vendor hook may drop the lock temporarily, so
+ * pass the rq flags to unpin lock. We expect the
+ * rq lock to be held after return.
+ */
+ trace_android_rvh_migrate_queued_task(rq, rf, p, new_cpu, &detached);
+ if (detached)
+ goto attach;
+
deactivate_task(rq, p, DEQUEUE_NOCLOCK);
set_task_cpu(p, new_cpu);
- rq_unlock(rq, rf);
+attach:
+ rq_unlock(rq, rf);
rq = cpu_rq(new_cpu);
rq_lock(rq, rf);
@@ -2364,8 +2422,8 @@ struct set_affinity_pending {
* So we race with normal scheduler movements, but that's OK, as long
* as the task is no longer on this CPU.
*/
-static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
- struct task_struct *p, int dest_cpu)
+struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
+ struct task_struct *p, int dest_cpu)
{
/* Affinity changed (again). */
if (!is_cpu_allowed(p, dest_cpu))
@@ -2376,6 +2434,7 @@ static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
return rq;
}
+EXPORT_SYMBOL_GPL(__migrate_task);
/*
* migration_cpu_stop - this will be executed by a highprio stopper thread
@@ -2924,6 +2983,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* immediately required to distribute the tasks within their new mask.
*/
dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask);
+ trace_android_rvh_set_cpus_allowed_by_task(cpu_valid_mask, ctx->new_mask, p, &dest_cpu);
if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL;
goto out;
@@ -3144,12 +3204,13 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
p->se.nr_migrations++;
rseq_migrate(p);
perf_event_task_migrate(p);
+ trace_android_rvh_set_task_cpu(p, new_cpu);
}
__set_task_cpu(p, new_cpu);
}
+EXPORT_SYMBOL_GPL(set_task_cpu);
-#ifdef CONFIG_NUMA_BALANCING
static void __migrate_swap_task(struct task_struct *p, int cpu)
{
if (task_on_rq_queued(p)) {
@@ -3264,7 +3325,7 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p,
out:
return ret;
}
-#endif /* CONFIG_NUMA_BALANCING */
+EXPORT_SYMBOL_GPL(migrate_swap);
/*
* wait_task_inactive - wait for a thread to unschedule.
@@ -3421,12 +3482,16 @@ EXPORT_SYMBOL_GPL(kick_process);
* select_task_rq() below may allow selection of !active CPUs in order
* to satisfy the above rules.
*/
-static int select_fallback_rq(int cpu, struct task_struct *p)
+int select_fallback_rq(int cpu, struct task_struct *p)
{
int nid = cpu_to_node(cpu);
const struct cpumask *nodemask = NULL;
enum { cpuset, possible, fail } state = cpuset;
- int dest_cpu;
+ int dest_cpu = -1;
+
+ trace_android_rvh_select_fallback_rq(cpu, p, &dest_cpu);
+ if (dest_cpu >= 0)
+ return dest_cpu;
/*
* If the node that the CPU is on has been offlined, cpu_to_node()
@@ -3491,6 +3556,7 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
return dest_cpu;
}
+EXPORT_SYMBOL_GPL(select_fallback_rq);
/*
* The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
@@ -3666,6 +3732,9 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
{
int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
+ if (wake_flags & WF_SYNC)
+ en_flags |= ENQUEUE_WAKEUP_SYNC;
+
lockdep_assert_rq_held(rq);
if (p->sched_contributes_to_load)
@@ -3811,6 +3880,7 @@ void wake_up_if_idle(int cpu)
out:
rcu_read_unlock();
}
+EXPORT_SYMBOL_GPL(wake_up_if_idle);
bool cpus_share_cache(int this_cpu, int that_cpu)
{
@@ -3862,7 +3932,12 @@ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
{
- if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) {
+ bool cond = false;
+
+ trace_android_rvh_ttwu_cond(cpu, &cond);
+
+ if ((sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) ||
+ cond) {
sched_clock_cpu(cpu); /* Sync clocks across CPUs */
__ttwu_queue_wakelist(p, cpu, wake_flags);
return true;
@@ -4130,6 +4205,9 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
goto unlock;
+ if (READ_ONCE(p->__state) & TASK_UNINTERRUPTIBLE)
+ trace_sched_blocked_reason(p);
+
#ifdef CONFIG_SMP
/*
* Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
@@ -4198,6 +4276,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
smp_cond_load_acquire(&p->on_cpu, !VAL);
+ trace_android_rvh_try_to_wake_up(p);
+
cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU);
if (task_cpu(p) != cpu) {
if (p->in_iowait) {
@@ -4217,8 +4297,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
unlock:
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
out:
- if (success)
+ if (success) {
+ trace_android_rvh_try_to_wake_up_success(p);
ttwu_stat(p, task_cpu(p), wake_flags);
+ }
preempt_enable();
return success;
@@ -4378,6 +4460,8 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
p->se.cfs_rq = NULL;
#endif
+ trace_android_rvh_sched_fork_init(p);
+
#ifdef CONFIG_SCHEDSTATS
/* Even if schedstat is disabled, there should not be garbage */
memset(&p->stats, 0, sizeof(p->stats));
@@ -4585,6 +4669,8 @@ late_initcall(sched_core_sysctl_init);
*/
int sched_fork(unsigned long clone_flags, struct task_struct *p)
{
+ trace_android_rvh_sched_fork(p);
+
__sched_fork(clone_flags, p);
/*
* We mark the process as NEW here. This guarantees that
@@ -4597,6 +4683,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
* Make sure we do not leak PI boosting priority to the child.
*/
p->prio = current->normal_prio;
+ trace_android_rvh_prepare_prio_fork(p);
uclamp_fork(p);
@@ -4629,6 +4716,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
p->sched_class = &fair_sched_class;
init_entity_runnable_average(&p->se);
+ trace_android_rvh_finish_prio_fork(p);
#ifdef CONFIG_SCHED_INFO
@@ -4708,6 +4796,8 @@ void wake_up_new_task(struct task_struct *p)
struct rq_flags rf;
struct rq *rq;
+ trace_android_rvh_wake_up_new_task(p);
+
raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
WRITE_ONCE(p->__state, TASK_RUNNING);
#ifdef CONFIG_SMP
@@ -4726,6 +4816,7 @@ void wake_up_new_task(struct task_struct *p)
rq = __task_rq_lock(p, &rf);
update_rq_clock(rq);
post_init_entity_util_avg(p);
+ trace_android_rvh_new_task_stats(p);
activate_task(rq, p, ENQUEUE_NOCLOCK);
trace_sched_wakeup_new(p);
@@ -4899,6 +4990,7 @@ struct balance_callback balance_push_callback = {
.next = NULL,
.func = balance_push,
};
+EXPORT_SYMBOL_GPL(balance_push_callback);
static inline struct balance_callback *
__splice_balance_callbacks(struct rq *rq, bool split)
@@ -4930,10 +5022,11 @@ static inline struct balance_callback *splice_balance_callbacks(struct rq *rq)
return __splice_balance_callbacks(rq, true);
}
-static void __balance_callbacks(struct rq *rq)
+void __balance_callbacks(struct rq *rq)
{
do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));
}
+EXPORT_SYMBOL_GPL(__balance_callbacks);
static inline void balance_callbacks(struct rq *rq, struct balance_callback *head)
{
@@ -5140,6 +5233,8 @@ static struct rq *finish_task_switch(struct task_struct *prev)
if (prev->sched_class->task_dead)
prev->sched_class->task_dead(prev);
+ trace_android_rvh_flush_task(prev);
+
/* Task is done with its stack. */
put_task_stack(prev);
@@ -5345,6 +5440,11 @@ void sched_exec(void)
struct task_struct *p = current;
unsigned long flags;
int dest_cpu;
+ bool cond = false;
+
+ trace_android_rvh_sched_exec(&cond);
+ if (cond)
+ return;
raw_spin_lock_irqsave(&p->pi_lock, flags);
dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
@@ -5499,6 +5599,8 @@ void scheduler_tick(void)
rq_lock(rq, &rf);
update_rq_clock(rq);
+ trace_android_rvh_tick_entry(rq);
+
thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure);
curr->sched_class->task_tick(rq, curr, 0);
@@ -5518,6 +5620,8 @@ void scheduler_tick(void)
rq->idle_balance = idle_cpu(cpu);
trigger_load_balance(rq);
#endif
+
+ trace_android_vh_scheduler_tick(rq);
}
#ifdef CONFIG_NO_HZ_FULL
@@ -5774,6 +5878,8 @@ static noinline void __schedule_bug(struct task_struct *prev)
if (panic_on_warn)
panic("scheduling while atomic\n");
+ trace_android_rvh_schedule_bug(prev);
+
dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
}
@@ -6513,6 +6619,7 @@ static void __sched notrace __schedule(unsigned int sched_mode)
rq->last_seen_need_resched_ns = 0;
#endif
+ trace_android_rvh_schedule(prev, next, rq);
if (likely(prev != next)) {
rq->nr_switches++;
/*
@@ -6918,6 +7025,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
struct rq_flags rf;
struct rq *rq;
+ trace_android_rvh_rtmutex_prepare_setprio(p, pi_task);
/* XXX used to be waiter->prio, not waiter->task->prio */
prio = __rt_effective_prio(pi_task, p->normal_prio);
@@ -7036,12 +7144,13 @@ static inline int rt_effective_prio(struct task_struct *p, int prio)
void set_user_nice(struct task_struct *p, long nice)
{
- bool queued, running;
+ bool queued, running, allowed = false;
int old_prio;
struct rq_flags rf;
struct rq *rq;
- if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
+ trace_android_rvh_set_user_nice(p, &nice, &allowed);
+ if ((task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) && !allowed)
return;
/*
* We have to be careful, if called from sys_setpriority(),
@@ -7206,6 +7315,7 @@ int available_idle_cpu(int cpu)
return 1;
}
+EXPORT_SYMBOL_GPL(available_idle_cpu);
/**
* idle_task - return the idle task for a given CPU.
@@ -7515,9 +7625,6 @@ static int __sched_setscheduler(struct task_struct *p,
return retval;
}
- if (pi)
- cpuset_read_lock();
-
/*
* Make sure no PI-waiters arrive (or leave) while we are
* changing the priority of the task:
@@ -7592,8 +7699,6 @@ static int __sched_setscheduler(struct task_struct *p,
if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
policy = oldpolicy = -1;
task_rq_unlock(rq, p, &rf);
- if (pi)
- cpuset_read_unlock();
goto recheck;
}
@@ -7659,10 +7764,8 @@ static int __sched_setscheduler(struct task_struct *p,
head = splice_balance_callbacks(rq);
task_rq_unlock(rq, p, &rf);
- if (pi) {
- cpuset_read_unlock();
+ if (pi)
rt_mutex_adjust_pi(p);
- }
/* Run balance callbacks after we've adjusted the PI chain: */
balance_callbacks(rq, head);
@@ -7672,8 +7775,6 @@ static int __sched_setscheduler(struct task_struct *p,
unlock:
task_rq_unlock(rq, p, &rf);
- if (pi)
- cpuset_read_unlock();
return retval;
}
@@ -7712,11 +7813,13 @@ int sched_setscheduler(struct task_struct *p, int policy,
{
return _sched_setscheduler(p, policy, param, true);
}
+EXPORT_SYMBOL_GPL(sched_setscheduler);
int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
{
return __sched_setscheduler(p, attr, true, true);
}
+EXPORT_SYMBOL_GPL(sched_setattr);
int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
{
@@ -7742,6 +7845,7 @@ int sched_setscheduler_nocheck(struct task_struct *p, int policy,
{
return _sched_setscheduler(p, policy, param, false);
}
+EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
/*
* SCHED_FIFO is a broken scheduler model; that is, it is fundamentally
@@ -7803,14 +7907,9 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
rcu_read_lock();
retval = -ESRCH;
p = find_process_by_pid(pid);
- if (likely(p))
- get_task_struct(p);
- rcu_read_unlock();
-
- if (likely(p)) {
+ if (p != NULL)
retval = sched_setscheduler(p, policy, &lparam);
- put_task_struct(p);
- }
+ rcu_read_unlock();
return retval;
}
@@ -8242,6 +8341,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
};
retval = __sched_setaffinity(p, &ac);
+ trace_android_rvh_sched_setaffinity(p, in_mask, &retval);
kfree(ac.user_mask);
out_put_task:
@@ -8303,6 +8403,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
raw_spin_lock_irqsave(&p->pi_lock, flags);
cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
+ trace_android_rvh_sched_getaffinity(p, mask);
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
out_unlock:
@@ -8358,6 +8459,8 @@ static void do_sched_yield(void)
schedstat_inc(rq->yld_count);
current->sched_class->yield_task(rq);
+ trace_android_rvh_do_sched_yield(rq);
+
preempt_disable();
rq_unlock_irq(rq, &rf);
sched_preempt_enable_no_resched();
@@ -9210,6 +9313,24 @@ void idle_task_exit(void)
/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
}
+struct task_struct *pick_migrate_task(struct rq *rq)
+{
+ const struct sched_class *class;
+ struct task_struct *next;
+
+ for_each_class(class) {
+ next = class->pick_next_task(rq);
+ if (next) {
+ next->sched_class->put_prev_task(rq, next);
+ return next;
+ }
+ }
+
+ /* The idle class should always have a runnable task */
+ BUG();
+}
+EXPORT_SYMBOL_GPL(pick_migrate_task);
+
static int __balance_push_cpu_stop(void *arg)
{
struct task_struct *p = arg;
@@ -9555,6 +9676,7 @@ int sched_cpu_starting(unsigned int cpu)
sched_core_cpu_starting(cpu);
sched_rq_cpu_starting(cpu);
sched_tick_start(cpu);
+ trace_android_rvh_sched_cpu_starting(cpu);
return 0;
}
@@ -9628,6 +9750,8 @@ int sched_cpu_dying(unsigned int cpu)
}
rq_unlock_irqrestore(rq, &rf);
+ trace_android_rvh_sched_cpu_dying(cpu);
+
calc_load_migrate(rq);
update_max_interval();
hrtick_clear(rq);
@@ -9688,7 +9812,9 @@ int in_sched_functions(unsigned long addr)
* Every task in system belongs to this group at bootup.
*/
struct task_group root_task_group;
+EXPORT_SYMBOL_GPL(root_task_group);
LIST_HEAD(task_groups);
+EXPORT_SYMBOL_GPL(task_groups);
/* Cacheline aligned slab cache for task_group */
static struct kmem_cache *task_group_cache __read_mostly;
@@ -9972,6 +10098,8 @@ void __might_resched(const char *file, int line, unsigned int offsets)
print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK,
preempt_disable_ip);
+ trace_android_rvh_schedule_bug(NULL);
+
dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
}
@@ -10351,6 +10479,7 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
mutex_unlock(&uclamp_mutex);
#endif
+ trace_android_rvh_cpu_cgroup_online(css);
return 0;
}
@@ -10392,6 +10521,8 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset)
cgroup_taskset_for_each(task, css, tset)
sched_move_task(task);
+
+ trace_android_rvh_cpu_cgroup_attach(tset);
}
#ifdef CONFIG_UCLAMP_TASK_GROUP
@@ -10569,6 +10700,27 @@ static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
cpu_uclamp_print(sf, UCLAMP_MAX);
return 0;
}
+
+static int cpu_uclamp_ls_write_u64(struct cgroup_subsys_state *css,
+ struct cftype *cftype, u64 ls)
+{
+ struct task_group *tg;
+
+ if (ls > 1)
+ return -EINVAL;
+ tg = css_tg(css);
+ tg->latency_sensitive = (unsigned int) ls;
+
+ return 0;
+}
+
+static u64 cpu_uclamp_ls_read_u64(struct cgroup_subsys_state *css,
+ struct cftype *cft)
+{
+ struct task_group *tg = css_tg(css);
+
+ return (u64) tg->latency_sensitive;
+}
#endif /* CONFIG_UCLAMP_TASK_GROUP */
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -11011,6 +11163,12 @@ static struct cftype cpu_legacy_files[] = {
.seq_show = cpu_uclamp_max_show,
.write = cpu_uclamp_max_write,
},
+ {
+ .name = "uclamp.latency_sensitive",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .read_u64 = cpu_uclamp_ls_read_u64,
+ .write_u64 = cpu_uclamp_ls_write_u64,
+ },
#endif
{ } /* Terminate */
};
@@ -11209,6 +11367,12 @@ static struct cftype cpu_files[] = {
.seq_show = cpu_uclamp_max_show,
.write = cpu_uclamp_max_write,
},
+ {
+ .name = "uclamp.latency_sensitive",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .read_u64 = cpu_uclamp_ls_read_u64,
+ .write_u64 = cpu_uclamp_ls_write_u64,
+ },
#endif
{ } /* terminate */
};
diff --git a/kernel/sched/cpufreq.c b/kernel/sched/cpufreq.c
index 5252fb1..b3a8120 100644
--- a/kernel/sched/cpufreq.c
+++ b/kernel/sched/cpufreq.c
@@ -72,3 +72,4 @@ bool cpufreq_this_cpu_can_update(struct cpufreq_policy *policy)
(policy->dvfs_possible_from_any_cpu &&
rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data)));
}
+EXPORT_SYMBOL_GPL(cpufreq_this_cpu_can_update);
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 1207c78..919f8e9 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -834,29 +834,3 @@ struct cpufreq_governor *cpufreq_default_governor(void)
#endif
cpufreq_governor_init(schedutil_gov);
-
-#ifdef CONFIG_ENERGY_MODEL
-static void rebuild_sd_workfn(struct work_struct *work)
-{
- rebuild_sched_domains_energy();
-}
-static DECLARE_WORK(rebuild_sd_work, rebuild_sd_workfn);
-
-/*
- * EAS shouldn't be attempted without sugov, so rebuild the sched_domains
- * on governor changes to make sure the scheduler knows about it.
- */
-void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
- struct cpufreq_governor *old_gov)
-{
- if (old_gov == &schedutil_gov || policy->governor == &schedutil_gov) {
- /*
- * When called from the cpufreq_register_driver() path, the
- * cpu_hotplug_lock is already held, so use a work item to
- * avoid nested locking in rebuild_sched_domains().
- */
- schedule_work(&rebuild_sd_work);
- }
-
-}
-#endif
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index a286e72..64e0cce 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -195,6 +195,7 @@ int cpupri_find_fitness(struct cpupri *cp, struct task_struct *p,
return 0;
}
+EXPORT_SYMBOL_GPL(cpupri_find_fitness);
/**
* cpupri_set - update the CPU priority setting
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 95fc778..a485f3b0 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -2,6 +2,8 @@
/*
* Simple CPU accounting cgroup controller
*/
+#include <linux/cpufreq_times.h>
+#include <trace/hooks/sched.h>
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -17,6 +19,7 @@
* compromise in place of having locks on each irq in account_system_time.
*/
DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
+EXPORT_PER_CPU_SYMBOL_GPL(cpu_irqtime);
static int sched_clock_irqtime;
@@ -52,6 +55,7 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
unsigned int pc;
s64 delta;
int cpu;
+ bool irq_start = true;
if (!sched_clock_irqtime)
return;
@@ -67,10 +71,15 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
* in that case, so as not to confuse scheduler with a special task
* that do not consume any time, but still wants to run.
*/
- if (pc & HARDIRQ_MASK)
+ if (pc & HARDIRQ_MASK) {
irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
- else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())
+ irq_start = false;
+ } else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) {
irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
+ irq_start = false;
+ }
+
+ trace_android_rvh_account_irq(curr, cpu, delta, irq_start);
}
static u64 irqtime_tick_accounted(u64 maxtime)
@@ -129,6 +138,9 @@ void account_user_time(struct task_struct *p, u64 cputime)
/* Account for user time used */
acct_account_cputime(p);
+
+ /* Account power usage for user time */
+ cpufreq_acct_update_power(p, cputime);
}
/*
@@ -173,6 +185,9 @@ void account_system_index_time(struct task_struct *p,
/* Account for system time used */
acct_account_cputime(p);
+
+ /* Account power usage for system time */
+ cpufreq_acct_update_power(p, cputime);
}
/*
@@ -472,6 +487,7 @@ void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
*ut = cputime.utime;
*st = cputime.stime;
}
+EXPORT_SYMBOL_GPL(thread_group_cputime_adjusted);
#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
@@ -642,6 +658,8 @@ void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
thread_group_cputime(p, &cputime);
cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
}
+EXPORT_SYMBOL_GPL(thread_group_cputime_adjusted);
+
#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 1637b65..3ceee0e 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -47,9 +47,10 @@ static unsigned long nsec_low(unsigned long long nsec)
#define SCHED_FEAT(name, enabled) \
#name ,
-static const char * const sched_feat_names[] = {
+const char * const sched_feat_names[] = {
#include "features.h"
};
+EXPORT_SYMBOL_GPL(sched_feat_names);
#undef SCHED_FEAT
@@ -78,6 +79,7 @@ static int sched_feat_show(struct seq_file *m, void *v)
struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
#include "features.h"
};
+EXPORT_SYMBOL_GPL(sched_feat_keys);
#undef SCHED_FEAT
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 0cd1d0f..20aec5f2 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -56,6 +56,8 @@
#include "stats.h"
#include "autogroup.h"
+#include <trace/hooks/sched.h>
+
/*
* Targeted preemption latency for CPU-bound tasks:
*
@@ -70,6 +72,7 @@
* (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
*/
unsigned int sysctl_sched_latency = 6000000ULL;
+EXPORT_SYMBOL_GPL(sysctl_sched_latency);
static unsigned int normalized_sysctl_sched_latency = 6000000ULL;
/*
@@ -627,11 +630,13 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
*/
static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
+ trace_android_rvh_enqueue_entity(cfs_rq, se);
rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less);
}
static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
+ trace_android_rvh_dequeue_entity(cfs_rq, se);
rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline);
}
@@ -4348,6 +4353,11 @@ static inline void util_est_update(struct cfs_rq *cfs_rq,
{
long last_ewma_diff, last_enqueued_diff;
struct util_est ue;
+ int ret = 0;
+
+ trace_android_rvh_util_est_update(cfs_rq, p, task_sleep, &ret);
+ if (ret)
+ return;
if (!sched_feat(UTIL_EST))
return;
@@ -4565,7 +4575,10 @@ static inline int task_fits_cpu(struct task_struct *p, int cpu)
static inline void update_misfit_status(struct task_struct *p, struct rq *rq)
{
- if (!sched_asym_cpucap_active())
+ bool need_update = true;
+
+ trace_android_rvh_update_misfit_status(p, rq, &need_update);
+ if (!sched_asym_cpucap_active() || !need_update)
return;
if (!p || p->nr_cpus_allowed == 1) {
@@ -4675,6 +4688,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
/* ensure we never gain time by being placed backwards. */
se->vruntime = max_vruntime(se->vruntime, vruntime);
+ trace_android_rvh_place_entity(cfs_rq, se, initial, &vruntime);
}
static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
@@ -4883,9 +4897,14 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
unsigned long ideal_runtime, delta_exec;
struct sched_entity *se;
s64 delta;
+ bool skip_preempt = false;
ideal_runtime = sched_slice(cfs_rq, curr);
delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
+ trace_android_rvh_check_preempt_tick(current, &ideal_runtime, &skip_preempt,
+ delta_exec, cfs_rq, curr, sysctl_sched_min_granularity);
+ if (skip_preempt)
+ return;
if (delta_exec > ideal_runtime) {
resched_curr(rq_of(cfs_rq));
/*
@@ -4914,8 +4933,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
resched_curr(rq_of(cfs_rq));
}
-static void
-set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
+void set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
clear_buddies(cfs_rq, se);
@@ -4951,6 +4969,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
se->prev_sum_exec_runtime = se->sum_exec_runtime;
}
+EXPORT_SYMBOL_GPL(set_next_entity);
static int
wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
@@ -4966,7 +4985,11 @@ static struct sched_entity *
pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
{
struct sched_entity *left = __pick_first_entity(cfs_rq);
- struct sched_entity *se;
+ struct sched_entity *se = NULL;
+
+ trace_android_rvh_pick_next_entity(cfs_rq, curr, &se);
+ if (se)
+ goto done;
/*
* If curr is set we have to see if its left of the leftmost entity
@@ -5008,6 +5031,7 @@ pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
se = cfs_rq->last;
}
+done:
return se;
}
@@ -5070,6 +5094,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
if (cfs_rq->nr_running > 1)
check_preempt_tick(cfs_rq, curr);
+ trace_android_rvh_entity_tick(cfs_rq, curr);
}
@@ -5995,6 +6020,11 @@ static inline bool cpu_overutilized(int cpu)
{
unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
+ int overutilized = -1;
+
+ trace_android_rvh_cpu_overutilized(cpu, &overutilized);
+ if (overutilized != -1)
+ return overutilized;
return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu);
}
@@ -6083,6 +6113,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
flags = ENQUEUE_WAKEUP;
}
+ trace_android_rvh_enqueue_task_fair(rq, p, flags);
for_each_sched_entity(se) {
cfs_rq = cfs_rq_of(se);
@@ -6173,6 +6204,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
flags |= DEQUEUE_SLEEP;
}
+ trace_android_rvh_dequeue_task_fair(rq, p, flags);
for_each_sched_entity(se) {
cfs_rq = cfs_rq_of(se);
@@ -7181,7 +7213,7 @@ compute_energy(struct energy_env *eenv, struct perf_domain *pd,
* other use-cases too. So, until someone finds a better way to solve this,
* let's keep things simple by re-using the existing slow path.
*/
-static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
+static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync)
{
struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
@@ -7193,11 +7225,21 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
struct perf_domain *pd;
struct energy_env eenv;
+ sync_entity_load_avg(&p->se);
+
rcu_read_lock();
pd = rcu_dereference(rd->pd);
if (!pd || READ_ONCE(rd->overutilized))
goto unlock;
+ cpu = smp_processor_id();
+ if (sync && cpu_rq(cpu)->nr_running == 1 &&
+ cpumask_test_cpu(cpu, p->cpus_ptr) &&
+ task_fits_cpu(p, cpu)) {
+ rcu_read_unlock();
+ return cpu;
+ }
+
/*
* Energy-aware wake-up happens on the lowest sched_domain starting
* from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
@@ -7358,9 +7400,18 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
int cpu = smp_processor_id();
int new_cpu = prev_cpu;
int want_affine = 0;
+ int target_cpu = -1;
/* SD_flags and WF_flags share the first nibble */
int sd_flag = wake_flags & 0xF;
+ if (trace_android_rvh_select_task_rq_fair_enabled() &&
+ !(sd_flag & SD_BALANCE_FORK))
+ sync_entity_load_avg(&p->se);
+ trace_android_rvh_select_task_rq_fair(p, prev_cpu, sd_flag,
+ wake_flags, &target_cpu);
+ if (target_cpu >= 0)
+ return target_cpu;
+
/*
* required for stable ->cpus_allowed
*/
@@ -7369,7 +7420,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
record_wakee(p);
if (sched_energy_enabled()) {
- new_cpu = find_energy_efficient_cpu(p, prev_cpu);
+ new_cpu = find_energy_efficient_cpu(p, prev_cpu, sync);
if (new_cpu >= 0)
return new_cpu;
new_cpu = prev_cpu;
@@ -7565,9 +7616,14 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
int scale = cfs_rq->nr_running >= sched_nr_latency;
int next_buddy_marked = 0;
int cse_is_idle, pse_is_idle;
+ bool ignore = false;
+ bool preempt = false;
if (unlikely(se == pse))
return;
+ trace_android_rvh_check_preempt_wakeup_ignore(curr, &ignore);
+ if (ignore)
+ return;
/*
* This is possible from callers such as attach_tasks(), in which we
@@ -7624,6 +7680,13 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
return;
update_curr(cfs_rq_of(se));
+ trace_android_rvh_check_preempt_wakeup(rq, p, &preempt, &ignore,
+ wake_flags, se, pse, next_buddy_marked, sysctl_sched_wakeup_granularity);
+ if (preempt)
+ goto preempt;
+ if (ignore)
+ return;
+
if (wakeup_preempt_entity(se, pse) == 1) {
/*
* Bias pick_next to pick the sched entity that is
@@ -7691,9 +7754,10 @@ struct task_struct *
pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
{
struct cfs_rq *cfs_rq = &rq->cfs;
- struct sched_entity *se;
- struct task_struct *p;
+ struct sched_entity *se = NULL;
+ struct task_struct *p = NULL;
int new_tasks;
+ bool repick = false;
again:
if (!sched_fair_runnable(rq))
@@ -7747,7 +7811,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
} while (cfs_rq);
p = task_of(se);
-
+ trace_android_rvh_replace_next_task_fair(rq, &p, &se, &repick, false, prev);
/*
* Since we haven't yet done put_prev_entity and if the selected task
* is a different task than we started out with, try and touch the
@@ -7780,6 +7844,10 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
if (prev)
put_prev_task(rq, prev);
+ trace_android_rvh_replace_next_task_fair(rq, &p, &se, &repick, true, prev);
+ if (repick)
+ goto done;
+
do {
se = pick_next_entity(cfs_rq, NULL);
set_next_entity(cfs_rq, se);
@@ -8021,7 +8089,8 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
* rewrite all of this once again.]
*/
-static unsigned long __read_mostly max_load_balance_interval = HZ/10;
+unsigned long __read_mostly max_load_balance_interval = HZ/10;
+EXPORT_SYMBOL_GPL(max_load_balance_interval);
enum fbq_type { regular, remote, all };
@@ -8101,6 +8170,7 @@ struct lb_env {
enum fbq_type fbq_type;
enum migration_type migration_type;
struct list_head tasks;
+ struct rq_flags *src_rq_rf;
};
/*
@@ -8215,9 +8285,14 @@ static
int can_migrate_task(struct task_struct *p, struct lb_env *env)
{
int tsk_cache_hot;
+ int can_migrate = 1;
lockdep_assert_rq_held(env->src_rq);
+ trace_android_rvh_can_migrate_task(p, env->dst_cpu, &can_migrate);
+ if (!can_migrate)
+ return 0;
+
/*
* We do not migrate tasks that are:
* 1) throttled_lb_pair, or
@@ -8305,8 +8380,20 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
*/
static void detach_task(struct task_struct *p, struct lb_env *env)
{
+ int detached = 0;
+
lockdep_assert_rq_held(env->src_rq);
+ /*
+ * The vendor hook may drop the lock temporarily, so
+ * pass the rq flags to unpin lock. We expect the
+ * rq lock to be held after return.
+ */
+ trace_android_rvh_migrate_queued_task(env->src_rq, env->src_rq_rf, p,
+ env->dst_cpu, &detached);
+ if (detached)
+ return;
+
deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
set_task_cpu(p, env->dst_cpu);
}
@@ -8839,7 +8926,9 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu)
if (!capacity)
capacity = 1;
+ trace_android_rvh_update_cpu_capacity(cpu, &capacity);
rq->cpu_capacity = capacity;
+ trace_sched_cpu_capacity_tp(rq);
/*
* Detect if the performance domain is in capacity inversion state.
@@ -10125,8 +10214,12 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
if (sched_energy_enabled()) {
struct root_domain *rd = env->dst_rq->rd;
+ int out_balance = 1;
- if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized))
+ trace_android_rvh_find_busiest_group(sds.busiest, env->dst_rq,
+ &out_balance);
+ if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)
+ && out_balance)
goto out_balanced;
}
@@ -10245,7 +10338,12 @@ static struct rq *find_busiest_queue(struct lb_env *env,
struct rq *busiest = NULL, *rq;
unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
unsigned int busiest_nr = 0;
- int i;
+ int i, done = 0;
+
+ trace_android_rvh_find_busiest_queue(env->dst_cpu, group, env->cpus,
+ &busiest, &done);
+ if (done)
+ return busiest;
for_each_cpu_and(i, sched_group_span(group), env->cpus) {
unsigned long capacity, load, util;
@@ -10547,6 +10645,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
more_balance:
rq_lock_irqsave(busiest, &rf);
+ env.src_rq_rf = &rf;
update_rq_clock(busiest);
/*
@@ -10842,6 +10941,7 @@ static int active_load_balance_cpu_stop(void *data)
.src_rq = busiest_rq,
.idle = CPU_IDLE,
.flags = LBF_ACTIVE_LB,
+ .src_rq_rf = &rf,
};
schedstat_inc(sd->alb_count);
@@ -10923,6 +11023,10 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
int need_serialize, need_decay = 0;
u64 max_cost = 0;
+ trace_android_rvh_sched_rebalance_domains(rq, &continue_balancing);
+ if (!continue_balancing)
+ return;
+
rcu_read_lock();
for_each_domain(cpu, sd) {
/*
@@ -11073,6 +11177,7 @@ static void nohz_balancer_kick(struct rq *rq)
struct sched_domain *sd;
int nr_busy, i, cpu = rq->cpu;
unsigned int flags = 0;
+ int done = 0;
if (unlikely(rq->idle_balance))
return;
@@ -11097,6 +11202,10 @@ static void nohz_balancer_kick(struct rq *rq)
if (time_before(now, nohz.next_balance))
goto out;
+ trace_android_rvh_sched_nohz_balancer_kick(rq, &flags, &done);
+ if (done)
+ goto out;
+
if (rq->nr_running >= 2) {
flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto out;
@@ -11502,6 +11611,11 @@ static int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
u64 t0, t1, curr_cost = 0;
struct sched_domain *sd;
int pulled_task = 0;
+ int done = 0;
+
+ trace_android_rvh_sched_newidle_balance(this_rq, rf, &pulled_task, &done);
+ if (done)
+ return pulled_task;
update_misfit_status(NULL, this_rq);
diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c
index 0f31076..7a64ade 100644
--- a/kernel/sched/pelt.c
+++ b/kernel/sched/pelt.c
@@ -302,6 +302,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se)
return 0;
}
+EXPORT_SYMBOL_GPL(__update_load_avg_blocked_se);
int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se)
{
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index ed2a47e..18ad8d7 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -4,6 +4,8 @@
* policies)
*/
+#include <trace/hooks/sched.h>
+
int sched_rr_timeslice = RR_TIMESLICE;
/* More than 4 hours if BW_SHIFT equals 20. */
static const u64 max_rt_runtime = MAX_BW;
@@ -1020,6 +1022,13 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
if (likely(rt_b->rt_runtime)) {
rt_rq->rt_throttled = 1;
printk_deferred_once("sched: RT throttling activated\n");
+
+ trace_android_vh_dump_throttled_rt_tasks(
+ raw_smp_processor_id(),
+ rq_clock(rq_of_rt_rq(rt_rq)),
+ sched_rt_period(rt_rq),
+ runtime,
+ hrtimer_get_expires_ns(&rt_b->rt_period_timer));
} else {
/*
* In case we did anyway, make it go away,
@@ -1528,6 +1537,27 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
enqueue_top_rt_rq(&rq->rt);
}
+#ifdef CONFIG_SMP
+static inline bool should_honor_rt_sync(struct rq *rq, struct task_struct *p,
+ bool sync)
+{
+ /*
+ * If the waker is CFS, then an RT sync wakeup would preempt the waker
+ * and force it to run for a likely small time after the RT wakee is
+ * done. So, only honor RT sync wakeups from RT wakers.
+ */
+ return sync && task_has_rt_policy(rq->curr) &&
+ p->prio <= rq->rt.highest_prio.next &&
+ rq->rt.rt_nr_running <= 2;
+}
+#else
+static inline bool should_honor_rt_sync(struct rq *rq, struct task_struct *p,
+ bool sync)
+{
+ return 0;
+}
+#endif
+
/*
* Adding/removing a task to/from a priority array:
*/
@@ -1535,6 +1565,7 @@ static void
enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
{
struct sched_rt_entity *rt_se = &p->rt;
+ bool sync = !!(flags & ENQUEUE_WAKEUP_SYNC);
if (flags & ENQUEUE_WAKEUP)
rt_se->timeout = 0;
@@ -1544,7 +1575,8 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
enqueue_rt_entity(rt_se, flags);
- if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
+ if (!task_current(rq, p) && p->nr_cpus_allowed > 1 &&
+ !should_honor_rt_sync(rq, p, sync))
enqueue_pushable_task(rq, p);
}
@@ -1600,7 +1632,16 @@ select_task_rq_rt(struct task_struct *p, int cpu, int flags)
{
struct task_struct *curr;
struct rq *rq;
+ struct rq *this_cpu_rq;
bool test;
+ int target_cpu = -1;
+ bool sync = !!(flags & WF_SYNC);
+ int this_cpu;
+
+ trace_android_rvh_select_task_rq_rt(p, cpu, flags & 0xF,
+ flags, &target_cpu);
+ if (target_cpu >= 0)
+ return target_cpu;
/* For anything but wake ups, just return the task_cpu */
if (!(flags & (WF_TTWU | WF_FORK)))
@@ -1610,6 +1651,8 @@ select_task_rq_rt(struct task_struct *p, int cpu, int flags)
rcu_read_lock();
curr = READ_ONCE(rq->curr); /* unlocked access */
+ this_cpu = smp_processor_id();
+ this_cpu_rq = cpu_rq(this_cpu);
/*
* If the current task on @p's runqueue is an RT task, then
@@ -1641,6 +1684,15 @@ select_task_rq_rt(struct task_struct *p, int cpu, int flags)
unlikely(rt_task(curr)) &&
(curr->nr_cpus_allowed < 2 || curr->prio <= p->prio);
+ /*
+ * Respect the sync flag as long as the task can run on this CPU.
+ */
+ if (should_honor_rt_sync(this_cpu_rq, p, sync) &&
+ cpumask_test_cpu(this_cpu, p->cpus_ptr)) {
+ cpu = this_cpu;
+ goto out_unlock;
+ }
+
if (test || !rt_task_fits_capacity(p, cpu)) {
int target = find_lowest_rq(p);
@@ -1697,6 +1749,8 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
{
if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
+ int done = 0;
+
/*
* This is OK, because current is on_cpu, which avoids it being
* picked for load-balance and preemption/IRQs are still
@@ -1704,7 +1758,9 @@ static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
* not yet started the picking loop.
*/
rq_unpin_lock(rq, rf);
- pull_rt_task(rq);
+ trace_android_rvh_sched_balance_rt(rq, p, &done);
+ if (!done)
+ pull_rt_task(rq);
rq_repin_lock(rq, rf);
}
@@ -1856,7 +1912,7 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
* Return the highest pushable rq's task, which is suitable to be executed
* on the CPU, NULL otherwise
*/
-static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
+struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
{
struct plist_head *head = &rq->rt.pushable_tasks;
struct task_struct *p;
@@ -1871,6 +1927,7 @@ static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
return NULL;
}
+EXPORT_SYMBOL_GPL(pick_highest_pushable_task);
static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
@@ -1879,7 +1936,7 @@ static int find_lowest_rq(struct task_struct *task)
struct sched_domain *sd;
struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
int this_cpu = smp_processor_id();
- int cpu = task_cpu(task);
+ int cpu = -1;
int ret;
/* Make sure the mask is initialized first */
@@ -1904,9 +1961,15 @@ static int find_lowest_rq(struct task_struct *task)
task, lowest_mask);
}
+ trace_android_rvh_find_lowest_rq(task, lowest_mask, ret, &cpu);
+ if (cpu >= 0)
+ return cpu;
+
if (!ret)
return -1; /* No targets found */
+ cpu = task_cpu(task);
+
/*
* At this point we have built a mask of CPUs representing the
* lowest priority tasks in the system. Now we want to elect
@@ -2238,6 +2301,9 @@ static int rto_next_cpu(struct root_domain *rd)
/* When rto_cpu is -1 this acts like cpumask_first() */
cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
+ /* this will be any CPU in the rd->rto_mask, and can be a halted cpu update it */
+ trace_android_rvh_rto_next_cpu(rd->rto_cpu, rd->rto_mask, &cpu);
+
rd->rto_cpu = cpu;
if (cpu < nr_cpu_ids)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 771f8dd..d44dbb5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -68,6 +68,8 @@
#include <linux/wait_api.h>
#include <linux/wait_bit.h>
#include <linux/workqueue_api.h>
+#include <linux/android_vendor.h>
+#include "android.h"
#include <trace/events/power.h>
#include <trace/events/sched.h>
@@ -416,6 +418,10 @@ struct task_group {
struct uclamp_se uclamp_req[UCLAMP_CNT];
/* Effective clamp values used for a task group */
struct uclamp_se uclamp[UCLAMP_CNT];
+ /* Latency-sensitive flag used for a task group */
+ unsigned int latency_sensitive;
+
+ ANDROID_VENDOR_DATA_ARRAY(1, 4);
#endif
};
@@ -892,6 +898,7 @@ extern void sched_put_rd(struct root_domain *rd);
#ifdef HAVE_RT_PUSH_IPI
extern void rto_push_irq_work_func(struct irq_work *work);
#endif
+extern struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu);
#endif /* CONFIG_SMP */
#ifdef CONFIG_UCLAMP_TASK
@@ -1554,6 +1561,11 @@ struct rq_flags {
#endif
};
+#ifdef CONFIG_SMP
+extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
+ struct task_struct *p, int dest_cpu);
+#endif
+
extern struct balance_callback balance_push_callback;
/*
@@ -1720,8 +1732,6 @@ enum numa_faults_stats {
};
extern void sched_setnuma(struct task_struct *p, int node);
extern int migrate_task_to(struct task_struct *p, int cpu);
-extern int migrate_swap(struct task_struct *p, struct task_struct *t,
- int cpu, int scpu);
extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p);
#else
static inline void
@@ -1732,6 +1742,9 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
#ifdef CONFIG_SMP
+extern int migrate_swap(struct task_struct *p, struct task_struct *t,
+ int cpu, int scpu);
+
static inline void
queue_balance_callback(struct rq *rq,
struct balance_callback *head,
@@ -2021,6 +2034,8 @@ static __always_inline bool static_branch_##name(struct static_key *key) \
#undef SCHED_FEAT
extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
+extern const char * const sched_feat_names[__SCHED_FEAT_NR];
+
#define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
#else /* !CONFIG_JUMP_LABEL */
@@ -2153,6 +2168,8 @@ extern const u32 sched_prio_to_wmult[40];
#define ENQUEUE_MIGRATED 0x00
#endif
+#define ENQUEUE_WAKEUP_SYNC 0x80
+
#define RETRY_TASK ((void *)-1UL)
struct affinity_context {
@@ -2323,6 +2340,7 @@ static inline struct task_struct *get_push_task(struct rq *rq)
extern int push_cpu_stop(void *arg);
+extern unsigned long __read_mostly max_load_balance_interval;
#endif
#ifdef CONFIG_CPU_IDLE
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 8739c2a..c480058 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -3,6 +3,8 @@
* Scheduler topology setup/handling methods
*/
+#include <trace/hooks/sched.h>
+
DEFINE_MUTEX(sched_domains_mutex);
/* Protected by sched_domains_mutex: */
@@ -347,8 +349,7 @@ static void sched_energy_set(bool has_eas)
* 2. the SD_ASYM_CPUCAPACITY flag is set in the sched_domain hierarchy.
* 3. no SMT is detected.
* 4. the EM complexity is low enough to keep scheduling overheads low;
- * 5. schedutil is driving the frequency of all CPUs of the rd;
- * 6. frequency invariance support is present;
+ * 5. frequency invariance support is present;
*
* The complexity of the Energy Model is defined as:
*
@@ -368,21 +369,23 @@ static void sched_energy_set(bool has_eas)
*/
#define EM_MAX_COMPLEXITY 2048
-extern struct cpufreq_governor schedutil_gov;
static bool build_perf_domains(const struct cpumask *cpu_map)
{
int i, nr_pd = 0, nr_ps = 0, nr_cpus = cpumask_weight(cpu_map);
struct perf_domain *pd = NULL, *tmp;
int cpu = cpumask_first(cpu_map);
struct root_domain *rd = cpu_rq(cpu)->rd;
- struct cpufreq_policy *policy;
- struct cpufreq_governor *gov;
+ bool eas_check = false;
if (!sysctl_sched_energy_aware)
goto free;
- /* EAS is enabled for asymmetric CPU capacity topologies. */
- if (!per_cpu(sd_asym_cpucapacity, cpu)) {
+ /*
+ * EAS is enabled for asymmetric CPU capacity topologies.
+ * Allow vendor to override if desired.
+ */
+ trace_android_rvh_build_perf_domains(&eas_check);
+ if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) {
if (sched_debug()) {
pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n",
cpumask_pr_args(cpu_map));
@@ -410,19 +413,6 @@ static bool build_perf_domains(const struct cpumask *cpu_map)
if (find_pd(pd, i))
continue;
- /* Do not attempt EAS if schedutil is not being used. */
- policy = cpufreq_cpu_get(i);
- if (!policy)
- goto free;
- gov = policy->governor;
- cpufreq_cpu_put(policy);
- if (gov != &schedutil_gov) {
- if (rd->pd)
- pr_warn("rd %*pbl: Disabling EAS, schedutil is mandatory\n",
- cpumask_pr_args(cpu_map));
- goto free;
- }
-
/* Create the new pd and add it to the local list. */
tmp = pd_init(i);
if (!tmp)
@@ -2388,6 +2378,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n",
cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
}
+ trace_android_vh_build_sched_domains(has_asym);
ret = 0;
error:
diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c
new file mode 100644
index 0000000..9fc1695
--- /dev/null
+++ b/kernel/sched/vendor_hooks.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* vendor_hook.c
+ *
+ * Copyright 2022 Google LLC
+ */
+#include <linux/sched/cputime.h>
+#include "sched.h"
+#include "pelt.h"
+#include "smp.h"
+
+#define CREATE_TRACE_POINTS
+#include <trace/hooks/vendor_hooks.h>
+#include <linux/tracepoint.h>
+#include <trace/hooks/sched.h>
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_select_task_rq_fair);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_select_task_rq_rt);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_select_fallback_rq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scheduler_tick);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_can_migrate_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_lowest_rq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_prepare_prio_fork);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_finish_prio_fork);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rtmutex_prepare_setprio);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_group);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dump_throttled_rt_tasks);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_jiffies_update);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_newidle_balance);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_nohz_balancer_kick);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_rebalance_domains);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_queue);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_migrate_queued_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_overutilized);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_setaffinity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_sched_domains);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_balance_rt);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_pick_next_entity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_cpus_allowed_by_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_uclamp_eff_get);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_after_enqueue_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_after_dequeue_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_entity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_entity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_entity_tick);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task_fair);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_setscheduler_uclamp);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_wake_up_new_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_new_task_stats);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_flush_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tick_entry);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_starting);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_dying);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_account_irq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_place_entity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_build_perf_domains);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpu_capacity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_misfit_status);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rto_next_cpu);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_is_cpu_allowed);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_get_nohz_timer_target);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_getaffinity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_sched_yield);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork_init);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_ttwu_cond);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule_bug);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_exec);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_update_topology_flags_workfn);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_thermal_stats);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index c8a6913..4a4625b 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -33,6 +33,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/irq.h>
+EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_entry);
+
/*
- No shared variables, all the data are CPU local.
- If a softirq needs serialization, let it serialize itself
@@ -59,6 +61,7 @@ EXPORT_PER_CPU_SYMBOL(irq_stat);
static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
+EXPORT_PER_CPU_SYMBOL_GPL(ksoftirqd);
const char * const softirq_to_name[NR_SOFTIRQS] = {
"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
index 9ed5ce9..af34650 100644
--- a/kernel/stacktrace.c
+++ b/kernel/stacktrace.c
@@ -151,6 +151,7 @@ unsigned int stack_trace_save_tsk(struct task_struct *tsk, unsigned long *store,
put_task_stack(tsk);
return c.len;
}
+EXPORT_SYMBOL_GPL(stack_trace_save_tsk);
/**
* stack_trace_save_regs - Save a stack trace based on pt_regs into a storage array
@@ -174,6 +175,7 @@ unsigned int stack_trace_save_regs(struct pt_regs *regs, unsigned long *store,
arch_stack_walk(consume_entry, &c, current, regs);
return c.len;
}
+EXPORT_SYMBOL_GPL(stack_trace_save_regs);
#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
/**
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index cedb17b..feffcfd 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -152,6 +152,7 @@ int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
wait_for_completion(&done.completion);
return done.ret;
}
+EXPORT_SYMBOL_GPL(stop_one_cpu);
/* This controls the threads on each CPU. */
enum multi_stop_state {
@@ -387,6 +388,7 @@ bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
return cpu_stop_queue_work(cpu, work_buf);
}
+EXPORT_SYMBOL_GPL(stop_one_cpu_nowait);
static bool queue_stop_cpus_work(const struct cpumask *cpumask,
cpu_stop_fn_t fn, void *arg,
diff --git a/kernel/sys.c b/kernel/sys.c
index 5fd54bf..2c1f8fd 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -76,6 +76,8 @@
#include "uid16.h"
+#include <trace/hooks/sys.h>
+
#ifndef SET_UNALIGN_CTL
# define SET_UNALIGN_CTL(a, b) (-EINVAL)
#endif
@@ -2630,6 +2632,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
error = -EINVAL;
break;
}
+ trace_android_vh_syscall_prctl_finished(option, me);
return error;
}
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index 8464c5a..2d945d3 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -17,6 +17,7 @@
#include <linux/sched_clock.h>
#include <linux/seqlock.h>
#include <linux/bitops.h>
+#include <trace/hooks/epoch.h>
#include "timekeeping.h"
@@ -269,6 +270,7 @@ int sched_clock_suspend(void)
update_sched_clock();
hrtimer_cancel(&sched_clock_timer);
rd->read_sched_clock = suspended_sched_clock_read;
+ trace_android_vh_show_suspend_epoch_val(rd->epoch_ns, rd->epoch_cyc);
return 0;
}
@@ -280,6 +282,7 @@ void sched_clock_resume(void)
rd->epoch_cyc = cd.actual_read_sched_clock();
hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
rd->read_sched_clock = cd.actual_read_sched_clock;
+ trace_android_vh_show_resume_epoch_val(rd->epoch_cyc);
}
static struct syscore_ops sched_clock_ops = {
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 4678935..5c47cc5 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -17,6 +17,7 @@
#include <linux/sched.h>
#include <linux/module.h>
#include <trace/events/power.h>
+#include <trace/hooks/sched.h>
#include <asm/irq_regs.h>
@@ -95,6 +96,7 @@ static void tick_periodic(int cpu)
write_seqcount_end(&jiffies_seq);
raw_spin_unlock(&jiffies_lock);
update_wall_time();
+ trace_android_vh_jiffies_update(NULL);
}
update_process_times(user_mode(get_irq_regs()));
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index b0e3c920..1e67c83 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -26,6 +26,7 @@
#include <linux/posix-timers.h>
#include <linux/context_tracking.h>
#include <linux/mm.h>
+#include <trace/hooks/sched.h>
#include <asm/irq_regs.h>
@@ -195,8 +196,10 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
#endif
/* Check, if the jiffies need an update */
- if (tick_do_timer_cpu == cpu)
+ if (tick_do_timer_cpu == cpu) {
tick_do_update_jiffies64(now);
+ trace_android_vh_jiffies_update(NULL);
+ }
/*
* If jiffies update stalled for too long (timekeeper in stop_machine()
@@ -1248,6 +1251,7 @@ ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
return ktime_sub(next_event, now);
}
+EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length);
/**
* tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
@@ -1261,6 +1265,7 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
return ts->idle_calls;
}
+EXPORT_SYMBOL_GPL(tick_nohz_get_idle_calls_cpu);
/**
* tick_nohz_get_idle_calls - return the current idle calls counter value
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 526257b..f2e92bd 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -686,6 +686,7 @@ u64 nsec_to_clock_t(u64 x)
return div_u64(x * 9, (9ull * NSEC_PER_SEC + (USER_HZ / 2)) / USER_HZ);
#endif
}
+EXPORT_SYMBOL_GPL(nsec_to_clock_t);
u64 jiffies64_to_nsecs(u64 j)
{
diff --git a/kernel/trace/power-traces.c b/kernel/trace/power-traces.c
index 21bb161..3ca551c 100644
--- a/kernel/trace/power-traces.c
+++ b/kernel/trace/power-traces.c
@@ -18,4 +18,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(suspend_resume);
EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_idle);
EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency);
EXPORT_TRACEPOINT_SYMBOL_GPL(powernv_throttle);
-
+EXPORT_TRACEPOINT_SYMBOL_GPL(device_pm_callback_start);
+EXPORT_TRACEPOINT_SYMBOL_GPL(device_pm_callback_end);
+EXPORT_TRACEPOINT_SYMBOL_GPL(clock_set_rate);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 5cfc95a5..68e21dd 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -49,6 +49,7 @@
#include <linux/fsnotify.h>
#include <linux/irq_work.h>
#include <linux/workqueue.h>
+#include <trace/hooks/ftrace_dump.h>
#include "trace.h"
#include "trace_output.h"
@@ -9858,8 +9859,17 @@ fs_initcall(tracer_init_tracefs);
static int trace_panic_handler(struct notifier_block *this,
unsigned long event, void *unused)
{
+ bool ftrace_check = false;
+
+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
+
+ if (ftrace_check)
+ return NOTIFY_OK;
+
if (ftrace_dump_on_oops)
ftrace_dump(ftrace_dump_on_oops);
+
+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
return NOTIFY_OK;
}
@@ -9873,6 +9883,13 @@ static int trace_die_handler(struct notifier_block *self,
unsigned long val,
void *data)
{
+ bool ftrace_check = false;
+
+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
+
+ if (ftrace_check)
+ return NOTIFY_OK;
+
switch (val) {
case DIE_OOPS:
if (ftrace_dump_on_oops)
@@ -9881,6 +9898,8 @@ static int trace_die_handler(struct notifier_block *self,
default:
break;
}
+
+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
return NOTIFY_OK;
}
@@ -9905,6 +9924,8 @@ static struct notifier_block trace_die_notifier = {
void
trace_printk_seq(struct trace_seq *s)
{
+ bool dump_printk = true;
+
/* Probably should print a warning here. */
if (s->seq.len >= TRACE_MAX_PRINT)
s->seq.len = TRACE_MAX_PRINT;
@@ -9920,7 +9941,9 @@ trace_printk_seq(struct trace_seq *s)
/* should be zero ended, but we are paranoid. */
s->buffer[s->seq.len] = 0;
- printk(KERN_TRACE "%s", s->buffer);
+ trace_android_vh_ftrace_dump_buffer(s, &dump_printk);
+ if (dump_printk)
+ printk(KERN_TRACE "%s", s->buffer);
trace_seq_init(s);
}
@@ -9959,6 +9982,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
unsigned int old_userobj;
unsigned long flags;
int cnt = 0, cpu;
+ bool ftrace_check = false;
+ unsigned long size;
/* Only allow one dump user at a time. */
if (atomic_inc_return(&dump_running) != 1) {
@@ -9983,6 +10008,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
for_each_tracing_cpu(cpu) {
atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
+ size = ring_buffer_size(iter.array_buffer->buffer, cpu);
+ trace_android_vh_ftrace_size_check(size, &ftrace_check);
}
old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
@@ -9990,6 +10017,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
/* don't look at user memory in panic mode */
tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
+ if (ftrace_check)
+ goto out_enable;
+
switch (oops_dump_mode) {
case DUMP_ALL:
iter.cpu_file = RING_BUFFER_ALL_CPUS;
@@ -10020,6 +10050,7 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
*/
while (!trace_empty(&iter)) {
+ ftrace_check = true;
if (!cnt)
printk(KERN_TRACE "---------------------------------\n");
@@ -10027,7 +10058,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
cnt++;
trace_iterator_reset(&iter);
- iter.iter_flags |= TRACE_FILE_LAT_FMT;
+ trace_android_vh_ftrace_format_check(&ftrace_check);
+ if (ftrace_check)
+ iter.iter_flags |= TRACE_FILE_LAT_FMT;
if (trace_find_next_entry_inc(&iter) != NULL) {
int ret;
diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
index 1e130da..070befb 100644
--- a/kernel/trace/trace_preemptirq.c
+++ b/kernel/trace/trace_preemptirq.c
@@ -14,6 +14,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/preemptirq.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/preemptirq.h>
#ifdef CONFIG_TRACE_IRQFLAGS
/* Per-cpu variable to prevent redundant calls when IRQs already off */
@@ -28,8 +30,11 @@ static DEFINE_PER_CPU(int, tracing_irq_cpu);
void trace_hardirqs_on_prepare(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -40,8 +45,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
void trace_hardirqs_on(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -63,8 +71,11 @@ void trace_hardirqs_off_finish(void)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_disable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
}
}
@@ -78,8 +89,11 @@ void trace_hardirqs_off(void)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_disable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
}
}
EXPORT_SYMBOL(trace_hardirqs_off);
@@ -88,8 +102,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_off);
__visible void trace_hardirqs_on_caller(unsigned long caller_addr)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ caller_addr);
+ }
tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -107,8 +124,11 @@ __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ caller_addr);
+ }
}
}
EXPORT_SYMBOL(trace_hardirqs_off_caller);
@@ -119,15 +139,19 @@ NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
void trace_preempt_on(unsigned long a0, unsigned long a1)
{
- if (!in_nmi())
+ if (!in_nmi()) {
trace_preempt_enable_rcuidle(a0, a1);
+ trace_android_rvh_preempt_enable(a0, a1);
+ }
tracer_preempt_on(a0, a1);
}
void trace_preempt_off(unsigned long a0, unsigned long a1)
{
- if (!in_nmi())
+ if (!in_nmi()) {
trace_preempt_disable_rcuidle(a0, a1);
+ trace_android_rvh_preempt_disable(a0, a1);
+ }
tracer_preempt_off(a0, a1);
}
#endif
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index f23144a..a644ecc 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -785,3 +785,82 @@ void syscall_unregfunc(void)
}
}
#endif
+
+#ifdef CONFIG_ANDROID_VENDOR_HOOKS
+
+static void *rvh_zalloc_funcs(int count)
+{
+ return kzalloc(sizeof(struct tracepoint_func) * count, GFP_KERNEL);
+}
+
+#define ANDROID_RVH_NR_PROBES_MAX 2
+static int rvh_func_add(struct tracepoint *tp, struct tracepoint_func *func)
+{
+ int i;
+
+ if (!static_key_enabled(&tp->key)) {
+ /* '+ 1' for the last NULL element */
+ tp->funcs = rvh_zalloc_funcs(ANDROID_RVH_NR_PROBES_MAX + 1);
+ if (!tp->funcs)
+ return ENOMEM;
+ }
+
+ for (i = 0; i < ANDROID_RVH_NR_PROBES_MAX; i++) {
+ if (!tp->funcs[i].func) {
+ if (!static_key_enabled(&tp->key))
+ tp->funcs[i].data = func->data;
+ WRITE_ONCE(tp->funcs[i].func, func->func);
+
+ return 0;
+ }
+ }
+
+ return -EBUSY;
+}
+
+static int android_rvh_add_func(struct tracepoint *tp, struct tracepoint_func *func)
+{
+ int ret;
+
+ if (tp->regfunc && !static_key_enabled(&tp->key)) {
+ ret = tp->regfunc();
+ if (ret < 0)
+ return ret;
+ }
+
+ ret = rvh_func_add(tp, func);
+ if (ret)
+ return ret;
+ tracepoint_update_call(tp, tp->funcs);
+ static_key_enable(&tp->key);
+
+ return 0;
+}
+
+int android_rvh_probe_register(struct tracepoint *tp, void *probe, void *data)
+{
+ struct tracepoint_func tp_func;
+ int ret;
+
+ /*
+ * Once the static key has been flipped, the array may be read
+ * concurrently. Although __traceiter_*() always checks .func first,
+ * it doesn't enforce read->read dependencies, and we can't strongly
+ * guarantee it will see the correct .data for the second element
+ * without adding smp_load_acquire() in the fast path. But this is a
+ * corner case which is unlikely to be needed by anybody in practice,
+ * so let's just forbid it and keep the fast path clean.
+ */
+ if (WARN_ON(static_key_enabled(&tp->key) && data))
+ return -EINVAL;
+
+ mutex_lock(&tracepoints_mutex);
+ tp_func.func = probe;
+ tp_func.data = data;
+ ret = android_rvh_add_func(tp, &tp_func);
+ mutex_unlock(&tracepoints_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(android_rvh_probe_register);
+#endif
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 07895de..a72db16 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -54,6 +54,10 @@
#include "workqueue_internal.h"
+#include <trace/hooks/wqlockup.h>
+/* events/workqueue.h uses default TRACE_INCLUDE_PATH */
+#undef TRACE_INCLUDE_PATH
+
enum {
/*
* worker_pool flags
@@ -5871,6 +5875,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused)
pr_cont_pool_info(pool);
pr_cont(" stuck for %us!\n",
jiffies_to_msecs(now - pool_ts) / 1000);
+ trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
}
}
diff --git a/mm/OWNERS b/mm/OWNERS
new file mode 100644
index 0000000..5f97cfd
--- /dev/null
+++ b/mm/OWNERS
@@ -0,0 +1,3 @@
+kaleshsingh@google.com
+surenb@google.com
+minchan@google.com
diff --git a/mm/cma.c b/mm/cma.c
index 4a978e0..332f3c6 100644
--- a/mm/cma.c
+++ b/mm/cma.c
@@ -24,6 +24,7 @@
#include <linux/memblock.h>
#include <linux/err.h>
#include <linux/mm.h>
+#include <linux/module.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/log2.h>
@@ -31,6 +32,8 @@
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/kmemleak.h>
+#include <linux/sched.h>
+#include <linux/jiffies.h>
#include <trace/events/cma.h>
#include "cma.h"
@@ -53,6 +56,7 @@ const char *cma_get_name(const struct cma *cma)
{
return cma->name;
}
+EXPORT_SYMBOL_GPL(cma_get_name);
static unsigned long cma_bitmap_aligned_mask(const struct cma *cma,
unsigned int align_order)
@@ -431,6 +435,8 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
unsigned long i;
struct page *page = NULL;
int ret = -ENOMEM;
+ int num_attempts = 0;
+ int max_retries = 5;
if (!cma || !cma->count || !cma->bitmap)
goto out;
@@ -457,8 +463,28 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
bitmap_maxno, start, bitmap_count, mask,
offset);
if (bitmap_no >= bitmap_maxno) {
- spin_unlock_irq(&cma->lock);
- break;
+ if ((num_attempts < max_retries) && (ret == -EBUSY)) {
+ spin_unlock_irq(&cma->lock);
+
+ if (fatal_signal_pending(current))
+ break;
+
+ /*
+ * Page may be momentarily pinned by some other
+ * process which has been scheduled out, e.g.
+ * in exit path, during unmap call, or process
+ * fork and so cannot be freed there. Sleep
+ * for 100ms and retry the allocation.
+ */
+ start = 0;
+ ret = -ENOMEM;
+ schedule_timeout_killable(msecs_to_jiffies(100));
+ num_attempts++;
+ continue;
+ } else {
+ spin_unlock_irq(&cma->lock);
+ break;
+ }
}
bitmap_set(cma->bitmap, bitmap_no, bitmap_count);
/*
@@ -522,6 +548,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
return page;
}
+EXPORT_SYMBOL_GPL(cma_alloc);
bool cma_pages_valid(struct cma *cma, const struct page *pages,
unsigned long count)
@@ -572,6 +599,7 @@ bool cma_release(struct cma *cma, const struct page *pages,
return true;
}
+EXPORT_SYMBOL_GPL(cma_release);
int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
{
@@ -586,3 +614,4 @@ int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
return 0;
}
+EXPORT_SYMBOL_GPL(cma_for_each_area);
diff --git a/mm/memblock.c b/mm/memblock.c
index 511d478..c79f4823 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -1671,6 +1671,7 @@ phys_addr_t __init_memblock memblock_end_of_DRAM(void)
return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
}
+EXPORT_SYMBOL_GPL(memblock_end_of_DRAM);
static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
{
diff --git a/mm/memory.c b/mm/memory.c
index 8c84209..dcb4298 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -166,6 +166,7 @@ void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
{
trace_rss_stat(mm, member, count);
}
+EXPORT_SYMBOL_GPL(mm_trace_rss_stat);
#if defined(SPLIT_RSS_COUNTING)
diff --git a/mm/mmap.c b/mm/mmap.c
index 54abd46..cf0fa6f 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1629,6 +1629,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
trace_vm_unmapped_area(addr, info);
return addr;
}
+EXPORT_SYMBOL_GPL(vm_unmapped_area);
/* Get an address range which is currently unmapped.
* For shmat() with addr=0.
diff --git a/mm/percpu.c b/mm/percpu.c
index acd78da..5251e64 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -2409,6 +2409,7 @@ phys_addr_t per_cpu_ptr_to_phys(void *addr)
return page_to_phys(pcpu_addr_to_page(addr)) +
offset_in_page(addr);
}
+EXPORT_SYMBOL_GPL(per_cpu_ptr_to_phys);
/**
* pcpu_alloc_alloc_info - allocate percpu allocation info
diff --git a/mm/util.c b/mm/util.c
index 12984e7..4afa7dd 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -29,6 +29,10 @@
#include "internal.h"
#include "swap.h"
+#ifndef __GENSYMS__
+#include <trace/hooks/syscall_check.h>
+#endif
+
/**
* kfree_const - conditionally free memory
* @x: pointer to the memory
@@ -524,6 +528,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
if (populate)
mm_populate(ret, populate);
}
+ trace_android_vh_check_mmap_file(file, prot, flag, ret);
return ret;
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 8fcc5fa..3fdce84 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -68,6 +68,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/vmscan.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/vmscan.h>
+
struct scan_control {
/* How many pages shrink_list() should reclaim */
unsigned long nr_to_reclaim;
@@ -2907,6 +2910,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
enum scan_balance scan_balance;
unsigned long ap, fp;
enum lru_list lru;
+ bool balance_anon_file_reclaim = false;
/* If we have no swap space, do not bother scanning anon folios. */
if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
@@ -2944,11 +2948,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
goto out;
}
+ trace_android_rvh_set_balance_anon_file_reclaim(&balance_anon_file_reclaim);
+
/*
* If there is enough inactive page cache, we do not reclaim
- * anything from the anonymous working right now.
+ * anything from the anonymous working right now. But when balancing
+ * anon and page cache files for reclaim, allow swapping of anon pages
+ * even if there are a number of inactive file cache pages.
*/
- if (sc->cache_trim_mode) {
+ if (!balance_anon_file_reclaim && sc->cache_trim_mode) {
scan_balance = SCAN_FILE;
goto out;
}
diff --git a/mm/vmstat.c b/mm/vmstat.c
index b2371d7..436d3ef 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1176,9 +1176,7 @@ const char * const vmstat_text[] = {
"nr_zone_write_pending",
"nr_mlock",
"nr_bounce",
-#if IS_ENABLED(CONFIG_ZSMALLOC)
"nr_zspages",
-#endif
"nr_free_cma",
/* enum numa_stat_item counters */
diff --git a/modules.bzl b/modules.bzl
new file mode 100644
index 0000000..6473dcb
--- /dev/null
+++ b/modules.bzl
@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2022 The Android Open Source Project
+
+"""
+This module contains a full list of kernel modules
+ compiled by GKI.
+"""
+
+COMMON_GKI_MODULES_LIST = [
+ # keep sorted
+ "drivers/block/zram/zram.ko",
+ "drivers/bluetooth/btbcm.ko",
+ "drivers/bluetooth/btqca.ko",
+ "drivers/bluetooth/btsdio.ko",
+ "drivers/bluetooth/hci_uart.ko",
+ "drivers/net/can/dev/can-dev.ko",
+ "drivers/net/can/slcan/slcan.ko",
+ "drivers/net/can/vcan.ko",
+ "drivers/net/ppp/bsd_comp.ko",
+ "drivers/net/ppp/ppp_deflate.ko",
+ "drivers/net/ppp/ppp_generic.ko",
+ "drivers/net/ppp/ppp_mppe.ko",
+ "drivers/net/ppp/pppox.ko",
+ "drivers/net/ppp/pptp.ko",
+ "drivers/net/slip/slhc.ko",
+ "drivers/usb/class/cdc-acm.ko",
+ "drivers/usb/serial/ftdi_sio.ko",
+ "drivers/usb/serial/usbserial.ko",
+ "lib/crypto/libarc4.ko",
+ "mm/zsmalloc.ko",
+ "net/6lowpan/6lowpan.ko",
+ "net/6lowpan/nhc_dest.ko",
+ "net/6lowpan/nhc_fragment.ko",
+ "net/6lowpan/nhc_hop.ko",
+ "net/6lowpan/nhc_ipv6.ko",
+ "net/6lowpan/nhc_mobility.ko",
+ "net/6lowpan/nhc_routing.ko",
+ "net/6lowpan/nhc_udp.ko",
+ "net/8021q/8021q.ko",
+ "net/bluetooth/bluetooth.ko",
+ "net/bluetooth/hidp/hidp.ko",
+ "net/bluetooth/rfcomm/rfcomm.ko",
+ "net/can/can.ko",
+ "net/can/can-bcm.ko",
+ "net/can/can-gw.ko",
+ "net/can/can-raw.ko",
+ "net/ieee802154/6lowpan/ieee802154_6lowpan.ko",
+ "net/ieee802154/ieee802154.ko",
+ "net/ieee802154/ieee802154_socket.ko",
+ "net/l2tp/l2tp_core.ko",
+ "net/l2tp/l2tp_ppp.ko",
+ "net/mac80211/mac80211.ko",
+ "net/mac802154/mac802154.ko",
+ "net/nfc/nfc.ko",
+ "net/rfkill/rfkill.ko",
+ "net/tipc/diag.ko",
+ "net/tipc/tipc.ko",
+ "net/wireless/cfg80211.ko",
+]
diff --git a/net/OWNERS b/net/OWNERS
new file mode 100644
index 0000000..cbbfa70
--- /dev/null
+++ b/net/OWNERS
@@ -0,0 +1,2 @@
+lorenzo@google.com
+maze@google.com
diff --git a/net/TEST_MAPPING b/net/TEST_MAPPING
new file mode 100644
index 0000000..1eb8d43
--- /dev/null
+++ b/net/TEST_MAPPING
@@ -0,0 +1,12 @@
+{
+ "presubmit": [
+ {
+ "name": "CtsNetTestCases",
+ "options": [
+ {
+ "exclude-annotation": "com.android.testutils.SkipPresubmit"
+ }
+ ]
+ }
+ ]
+}
diff --git a/net/core/dev.c b/net/core/dev.c
index 3be2560..674872a 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -150,6 +150,7 @@
#include <linux/pm_runtime.h>
#include <linux/prandom.h>
#include <linux/once_lite.h>
+#include <trace/hooks/net.h>
#include "dev.h"
#include "net-sysfs.h"
@@ -535,6 +536,12 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
static inline struct list_head *ptype_head(const struct packet_type *pt)
{
+ struct list_head vendor_pt = { .next = NULL, };
+
+ trace_android_vh_ptype_head(pt, &vendor_pt);
+ if (vendor_pt.next)
+ return vendor_pt.next;
+
if (pt->type == htons(ETH_P_ALL))
return pt->dev ? &pt->dev->ptype_all : &ptype_all;
else
diff --git a/net/core/net-traces.c b/net/core/net-traces.c
index c40cd8dd..5e07acb 100644
--- a/net/core/net-traces.c
+++ b/net/core/net-traces.c
@@ -35,13 +35,11 @@
#include <trace/events/tcp.h>
#include <trace/events/fib.h>
#include <trace/events/qdisc.h>
-#if IS_ENABLED(CONFIG_BRIDGE)
#include <trace/events/bridge.h>
EXPORT_TRACEPOINT_SYMBOL_GPL(br_fdb_add);
EXPORT_TRACEPOINT_SYMBOL_GPL(br_fdb_external_learn_add);
EXPORT_TRACEPOINT_SYMBOL_GPL(fdb_delete);
EXPORT_TRACEPOINT_SYMBOL_GPL(br_fdb_update);
-#endif
#if IS_ENABLED(CONFIG_PAGE_POOL)
#include <trace/events/page_pool.h>
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index d720f6f..8efe085 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -211,6 +211,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = {
.accept_ra_rt_info_max_plen = 0,
#endif
#endif
+ .accept_ra_rt_table = 0,
.proxy_ndp = 0,
.accept_source_route = 0, /* we do not accept RH0 by default. */
.disable_ipv6 = 0,
@@ -271,6 +272,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = {
.accept_ra_rt_info_max_plen = 0,
#endif
#endif
+ .accept_ra_rt_table = 0,
.proxy_ndp = 0,
.accept_source_route = 0, /* we do not accept RH0 by default. */
.disable_ipv6 = 0,
@@ -2390,6 +2392,26 @@ static void ipv6_gen_rnd_iid(struct in6_addr *addr)
goto regen;
}
+u32 addrconf_rt_table(const struct net_device *dev, u32 default_table)
+{
+ struct inet6_dev *idev = in6_dev_get(dev);
+ int sysctl;
+ u32 table;
+
+ if (!idev)
+ return default_table;
+ sysctl = idev->cnf.accept_ra_rt_table;
+ if (sysctl == 0) {
+ table = default_table;
+ } else if (sysctl > 0) {
+ table = (u32) sysctl;
+ } else {
+ table = (unsigned) dev->ifindex + (-sysctl);
+ }
+ in6_dev_put(idev);
+ return table;
+}
+
/*
* Add prefix route.
*/
@@ -2400,7 +2422,7 @@ addrconf_prefix_route(struct in6_addr *pfx, int plen, u32 metric,
u32 flags, gfp_t gfp_flags)
{
struct fib6_config cfg = {
- .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_PREFIX,
+ .fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_PREFIX),
.fc_metric = metric ? : IP6_RT_PRIO_ADDRCONF,
.fc_ifindex = dev->ifindex,
.fc_expires = expires,
@@ -2435,7 +2457,7 @@ static struct fib6_info *addrconf_get_prefix_route(const struct in6_addr *pfx,
struct fib6_node *fn;
struct fib6_info *rt = NULL;
struct fib6_table *table;
- u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_PREFIX;
+ u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_PREFIX);
table = fib6_get_table(dev_net(dev), tb_id);
if (!table)
@@ -6824,6 +6846,13 @@ static const struct ctl_table addrconf_sysctl[] = {
#endif
#endif
{
+ .procname = "accept_ra_rt_table",
+ .data = &ipv6_devconf.accept_ra_rt_table,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ },
+ {
.procname = "proxy_ndp",
.data = &ipv6_devconf.proxy_ndp,
.maxlen = sizeof(int),
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index e74e036..25047dd 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -4251,7 +4251,7 @@ static struct fib6_info *rt6_get_route_info(struct net *net,
const struct in6_addr *gwaddr,
struct net_device *dev)
{
- u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
+ u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_INFO);
int ifindex = dev->ifindex;
struct fib6_node *fn;
struct fib6_info *rt = NULL;
@@ -4305,7 +4305,7 @@ static struct fib6_info *rt6_add_route_info(struct net *net,
.fc_nlinfo.nl_net = net,
};
- cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
+ cfg.fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_INFO);
cfg.fc_dst = *prefix;
cfg.fc_gateway = *gwaddr;
@@ -4323,7 +4323,7 @@ struct fib6_info *rt6_get_dflt_router(struct net *net,
const struct in6_addr *addr,
struct net_device *dev)
{
- u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
+ u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_DFLT);
struct fib6_info *rt;
struct fib6_table *table;
@@ -4358,7 +4358,7 @@ struct fib6_info *rt6_add_dflt_router(struct net *net,
u32 defrtr_usr_metric)
{
struct fib6_config cfg = {
- .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
+ .fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_DFLT),
.fc_metric = defrtr_usr_metric,
.fc_ifindex = dev->ifindex,
.fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
@@ -4383,47 +4383,24 @@ struct fib6_info *rt6_add_dflt_router(struct net *net,
return rt6_get_dflt_router(net, gwaddr, dev);
}
-static void __rt6_purge_dflt_routers(struct net *net,
- struct fib6_table *table)
+static int rt6_addrconf_purge(struct fib6_info *rt, void *arg)
{
- struct fib6_info *rt;
+ struct net_device *dev = fib6_info_nh_dev(rt);
+ struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
-restart:
- rcu_read_lock();
- for_each_fib6_node_rt_rcu(&table->tb6_root) {
- struct net_device *dev = fib6_info_nh_dev(rt);
- struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
-
- if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
- (!idev || idev->cnf.accept_ra != 2) &&
- fib6_info_hold_safe(rt)) {
- rcu_read_unlock();
- ip6_del_rt(net, rt, false);
- goto restart;
- }
+ if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
+ (!idev || idev->cnf.accept_ra != 2)) {
+ /* Delete this route. See fib6_clean_tree() */
+ return -1;
}
- rcu_read_unlock();
- table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
+ /* Continue walking */
+ return 0;
}
void rt6_purge_dflt_routers(struct net *net)
{
- struct fib6_table *table;
- struct hlist_head *head;
- unsigned int h;
-
- rcu_read_lock();
-
- for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
- head = &net->ipv6.fib_table_hash[h];
- hlist_for_each_entry_rcu(table, head, tb6_hlist) {
- if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
- __rt6_purge_dflt_routers(net, table);
- }
- }
-
- rcu_read_unlock();
+ fib6_clean_all(net, rt6_addrconf_purge, NULL);
}
static void rtmsg_to_fib6_config(struct net *net,
diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig
index 4b8d046..09b2c54 100644
--- a/net/netfilter/Kconfig
+++ b/net/netfilter/Kconfig
@@ -1523,6 +1523,29 @@
If you want to compile it as a module, say M here and read
<file:Documentation/kbuild/modules.rst>. If unsure, say `N'.
+config NETFILTER_XT_MATCH_QUOTA2
+ tristate '"quota2" match support'
+ depends on NETFILTER_ADVANCED
+ help
+ This option adds a `quota2' match, which allows to match on a
+ byte counter correctly and not per CPU.
+ It allows naming the quotas.
+ This is based on http://xtables-addons.git.sourceforge.net
+
+ If you want to compile it as a module, say M here and read
+ <file:Documentation/kbuild/modules.txt>. If unsure, say `N'.
+
+config NETFILTER_XT_MATCH_QUOTA2_LOG
+ bool '"quota2" Netfilter LOG support'
+ depends on NETFILTER_XT_MATCH_QUOTA2
+ default n
+ help
+ This option allows `quota2' to log ONCE when a quota limit
+ is passed. It logs via NETLINK using the NETLINK_NFLOG family.
+ It logs similarly to how ipt_ULOG would without data.
+
+ If unsure, say `N'.
+
config NETFILTER_XT_MATCH_RATEEST
tristate '"rateest" match support'
depends on NETFILTER_ADVANCED
diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile
index 0f060d10..7756b2b 100644
--- a/net/netfilter/Makefile
+++ b/net/netfilter/Makefile
@@ -206,6 +206,7 @@
obj-$(CONFIG_NETFILTER_XT_MATCH_PKTTYPE) += xt_pkttype.o
obj-$(CONFIG_NETFILTER_XT_MATCH_POLICY) += xt_policy.o
obj-$(CONFIG_NETFILTER_XT_MATCH_QUOTA) += xt_quota.o
+obj-$(CONFIG_NETFILTER_XT_MATCH_QUOTA2) += xt_quota2.o
obj-$(CONFIG_NETFILTER_XT_MATCH_RATEEST) += xt_rateest.o
obj-$(CONFIG_NETFILTER_XT_MATCH_REALM) += xt_realm.o
obj-$(CONFIG_NETFILTER_XT_MATCH_RECENT) += xt_recent.o
diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c
index 0f8bb0b..30ba8d7 100644
--- a/net/netfilter/xt_IDLETIMER.c
+++ b/net/netfilter/xt_IDLETIMER.c
@@ -28,6 +28,11 @@
#include <linux/kobject.h>
#include <linux/workqueue.h>
#include <linux/sysfs.h>
+#include <linux/suspend.h>
+#include <net/sock.h>
+#include <net/inet_sock.h>
+
+#define NLMSG_MAX_SIZE 64
struct idletimer_tg {
struct list_head entry;
@@ -38,15 +43,112 @@ struct idletimer_tg {
struct kobject *kobj;
struct device_attribute attr;
+ struct timespec64 delayed_timer_trigger;
+ struct timespec64 last_modified_timer;
+ struct timespec64 last_suspend_time;
+ struct notifier_block pm_nb;
+
+ int timeout;
unsigned int refcnt;
u8 timer_type;
+
+ bool work_pending;
+ bool send_nl_msg;
+ bool active;
+ uid_t uid;
+ bool suspend_time_valid;
};
static LIST_HEAD(idletimer_tg_list);
static DEFINE_MUTEX(list_mutex);
+static DEFINE_SPINLOCK(timestamp_lock);
static struct kobject *idletimer_tg_kobj;
+static bool check_for_delayed_trigger(struct idletimer_tg *timer,
+ struct timespec64 *ts)
+{
+ bool state;
+ struct timespec64 temp;
+ spin_lock_bh(×tamp_lock);
+ timer->work_pending = false;
+ if ((ts->tv_sec - timer->last_modified_timer.tv_sec) > timer->timeout ||
+ timer->delayed_timer_trigger.tv_sec != 0) {
+ state = false;
+ temp.tv_sec = timer->timeout;
+ temp.tv_nsec = 0;
+ if (timer->delayed_timer_trigger.tv_sec != 0) {
+ temp = timespec64_add(timer->delayed_timer_trigger,
+ temp);
+ ts->tv_sec = temp.tv_sec;
+ ts->tv_nsec = temp.tv_nsec;
+ timer->delayed_timer_trigger.tv_sec = 0;
+ timer->work_pending = true;
+ schedule_work(&timer->work);
+ } else {
+ temp = timespec64_add(timer->last_modified_timer, temp);
+ ts->tv_sec = temp.tv_sec;
+ ts->tv_nsec = temp.tv_nsec;
+ }
+ } else {
+ state = timer->active;
+ }
+ spin_unlock_bh(×tamp_lock);
+ return state;
+}
+
+static void notify_netlink_uevent(const char *iface, struct idletimer_tg *timer)
+{
+ char iface_msg[NLMSG_MAX_SIZE];
+ char state_msg[NLMSG_MAX_SIZE];
+ char timestamp_msg[NLMSG_MAX_SIZE];
+ char uid_msg[NLMSG_MAX_SIZE];
+ char *envp[] = { iface_msg, state_msg, timestamp_msg, uid_msg, NULL };
+ int res;
+ struct timespec64 ts;
+ u64 time_ns;
+ bool state;
+
+ res = snprintf(iface_msg, NLMSG_MAX_SIZE, "INTERFACE=%s",
+ iface);
+ if (NLMSG_MAX_SIZE <= res) {
+ pr_err("message too long (%d)\n", res);
+ return;
+ }
+
+ ts = ktime_to_timespec64(ktime_get_boottime());
+ state = check_for_delayed_trigger(timer, &ts);
+ res = snprintf(state_msg, NLMSG_MAX_SIZE, "STATE=%s",
+ state ? "active" : "inactive");
+
+ if (NLMSG_MAX_SIZE <= res) {
+ pr_err("message too long (%d)\n", res);
+ return;
+ }
+
+ if (state) {
+ res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID=%u", timer->uid);
+ if (NLMSG_MAX_SIZE <= res)
+ pr_err("message too long (%d)\n", res);
+ } else {
+ res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID=");
+ if (NLMSG_MAX_SIZE <= res)
+ pr_err("message too long (%d)\n", res);
+ }
+
+ time_ns = timespec64_to_ns(&ts);
+ res = snprintf(timestamp_msg, NLMSG_MAX_SIZE, "TIME_NS=%llu", time_ns);
+ if (NLMSG_MAX_SIZE <= res) {
+ timestamp_msg[0] = '\0';
+ pr_err("message too long (%d)\n", res);
+ }
+
+ pr_debug("putting nlmsg: <%s> <%s> <%s> <%s>\n", iface_msg, state_msg,
+ timestamp_msg, uid_msg);
+ kobject_uevent_env(idletimer_tg_kobj, KOBJ_CHANGE, envp);
+ return;
+}
+
static
struct idletimer_tg *__idletimer_tg_find_by_label(const char *label)
{
@@ -67,6 +169,7 @@ static ssize_t idletimer_tg_show(struct device *dev,
unsigned long expires = 0;
struct timespec64 ktimespec = {};
long time_diff = 0;
+ unsigned long now = jiffies;
mutex_lock(&list_mutex);
@@ -78,15 +181,19 @@ static ssize_t idletimer_tg_show(struct device *dev,
time_diff = ktimespec.tv_sec;
} else {
expires = timer->timer.expires;
- time_diff = jiffies_to_msecs(expires - jiffies) / 1000;
+ time_diff = jiffies_to_msecs(expires - now) / 1000;
}
}
mutex_unlock(&list_mutex);
- if (time_after(expires, jiffies) || ktimespec.tv_sec > 0)
+ if (time_after(expires, now) || ktimespec.tv_sec > 0)
return sysfs_emit(buf, "%ld\n", time_diff);
+ if (timer->send_nl_msg)
+ return sysfs_emit(buf, "0 %d\n",
+ jiffies_to_msecs(now - expires) / 1000);
+
return sysfs_emit(buf, "0\n");
}
@@ -96,6 +203,9 @@ static void idletimer_tg_work(struct work_struct *work)
work);
sysfs_notify(idletimer_tg_kobj, NULL, timer->attr.attr.name);
+
+ if (timer->send_nl_msg)
+ notify_netlink_uevent(timer->attr.attr.name, timer);
}
static void idletimer_tg_expired(struct timer_list *t)
@@ -104,7 +214,62 @@ static void idletimer_tg_expired(struct timer_list *t)
pr_debug("timer %s expired\n", timer->attr.attr.name);
+ spin_lock_bh(×tamp_lock);
+ timer->active = false;
+ timer->work_pending = true;
schedule_work(&timer->work);
+ spin_unlock_bh(×tamp_lock);
+}
+
+static int idletimer_resume(struct notifier_block *notifier,
+ unsigned long pm_event, void *unused)
+{
+ struct timespec64 ts;
+ unsigned long time_diff, now = jiffies;
+ struct idletimer_tg *timer = container_of(notifier,
+ struct idletimer_tg, pm_nb);
+ if (!timer)
+ return NOTIFY_DONE;
+
+ switch (pm_event) {
+ case PM_SUSPEND_PREPARE:
+ timer->last_suspend_time =
+ ktime_to_timespec64(ktime_get_boottime());
+ timer->suspend_time_valid = true;
+ break;
+ case PM_POST_SUSPEND:
+ if (!timer->suspend_time_valid)
+ break;
+ timer->suspend_time_valid = false;
+
+ spin_lock_bh(×tamp_lock);
+ if (!timer->active) {
+ spin_unlock_bh(×tamp_lock);
+ break;
+ }
+ /* since jiffies are not updated when suspended now represents
+ * the time it would have suspended */
+ if (time_after(timer->timer.expires, now)) {
+ ts = ktime_to_timespec64(ktime_get_boottime());
+ ts = timespec64_sub(ts, timer->last_suspend_time);
+ time_diff = timespec64_to_jiffies(&ts);
+ if (timer->timer.expires > (time_diff + now)) {
+ mod_timer_pending(&timer->timer,
+ (timer->timer.expires - time_diff));
+ } else {
+ del_timer(&timer->timer);
+ timer->timer.expires = 0;
+ timer->active = false;
+ timer->work_pending = true;
+ schedule_work(&timer->work);
+ }
+ }
+ spin_unlock_bh(×tamp_lock);
+ break;
+ default:
+ break;
+ }
+ return NOTIFY_DONE;
}
static enum alarmtimer_restart idletimer_tg_alarmproc(struct alarm *alarm,
@@ -158,17 +323,34 @@ static int idletimer_tg_create(struct idletimer_tg_info *info)
ret = sysfs_create_file(idletimer_tg_kobj, &info->timer->attr.attr);
if (ret < 0) {
- pr_debug("couldn't add file to sysfs");
+ pr_debug("couldn't add file to sysfs\n");
goto out_free_attr;
}
list_add(&info->timer->entry, &idletimer_tg_list);
-
- timer_setup(&info->timer->timer, idletimer_tg_expired, 0);
+ pr_debug("timer type value is 0.\n");
+ info->timer->timer_type = 0;
info->timer->refcnt = 1;
+ info->timer->send_nl_msg = false;
+ info->timer->active = true;
+ info->timer->timeout = info->timeout;
+
+ info->timer->delayed_timer_trigger.tv_sec = 0;
+ info->timer->delayed_timer_trigger.tv_nsec = 0;
+ info->timer->work_pending = false;
+ info->timer->uid = 0;
+ info->timer->last_modified_timer =
+ ktime_to_timespec64(ktime_get_boottime());
+
+ info->timer->pm_nb.notifier_call = idletimer_resume;
+ ret = register_pm_notifier(&info->timer->pm_nb);
+ if (ret)
+ printk(KERN_WARNING "[%s] Failed to register pm notifier %d\n",
+ __func__, ret);
INIT_WORK(&info->timer->work, idletimer_tg_work);
+ timer_setup(&info->timer->timer, idletimer_tg_expired, 0);
mod_timer(&info->timer->timer,
msecs_to_jiffies(info->timeout * 1000) + jiffies);
@@ -186,7 +368,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
{
int ret;
- info->timer = kmalloc(sizeof(*info->timer), GFP_KERNEL);
+ info->timer = kzalloc(sizeof(*info->timer), GFP_KERNEL);
if (!info->timer) {
ret = -ENOMEM;
goto out;
@@ -207,7 +389,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
ret = sysfs_create_file(idletimer_tg_kobj, &info->timer->attr.attr);
if (ret < 0) {
- pr_debug("couldn't add file to sysfs");
+ pr_debug("couldn't add file to sysfs\n");
goto out_free_attr;
}
@@ -215,9 +397,25 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
kobject_uevent(idletimer_tg_kobj,KOBJ_ADD);
list_add(&info->timer->entry, &idletimer_tg_list);
- pr_debug("timer type value is %u", info->timer_type);
+ pr_debug("timer type value is %u\n", info->timer_type);
info->timer->timer_type = info->timer_type;
info->timer->refcnt = 1;
+ info->timer->send_nl_msg = (info->send_nl_msg != 0);
+ info->timer->active = true;
+ info->timer->timeout = info->timeout;
+
+ info->timer->delayed_timer_trigger.tv_sec = 0;
+ info->timer->delayed_timer_trigger.tv_nsec = 0;
+ info->timer->work_pending = false;
+ info->timer->uid = 0;
+ info->timer->last_modified_timer =
+ ktime_to_timespec64(ktime_get_boottime());
+
+ info->timer->pm_nb.notifier_call = idletimer_resume;
+ ret = register_pm_notifier(&info->timer->pm_nb);
+ if (ret)
+ printk(KERN_WARNING "[%s] Failed to register pm notifier %d\n",
+ __func__, ret);
INIT_WORK(&info->timer->work, idletimer_tg_work);
@@ -231,7 +429,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
} else {
timer_setup(&info->timer->timer, idletimer_tg_expired, 0);
mod_timer(&info->timer->timer,
- msecs_to_jiffies(info->timeout * 1000) + jiffies);
+ msecs_to_jiffies(info->timeout * 1000) + jiffies);
}
return 0;
@@ -244,6 +442,41 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
return ret;
}
+static void reset_timer(struct idletimer_tg * const info_timer,
+ const __u32 info_timeout,
+ struct sk_buff *skb)
+{
+ unsigned long now = jiffies;
+ bool timer_prev;
+
+ spin_lock_bh(×tamp_lock);
+ timer_prev = info_timer->active;
+ info_timer->active = true;
+ /* timer_prev is used to guard overflow problem in time_before*/
+ if (!timer_prev || time_before(info_timer->timer.expires, now)) {
+ pr_debug("Starting Checkentry timer (Expired, Jiffies): %lu, %lu\n",
+ info_timer->timer.expires, now);
+
+ /* Stores the uid resposible for waking up the radio */
+ if (skb && (skb->sk)) {
+ info_timer->uid = from_kuid_munged(current_user_ns(),
+ sock_i_uid(skb_to_full_sk(skb)));
+ }
+
+ /* checks if there is a pending inactive notification*/
+ if (info_timer->work_pending)
+ info_timer->delayed_timer_trigger = info_timer->last_modified_timer;
+ else {
+ info_timer->work_pending = true;
+ schedule_work(&info_timer->work);
+ }
+ }
+
+ info_timer->last_modified_timer = ktime_to_timespec64(ktime_get_boottime());
+ mod_timer(&info_timer->timer, msecs_to_jiffies(info_timeout * 1000) + now);
+ spin_unlock_bh(×tamp_lock);
+}
+
/*
* The actual xt_tables plugin.
*/
@@ -251,12 +484,21 @@ static unsigned int idletimer_tg_target(struct sk_buff *skb,
const struct xt_action_param *par)
{
const struct idletimer_tg_info *info = par->targinfo;
+ unsigned long now = jiffies;
pr_debug("resetting timer %s, timeout period %u\n",
info->label, info->timeout);
- mod_timer(&info->timer->timer,
- msecs_to_jiffies(info->timeout * 1000) + jiffies);
+ info->timer->active = true;
+
+ if (time_before(info->timer->timer.expires, now)) {
+ schedule_work(&info->timer->work);
+ pr_debug("Starting timer %s (Expired, Jiffies): %lu, %lu\n",
+ info->label, info->timer->timer.expires, now);
+ }
+
+ /* TODO: Avoid modifying timers on each packet */
+ reset_timer(info->timer, info->timeout, skb);
return XT_CONTINUE;
}
@@ -268,6 +510,7 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb,
const struct xt_action_param *par)
{
const struct idletimer_tg_info_v1 *info = par->targinfo;
+ unsigned long now = jiffies;
pr_debug("resetting timer %s, timeout period %u\n",
info->label, info->timeout);
@@ -276,8 +519,16 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb,
ktime_t tout = ktime_set(info->timeout, 0);
alarm_start_relative(&info->timer->alarm, tout);
} else {
- mod_timer(&info->timer->timer,
- msecs_to_jiffies(info->timeout * 1000) + jiffies);
+ info->timer->active = true;
+
+ if (time_before(info->timer->timer.expires, now)) {
+ schedule_work(&info->timer->work);
+ pr_debug("Starting timer %s (Expired, Jiffies): %lu, %lu\n",
+ info->label, info->timer->timer.expires, now);
+ }
+
+ /* TODO: Avoid modifying timers on each packet */
+ reset_timer(info->timer, info->timeout, skb);
}
return XT_CONTINUE;
@@ -321,9 +572,7 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par)
info->timer = __idletimer_tg_find_by_label(info->label);
if (info->timer) {
info->timer->refcnt++;
- mod_timer(&info->timer->timer,
- msecs_to_jiffies(info->timeout * 1000) + jiffies);
-
+ reset_timer(info->timer, info->timeout, NULL);
pr_debug("increased refcnt of timer %s to %u\n",
info->label, info->timer->refcnt);
} else {
@@ -346,9 +595,6 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par)
pr_debug("checkentry targinfo%s\n", info->label);
- if (info->send_nl_msg)
- return -EOPNOTSUPP;
-
ret = idletimer_tg_helper((struct idletimer_tg_info *)info);
if(ret < 0)
{
@@ -361,6 +607,11 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par)
return -EINVAL;
}
+ if (info->send_nl_msg > 1) {
+ pr_debug("invalid value for send_nl_msg\n");
+ return -EINVAL;
+ }
+
mutex_lock(&list_mutex);
info->timer = __idletimer_tg_find_by_label(info->label);
@@ -383,8 +634,7 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par)
alarm_start_relative(&info->timer->alarm, tout);
}
} else {
- mod_timer(&info->timer->timer,
- msecs_to_jiffies(info->timeout * 1000) + jiffies);
+ reset_timer(info->timer, info->timeout, NULL);
}
pr_debug("increased refcnt of timer %s to %u\n",
info->label, info->timer->refcnt);
@@ -414,8 +664,9 @@ static void idletimer_tg_destroy(const struct xt_tgdtor_param *par)
list_del(&info->timer->entry);
del_timer_sync(&info->timer->timer);
- cancel_work_sync(&info->timer->work);
sysfs_remove_file(idletimer_tg_kobj, &info->timer->attr.attr);
+ unregister_pm_notifier(&info->timer->pm_nb);
+ cancel_work_sync(&info->timer->work);
kfree(info->timer->attr.attr.name);
kfree(info->timer);
} else {
@@ -443,8 +694,9 @@ static void idletimer_tg_destroy_v1(const struct xt_tgdtor_param *par)
} else {
del_timer_sync(&info->timer->timer);
}
- cancel_work_sync(&info->timer->work);
sysfs_remove_file(idletimer_tg_kobj, &info->timer->attr.attr);
+ unregister_pm_notifier(&info->timer->pm_nb);
+ cancel_work_sync(&info->timer->work);
kfree(info->timer->attr.attr.name);
kfree(info->timer);
} else {
@@ -540,3 +792,4 @@ MODULE_DESCRIPTION("Xtables: idle time monitor");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS("ipt_IDLETIMER");
MODULE_ALIAS("ip6t_IDLETIMER");
+MODULE_ALIAS("arpt_IDLETIMER");
diff --git a/net/netfilter/xt_quota2.c b/net/netfilter/xt_quota2.c
new file mode 100644
index 0000000..27acc86
--- /dev/null
+++ b/net/netfilter/xt_quota2.c
@@ -0,0 +1,397 @@
+/*
+ * xt_quota2 - enhanced xt_quota that can count upwards and in packets
+ * as a minimal accounting match.
+ * by Jan Engelhardt <jengelh@medozas.de>, 2008
+ *
+ * Originally based on xt_quota.c:
+ * netfilter module to enforce network quotas
+ * Sam Johnston <samj@samj.net>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License; either
+ * version 2 of the License, as published by the Free Software Foundation.
+ */
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/skbuff.h>
+#include <linux/spinlock.h>
+#include <asm/atomic.h>
+#include <net/netlink.h>
+
+#include <linux/netfilter/x_tables.h>
+#include <linux/netfilter/xt_quota2.h>
+
+#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG
+/* For compatibility, these definitions are copied from the
+ * deprecated header file <linux/netfilter_ipv4/ipt_ULOG.h> */
+#define ULOG_MAC_LEN 80
+#define ULOG_PREFIX_LEN 32
+
+/* Format of the ULOG packets passed through netlink */
+typedef struct ulog_packet_msg {
+ unsigned long mark;
+ long timestamp_sec;
+ long timestamp_usec;
+ unsigned int hook;
+ char indev_name[IFNAMSIZ];
+ char outdev_name[IFNAMSIZ];
+ size_t data_len;
+ char prefix[ULOG_PREFIX_LEN];
+ unsigned char mac_len;
+ unsigned char mac[ULOG_MAC_LEN];
+ unsigned char payload[0];
+} ulog_packet_msg_t;
+#endif
+
+/**
+ * @lock: lock to protect quota writers from each other
+ */
+struct xt_quota_counter {
+ u_int64_t quota;
+ spinlock_t lock;
+ struct list_head list;
+ atomic_t ref;
+ char name[sizeof(((struct xt_quota_mtinfo2 *)NULL)->name)];
+ struct proc_dir_entry *procfs_entry;
+};
+
+#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG
+/* Harald's favorite number +1 :D From ipt_ULOG.C */
+static int qlog_nl_event = 112;
+module_param_named(event_num, qlog_nl_event, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(event_num,
+ "Event number for NETLINK_NFLOG message. 0 disables log."
+ "111 is what ipt_ULOG uses.");
+static struct sock *nflognl;
+#endif
+
+static LIST_HEAD(counter_list);
+static DEFINE_SPINLOCK(counter_list_lock);
+
+static struct proc_dir_entry *proc_xt_quota;
+static unsigned int quota_list_perms = S_IRUGO | S_IWUSR;
+static kuid_t quota_list_uid = KUIDT_INIT(0);
+static kgid_t quota_list_gid = KGIDT_INIT(0);
+module_param_named(perms, quota_list_perms, uint, S_IRUGO | S_IWUSR);
+
+#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG
+static void quota2_log(unsigned int hooknum,
+ const struct sk_buff *skb,
+ const struct net_device *in,
+ const struct net_device *out,
+ const char *prefix)
+{
+ ulog_packet_msg_t *pm;
+ struct sk_buff *log_skb;
+ size_t size;
+ struct nlmsghdr *nlh;
+
+ if (!qlog_nl_event)
+ return;
+
+ size = NLMSG_SPACE(sizeof(*pm));
+ size = max(size, (size_t)NLMSG_GOODSIZE);
+ log_skb = alloc_skb(size, GFP_ATOMIC);
+ if (!log_skb) {
+ pr_err("xt_quota2: cannot alloc skb for logging\n");
+ return;
+ }
+
+ nlh = nlmsg_put(log_skb, /*pid*/0, /*seq*/0, qlog_nl_event,
+ sizeof(*pm), 0);
+ if (!nlh) {
+ pr_err("xt_quota2: nlmsg_put failed\n");
+ kfree_skb(log_skb);
+ return;
+ }
+ pm = nlmsg_data(nlh);
+ memset(pm, 0, sizeof(*pm));
+ if (skb->tstamp == 0)
+ __net_timestamp((struct sk_buff *)skb);
+ pm->hook = hooknum;
+ if (prefix != NULL)
+ strlcpy(pm->prefix, prefix, sizeof(pm->prefix));
+ if (in)
+ strlcpy(pm->indev_name, in->name, sizeof(pm->indev_name));
+ if (out)
+ strlcpy(pm->outdev_name, out->name, sizeof(pm->outdev_name));
+
+ NETLINK_CB(log_skb).dst_group = 1;
+ pr_debug("throwing 1 packets to netlink group 1\n");
+ netlink_broadcast(nflognl, log_skb, 0, 1, GFP_ATOMIC);
+}
+#else
+static void quota2_log(unsigned int hooknum,
+ const struct sk_buff *skb,
+ const struct net_device *in,
+ const struct net_device *out,
+ const char *prefix)
+{
+}
+#endif /* if+else CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG */
+
+static ssize_t quota_proc_read(struct file *file, char __user *buf,
+ size_t size, loff_t *ppos)
+{
+ struct xt_quota_counter *e = pde_data(file_inode(file));
+ char tmp[24];
+ size_t tmp_size;
+
+ spin_lock_bh(&e->lock);
+ tmp_size = scnprintf(tmp, sizeof(tmp), "%llu\n", e->quota);
+ spin_unlock_bh(&e->lock);
+ return simple_read_from_buffer(buf, size, ppos, tmp, tmp_size);
+}
+
+static ssize_t quota_proc_write(struct file *file, const char __user *input,
+ size_t size, loff_t *ppos)
+{
+ struct xt_quota_counter *e = pde_data(file_inode(file));
+ char buf[sizeof("18446744073709551616")];
+
+ if (size > sizeof(buf))
+ size = sizeof(buf);
+ if (copy_from_user(buf, input, size) != 0)
+ return -EFAULT;
+ buf[sizeof(buf)-1] = '\0';
+ if (size < sizeof(buf))
+ buf[size] = '\0';
+
+ spin_lock_bh(&e->lock);
+ e->quota = simple_strtoull(buf, NULL, 0);
+ spin_unlock_bh(&e->lock);
+ return size;
+}
+
+static const struct proc_ops q2_counter_fops = {
+ .proc_read = quota_proc_read,
+ .proc_write = quota_proc_write,
+ .proc_lseek = default_llseek,
+};
+
+static struct xt_quota_counter *
+q2_new_counter(const struct xt_quota_mtinfo2 *q, bool anon)
+{
+ struct xt_quota_counter *e;
+ unsigned int size;
+
+ /* Do not need all the procfs things for anonymous counters. */
+ size = anon ? offsetof(typeof(*e), list) : sizeof(*e);
+ e = kmalloc(size, GFP_KERNEL);
+ if (e == NULL)
+ return NULL;
+
+ e->quota = q->quota;
+ spin_lock_init(&e->lock);
+ if (!anon) {
+ INIT_LIST_HEAD(&e->list);
+ atomic_set(&e->ref, 1);
+ strlcpy(e->name, q->name, sizeof(e->name));
+ }
+ return e;
+}
+
+/**
+ * q2_get_counter - get ref to counter or create new
+ * @name: name of counter
+ */
+static struct xt_quota_counter *
+q2_get_counter(const struct xt_quota_mtinfo2 *q)
+{
+ struct proc_dir_entry *p;
+ struct xt_quota_counter *e = NULL;
+ struct xt_quota_counter *new_e;
+
+ if (*q->name == '\0')
+ return q2_new_counter(q, true);
+
+ /* No need to hold a lock while getting a new counter */
+ new_e = q2_new_counter(q, false);
+ if (new_e == NULL)
+ goto out;
+
+ spin_lock_bh(&counter_list_lock);
+ list_for_each_entry(e, &counter_list, list)
+ if (strcmp(e->name, q->name) == 0) {
+ atomic_inc(&e->ref);
+ spin_unlock_bh(&counter_list_lock);
+ kfree(new_e);
+ pr_debug("xt_quota2: old counter name=%s", e->name);
+ return e;
+ }
+ e = new_e;
+ pr_debug("xt_quota2: new_counter name=%s", e->name);
+ list_add_tail(&e->list, &counter_list);
+ /* The entry having a refcount of 1 is not directly destructible.
+ * This func has not yet returned the new entry, thus iptables
+ * has not references for destroying this entry.
+ * For another rule to try to destroy it, it would 1st need for this
+ * func* to be re-invoked, acquire a new ref for the same named quota.
+ * Nobody will access the e->procfs_entry either.
+ * So release the lock. */
+ spin_unlock_bh(&counter_list_lock);
+
+ /* create_proc_entry() is not spin_lock happy */
+ p = e->procfs_entry = proc_create_data(e->name, quota_list_perms,
+ proc_xt_quota, &q2_counter_fops, e);
+
+ if (IS_ERR_OR_NULL(p)) {
+ spin_lock_bh(&counter_list_lock);
+ list_del(&e->list);
+ spin_unlock_bh(&counter_list_lock);
+ goto out;
+ }
+ proc_set_user(p, quota_list_uid, quota_list_gid);
+ return e;
+
+ out:
+ kfree(e);
+ return NULL;
+}
+
+static int quota_mt2_check(const struct xt_mtchk_param *par)
+{
+ struct xt_quota_mtinfo2 *q = par->matchinfo;
+
+ pr_debug("xt_quota2: check() flags=0x%04x", q->flags);
+
+ if (q->flags & ~XT_QUOTA_MASK)
+ return -EINVAL;
+
+ q->name[sizeof(q->name)-1] = '\0';
+ if (*q->name == '.' || strchr(q->name, '/') != NULL) {
+ printk(KERN_ERR "xt_quota.3: illegal name\n");
+ return -EINVAL;
+ }
+
+ q->master = q2_get_counter(q);
+ if (q->master == NULL) {
+ printk(KERN_ERR "xt_quota.3: memory alloc failure\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void quota_mt2_destroy(const struct xt_mtdtor_param *par)
+{
+ struct xt_quota_mtinfo2 *q = par->matchinfo;
+ struct xt_quota_counter *e = q->master;
+
+ if (*q->name == '\0') {
+ kfree(e);
+ return;
+ }
+
+ spin_lock_bh(&counter_list_lock);
+ if (!atomic_dec_and_test(&e->ref)) {
+ spin_unlock_bh(&counter_list_lock);
+ return;
+ }
+
+ list_del(&e->list);
+ spin_unlock_bh(&counter_list_lock);
+ remove_proc_entry(e->name, proc_xt_quota);
+ kfree(e);
+}
+
+static bool
+quota_mt2(const struct sk_buff *skb, struct xt_action_param *par)
+{
+ struct xt_quota_mtinfo2 *q = (void *)par->matchinfo;
+ struct xt_quota_counter *e = q->master;
+ int charge = (q->flags & XT_QUOTA_PACKET) ? 1 : skb->len;
+ bool no_change = q->flags & XT_QUOTA_NO_CHANGE;
+ bool ret = q->flags & XT_QUOTA_INVERT;
+
+ spin_lock_bh(&e->lock);
+ if (q->flags & XT_QUOTA_GROW) {
+ /*
+ * While no_change is pointless in "grow" mode, we will
+ * implement it here simply to have a consistent behavior.
+ */
+ if (!no_change)
+ e->quota += charge;
+ ret = true; /* note: does not respect inversion (bug??) */
+ } else {
+ if (e->quota > charge) {
+ if (!no_change)
+ e->quota -= charge;
+ ret = !ret;
+ } else if (e->quota) {
+ /* We are transitioning, log that fact. */
+ quota2_log(xt_hooknum(par),
+ skb,
+ xt_in(par),
+ xt_out(par),
+ q->name);
+ /* we do not allow even small packets from now on */
+ e->quota = 0;
+ }
+ }
+ spin_unlock_bh(&e->lock);
+ return ret;
+}
+
+static struct xt_match quota_mt2_reg[] __read_mostly = {
+ {
+ .name = "quota2",
+ .revision = 3,
+ .family = NFPROTO_IPV4,
+ .checkentry = quota_mt2_check,
+ .match = quota_mt2,
+ .destroy = quota_mt2_destroy,
+ .matchsize = sizeof(struct xt_quota_mtinfo2),
+ .usersize = offsetof(struct xt_quota_mtinfo2, master),
+ .me = THIS_MODULE,
+ },
+ {
+ .name = "quota2",
+ .revision = 3,
+ .family = NFPROTO_IPV6,
+ .checkentry = quota_mt2_check,
+ .match = quota_mt2,
+ .destroy = quota_mt2_destroy,
+ .matchsize = sizeof(struct xt_quota_mtinfo2),
+ .usersize = offsetof(struct xt_quota_mtinfo2, master),
+ .me = THIS_MODULE,
+ },
+};
+
+static int __init quota_mt2_init(void)
+{
+ int ret;
+ pr_debug("xt_quota2: init()");
+
+#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG
+ nflognl = netlink_kernel_create(&init_net, NETLINK_NFLOG, NULL);
+ if (!nflognl)
+ return -ENOMEM;
+#endif
+
+ proc_xt_quota = proc_mkdir("xt_quota", init_net.proc_net);
+ if (proc_xt_quota == NULL)
+ return -EACCES;
+
+ ret = xt_register_matches(quota_mt2_reg, ARRAY_SIZE(quota_mt2_reg));
+ if (ret < 0)
+ remove_proc_entry("xt_quota", init_net.proc_net);
+ pr_debug("xt_quota2: init() %d", ret);
+ return ret;
+}
+
+static void __exit quota_mt2_exit(void)
+{
+ xt_unregister_matches(quota_mt2_reg, ARRAY_SIZE(quota_mt2_reg));
+ remove_proc_entry("xt_quota", init_net.proc_net);
+}
+
+module_init(quota_mt2_init);
+module_exit(quota_mt2_exit);
+MODULE_DESCRIPTION("Xtables: countdown quota match; up counter");
+MODULE_AUTHOR("Sam Johnston <samj@samj.net>");
+MODULE_AUTHOR("Jan Engelhardt <jengelh@medozas.de>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("ipt_quota2");
+MODULE_ALIAS("ip6t_quota2");
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index a9980e9b..b6ab3c0 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -26,6 +26,10 @@
/* Threshold for detecting small packets to copy */
#define GOOD_COPY_LEN 128
+uint virtio_transport_max_vsock_pkt_buf_size = 64 * 1024;
+module_param(virtio_transport_max_vsock_pkt_buf_size, uint, 0444);
+EXPORT_SYMBOL_GPL(virtio_transport_max_vsock_pkt_buf_size);
+
static const struct virtio_transport *
virtio_transport_get_ops(struct vsock_sock *vsk)
{
diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c
index 094734f..8ff6e87 100644
--- a/net/xfrm/xfrm_algo.c
+++ b/net/xfrm/xfrm_algo.c
@@ -237,7 +237,7 @@ static struct xfrm_algo_desc aalg_list[] = {
.uinfo = {
.auth = {
- .icv_truncbits = 96,
+ .icv_truncbits = IS_ENABLED(CONFIG_GKI_NET_XFRM_HACKS) ? 128 : 96,
.icv_fullbits = 256,
}
},
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
index 9a5e79a..93e790d 100644
--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -480,13 +480,11 @@ static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
return -EOPNOTSUPP;
}
-#if IS_ENABLED(CONFIG_NET_PKTGEN)
int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
{
return xfrm_outer_mode_output(x, skb);
}
EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
-#endif
static int xfrm_output_one(struct sk_buff *skb, int err)
{
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index d63a364..fa7663401 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -2442,19 +2442,22 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
if (IS_ERR(data))
return PTR_ERR(data);
- if (in_compat_syscall()) {
- struct xfrm_translator *xtr = xfrm_get_translator();
+ /* Use the 64-bit / untranslated format on Android, even for compat */
+ if (!IS_ENABLED(CONFIG_GKI_NET_XFRM_HACKS) || IS_ENABLED(CONFIG_XFRM_USER_COMPAT)) {
+ if (in_compat_syscall()) {
+ struct xfrm_translator *xtr = xfrm_get_translator();
- if (!xtr) {
- kfree(data);
- return -EOPNOTSUPP;
- }
+ if (!xtr) {
+ kfree(data);
+ return -EOPNOTSUPP;
+ }
- err = xtr->xlate_user_policy_sockptr(&data, optlen);
- xfrm_put_translator(xtr);
- if (err) {
- kfree(data);
- return err;
+ err = xtr->xlate_user_policy_sockptr(&data, optlen);
+ xfrm_put_translator(xtr);
+ if (err) {
+ kfree(data);
+ return err;
+ }
}
}
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index e73f9ef..9605896 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -3007,19 +3007,22 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
if (!netlink_net_capable(skb, CAP_NET_ADMIN))
return -EPERM;
- if (in_compat_syscall()) {
- struct xfrm_translator *xtr = xfrm_get_translator();
+ /* Use the 64-bit / untranslated format on Android, even for compat */
+ if (!IS_ENABLED(CONFIG_GKI_NET_XFRM_HACKS) || IS_ENABLED(CONFIG_XFRM_USER_COMPAT)) {
+ if (in_compat_syscall()) {
+ struct xfrm_translator *xtr = xfrm_get_translator();
- if (!xtr)
- return -EOPNOTSUPP;
+ if (!xtr)
+ return -EOPNOTSUPP;
- nlh64 = xtr->rcv_msg_compat(nlh, link->nla_max,
- link->nla_pol, extack);
- xfrm_put_translator(xtr);
- if (IS_ERR(nlh64))
- return PTR_ERR(nlh64);
- if (nlh64)
- nlh = nlh64;
+ nlh64 = xtr->rcv_msg_compat(nlh, link->nla_max,
+ link->nla_pol, extack);
+ xfrm_put_translator(xtr);
+ if (IS_ERR(nlh64))
+ return PTR_ERR(nlh64);
+ if (nlh64)
+ nlh = nlh64;
+ }
}
if ((type == (XFRM_MSG_GETSA - XFRM_MSG_BASE) ||
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 250b9fd..a384680 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -242,7 +242,9 @@
ld_flags = $(KBUILD_LDFLAGS) $(ldflags-y) $(LDFLAGS_$(@F))
-DTC_INCLUDE := $(srctree)/scripts/dtc/include-prefixes
+# ANDROID: Allow DTC_INCLUDE to be set by the BUILD_CONFIG. This allows one to
+# compile an out-of-tree device tree.
+DTC_INCLUDE += $(srctree)/scripts/dtc/include-prefixes
dtc_cpp_flags = -Wp,-MMD,$(depfile).pre.tmp -nostdinc \
$(addprefix -I,$(DTC_INCLUDE)) \
@@ -476,10 +478,10 @@
cmd_lzo_with_size = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout > $@
+ cmd_lz4 = cat $(real-prereqs) | $(LZ4) -l -12 --favor-decSpeed stdin stdout > $@
quiet_cmd_lz4_with_size = LZ4 $@
- cmd_lz4_with_size = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
+ cmd_lz4_with_size = { cat $(real-prereqs) | $(LZ4) -l -12 --favor-decSpeed stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal
index 25bedd8..440f0b28 100644
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -12,6 +12,8 @@
# for c_flags
include $(srctree)/scripts/Makefile.lib
+mixed-build-prefix = $(if $(KBUILD_MIXED_TREE),$(KBUILD_MIXED_TREE)/)
+
# find all modules listed in modules.order
modules := $(sort $(shell cat $(MODORDER)))
@@ -39,13 +41,13 @@
quiet_cmd_btf_ko = BTF [M] $@
cmd_btf_ko = \
- if [ ! -f vmlinux ]; then \
- printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \
+ if [ ! -f $(mixed-build-prefix)vmlinux ]; then \
+ printf "Skipping BTF generation for %s due to unavailability of $(mixed-build-prefix)vmlinux\n" $@ 1>&2; \
elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then \
printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \
else \
- LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \
- $(RESOLVE_BTFIDS) -b vmlinux $@; \
+ LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base $(mixed-build-prefix)vmlinux $@; \
+ $(RESOLVE_BTFIDS) -b $(mixed-build-prefix)vmlinux $@; \
fi;
# Same as newer-prereqs, but allows to exclude specified extra dependencies
@@ -57,8 +59,8 @@
printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:)
# Re-generate module BTFs if either module's .ko or vmlinux changed
-$(modules): %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),vmlinux) FORCE
- +$(call if_changed_except,ld_ko_o,vmlinux)
+$(modules): %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),$(mixed-build-prefix)vmlinux) FORCE
+ +$(call if_changed_except,ld_ko_o,$(mixed-build-prefix)vmlinux)
ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+$(if $(newer-prereqs),$(call cmd,btf_ko))
endif
diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst
index a4c987c..917365d 100644
--- a/scripts/Makefile.modinst
+++ b/scripts/Makefile.modinst
@@ -27,6 +27,10 @@
suffix-$(CONFIG_MODULE_COMPRESS_ZSTD) := .zst
modules := $(patsubst $(extmod_prefix)%, $(dst)/%$(suffix-y), $(modules))
+ifneq ($(KBUILD_EXTMOD),)
+extmod_suffix := $(shell echo "${KBUILD_EXTMOD}" | md5sum | cut -d " " -f 1)
+modules += $(dst)/modules.order.$(extmod_suffix)
+endif
__modinst: $(modules)
@:
@@ -82,6 +86,12 @@
$(call cmd,strip)
$(call cmd,sign)
+ifneq ($(KBUILD_EXTMOD),)
+$(dst)/modules.order.$(extmod_suffix): $(MODORDER) FORCE
+ $(call cmd,install)
+ @sed -i "s:^$(KBUILD_EXTMOD):$(INSTALL_MOD_DIR):g" $@
+endif
+
else
$(dst)/%.ko: FORCE
diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost
index e41dee6..9aaf186 100644
--- a/scripts/Makefile.modpost
+++ b/scripts/Makefile.modpost
@@ -38,6 +38,8 @@
include include/config/auto.conf
include $(srctree)/scripts/Kbuild.include
+mixed-build-prefix = $(if $(KBUILD_MIXED_TREE),$(KBUILD_MIXED_TREE)/)
+
modpost-args = \
$(if $(CONFIG_MODVERSIONS),-m) \
$(if $(CONFIG_MODULE_SRCVERSION_ALL),-a) \
@@ -58,8 +60,10 @@
# This is used to retrieve symbol versions generated by genksyms.
ifdef CONFIG_MODVERSIONS
+ifndef KBUILD_MIXED_TREE
vmlinux.symvers Module.symvers: .vmlinux.objs
endif
+endif
# Ignore libgcc.a
# Some architectures do '$(CC) --print-libgcc-file-name' to borrow libgcc.a
@@ -74,9 +78,17 @@
esac \
done > $@
+quiet_cmd_vmlinux_symvers = GEN $@
+ cmd_vmlinux_symvers = grep "\<vmlinux\s\+EXPORT" $< > $@
+
+quiet_cmd_cat_symvers = GEN $@
+ cmd_cat_symvers = cat $(real-prereqs) > $@
+
+ifndef KBUILD_MIXED_TREE
targets += .vmlinux.objs
.vmlinux.objs: vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE
$(call if_changed,vmlinux_objs)
+endif
vmlinux.o-if-present := $(wildcard vmlinux.o)
output-symdump := vmlinux.symvers
@@ -84,6 +96,27 @@
ifdef KBUILD_MODULES
output-symdump := $(if $(vmlinux.o-if-present), Module.symvers, modules-only.symvers)
missing-input := $(filter-out $(vmlinux.o-if-present),vmlinux.o)
+
+targets += vmlinux.symvers
+vmlinux.symvers: Module.symvers FORCE
+ $(call if_changed,vmlinux_symvers)
+
+__modpost: $(if $(vmlinux.o-if-present),vmlinux.symvers,)
+endif
+
+# Overwrite values for mixed building (overwritting makes merges easier)
+ifdef KBUILD_MIXED_TREE
+targets += Module.symvers
+Module.symvers: $(mixed-build-prefix)vmlinux.symvers modules-only.symvers FORCE
+ $(call if_changed,cat_symvers)
+
+__modpost: Module.symvers
+
+vmlinux.o-if-present :=
+vmlinux.symvers-if-present := $(wildcard $(mixed-build-prefix)vmlinux.symvers)
+modpost-args += $(addprefix -i ,$(vmlinux.symvers-if-present))
+output-symdump := modules-only.symvers
+missing-input := $(filter-out $(vmlinux.symvers-if-present),$(mixed-build-prefix)vmlinux.symvers)
endif
else
diff --git a/scripts/depmod.sh b/scripts/depmod.sh
index 3643b4f..2de598d 100755
--- a/scripts/depmod.sh
+++ b/scripts/depmod.sh
@@ -3,14 +3,15 @@
#
# A depmod wrapper used by the toplevel Makefile
-if test $# -ne 2; then
- echo "Usage: $0 /sbin/depmod <kernelrelease>" >&2
+if test $# -ne 2 -a $# -ne 3; then
+ echo "Usage: $0 /sbin/depmod <kernelrelease> [System.map folder]" >&2
exit 1
fi
DEPMOD=$1
KERNELRELEASE=$2
+KBUILD_MIXED_TREE=$3
-if ! test -r System.map ; then
+if ! test -r ${KBUILD_MIXED_TREE}System.map ; then
echo "Warning: modules_install: missing 'System.map' file. Skipping depmod." >&2
exit 0
fi
@@ -41,7 +42,7 @@
KERNELRELEASE=99.98.$KERNELRELEASE
fi
-set -- -ae -F System.map
+set -- -ae -F ${KBUILD_MIXED_TREE}System.map
if test -n "$INSTALL_MOD_PATH"; then
set -- "$@" -b "$INSTALL_MOD_PATH"
fi
diff --git a/scripts/gen_gki_modules_headers.sh b/scripts/gen_gki_modules_headers.sh
new file mode 100755
index 0000000..c1074ebd
--- /dev/null
+++ b/scripts/gen_gki_modules_headers.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright 2022 Google LLC
+# Author: ramjiyani@google.com (Ramji Jiyani)
+#
+
+#
+# Generates header file with list of unprotected symbols
+#
+# Called By: KERNEL_SRC/kernel/Makefile if CONFIG_MODULE_SIG_PROTECT=y
+#
+# gki_module_unprotected.h: Symbols allowed to _access_ by unsigned modules
+#
+# If valid symbol file doesn't exists then still generates valid C header files for
+# compilation to proceed with no symbols to protect
+#
+
+# Collect arguments from Makefile
+TARGET=$1
+SRCTREE=$2
+SYMBOL_LIST=$3
+
+set -e
+
+#
+# Common Definitions
+#
+# Use "make V=1" to debug this script.
+case "$KBUILD_VERBOSE" in
+*1*)
+ set -x
+ ;;
+esac
+
+#
+# generate_header():
+# Args: $1 = Name of the header file
+# $2 = Input symbol list
+# $3 = Symbol type ("unprotected")
+#
+generate_header() {
+ local header_file=$1
+ local symbol_file=$2
+ local symbol_type=$3
+
+ if [ -f "${header_file}" ]; then
+ rm -f -- "${header_file}"
+ fi
+
+ # If symbol_file exist preprocess it and find maximum name length
+ if [ -s "${symbol_file}" ]; then
+ # Remove White Spaces, empty lines and symbol list markers if any
+ sed -i '/^[[:space:]]*$/d; /^#/d; /\[abi_symbol_list\]/d' "${symbol_file}"
+
+ # Sort in byte order for kernel binary search at runtime
+ LC_ALL=C sort -o "${symbol_file}" "${symbol_file}"
+
+ # Trim white spaces & +1 for null termination
+ local max_name_len=$(awk '
+ {
+ $1=$1;
+ if ( length > L ) {
+ L=length
+ }
+ } END { print ++L }' "${symbol_file}")
+ else
+ # Set to 1 to generate valid C header file
+ local max_name_len=1
+ fi
+
+ # Header generation
+ cat > "${header_file}" <<- EOT
+ /*
+ * DO NOT EDIT
+ *
+ * Build generated header file with ${symbol_type}
+ */
+
+ #define NR_$(printf ${symbol_type} | tr [:lower:] [:upper:])_SYMBOLS \\
+ $(printf '\t')(ARRAY_SIZE(gki_${symbol_type}_symbols))
+ #define MAX_$(printf ${symbol_type} | tr [:lower:] [:upper:])_NAME_LEN (${max_name_len})
+
+ static const char gki_${symbol_type}_symbols[][MAX_$(printf ${symbol_type} |
+ tr [:lower:] [:upper:])_NAME_LEN] = {
+ EOT
+
+ # If a valid symbol_file present add symbols in an array except the 1st line
+ if [ -s "${symbol_file}" ]; then
+ sed -e 's/^[ \t]*/\t"/;s/[ \t]*$/",/' "${symbol_file}" >> "${header_file}"
+ fi
+
+ # Terminate the file
+ echo "};" >> "${header_file}"
+}
+
+if [ "$(basename "${TARGET}")" = "gki_module_unprotected.h" ]; then
+ # Union of vendor symbol lists
+ GKI_VENDOR_SYMBOLS="${SYMBOL_LIST}"
+ generate_header "${TARGET}" "${GKI_VENDOR_SYMBOLS}" "unprotected"
+else
+ # Sorted list of exported symbols
+ GKI_EXPORTED_SYMBOLS="${SYMBOL_LIST}"
+
+ generate_header "${TARGET}" "${GKI_EXPORTED_SYMBOLS}" "protected_exports"
+fi
+
diff --git a/scripts/setlocalversion b/scripts/setlocalversion
index af4754a..1b733ae 100755
--- a/scripts/setlocalversion
+++ b/scripts/setlocalversion
@@ -11,12 +11,14 @@
#
usage() {
- echo "Usage: $0 [--save-scmversion] [srctree]" >&2
+ echo "Usage: $0 [--save-scmversion] [srctree] [branch] [kmi-generation]" >&2
exit 1
}
scm_only=false
srctree=.
+android_release=
+kmi_generation=
if test "$1" = "--save-scmversion"; then
scm_only=true
shift
@@ -25,6 +27,22 @@
srctree=$1
shift
fi
+if test $# -gt 0; then
+ # Extract the Android release version. If there is no match, then return 255
+ # and clear the var $android_release
+ android_release=`echo "$1" | sed -e '/android[0-9]\{2,\}/!{q255}; \
+ s/^\(android[0-9]\{2,\}\)-.*/\1/'`
+ if test $? -ne 0; then
+ android_release=
+ fi
+ shift
+
+ if test $# -gt 0; then
+ kmi_generation=$1
+ [ $(expr $kmi_generation : '^[0-9]\+$') -eq 0 ] && usage
+ shift
+ fi
+fi
if test $# -gt 0 -o ! -d "$srctree"; then
usage
fi
@@ -44,8 +62,13 @@
fi
# Check for git and a git repo.
- if test -z "$(git rev-parse --show-cdup 2>/dev/null)" &&
- head=$(git rev-parse --verify HEAD 2>/dev/null); then
+ if head=$(git rev-parse --verify HEAD 2>/dev/null); then
+
+ if [ -n "$android_release" ] && [ -n "$kmi_generation" ]; then
+ printf '%s' "-$android_release-$kmi_generation"
+ elif [ -n "$android_release" ]; then
+ printf '%s' "-$android_release"
+ fi
# If we are at a tagged commit (like "v2.6.30-rc6"), we ignore
# it, because this version is defined in the top level Makefile.
@@ -142,4 +165,9 @@
res="$res${scm:++}"
fi
+# finally, add the abXXX number if BUILD_NUMBER is set
+if test -n "${BUILD_NUMBER}"; then
+ res="$res-ab${BUILD_NUMBER}"
+fi
+
echo "$res"
diff --git a/scripts/sign-file.c b/scripts/sign-file.c
index 598ef54..cadbc26 100644
--- a/scripts/sign-file.c
+++ b/scripts/sign-file.c
@@ -99,6 +99,7 @@ static void display_openssl_errors(int l)
}
}
+#ifndef OPENSSL_NO_ENGINE
static void drain_openssl_errors(void)
{
const char *file;
@@ -108,6 +109,7 @@ static void drain_openssl_errors(void)
return;
while (ERR_get_error_line(&file, &line)) {}
}
+#endif
#define ERR(cond, fmt, ...) \
do { \
@@ -142,7 +144,9 @@ static int pem_pw_cb(char *buf, int len, int w, void *v)
static EVP_PKEY *read_private_key(const char *private_key_name)
{
EVP_PKEY *private_key;
+ BIO *b;
+#ifndef OPENSSL_NO_ENGINE
if (!strncmp(private_key_name, "pkcs11:", 7)) {
ENGINE *e;
@@ -160,17 +164,16 @@ static EVP_PKEY *read_private_key(const char *private_key_name)
private_key = ENGINE_load_private_key(e, private_key_name,
NULL, NULL);
ERR(!private_key, "%s", private_key_name);
- } else {
- BIO *b;
-
- b = BIO_new_file(private_key_name, "rb");
- ERR(!b, "%s", private_key_name);
- private_key = PEM_read_bio_PrivateKey(b, NULL, pem_pw_cb,
- NULL);
- ERR(!private_key, "%s", private_key_name);
- BIO_free(b);
+ return private_key;
}
+#endif
+ b = BIO_new_file(private_key_name, "rb");
+ ERR(!b, "%s", private_key_name);
+ private_key = PEM_read_bio_PrivateKey(b, NULL, pem_pw_cb,
+ NULL);
+ ERR(!private_key, "%s", private_key_name);
+ BIO_free(b);
return private_key;
}
diff --git a/security/selinux/avc.c b/security/selinux/avc.c
index 9a43af0..e897b1c 100644
--- a/security/selinux/avc.c
+++ b/security/selinux/avc.c
@@ -44,6 +44,9 @@
#define avc_cache_stats_incr(field) do {} while (0)
#endif
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/avc.h>
+
struct avc_entry {
u32 ssid;
u32 tsid;
@@ -441,6 +444,7 @@ static void avc_node_free(struct rcu_head *rhead)
static void avc_node_delete(struct selinux_avc *avc, struct avc_node *node)
{
+ trace_android_rvh_selinux_avc_node_delete(node);
hlist_del_rcu(&node->list);
call_rcu(&node->rhead, avc_node_free);
atomic_dec(&avc->avc_cache.active_nodes);
@@ -457,6 +461,7 @@ static void avc_node_kill(struct selinux_avc *avc, struct avc_node *node)
static void avc_node_replace(struct selinux_avc *avc,
struct avc_node *new, struct avc_node *old)
{
+ trace_android_rvh_selinux_avc_node_replace(old, new);
hlist_replace_rcu(&old->list, &new->list);
call_rcu(&old->rhead, avc_node_free);
atomic_dec(&avc->avc_cache.active_nodes);
@@ -566,8 +571,10 @@ static struct avc_node *avc_lookup(struct selinux_avc *avc,
avc_cache_stats_incr(lookups);
node = avc_search_node(avc, ssid, tsid, tclass);
- if (node)
+ if (node) {
+ trace_android_rvh_selinux_avc_lookup(node, ssid, tsid, tclass);
return node;
+ }
avc_cache_stats_incr(misses);
return NULL;
@@ -652,6 +659,7 @@ static struct avc_node *avc_insert(struct selinux_avc *avc,
}
}
hlist_add_head_rcu(&node->list, head);
+ trace_android_rvh_selinux_avc_insert(node);
found:
spin_unlock_irqrestore(lock, flag);
return node;
diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h
index a3c3807..d743c30 100644
--- a/security/selinux/include/classmap.h
+++ b/security/selinux/include/classmap.h
@@ -117,7 +117,8 @@ const struct security_class_mapping secclass_map[] = {
{ COMMON_IPC_PERMS, NULL } },
{ "netlink_route_socket",
{ COMMON_SOCK_PERMS,
- "nlmsg_read", "nlmsg_write", NULL } },
+ "nlmsg_read", "nlmsg_write", "nlmsg_readpriv", "nlmsg_getneigh",
+ NULL } },
{ "netlink_tcpdiag_socket",
{ COMMON_SOCK_PERMS,
"nlmsg_read", "nlmsg_write", NULL } },
diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h
index 393aff4..ad59c4c0 100644
--- a/security/selinux/include/security.h
+++ b/security/selinux/include/security.h
@@ -99,6 +99,8 @@ struct selinux_state {
bool checkreqprot;
bool initialized;
bool policycap[__POLICYDB_CAP_MAX];
+ bool android_netlink_route;
+ bool android_netlink_getneigh;
struct page *status_page;
struct mutex status_lock;
@@ -230,6 +232,20 @@ static inline bool selinux_policycap_ioctl_skip_cloexec(void)
return READ_ONCE(state->policycap[POLICYDB_CAP_IOCTL_SKIP_CLOEXEC]);
}
+static inline bool selinux_android_nlroute_getlink(void)
+{
+ struct selinux_state *state = &selinux_state;
+
+ return state->android_netlink_route;
+}
+
+static inline bool selinux_android_nlroute_getneigh(void)
+{
+ struct selinux_state *state = &selinux_state;
+
+ return state->android_netlink_getneigh;
+}
+
struct selinux_policy_convert_data;
struct selinux_load_state {
@@ -463,5 +479,6 @@ extern void avtab_cache_init(void);
extern void ebitmap_cache_init(void);
extern void hashtab_cache_init(void);
extern int security_sidtab_hash_stats(struct selinux_state *state, char *page);
+extern void selinux_nlmsg_init(void);
#endif /* _SELINUX_SECURITY_H_ */
diff --git a/security/selinux/nlmsgtab.c b/security/selinux/nlmsgtab.c
index 2ee7b4ed..32200ec 100644
--- a/security/selinux/nlmsgtab.c
+++ b/security/selinux/nlmsgtab.c
@@ -25,7 +25,7 @@ struct nlmsg_perm {
u32 perm;
};
-static const struct nlmsg_perm nlmsg_route_perms[] = {
+static struct nlmsg_perm nlmsg_route_perms[] = {
{ RTM_NEWLINK, NETLINK_ROUTE_SOCKET__NLMSG_WRITE },
{ RTM_DELLINK, NETLINK_ROUTE_SOCKET__NLMSG_WRITE },
{ RTM_GETLINK, NETLINK_ROUTE_SOCKET__NLMSG_READ },
@@ -216,3 +216,43 @@ int selinux_nlmsg_lookup(u16 sclass, u16 nlmsg_type, u32 *perm)
return err;
}
+
+static void nlmsg_set_perm_for_type(u32 perm, u16 type)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(nlmsg_route_perms); i++) {
+ if (nlmsg_route_perms[i].nlmsg_type == type) {
+ nlmsg_route_perms[i].perm = perm;
+ break;
+ }
+ }
+}
+
+/**
+ * Use nlmsg_readpriv as the permission for RTM_GETLINK messages if the
+ * netlink_route_getlink policy capability is set. Otherwise use nlmsg_read.
+ * Similarly, use nlmsg_getneigh for RTM_GETNEIGH and RTM_GETNEIGHTBL if the
+ * netlink_route_getneigh policy capability is set. Otherwise use nlmsg_read.
+ */
+void selinux_nlmsg_init(void)
+{
+ if (selinux_android_nlroute_getlink())
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_READPRIV,
+ RTM_GETLINK);
+ else
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_READ,
+ RTM_GETLINK);
+
+ if (selinux_android_nlroute_getneigh()) {
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_GETNEIGH,
+ RTM_GETNEIGH);
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_GETNEIGH,
+ RTM_GETNEIGHTBL);
+ } else {
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_READ,
+ RTM_GETNEIGH);
+ nlmsg_set_perm_for_type(NETLINK_ROUTE_SOCKET__NLMSG_READ,
+ RTM_GETNEIGHTBL);
+ }
+}
diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
index adcfb63..a36825a 100644
--- a/security/selinux/ss/policydb.c
+++ b/security/selinux/ss/policydb.c
@@ -2485,6 +2485,14 @@ int policydb_read(struct policydb *p, void *fp)
p->reject_unknown = !!(le32_to_cpu(buf[1]) & REJECT_UNKNOWN);
p->allow_unknown = !!(le32_to_cpu(buf[1]) & ALLOW_UNKNOWN);
+ if ((le32_to_cpu(buf[1]) & POLICYDB_CONFIG_ANDROID_NETLINK_ROUTE)) {
+ p->android_netlink_route = 1;
+ }
+
+ if ((le32_to_cpu(buf[1]) & POLICYDB_CONFIG_ANDROID_NETLINK_GETNEIGH)) {
+ p->android_netlink_getneigh = 1;
+ }
+
if (p->policyvers >= POLICYDB_VERSION_POLCAP) {
rc = ebitmap_read(&p->policycaps, fp);
if (rc)
diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h
index ffc4e7b..1d75811 100644
--- a/security/selinux/ss/policydb.h
+++ b/security/selinux/ss/policydb.h
@@ -238,6 +238,8 @@ struct genfs {
/* The policy database */
struct policydb {
int mls_enabled;
+ int android_netlink_route;
+ int android_netlink_getneigh;
/* symbol tables */
struct symtab symtab[SYM_NUM];
@@ -334,6 +336,8 @@ extern struct role_trans_datum *policydb_roletr_search(
struct policydb *p, struct role_trans_key *key);
#define POLICYDB_CONFIG_MLS 1
+#define POLICYDB_CONFIG_ANDROID_NETLINK_ROUTE (1 << 31)
+#define POLICYDB_CONFIG_ANDROID_NETLINK_GETNEIGH (1 << 30)
/* the config flags related to unknown classes/perms are bits 2 and 3 */
#define REJECT_UNKNOWN 0x00000002
diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
index 0092b29..3f5834a 100644
--- a/security/selinux/ss/services.c
+++ b/security/selinux/ss/services.c
@@ -68,6 +68,8 @@
#include "policycap_names.h"
#include "ima.h"
+#include <trace/hooks/selinux.h>
+
struct selinux_policy_convert_data {
struct convert_context_args args;
struct sidtab_convert_params sidtab_params;
@@ -2158,6 +2160,10 @@ static void security_load_policycaps(struct selinux_state *state,
pr_info("SELinux: unknown policy capability %u\n",
i);
}
+
+ state->android_netlink_route = p->android_netlink_route;
+ state->android_netlink_getneigh = p->android_netlink_getneigh;
+ selinux_nlmsg_init();
}
static int security_preserve_bools(struct selinux_policy *oldpolicy,
@@ -2251,6 +2257,7 @@ void selinux_policy_commit(struct selinux_state *state,
*/
selinux_mark_initialized(state);
selinux_complete_init();
+ trace_android_rvh_selinux_is_initialized(state);
}
/* Free the old policy */
diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index e613698..4905977 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -3225,6 +3225,39 @@ int snd_soc_get_dai_id(struct device_node *ep)
}
EXPORT_SYMBOL_GPL(snd_soc_get_dai_id);
+/**
+ * snd_soc_info_multi_ext - external single mixer info callback
+ * @kcontrol: mixer control
+ * @uinfo: control element information
+ *
+ * Callback to provide information about a single external mixer control.
+ * that accepts multiple input.
+ *
+ * Returns 0 for success.
+ */
+int snd_soc_info_multi_ext(struct snd_kcontrol *kcontrol,
+ struct snd_ctl_elem_info *uinfo)
+{
+ struct soc_multi_mixer_control *mc =
+ (struct soc_multi_mixer_control *)kcontrol->private_value;
+ int platform_max;
+
+ if (!mc->platform_max)
+ mc->platform_max = mc->max;
+ platform_max = mc->platform_max;
+
+ if (platform_max == 1 && !strnstr(kcontrol->id.name, " Volume", 30))
+ uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
+ else
+ uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+
+ uinfo->count = mc->count;
+ uinfo->value.integer.min = 0;
+ uinfo->value.integer.max = platform_max;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(snd_soc_info_multi_ext);
+
int snd_soc_get_dai_name(const struct of_phandle_args *args,
const char **dai_name)
{
diff --git a/tools/testing/selftests/filesystems/incfs/.gitignore b/tools/testing/selftests/filesystems/incfs/.gitignore
new file mode 100644
index 0000000..f0e3cd9
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/.gitignore
@@ -0,0 +1,3 @@
+incfs_test
+incfs_stress
+incfs_perf
diff --git a/tools/testing/selftests/filesystems/incfs/Makefile b/tools/testing/selftests/filesystems/incfs/Makefile
new file mode 100644
index 0000000..5a2f630
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/Makefile
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../.. -fno-omit-frame-pointer -fsanitize=address -g
+LDLIBS := -llz4 -lzstd -lcrypto -lpthread -fsanitize=address
+TEST_GEN_PROGS := incfs_test incfs_stress incfs_perf
+
+include ../../lib.mk
+
+# Put after include ../../lib.mk since that changes $(TEST_GEN_PROGS)
+# Otherwise you get multiple targets, this becomes the default, and it's a mess
+EXTRA_SOURCES := utils.c
+$(TEST_GEN_PROGS) : $(EXTRA_SOURCES)
diff --git a/tools/testing/selftests/filesystems/incfs/OWNERS b/tools/testing/selftests/filesystems/incfs/OWNERS
new file mode 100644
index 0000000..f26e11c
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/OWNERS
@@ -0,0 +1 @@
+file:/fs/incfs/OWNERS
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_perf.c b/tools/testing/selftests/filesystems/incfs/incfs_perf.c
new file mode 100644
index 0000000..ed36bbd
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/incfs_perf.c
@@ -0,0 +1,717 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Google LLC
+ */
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <lz4.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <ctype.h>
+#include <unistd.h>
+
+#include "utils.h"
+
+#define err_msg(...) \
+ do { \
+ fprintf(stderr, "%s: (%d) ", TAG, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, " (%s)\n", strerror(errno)); \
+ } while (false)
+
+#define TAG "incfs_perf"
+
+struct options {
+ int blocks; /* -b number of diff block sizes */
+ bool no_cleanup; /* -c don't clean up after */
+ const char *test_dir; /* -d working directory */
+ const char *file_types; /* -f sScCvV */
+ bool no_native; /* -n don't test native files */
+ bool no_random; /* -r don't do random reads*/
+ bool no_linear; /* -R random reads only */
+ size_t size; /* -s file size as power of 2 */
+ int tries; /* -t times to run test*/
+};
+
+enum flags {
+ SHUFFLE = 1,
+ COMPRESS = 2,
+ VERIFY = 4,
+ LAST_FLAG = 8,
+};
+
+void print_help(void)
+{
+ puts(
+ "incfs_perf. Performance test tool for incfs\n"
+ "\tTests read performance of incfs by creating files of various types\n"
+ "\tflushing caches and then reading them back.\n"
+ "\tEach file is read with different block sizes and average\n"
+ "\tthroughput in megabytes/second and memory usage are reported for\n"
+ "\teach block size\n"
+ "\tNative files are tested for comparison\n"
+ "\tNative files are created in native folder, incfs files are created\n"
+ "\tin src folder which is mounted on dst folder\n"
+ "\n"
+ "\t-bn (default 8) number of different block sizes, starting at 4096\n"
+ "\t and doubling\n"
+ "\t-c don't Clean up - leave files and mount point\n"
+ "\t-d dir create directories in dir\n"
+ "\t-fs|Sc|Cv|V restrict which files are created.\n"
+ "\t s blocks not shuffled, S blocks shuffled\n"
+ "\t c blocks not compress, C blocks compressed\n"
+ "\t v files not verified, V files verified\n"
+ "\t If a letter is omitted, both options are tested\n"
+ "\t If no letter are given, incfs is not tested\n"
+ "\t-n Don't test native files\n"
+ "\t-r No random reads (sequential only)\n"
+ "\t-R Random reads only (no sequential)\n"
+ "\t-sn (default 30)File size as power of 2\n"
+ "\t-tn (default 5) Number of tries per file. Results are averaged\n"
+ );
+}
+
+int parse_options(int argc, char *const *argv, struct options *options)
+{
+ signed char c;
+
+ /* Set defaults here */
+ *options = (struct options){
+ .blocks = 8,
+ .test_dir = ".",
+ .tries = 5,
+ .size = 30,
+ };
+
+ /* Load options from command line here */
+ while ((c = getopt(argc, argv, "b:cd:f::hnrRs:t:")) != -1) {
+ switch (c) {
+ case 'b':
+ options->blocks = strtol(optarg, NULL, 10);
+ break;
+
+ case 'c':
+ options->no_cleanup = true;
+ break;
+
+ case 'd':
+ options->test_dir = optarg;
+ break;
+
+ case 'f':
+ if (optarg)
+ options->file_types = optarg;
+ else
+ options->file_types = "sS";
+ break;
+
+ case 'h':
+ print_help();
+ exit(0);
+
+ case 'n':
+ options->no_native = true;
+ break;
+
+ case 'r':
+ options->no_random = true;
+ break;
+
+ case 'R':
+ options->no_linear = true;
+ break;
+
+ case 's':
+ options->size = strtol(optarg, NULL, 10);
+ break;
+
+ case 't':
+ options->tries = strtol(optarg, NULL, 10);
+ break;
+
+ default:
+ print_help();
+ return -EINVAL;
+ }
+ }
+
+ options->size = 1L << options->size;
+
+ return 0;
+}
+
+void shuffle(size_t *buffer, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; ++i) {
+ size_t j = random() * (size - i - 1) / RAND_MAX;
+ size_t temp = buffer[i];
+
+ buffer[i] = buffer[j];
+ buffer[j] = temp;
+ }
+}
+
+int get_free_memory(void)
+{
+ FILE *meminfo = fopen("/proc/meminfo", "re");
+ char field[256];
+ char value[256] = {};
+
+ if (!meminfo)
+ return -ENOENT;
+
+ while (fscanf(meminfo, "%[^:]: %s kB\n", field, value) == 2) {
+ if (!strcmp(field, "MemFree"))
+ break;
+ *value = 0;
+ }
+
+ fclose(meminfo);
+
+ if (!*value)
+ return -ENOENT;
+
+ return strtol(value, NULL, 10);
+}
+
+int write_data(int cmd_fd, int dir_fd, const char *name, size_t size, int flags)
+{
+ int fd = openat(dir_fd, name, O_RDWR | O_CLOEXEC);
+ struct incfs_permit_fill permit_fill = {
+ .file_descriptor = fd,
+ };
+ int block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ size_t *blocks = malloc(sizeof(size_t) * block_count);
+ int error = 0;
+ size_t i;
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ uint8_t compressed_data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ struct incfs_fill_block fill_block = {
+ .compression = COMPRESSION_NONE,
+ .data_len = sizeof(data),
+ .data = ptr_to_u64(data),
+ };
+
+ if (!blocks) {
+ err_msg("Out of memory");
+ error = -errno;
+ goto out;
+ }
+
+ if (fd == -1) {
+ err_msg("Could not open file for writing %s", name);
+ error = -errno;
+ goto out;
+ }
+
+ if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ err_msg("Failed to call PERMIT_FILL");
+ error = -errno;
+ goto out;
+ }
+
+ for (i = 0; i < block_count; ++i)
+ blocks[i] = i;
+
+ if (flags & SHUFFLE)
+ shuffle(blocks, block_count);
+
+ if (flags & COMPRESS) {
+ size_t comp_size = LZ4_compress_default(
+ (char *)data, (char *)compressed_data, sizeof(data),
+ ARRAY_SIZE(compressed_data));
+
+ if (comp_size <= 0) {
+ error = -EBADMSG;
+ goto out;
+ }
+ fill_block.compression = COMPRESSION_LZ4;
+ fill_block.data = ptr_to_u64(compressed_data);
+ fill_block.data_len = comp_size;
+ }
+
+ for (i = 0; i < block_count; ++i) {
+ struct incfs_fill_blocks fill_blocks = {
+ .count = 1,
+ .fill_blocks = ptr_to_u64(&fill_block),
+ };
+
+ fill_block.block_index = blocks[i];
+ int written = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+
+ if (written != 1) {
+ error = -errno;
+ err_msg("Failed to write block %lu in file %s", i,
+ name);
+ break;
+ }
+ }
+
+out:
+ free(blocks);
+ close(fd);
+ sync();
+ return error;
+}
+
+int measure_read_throughput_internal(const char *tag, int dir, const char *name,
+ const struct options *options, bool random)
+{
+ int block;
+
+ if (random)
+ printf("%32s(random)", tag);
+ else
+ printf("%40s", tag);
+
+ for (block = 0; block < options->blocks; ++block) {
+ size_t buffer_size;
+ char *buffer;
+ int try;
+ double time = 0;
+ double throughput;
+ int memory = 0;
+
+ buffer_size = 1 << (block + 12);
+ buffer = malloc(buffer_size);
+
+ for (try = 0; try < options->tries; ++try) {
+ int err;
+ struct timespec start_time, end_time;
+ off_t i;
+ int fd;
+ size_t offsets_size = options->size / buffer_size;
+ size_t *offsets =
+ malloc(offsets_size * sizeof(*offsets));
+ int start_memory, end_memory;
+
+ if (!offsets) {
+ err_msg("Not enough memory");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < offsets_size; ++i)
+ offsets[i] = i * buffer_size;
+
+ if (random)
+ shuffle(offsets, offsets_size);
+
+ err = drop_caches();
+ if (err) {
+ err_msg("Failed to drop caches");
+ return err;
+ }
+
+ start_memory = get_free_memory();
+ if (start_memory < 0) {
+ err_msg("Failed to get start memory");
+ return start_memory;
+ }
+
+ fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
+ if (fd == -1) {
+ err_msg("Failed to open file");
+ return err;
+ }
+
+ err = clock_gettime(CLOCK_MONOTONIC, &start_time);
+ if (err) {
+ err_msg("Failed to get start time");
+ return err;
+ }
+
+ for (i = 0; i < offsets_size; ++i)
+ if (pread(fd, buffer, buffer_size,
+ offsets[i]) != buffer_size) {
+ err_msg("Failed to read file");
+ err = -errno;
+ goto fail;
+ }
+
+ err = clock_gettime(CLOCK_MONOTONIC, &end_time);
+ if (err) {
+ err_msg("Failed to get start time");
+ goto fail;
+ }
+
+ end_memory = get_free_memory();
+ if (end_memory < 0) {
+ err_msg("Failed to get end memory");
+ return end_memory;
+ }
+
+ time += end_time.tv_sec - start_time.tv_sec;
+ time += (end_time.tv_nsec - start_time.tv_nsec) / 1e9;
+
+ close(fd);
+ fd = -1;
+ memory += start_memory - end_memory;
+
+fail:
+ free(offsets);
+ close(fd);
+ if (err)
+ return err;
+ }
+
+ throughput = options->size * options->tries / time;
+ printf("%10.3e %10d", throughput, memory / options->tries);
+ free(buffer);
+ }
+
+ printf("\n");
+ return 0;
+}
+
+int measure_read_throughput(const char *tag, int dir, const char *name,
+ const struct options *options)
+{
+ int err = 0;
+
+ if (!options->no_linear)
+ err = measure_read_throughput_internal(tag, dir, name, options,
+ false);
+
+ if (!err && !options->no_random)
+ err = measure_read_throughput_internal(tag, dir, name, options,
+ true);
+ return err;
+}
+
+int test_native_file(int dir, const struct options *options)
+{
+ const char *name = "file";
+ int fd;
+ char buffer[4096] = {};
+ off_t i;
+ int err;
+
+ fd = openat(dir, name, O_CREAT | O_WRONLY | O_CLOEXEC, 0600);
+ if (fd == -1) {
+ err_msg("Could not open native file");
+ return -errno;
+ }
+
+ for (i = 0; i < options->size; i += sizeof(buffer))
+ if (pwrite(fd, buffer, sizeof(buffer), i) != sizeof(buffer)) {
+ err_msg("Failed to write file");
+ err = -errno;
+ goto fail;
+ }
+
+ close(fd);
+ sync();
+ fd = -1;
+
+ err = measure_read_throughput("native", dir, name, options);
+
+fail:
+ close(fd);
+ return err;
+}
+
+struct hash_block {
+ char data[INCFS_DATA_FILE_BLOCK_SIZE];
+};
+
+static struct hash_block *build_mtree(size_t size, char *root_hash,
+ int *mtree_block_count)
+{
+ char data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ const int digest_size = SHA256_DIGEST_SIZE;
+ const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
+ int block_count = 0;
+ int hash_block_count = 0;
+ int total_tree_block_count = 0;
+ int tree_lvl_index[INCFS_MAX_MTREE_LEVELS] = {};
+ int tree_lvl_count[INCFS_MAX_MTREE_LEVELS] = {};
+ int levels_count = 0;
+ int i, level;
+ struct hash_block *mtree;
+
+ if (size == 0)
+ return 0;
+
+ block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ hash_block_count = block_count;
+ for (i = 0; hash_block_count > 1; i++) {
+ hash_block_count = (hash_block_count + hash_per_block - 1) /
+ hash_per_block;
+ tree_lvl_count[i] = hash_block_count;
+ total_tree_block_count += hash_block_count;
+ }
+ levels_count = i;
+
+ for (i = 0; i < levels_count; i++) {
+ int prev_lvl_base = (i == 0) ? total_tree_block_count :
+ tree_lvl_index[i - 1];
+
+ tree_lvl_index[i] = prev_lvl_base - tree_lvl_count[i];
+ }
+
+ *mtree_block_count = total_tree_block_count;
+ mtree = calloc(total_tree_block_count, sizeof(*mtree));
+ /* Build level 0 hashes. */
+ for (i = 0; i < block_count; i++) {
+ int block_index = tree_lvl_index[0] + i / hash_per_block;
+ int block_off = (i % hash_per_block) * digest_size;
+ char *hash_ptr = mtree[block_index].data + block_off;
+
+ sha256(data, INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
+ }
+
+ /* Build higher levels of hash tree. */
+ for (level = 1; level < levels_count; level++) {
+ int prev_lvl_base = tree_lvl_index[level - 1];
+ int prev_lvl_count = tree_lvl_count[level - 1];
+
+ for (i = 0; i < prev_lvl_count; i++) {
+ int block_index =
+ i / hash_per_block + tree_lvl_index[level];
+ int block_off = (i % hash_per_block) * digest_size;
+ char *hash_ptr = mtree[block_index].data + block_off;
+
+ sha256(mtree[i + prev_lvl_base].data,
+ INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
+ }
+ }
+
+ /* Calculate root hash from the top block */
+ sha256(mtree[0].data, INCFS_DATA_FILE_BLOCK_SIZE, root_hash);
+
+ return mtree;
+}
+
+static int load_hash_tree(int cmd_fd, int dir, const char *name,
+ struct hash_block *mtree, int mtree_block_count)
+{
+ int err;
+ int i;
+ int fd;
+ struct incfs_fill_block *fill_block_array =
+ calloc(mtree_block_count, sizeof(struct incfs_fill_block));
+ struct incfs_fill_blocks fill_blocks = {
+ .count = mtree_block_count,
+ .fill_blocks = ptr_to_u64(fill_block_array),
+ };
+ struct incfs_permit_fill permit_fill;
+
+ if (!fill_block_array)
+ return -ENOMEM;
+
+ for (i = 0; i < fill_blocks.count; i++) {
+ fill_block_array[i] = (struct incfs_fill_block){
+ .block_index = i,
+ .data_len = INCFS_DATA_FILE_BLOCK_SIZE,
+ .data = ptr_to_u64(mtree[i].data),
+ .flags = INCFS_BLOCK_FLAGS_HASH
+ };
+ }
+
+ fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
+ if (fd < 0) {
+ err = errno;
+ goto failure;
+ }
+
+ permit_fill.file_descriptor = fd;
+ if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ err_msg("Failed to call PERMIT_FILL");
+ err = -errno;
+ goto failure;
+ }
+
+ err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ close(fd);
+ if (err < fill_blocks.count)
+ err = errno;
+ else
+ err = 0;
+
+failure:
+ free(fill_block_array);
+ return err;
+}
+
+int test_incfs_file(int dst_dir, const struct options *options, int flags)
+{
+ int cmd_file = openat(dst_dir, INCFS_PENDING_READS_FILENAME,
+ O_RDONLY | O_CLOEXEC);
+ int err;
+ char name[4];
+ incfs_uuid_t id;
+ char tag[256];
+
+ snprintf(name, sizeof(name), "%c%c%c",
+ flags & SHUFFLE ? 'S' : 's',
+ flags & COMPRESS ? 'C' : 'c',
+ flags & VERIFY ? 'V' : 'v');
+
+ if (cmd_file == -1) {
+ err_msg("Could not open command file");
+ return -errno;
+ }
+
+ if (flags & VERIFY) {
+ char root_hash[INCFS_MAX_HASH_SIZE];
+ int mtree_block_count;
+ struct hash_block *mtree = build_mtree(options->size, root_hash,
+ &mtree_block_count);
+
+ if (!mtree) {
+ err_msg("Failed to build hash tree");
+ err = -ENOMEM;
+ goto fail;
+ }
+
+ err = crypto_emit_file(cmd_file, NULL, name, &id, options->size,
+ root_hash, "add_data");
+
+ if (!err)
+ err = load_hash_tree(cmd_file, dst_dir, name, mtree,
+ mtree_block_count);
+
+ free(mtree);
+ } else
+ err = emit_file(cmd_file, NULL, name, &id, options->size, NULL);
+
+ if (err) {
+ err_msg("Failed to create file %s", name);
+ goto fail;
+ }
+
+ if (write_data(cmd_file, dst_dir, name, options->size, flags))
+ goto fail;
+
+ snprintf(tag, sizeof(tag), "incfs%s%s%s",
+ flags & SHUFFLE ? "(shuffle)" : "",
+ flags & COMPRESS ? "(compress)" : "",
+ flags & VERIFY ? "(verify)" : "");
+
+ err = measure_read_throughput(tag, dst_dir, name, options);
+
+fail:
+ close(cmd_file);
+ return err;
+}
+
+bool skip(struct options const *options, int flag, char c)
+{
+ if (!options->file_types)
+ return false;
+
+ if (flag && strchr(options->file_types, tolower(c)))
+ return true;
+
+ if (!flag && strchr(options->file_types, toupper(c)))
+ return true;
+
+ return false;
+}
+
+int main(int argc, char *const *argv)
+{
+ struct options options;
+ int err;
+ const char *native_dir = "native";
+ const char *src_dir = "src";
+ const char *dst_dir = "dst";
+ int native_dir_fd = -1;
+ int src_dir_fd = -1;
+ int dst_dir_fd = -1;
+ int block;
+ int flags;
+
+ err = parse_options(argc, argv, &options);
+ if (err)
+ return err;
+
+ err = chdir(options.test_dir);
+ if (err) {
+ err_msg("Failed to change to %s", options.test_dir);
+ return -errno;
+ }
+
+ /* Clean up any interrupted previous runs */
+ while (!umount(dst_dir))
+ ;
+
+ err = remove_dir(native_dir) || remove_dir(src_dir) ||
+ remove_dir(dst_dir);
+ if (err)
+ return err;
+
+ err = mkdir(native_dir, 0700);
+ if (err) {
+ err_msg("Failed to make directory %s", src_dir);
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = mkdir(src_dir, 0700);
+ if (err) {
+ err_msg("Failed to make directory %s", src_dir);
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = mkdir(dst_dir, 0700);
+ if (err) {
+ err_msg("Failed to make directory %s", src_dir);
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = mount_fs_opt(dst_dir, src_dir, "readahead=0,rlog_pages=0", 0);
+ if (err) {
+ err_msg("Failed to mount incfs");
+ goto cleanup;
+ }
+
+ native_dir_fd = open(native_dir, O_RDONLY | O_CLOEXEC);
+ src_dir_fd = open(src_dir, O_RDONLY | O_CLOEXEC);
+ dst_dir_fd = open(dst_dir, O_RDONLY | O_CLOEXEC);
+ if (native_dir_fd == -1 || src_dir_fd == -1 || dst_dir_fd == -1) {
+ err_msg("Failed to open native, src or dst dir");
+ err = -errno;
+ goto cleanup;
+ }
+
+ printf("%40s", "");
+ for (block = 0; block < options.blocks; ++block)
+ printf("%21d", 1 << (block + 12));
+ printf("\n");
+
+ if (!err && !options.no_native)
+ err = test_native_file(native_dir_fd, &options);
+
+ for (flags = 0; flags < LAST_FLAG && !err; ++flags) {
+ if (skip(&options, flags & SHUFFLE, 's') ||
+ skip(&options, flags & COMPRESS, 'c') ||
+ skip(&options, flags & VERIFY, 'v'))
+ continue;
+ err = test_incfs_file(dst_dir_fd, &options, flags);
+ }
+
+cleanup:
+ close(native_dir_fd);
+ close(src_dir_fd);
+ close(dst_dir_fd);
+ if (!options.no_cleanup) {
+ umount(dst_dir);
+ remove_dir(native_dir);
+ remove_dir(dst_dir);
+ remove_dir(src_dir);
+ }
+
+ return err;
+}
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_stress.c b/tools/testing/selftests/filesystems/incfs/incfs_stress.c
new file mode 100644
index 0000000..a1d4917
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/incfs_stress.c
@@ -0,0 +1,322 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Google LLC
+ */
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "utils.h"
+
+#define err_msg(...) \
+ do { \
+ fprintf(stderr, "%s: (%d) ", TAG, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, " (%s)\n", strerror(errno)); \
+ } while (false)
+
+#define TAG "incfs_stress"
+
+struct options {
+ bool no_cleanup; /* -c */
+ const char *test_dir; /* -d */
+ unsigned int rng_seed; /* -g */
+ int num_reads; /* -n */
+ int readers; /* -r */
+ int size; /* -s */
+ int timeout; /* -t */
+};
+
+struct read_data {
+ const char *filename;
+ int dir_fd;
+ size_t filesize;
+ int num_reads;
+ unsigned int rng_seed;
+};
+
+int cancel_threads;
+
+int parse_options(int argc, char *const *argv, struct options *options)
+{
+ signed char c;
+
+ /* Set defaults here */
+ *options = (struct options){
+ .test_dir = ".",
+ .num_reads = 1000,
+ .readers = 10,
+ .size = 10,
+ };
+
+ /* Load options from command line here */
+ while ((c = getopt(argc, argv, "cd:g:n:r:s:t:")) != -1) {
+ switch (c) {
+ case 'c':
+ options->no_cleanup = true;
+ break;
+
+ case 'd':
+ options->test_dir = optarg;
+ break;
+
+ case 'g':
+ options->rng_seed = strtol(optarg, NULL, 10);
+ break;
+
+ case 'n':
+ options->num_reads = strtol(optarg, NULL, 10);
+ break;
+
+ case 'r':
+ options->readers = strtol(optarg, NULL, 10);
+ break;
+
+ case 's':
+ options->size = strtol(optarg, NULL, 10);
+ break;
+
+ case 't':
+ options->timeout = strtol(optarg, NULL, 10);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+void *reader(void *data)
+{
+ struct read_data *read_data = (struct read_data *)data;
+ int i;
+ int fd = -1;
+ void *buffer = malloc(read_data->filesize);
+
+ if (!buffer) {
+ err_msg("Failed to alloc read buffer");
+ goto out;
+ }
+
+ fd = openat(read_data->dir_fd, read_data->filename,
+ O_RDONLY | O_CLOEXEC);
+ if (fd == -1) {
+ err_msg("Failed to open file");
+ goto out;
+ }
+
+ for (i = 0; i < read_data->num_reads && !cancel_threads; ++i) {
+ off_t offset = rnd(read_data->filesize, &read_data->rng_seed);
+ size_t count =
+ rnd(read_data->filesize - offset, &read_data->rng_seed);
+ ssize_t err = pread(fd, buffer, count, offset);
+
+ if (err != count)
+ err_msg("failed to read with value %lu", err);
+ }
+
+out:
+ close(fd);
+ free(read_data);
+ free(buffer);
+ return NULL;
+}
+
+int write_data(int cmd_fd, int dir_fd, const char *name, size_t size)
+{
+ int fd = openat(dir_fd, name, O_RDWR | O_CLOEXEC);
+ struct incfs_permit_fill permit_fill = {
+ .file_descriptor = fd,
+ };
+ int error = 0;
+ int i;
+ int block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+
+ if (fd == -1) {
+ err_msg("Could not open file for writing %s", name);
+ return -errno;
+ }
+
+ if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ err_msg("Failed to call PERMIT_FILL");
+ error = -errno;
+ goto out;
+ }
+
+ for (i = 0; i < block_count; ++i) {
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ size_t block_size =
+ size > i * INCFS_DATA_FILE_BLOCK_SIZE ?
+ INCFS_DATA_FILE_BLOCK_SIZE :
+ size - (i * INCFS_DATA_FILE_BLOCK_SIZE);
+ struct incfs_fill_block fill_block = {
+ .compression = COMPRESSION_NONE,
+ .block_index = i,
+ .data_len = block_size,
+ .data = ptr_to_u64(data),
+ };
+ struct incfs_fill_blocks fill_blocks = {
+ .count = 1,
+ .fill_blocks = ptr_to_u64(&fill_block),
+ };
+ int written = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+
+ if (written != 1) {
+ error = -errno;
+ err_msg("Failed to write block %d in file %s", i, name);
+ break;
+ }
+ }
+out:
+ close(fd);
+ return error;
+}
+
+int test_files(int src_dir, int dst_dir, struct options const *options)
+{
+ unsigned int seed = options->rng_seed;
+ int cmd_file = openat(dst_dir, INCFS_PENDING_READS_FILENAME,
+ O_RDONLY | O_CLOEXEC);
+ int err;
+ const char *name = "001";
+ incfs_uuid_t id;
+ size_t size;
+ int i;
+ pthread_t *threads = NULL;
+
+ size = 1 << (rnd(options->size, &seed) + 12);
+ size += rnd(size, &seed);
+
+ if (cmd_file == -1) {
+ err_msg("Could not open command file");
+ return -errno;
+ }
+
+ err = emit_file(cmd_file, NULL, name, &id, size, NULL);
+ if (err) {
+ err_msg("Failed to create file %s", name);
+ return err;
+ }
+
+ threads = malloc(sizeof(pthread_t) * options->readers);
+ if (!threads) {
+ err_msg("Could not allocate memory for threads");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < options->readers; ++i) {
+ struct read_data *read_data = malloc(sizeof(*read_data));
+
+ if (!read_data) {
+ err_msg("Failed to allocate read_data");
+ err = -ENOMEM;
+ break;
+ }
+
+ *read_data = (struct read_data){
+ .filename = name,
+ .dir_fd = dst_dir,
+ .filesize = size,
+ .num_reads = options->num_reads,
+ .rng_seed = seed,
+ };
+
+ rnd(0, &seed);
+
+ err = pthread_create(threads + i, 0, reader, read_data);
+ if (err) {
+ err_msg("Failed to create thread");
+ free(read_data);
+ break;
+ }
+ }
+
+ if (err)
+ cancel_threads = 1;
+ else
+ err = write_data(cmd_file, dst_dir, name, size);
+
+ for (; i > 0; --i) {
+ if (pthread_join(threads[i - 1], NULL)) {
+ err_msg("FATAL: failed to join thread");
+ exit(-errno);
+ }
+ }
+
+ free(threads);
+ close(cmd_file);
+ return err;
+}
+
+int main(int argc, char *const *argv)
+{
+ struct options options;
+ int err;
+ const char *src_dir = "src";
+ const char *dst_dir = "dst";
+ int src_dir_fd = -1;
+ int dst_dir_fd = -1;
+
+ err = parse_options(argc, argv, &options);
+ if (err)
+ return err;
+
+ err = chdir(options.test_dir);
+ if (err) {
+ err_msg("Failed to change to %s", options.test_dir);
+ return -errno;
+ }
+
+ err = remove_dir(src_dir) || remove_dir(dst_dir);
+ if (err)
+ return err;
+
+ err = mkdir(src_dir, 0700);
+ if (err) {
+ err_msg("Failed to make directory %s", src_dir);
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = mkdir(dst_dir, 0700);
+ if (err) {
+ err_msg("Failed to make directory %s", src_dir);
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = mount_fs(dst_dir, src_dir, options.timeout);
+ if (err) {
+ err_msg("Failed to mount incfs");
+ goto cleanup;
+ }
+
+ src_dir_fd = open(src_dir, O_RDONLY | O_CLOEXEC);
+ dst_dir_fd = open(dst_dir, O_RDONLY | O_CLOEXEC);
+ if (src_dir_fd == -1 || dst_dir_fd == -1) {
+ err_msg("Failed to open src or dst dir");
+ err = -errno;
+ goto cleanup;
+ }
+
+ err = test_files(src_dir_fd, dst_dir_fd, &options);
+
+cleanup:
+ close(src_dir_fd);
+ close(dst_dir_fd);
+ if (!options.no_cleanup) {
+ umount(dst_dir);
+ remove_dir(dst_dir);
+ remove_dir(src_dir);
+ }
+
+ return err;
+}
diff --git a/tools/testing/selftests/filesystems/incfs/incfs_test.c b/tools/testing/selftests/filesystems/incfs/incfs_test.c
new file mode 100644
index 0000000..10c15fa
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/incfs_test.c
@@ -0,0 +1,4803 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+#define _GNU_SOURCE
+
+#include <alloca.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <lz4.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <sys/inotify.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/xattr.h>
+#include <sys/statvfs.h>
+
+#include <linux/random.h>
+#include <linux/stat.h>
+#include <linux/unistd.h>
+
+#include <openssl/pem.h>
+#include <openssl/x509.h>
+
+#include <kselftest.h>
+#include <include/uapi/linux/fsverity.h>
+
+#include "utils.h"
+
+/* Can't include uapi/linux/fs.h because it clashes with mount.h */
+#define FS_IOC_GETFLAGS _IOR('f', 1, long)
+#define FS_VERITY_FL 0x00100000 /* Verity protected inode */
+
+#define TEST_SKIP 2
+#define TEST_FAILURE 1
+#define TEST_SUCCESS 0
+
+#define INCFS_ROOT_INODE 0
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define le16_to_cpu(x) (x)
+#define le32_to_cpu(x) (x)
+#define le64_to_cpu(x) (x)
+#else
+#error Big endian not supported!
+#endif
+
+struct {
+ int file;
+ int test;
+ bool verbose;
+} options;
+
+#define TESTCOND(condition) \
+ do { \
+ if (!(condition)) { \
+ ksft_print_msg("%s failed %d\n", \
+ __func__, __LINE__); \
+ goto out; \
+ } else if (options.verbose) \
+ ksft_print_msg("%s succeeded %d\n", \
+ __func__, __LINE__); \
+ } while (false)
+
+#define TEST(statement, condition) \
+ do { \
+ statement; \
+ TESTCOND(condition); \
+ } while (false)
+
+#define TESTEQUAL(statement, res) \
+ TESTCOND((statement) == (res))
+
+#define TESTNE(statement, res) \
+ TESTCOND((statement) != (res))
+
+#define TESTSYSCALL(statement) \
+ do { \
+ int res = statement; \
+ \
+ if (res) \
+ ksft_print_msg("Failed: %s (%d)\n", \
+ strerror(errno), errno); \
+ TESTEQUAL(res, 0); \
+ } while (false)
+
+void print_bytes(const void *data, size_t size)
+{
+ const uint8_t *bytes = data;
+ int i;
+
+ for (i = 0; i < size; ++i) {
+ if (i % 0x10 == 0)
+ printf("%08x:", i);
+ printf("%02x ", (unsigned int) bytes[i]);
+ if (i % 0x10 == 0x0f)
+ printf("\n");
+ }
+
+ if (i % 0x10 != 0)
+ printf("\n");
+}
+
+struct hash_block {
+ char data[INCFS_DATA_FILE_BLOCK_SIZE];
+};
+
+struct test_signature {
+ void *data;
+ size_t size;
+
+ char add_data[100];
+ size_t add_data_size;
+};
+
+struct test_file {
+ int index;
+ incfs_uuid_t id;
+ char *name;
+ off_t size;
+ char root_hash[INCFS_MAX_HASH_SIZE];
+ struct hash_block *mtree;
+ int mtree_block_count;
+ struct test_signature sig;
+ unsigned char *verity_sig;
+ size_t verity_sig_size;
+};
+
+struct test_files_set {
+ struct test_file *files;
+ int files_count;
+};
+
+struct linux_dirent64 {
+ uint64_t d_ino;
+ int64_t d_off;
+ unsigned short d_reclen;
+ unsigned char d_type;
+ char d_name[0];
+} __packed;
+
+struct test_files_set get_test_files_set(void)
+{
+ static struct test_file files[] = {
+ { .index = 0, .name = "file_one_byte", .size = 1 },
+ { .index = 1,
+ .name = "file_one_block",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE },
+ { .index = 2,
+ .name = "file_one_and_a_half_blocks",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE +
+ INCFS_DATA_FILE_BLOCK_SIZE / 2 },
+ { .index = 3,
+ .name = "file_three",
+ .size = 300 * INCFS_DATA_FILE_BLOCK_SIZE + 3 },
+ { .index = 4,
+ .name = "file_four",
+ .size = 400 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 5,
+ .name = "file_five",
+ .size = 500 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 6,
+ .name = "file_six",
+ .size = 600 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 7,
+ .name = "file_seven",
+ .size = 700 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 8,
+ .name = "file_eight",
+ .size = 800 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 9,
+ .name = "file_nine",
+ .size = 900 * INCFS_DATA_FILE_BLOCK_SIZE + 7 },
+ { .index = 10, .name = "file_big", .size = 500 * 1024 * 1024 }
+ };
+
+ if (options.file)
+ return (struct test_files_set) {
+ .files = files + options.file - 1,
+ .files_count = 1,
+ };
+
+ return (struct test_files_set){ .files = files,
+ .files_count = ARRAY_SIZE(files) };
+}
+
+struct test_files_set get_small_test_files_set(void)
+{
+ static struct test_file files[] = {
+ { .index = 0, .name = "file_one_byte", .size = 1 },
+ { .index = 1,
+ .name = "file_one_block",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE },
+ { .index = 2,
+ .name = "file_one_and_a_half_blocks",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE +
+ INCFS_DATA_FILE_BLOCK_SIZE / 2 },
+ { .index = 3,
+ .name = "file_three",
+ .size = 300 * INCFS_DATA_FILE_BLOCK_SIZE + 3 },
+ { .index = 4,
+ .name = "file_four",
+ .size = 400 * INCFS_DATA_FILE_BLOCK_SIZE + 7 }
+ };
+ return (struct test_files_set){ .files = files,
+ .files_count = ARRAY_SIZE(files) };
+}
+
+static int get_file_block_seed(int file, int block)
+{
+ return 7919 * file + block;
+}
+
+static loff_t min(loff_t a, loff_t b)
+{
+ return a < b ? a : b;
+}
+
+static int ilog2(size_t n)
+{
+ int l = 0;
+
+ while (n > 1) {
+ ++l;
+ n >>= 1;
+ }
+ return l;
+}
+
+static pid_t flush_and_fork(void)
+{
+ fflush(stdout);
+ return fork();
+}
+
+static void print_error(char *msg)
+{
+ ksft_print_msg("%s: %s\n", msg, strerror(errno));
+}
+
+static int wait_for_process(pid_t pid)
+{
+ int status;
+ int wait_res;
+
+ wait_res = waitpid(pid, &status, 0);
+ if (wait_res <= 0) {
+ print_error("Can't wait for the child");
+ return -EINVAL;
+ }
+ if (!WIFEXITED(status)) {
+ ksft_print_msg("Unexpected child status pid=%d\n", pid);
+ return -EINVAL;
+ }
+ status = WEXITSTATUS(status);
+ if (status != 0)
+ return status;
+ return 0;
+}
+
+static void rnd_buf(uint8_t *data, size_t len, unsigned int seed)
+{
+ int i;
+
+ for (i = 0; i < len; i++) {
+ seed = 1103515245 * seed + 12345;
+ data[i] = (uint8_t)(seed >> (i % 13));
+ }
+}
+
+char *bin2hex(char *dst, const void *src, size_t count)
+{
+ const unsigned char *_src = src;
+ static const char hex_asc[] = "0123456789abcdef";
+
+ while (count--) {
+ unsigned char x = *_src++;
+
+ *dst++ = hex_asc[(x & 0xf0) >> 4];
+ *dst++ = hex_asc[(x & 0x0f)];
+ }
+ *dst = 0;
+ return dst;
+}
+
+static char *get_index_filename(const char *mnt_dir, incfs_uuid_t id)
+{
+ char path[FILENAME_MAX];
+ char str_id[1 + 2 * sizeof(id)];
+
+ bin2hex(str_id, id.bytes, sizeof(id.bytes));
+ snprintf(path, ARRAY_SIZE(path), "%s/.index/%s", mnt_dir, str_id);
+
+ return strdup(path);
+}
+
+static char *get_incomplete_filename(const char *mnt_dir, incfs_uuid_t id)
+{
+ char path[FILENAME_MAX];
+ char str_id[1 + 2 * sizeof(id)];
+
+ bin2hex(str_id, id.bytes, sizeof(id.bytes));
+ snprintf(path, ARRAY_SIZE(path), "%s/.incomplete/%s", mnt_dir, str_id);
+
+ return strdup(path);
+}
+
+int open_file_by_id(const char *mnt_dir, incfs_uuid_t id, bool use_ioctl)
+{
+ char *path = get_index_filename(mnt_dir, id);
+ int cmd_fd = open_commands_file(mnt_dir);
+ int fd = open(path, O_RDWR | O_CLOEXEC);
+ struct incfs_permit_fill permit_fill = {
+ .file_descriptor = fd,
+ };
+ int error = 0;
+
+ if (fd < 0) {
+ print_error("Can't open file by id.");
+ error = -errno;
+ goto out;
+ }
+
+ if (use_ioctl && ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ print_error("Failed to call PERMIT_FILL");
+ error = -errno;
+ goto out;
+ }
+
+ if (ioctl(fd, INCFS_IOC_PERMIT_FILL, &permit_fill) != -1) {
+ print_error(
+ "Successfully called PERMIT_FILL on non pending_read file");
+ return -errno;
+ goto out;
+ }
+
+out:
+ free(path);
+ close(cmd_fd);
+
+ if (error) {
+ close(fd);
+ return error;
+ }
+
+ return fd;
+}
+
+int get_file_attr(const char *mnt_dir, incfs_uuid_t id, char *value, int size)
+{
+ char *path = get_index_filename(mnt_dir, id);
+ int res;
+
+ res = getxattr(path, INCFS_XATTR_METADATA_NAME, value, size);
+ if (res < 0)
+ res = -errno;
+
+ free(path);
+ return res;
+}
+
+static bool same_id(incfs_uuid_t *id1, incfs_uuid_t *id2)
+{
+ return !memcmp(id1->bytes, id2->bytes, sizeof(id1->bytes));
+}
+
+ssize_t ZSTD_compress_default(char *data, char *comp_data, size_t data_size,
+ size_t comp_size)
+{
+ return ZSTD_compress(comp_data, comp_size, data, data_size, 1);
+}
+
+static int emit_test_blocks(const char *mnt_dir, struct test_file *file,
+ int blocks[], int count)
+{
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+ uint8_t comp_data[2 * INCFS_DATA_FILE_BLOCK_SIZE];
+ int block_count = (count > 32) ? 32 : count;
+ int data_buf_size = 2 * INCFS_DATA_FILE_BLOCK_SIZE * block_count;
+ uint8_t *data_buf = malloc(data_buf_size);
+ uint8_t *current_data = data_buf;
+ uint8_t *data_end = data_buf + data_buf_size;
+ struct incfs_fill_block *block_buf =
+ calloc(block_count, sizeof(struct incfs_fill_block));
+ struct incfs_fill_blocks fill_blocks = {
+ .count = block_count,
+ .fill_blocks = ptr_to_u64(block_buf),
+ };
+ ssize_t write_res = 0;
+ int fd = -1;
+ int error = 0;
+ int i = 0;
+ int blocks_written = 0;
+
+ for (i = 0; i < block_count; i++) {
+ int block_index = blocks[i];
+ bool compress_zstd = (file->index + block_index) % 4 == 2;
+ bool compress_lz4 = (file->index + block_index) % 4 == 0;
+ int seed = get_file_block_seed(file->index, block_index);
+ off_t block_offset =
+ ((off_t)block_index) * INCFS_DATA_FILE_BLOCK_SIZE;
+ size_t block_size = 0;
+
+ if (block_offset > file->size) {
+ error = -EINVAL;
+ break;
+ }
+ if (file->size - block_offset >
+ INCFS_DATA_FILE_BLOCK_SIZE)
+ block_size = INCFS_DATA_FILE_BLOCK_SIZE;
+ else
+ block_size = file->size - block_offset;
+
+ rnd_buf(data, block_size, seed);
+ if (compress_lz4) {
+ size_t comp_size = LZ4_compress_default((char *)data,
+ (char *)comp_data, block_size,
+ ARRAY_SIZE(comp_data));
+
+ if (comp_size <= 0) {
+ error = -EBADMSG;
+ break;
+ }
+ if (current_data + comp_size > data_end) {
+ error = -ENOMEM;
+ break;
+ }
+ memcpy(current_data, comp_data, comp_size);
+ block_size = comp_size;
+ block_buf[i].compression = COMPRESSION_LZ4;
+ } else if (compress_zstd) {
+ size_t comp_size = ZSTD_compress(comp_data,
+ ARRAY_SIZE(comp_data), data, block_size,
+ 1);
+
+ if (comp_size <= 0) {
+ error = -EBADMSG;
+ break;
+ }
+ if (current_data + comp_size > data_end) {
+ error = -ENOMEM;
+ break;
+ }
+ memcpy(current_data, comp_data, comp_size);
+ block_size = comp_size;
+ block_buf[i].compression = COMPRESSION_ZSTD;
+ } else {
+ if (current_data + block_size > data_end) {
+ error = -ENOMEM;
+ break;
+ }
+ memcpy(current_data, data, block_size);
+ block_buf[i].compression = COMPRESSION_NONE;
+ }
+
+ block_buf[i].block_index = block_index;
+ block_buf[i].data_len = block_size;
+ block_buf[i].data = ptr_to_u64(current_data);
+ current_data += block_size;
+ }
+
+ if (!error) {
+ fd = open_file_by_id(mnt_dir, file->id, false);
+ if (fd < 0) {
+ error = -errno;
+ goto out;
+ }
+ write_res = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ if (write_res >= 0) {
+ ksft_print_msg("Wrote to file via normal fd error\n");
+ error = -EPERM;
+ goto out;
+ }
+
+ close(fd);
+ fd = open_file_by_id(mnt_dir, file->id, true);
+ if (fd < 0) {
+ error = -errno;
+ goto out;
+ }
+ write_res = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ if (write_res < 0)
+ error = -errno;
+ else
+ blocks_written = write_res;
+ }
+ if (error) {
+ ksft_print_msg(
+ "Writing data block error. Write returned: %d. Error:%s\n",
+ write_res, strerror(-error));
+ }
+
+out:
+ free(block_buf);
+ free(data_buf);
+ close(fd);
+ return (error < 0) ? error : blocks_written;
+}
+
+static int emit_test_block(const char *mnt_dir, struct test_file *file,
+ int block_index)
+{
+ int res = emit_test_blocks(mnt_dir, file, &block_index, 1);
+
+ if (res == 0)
+ return -EINVAL;
+ if (res == 1)
+ return 0;
+ return res;
+}
+
+static void shuffle(int array[], int count, unsigned int seed)
+{
+ int i;
+
+ for (i = 0; i < count - 1; i++) {
+ int items_left = count - i;
+ int shuffle_index;
+ int v;
+
+ seed = 1103515245 * seed + 12345;
+ shuffle_index = i + seed % items_left;
+
+ v = array[shuffle_index];
+ array[shuffle_index] = array[i];
+ array[i] = v;
+ }
+}
+
+static int emit_test_file_data(const char *mount_dir, struct test_file *file)
+{
+ int i;
+ int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ int *block_indexes = NULL;
+ int result = 0;
+ int blocks_written = 0;
+
+ if (file->size == 0)
+ return 0;
+
+ block_indexes = calloc(block_cnt, sizeof(*block_indexes));
+ for (i = 0; i < block_cnt; i++)
+ block_indexes[i] = i;
+ shuffle(block_indexes, block_cnt, file->index);
+
+ for (i = 0; i < block_cnt; i += blocks_written) {
+ blocks_written = emit_test_blocks(mount_dir, file,
+ block_indexes + i, block_cnt - i);
+ if (blocks_written < 0) {
+ result = blocks_written;
+ goto out;
+ }
+ if (blocks_written == 0) {
+ result = -EIO;
+ goto out;
+ }
+ }
+out:
+ free(block_indexes);
+ return result;
+}
+
+static loff_t read_whole_file(const char *filename)
+{
+ int fd = -1;
+ loff_t result;
+ loff_t bytes_read = 0;
+ uint8_t buff[16 * 1024];
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ if (fd <= 0)
+ return fd;
+
+ while (1) {
+ int read_result = read(fd, buff, ARRAY_SIZE(buff));
+
+ if (read_result < 0) {
+ print_error("Error during reading from a file.");
+ result = -errno;
+ goto cleanup;
+ } else if (read_result == 0)
+ break;
+
+ bytes_read += read_result;
+ }
+ result = bytes_read;
+
+cleanup:
+ close(fd);
+ return result;
+}
+
+static int read_test_file(uint8_t *buf, size_t len, char *filename,
+ int block_idx)
+{
+ int fd = -1;
+ int result;
+ int bytes_read = 0;
+ size_t bytes_to_read = len;
+ off_t offset = ((off_t)block_idx) * INCFS_DATA_FILE_BLOCK_SIZE;
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ if (fd <= 0)
+ return fd;
+
+ if (lseek(fd, offset, SEEK_SET) != offset) {
+ print_error("Seek error");
+ return -errno;
+ }
+
+ while (bytes_read < bytes_to_read) {
+ int read_result =
+ read(fd, buf + bytes_read, bytes_to_read - bytes_read);
+ if (read_result < 0) {
+ result = -errno;
+ goto cleanup;
+ } else if (read_result == 0)
+ break;
+
+ bytes_read += read_result;
+ }
+ result = bytes_read;
+
+cleanup:
+ close(fd);
+ return result;
+}
+
+static char *create_backing_dir(const char *mount_dir)
+{
+ struct stat st;
+ char backing_dir_name[255];
+
+ snprintf(backing_dir_name, ARRAY_SIZE(backing_dir_name), "%s-src",
+ mount_dir);
+
+ if (stat(backing_dir_name, &st) == 0) {
+ if (S_ISDIR(st.st_mode)) {
+ int error = delete_dir_tree(backing_dir_name);
+
+ if (error) {
+ ksft_print_msg(
+ "Can't delete existing backing dir. %d\n",
+ error);
+ return NULL;
+ }
+ } else {
+ if (unlink(backing_dir_name)) {
+ print_error("Can't clear backing dir");
+ return NULL;
+ }
+ }
+ }
+
+ if (mkdir(backing_dir_name, 0777)) {
+ if (errno != EEXIST) {
+ print_error("Can't open/create backing dir");
+ return NULL;
+ }
+ }
+
+ return strdup(backing_dir_name);
+}
+
+static int validate_test_file_content_with_seed(const char *mount_dir,
+ struct test_file *file,
+ unsigned int shuffle_seed)
+{
+ int error = -1;
+ char *filename = concat_file_name(mount_dir, file->name);
+ off_t size = file->size;
+ loff_t actual_size = get_file_size(filename);
+ int block_cnt = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ int *block_indexes = NULL;
+ int i;
+
+ block_indexes = alloca(sizeof(int) * block_cnt);
+ for (i = 0; i < block_cnt; i++)
+ block_indexes[i] = i;
+
+ if (shuffle_seed != 0)
+ shuffle(block_indexes, block_cnt, shuffle_seed);
+
+ if (actual_size != size) {
+ ksft_print_msg(
+ "File size doesn't match. name: %s expected size:%ld actual size:%ld\n",
+ filename, size, actual_size);
+ error = -1;
+ goto failure;
+ }
+
+ for (i = 0; i < block_cnt; i++) {
+ int block_idx = block_indexes[i];
+ uint8_t expected_block[INCFS_DATA_FILE_BLOCK_SIZE];
+ uint8_t actual_block[INCFS_DATA_FILE_BLOCK_SIZE];
+ int seed = get_file_block_seed(file->index, block_idx);
+ size_t bytes_to_compare = min(
+ (off_t)INCFS_DATA_FILE_BLOCK_SIZE,
+ size - ((off_t)block_idx) * INCFS_DATA_FILE_BLOCK_SIZE);
+ int read_result =
+ read_test_file(actual_block, INCFS_DATA_FILE_BLOCK_SIZE,
+ filename, block_idx);
+ if (read_result < 0) {
+ ksft_print_msg(
+ "Error reading block %d from file %s. Error: %s\n",
+ block_idx, filename, strerror(-read_result));
+ error = read_result;
+ goto failure;
+ }
+ rnd_buf(expected_block, INCFS_DATA_FILE_BLOCK_SIZE, seed);
+ if (memcmp(expected_block, actual_block, bytes_to_compare)) {
+ ksft_print_msg(
+ "File contents don't match. name: %s block:%d\n",
+ file->name, block_idx);
+ error = -2;
+ goto failure;
+ }
+ }
+ free(filename);
+ return 0;
+
+failure:
+ free(filename);
+ return error;
+}
+
+static int validate_test_file_content(const char *mount_dir,
+ struct test_file *file)
+{
+ return validate_test_file_content_with_seed(mount_dir, file, 0);
+}
+
+static int data_producer(const char *mount_dir, struct test_files_set *test_set)
+{
+ int ret = 0;
+ int timeout_ms = 1000;
+ struct incfs_pending_read_info prs[100] = {};
+ int prs_size = ARRAY_SIZE(prs);
+ int fd = open_commands_file(mount_dir);
+
+ if (fd < 0)
+ return -errno;
+
+ while ((ret = wait_for_pending_reads(fd, timeout_ms, prs, prs_size)) >
+ 0) {
+ int read_count = ret;
+ int i;
+
+ for (i = 0; i < read_count; i++) {
+ int j = 0;
+ struct test_file *file = NULL;
+
+ for (j = 0; j < test_set->files_count; j++) {
+ bool same = same_id(&(test_set->files[j].id),
+ &(prs[i].file_id));
+
+ if (same) {
+ file = &test_set->files[j];
+ break;
+ }
+ }
+ if (!file) {
+ ksft_print_msg(
+ "Unknown file in pending reads.\n");
+ break;
+ }
+
+ ret = emit_test_block(mount_dir, file,
+ prs[i].block_index);
+ if (ret < 0) {
+ ksft_print_msg("Emitting test data error: %s\n",
+ strerror(-ret));
+ break;
+ }
+ }
+ }
+ close(fd);
+ return ret;
+}
+
+static int data_producer2(const char *mount_dir,
+ struct test_files_set *test_set)
+{
+ int ret = 0;
+ int timeout_ms = 1000;
+ struct incfs_pending_read_info2 prs[100] = {};
+ int prs_size = ARRAY_SIZE(prs);
+ int fd = open_commands_file(mount_dir);
+
+ if (fd < 0)
+ return -errno;
+
+ while ((ret = wait_for_pending_reads2(fd, timeout_ms, prs, prs_size)) >
+ 0) {
+ int read_count = ret;
+ int i;
+
+ for (i = 0; i < read_count; i++) {
+ int j = 0;
+ struct test_file *file = NULL;
+
+ for (j = 0; j < test_set->files_count; j++) {
+ bool same = same_id(&(test_set->files[j].id),
+ &(prs[i].file_id));
+
+ if (same) {
+ file = &test_set->files[j];
+ break;
+ }
+ }
+ if (!file) {
+ ksft_print_msg(
+ "Unknown file in pending reads.\n");
+ break;
+ }
+
+ ret = emit_test_block(mount_dir, file,
+ prs[i].block_index);
+ if (ret < 0) {
+ ksft_print_msg("Emitting test data error: %s\n",
+ strerror(-ret));
+ break;
+ }
+ }
+ }
+ close(fd);
+ return ret;
+}
+
+static int build_mtree(struct test_file *file)
+{
+ char data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ const int digest_size = SHA256_DIGEST_SIZE;
+ const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
+ int block_count = 0;
+ int hash_block_count = 0;
+ int total_tree_block_count = 0;
+ int tree_lvl_index[INCFS_MAX_MTREE_LEVELS] = {};
+ int tree_lvl_count[INCFS_MAX_MTREE_LEVELS] = {};
+ int levels_count = 0;
+ int i, level;
+
+ if (file->size == 0)
+ return 0;
+
+ block_count = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ hash_block_count = block_count;
+ for (i = 0; hash_block_count > 1; i++) {
+ hash_block_count = (hash_block_count + hash_per_block - 1)
+ / hash_per_block;
+ tree_lvl_count[i] = hash_block_count;
+ total_tree_block_count += hash_block_count;
+ }
+ levels_count = i;
+
+ for (i = 0; i < levels_count; i++) {
+ int prev_lvl_base = (i == 0) ? total_tree_block_count :
+ tree_lvl_index[i - 1];
+
+ tree_lvl_index[i] = prev_lvl_base - tree_lvl_count[i];
+ }
+
+ file->mtree_block_count = total_tree_block_count;
+ if (block_count == 1) {
+ int seed = get_file_block_seed(file->index, 0);
+
+ memset(data, 0, INCFS_DATA_FILE_BLOCK_SIZE);
+ rnd_buf((uint8_t *)data, file->size, seed);
+ sha256(data, INCFS_DATA_FILE_BLOCK_SIZE, file->root_hash);
+ return 0;
+ }
+
+ file->mtree = calloc(total_tree_block_count, sizeof(*file->mtree));
+ /* Build level 0 hashes. */
+ for (i = 0; i < block_count; i++) {
+ off_t offset = i * INCFS_DATA_FILE_BLOCK_SIZE;
+ size_t block_size = INCFS_DATA_FILE_BLOCK_SIZE;
+ int block_index = tree_lvl_index[0] +
+ i / hash_per_block;
+ int block_off = (i % hash_per_block) * digest_size;
+ int seed = get_file_block_seed(file->index, i);
+ char *hash_ptr = file->mtree[block_index].data + block_off;
+
+ if (file->size - offset < block_size) {
+ block_size = file->size - offset;
+ memset(data, 0, INCFS_DATA_FILE_BLOCK_SIZE);
+ }
+
+ rnd_buf((uint8_t *)data, block_size, seed);
+ sha256(data, INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
+ }
+
+ /* Build higher levels of hash tree. */
+ for (level = 1; level < levels_count; level++) {
+ int prev_lvl_base = tree_lvl_index[level - 1];
+ int prev_lvl_count = tree_lvl_count[level - 1];
+
+ for (i = 0; i < prev_lvl_count; i++) {
+ int block_index =
+ i / hash_per_block + tree_lvl_index[level];
+ int block_off = (i % hash_per_block) * digest_size;
+ char *hash_ptr =
+ file->mtree[block_index].data + block_off;
+
+ sha256(file->mtree[i + prev_lvl_base].data,
+ INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
+ }
+ }
+
+ /* Calculate root hash from the top block */
+ sha256(file->mtree[0].data,
+ INCFS_DATA_FILE_BLOCK_SIZE, file->root_hash);
+
+ return 0;
+}
+
+static int load_hash_tree(const char *mount_dir, struct test_file *file)
+{
+ int err;
+ int i;
+ int fd;
+ struct incfs_fill_blocks fill_blocks = {
+ .count = file->mtree_block_count,
+ };
+ struct incfs_fill_block *fill_block_array =
+ calloc(fill_blocks.count, sizeof(struct incfs_fill_block));
+
+ if (fill_blocks.count == 0)
+ return 0;
+
+ if (!fill_block_array)
+ return -ENOMEM;
+ fill_blocks.fill_blocks = ptr_to_u64(fill_block_array);
+
+ for (i = 0; i < fill_blocks.count; i++) {
+ fill_block_array[i] = (struct incfs_fill_block){
+ .block_index = i,
+ .data_len = INCFS_DATA_FILE_BLOCK_SIZE,
+ .data = ptr_to_u64(file->mtree[i].data),
+ .flags = INCFS_BLOCK_FLAGS_HASH
+ };
+ }
+
+ fd = open_file_by_id(mount_dir, file->id, false);
+ if (fd < 0) {
+ err = errno;
+ goto failure;
+ }
+
+ err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ close(fd);
+ if (err >= 0) {
+ err = -EPERM;
+ goto failure;
+ }
+
+ fd = open_file_by_id(mount_dir, file->id, true);
+ if (fd < 0) {
+ err = errno;
+ goto failure;
+ }
+
+ err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ close(fd);
+ if (err < fill_blocks.count)
+ err = errno;
+ else
+ err = 0;
+
+failure:
+ free(fill_block_array);
+ return err;
+}
+
+static int cant_touch_index_test(const char *mount_dir)
+{
+ char *file_name = "test_file";
+ int file_size = 123;
+ incfs_uuid_t file_id;
+ char *index_path = concat_file_name(mount_dir, ".index");
+ char *subdir = concat_file_name(index_path, "subdir");
+ char *dst_name = concat_file_name(mount_dir, "something");
+ char *filename_in_index = NULL;
+ char *file_path = concat_file_name(mount_dir, file_name);
+ char *backing_dir;
+ int cmd_fd = -1;
+ int err;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+ free(backing_dir);
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+
+ err = mkdir(subdir, 0777);
+ if (err == 0 || errno != EBUSY) {
+ print_error("Shouldn't be able to crate subdir in index\n");
+ goto failure;
+ }
+
+ err = rmdir(index_path);
+ if (err == 0 || errno != EBUSY) {
+ print_error(".index directory should not be removed\n");
+ goto failure;
+ }
+
+ err = emit_file(cmd_fd, ".index", file_name, &file_id,
+ file_size, NULL);
+ if (err != -EBUSY) {
+ print_error("Shouldn't be able to crate a file in index\n");
+ goto failure;
+ }
+
+ err = emit_file(cmd_fd, NULL, file_name, &file_id,
+ file_size, NULL);
+ if (err < 0)
+ goto failure;
+ filename_in_index = get_index_filename(mount_dir, file_id);
+
+ err = unlink(filename_in_index);
+ if (err == 0 || errno != EBUSY) {
+ print_error("Shouldn't be delete from index\n");
+ goto failure;
+ }
+
+
+ err = rename(filename_in_index, dst_name);
+ if (err == 0 || errno != EBUSY) {
+ print_error("Shouldn't be able to move from index\n");
+ goto failure;
+ }
+
+ free(filename_in_index);
+ filename_in_index = concat_file_name(index_path, "abc");
+ err = link(file_path, filename_in_index);
+ if (err == 0 || errno != EBUSY) {
+ print_error("Shouldn't be able to link inside index\n");
+ goto failure;
+ }
+
+ err = rename(index_path, dst_name);
+ if (err == 0 || errno != EBUSY) {
+ print_error("Shouldn't rename .index directory\n");
+ goto failure;
+ }
+
+ close(cmd_fd);
+ free(subdir);
+ free(index_path);
+ free(dst_name);
+ free(filename_in_index);
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ free(subdir);
+ free(dst_name);
+ free(index_path);
+ free(filename_in_index);
+ close(cmd_fd);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static bool iterate_directory(const char *dir_to_iterate, bool root,
+ int file_count)
+{
+ struct expected_name {
+ const char *name;
+ bool root_only;
+ bool found;
+ } names[] = {
+ {INCFS_LOG_FILENAME, true, false},
+ {INCFS_PENDING_READS_FILENAME, true, false},
+ {INCFS_BLOCKS_WRITTEN_FILENAME, true, false},
+ {".index", true, false},
+ {".incomplete", true, false},
+ {"..", false, false},
+ {".", false, false},
+ };
+
+ bool pass = true, found;
+ int i;
+
+ /* Test directory iteration */
+ int fd = open(dir_to_iterate, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
+
+ if (fd < 0) {
+ print_error("Can't open directory\n");
+ return false;
+ }
+
+ for (;;) {
+ /* Enough space for one dirent - no name over 30 */
+ char buf[sizeof(struct linux_dirent64) + NAME_MAX];
+ struct linux_dirent64 *dirent = (struct linux_dirent64 *) buf;
+ int nread;
+ int i;
+
+ for (i = 0; i < NAME_MAX; ++i) {
+ nread = syscall(__NR_getdents64, fd, buf,
+ sizeof(struct linux_dirent64) + i);
+
+ if (nread >= 0)
+ break;
+ if (errno != EINVAL)
+ break;
+ }
+
+ if (nread == 0)
+ break;
+ if (nread < 0) {
+ print_error("Error iterating directory\n");
+ pass = false;
+ goto failure;
+ }
+
+ /* Expected size is rounded up to 8 byte boundary. Not sure if
+ * this is universal truth or just happenstance, but useful test
+ * for the moment
+ */
+ if (nread != (((sizeof(struct linux_dirent64)
+ + strlen(dirent->d_name) + 1) + 7) & ~7)) {
+ print_error("Wrong dirent size");
+ pass = false;
+ goto failure;
+ }
+
+ found = false;
+ for (i = 0; i < sizeof(names) / sizeof(*names); ++i)
+ if (!strcmp(dirent->d_name, names[i].name)) {
+ if (names[i].root_only && !root) {
+ print_error("Root file error");
+ pass = false;
+ goto failure;
+ }
+
+ if (names[i].found) {
+ print_error("File appears twice");
+ pass = false;
+ goto failure;
+ }
+
+ names[i].found = true;
+ found = true;
+ break;
+ }
+
+ if (!found)
+ --file_count;
+ }
+
+ for (i = 0; i < sizeof(names) / sizeof(*names); ++i) {
+ if (!names[i].found)
+ if (root || !names[i].root_only) {
+ print_error("Expected file not present");
+ pass = false;
+ goto failure;
+ }
+ }
+
+ if (file_count) {
+ print_error("Wrong number of files\n");
+ pass = false;
+ goto failure;
+ }
+
+failure:
+ close(fd);
+ return pass;
+}
+
+static int basic_file_ops_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ char *subdir1 = concat_file_name(mount_dir, "subdir1");
+ char *subdir2 = concat_file_name(mount_dir, "subdir2");
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i, err;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+ free(backing_dir);
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ err = mkdir(subdir1, 0777);
+ if (err < 0 && errno != EEXIST) {
+ print_error("Can't create subdir1\n");
+ goto failure;
+ }
+
+ err = mkdir(subdir2, 0777);
+ if (err < 0 && errno != EEXIST) {
+ print_error("Can't create subdir2\n");
+ goto failure;
+ }
+
+ /* Create all test files in subdir1 directory */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ loff_t size;
+ char *file_path = concat_file_name(subdir1, file->name);
+
+ err = emit_file(cmd_fd, "subdir1", file->name, &file->id,
+ file->size, NULL);
+ if (err < 0)
+ goto failure;
+
+ size = get_file_size(file_path);
+ free(file_path);
+ if (size != file->size) {
+ ksft_print_msg("Wrong size %lld of %s.\n",
+ size, file->name);
+ goto failure;
+ }
+ }
+
+ if (!iterate_directory(subdir1, false, file_num))
+ goto failure;
+
+ /* Link the files to subdir2 */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *src_name = concat_file_name(subdir1, file->name);
+ char *dst_name = concat_file_name(subdir2, file->name);
+ loff_t size;
+
+ err = link(src_name, dst_name);
+ if (err < 0) {
+ print_error("Can't move file\n");
+ goto failure;
+ }
+
+ size = get_file_size(dst_name);
+ if (size != file->size) {
+ ksft_print_msg("Wrong size %lld of %s.\n",
+ size, file->name);
+ goto failure;
+ }
+ free(src_name);
+ free(dst_name);
+ }
+
+ /* Move the files from subdir2 to the mount dir */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *src_name = concat_file_name(subdir2, file->name);
+ char *dst_name = concat_file_name(mount_dir, file->name);
+ loff_t size;
+
+ err = rename(src_name, dst_name);
+ if (err < 0) {
+ print_error("Can't move file\n");
+ goto failure;
+ }
+
+ size = get_file_size(dst_name);
+ if (size != file->size) {
+ ksft_print_msg("Wrong size %lld of %s.\n",
+ size, file->name);
+ goto failure;
+ }
+ free(src_name);
+ free(dst_name);
+ }
+
+ /* +2 because there are 2 subdirs */
+ if (!iterate_directory(mount_dir, true, file_num + 2))
+ goto failure;
+
+ /* Open and close all files from the mount dir */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *path = concat_file_name(mount_dir, file->name);
+ int fd;
+
+ fd = open(path, O_RDWR | O_CLOEXEC);
+ free(path);
+ if (fd <= 0) {
+ print_error("Can't open file");
+ goto failure;
+ }
+ if (close(fd)) {
+ print_error("Can't close file");
+ goto failure;
+ }
+ }
+
+ /* Delete all files from the mount dir */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *path = concat_file_name(mount_dir, file->name);
+
+ err = unlink(path);
+ free(path);
+ if (err < 0) {
+ print_error("Can't unlink file");
+ goto failure;
+ }
+ }
+
+ err = delete_dir_tree(subdir1);
+ if (err) {
+ ksft_print_msg("Error deleting subdir1 %d", err);
+ goto failure;
+ }
+
+ err = rmdir(subdir2);
+ if (err) {
+ print_error("Error deleting subdir2");
+ goto failure;
+ }
+
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int dynamic_files_and_data_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ const int missing_file_idx = 5;
+ int cmd_fd = -1;
+ char *backing_dir;
+ int i;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+ free(backing_dir);
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Check that test files don't exist in the filesystem. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *filename = concat_file_name(mount_dir, file->name);
+
+ if (access(filename, F_OK) != -1) {
+ ksft_print_msg(
+ "File %s somehow already exists in a clean FS.\n",
+ filename);
+ goto failure;
+ }
+ free(filename);
+ }
+
+ /* Write test data into the command file. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ int res;
+
+ res = emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL);
+ if (res < 0) {
+ ksft_print_msg("Error %s emiting file %s.\n",
+ strerror(-res), file->name);
+ goto failure;
+ }
+
+ /* Skip writing data to one file so we can check */
+ /* that it's missing later. */
+ if (i == missing_file_idx)
+ continue;
+
+ res = emit_test_file_data(mount_dir, file);
+ if (res) {
+ ksft_print_msg("Error %s emiting data for %s.\n",
+ strerror(-res), file->name);
+ goto failure;
+ }
+ }
+
+ /* Validate contents of the FS */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (i == missing_file_idx) {
+ /* No data has been written to this file. */
+ /* Check for read error; */
+ uint8_t buf;
+ char *filename =
+ concat_file_name(mount_dir, file->name);
+ int res = read_test_file(&buf, 1, filename, 0);
+
+ free(filename);
+ if (res > 0) {
+ ksft_print_msg(
+ "Data present, even though never writtern.\n");
+ goto failure;
+ }
+ if (res != -ETIME) {
+ ksft_print_msg("Wrong error code: %d.\n", res);
+ goto failure;
+ }
+ } else {
+ if (validate_test_file_content(mount_dir, file) < 0)
+ goto failure;
+ }
+ }
+
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int concurrent_reads_and_writes_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ /* Validate each file from that many child processes. */
+ const int child_multiplier = 3;
+ int cmd_fd = -1;
+ char *backing_dir;
+ int status;
+ int i;
+ pid_t producer_pid;
+ pid_t *child_pids = alloca(child_multiplier * file_num * sizeof(pid_t));
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 500) != 0)
+ goto failure;
+ free(backing_dir);
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Tell FS about the files, without actually providing the data. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ int res;
+
+ res = emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL);
+ if (res)
+ goto failure;
+ }
+
+ /* Start child processes acessing data in the files */
+ for (i = 0; i < file_num * child_multiplier; i++) {
+ struct test_file *file = &test.files[i / child_multiplier];
+ pid_t child_pid = flush_and_fork();
+
+ if (child_pid == 0) {
+ /* This is a child process, do the data validation. */
+ int ret = validate_test_file_content_with_seed(
+ mount_dir, file, i);
+ if (ret >= 0) {
+ /* Zero exit status if data is valid. */
+ exit(0);
+ }
+
+ /* Positive status if validation error found. */
+ exit(-ret);
+ } else if (child_pid > 0) {
+ child_pids[i] = child_pid;
+ } else {
+ print_error("Fork error");
+ goto failure;
+ }
+ }
+
+ producer_pid = flush_and_fork();
+ if (producer_pid == 0) {
+ int ret;
+ /*
+ * This is a child that should provide data to
+ * pending reads.
+ */
+
+ ret = data_producer(mount_dir, &test);
+ exit(-ret);
+ } else {
+ status = wait_for_process(producer_pid);
+ if (status != 0) {
+ ksft_print_msg("Data produces failed. %d(%s) ", status,
+ strerror(status));
+ goto failure;
+ }
+ }
+
+ /* Check that all children has finished with 0 exit status */
+ for (i = 0; i < file_num * child_multiplier; i++) {
+ struct test_file *file = &test.files[i / child_multiplier];
+
+ status = wait_for_process(child_pids[i]);
+ if (status != 0) {
+ ksft_print_msg(
+ "Validation for the file %s failed with code %d (%s)\n",
+ file->name, status, strerror(status));
+ goto failure;
+ }
+ }
+
+ /* Check that there are no pending reads left */
+ {
+ struct incfs_pending_read_info prs[1] = {};
+ int timeout = 0;
+ int read_count = wait_for_pending_reads(cmd_fd, timeout, prs,
+ ARRAY_SIZE(prs));
+
+ if (read_count) {
+ ksft_print_msg(
+ "Pending reads pending when all data written\n");
+ goto failure;
+ }
+ }
+
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int work_after_remount_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ const int file_num_stage1 = file_num / 2;
+ const int file_num_stage2 = file_num;
+ char *backing_dir = NULL;
+ int i = 0;
+ int cmd_fd = -1;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Write first half of the data into the command file. (stage 1) */
+ for (i = 0; i < file_num_stage1; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL))
+ goto failure;
+
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
+ }
+
+ /* Unmount and mount again, to see that data is persistent. */
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Write the second half of the data into the command file. (stage 2) */
+ for (; i < file_num_stage2; i++) {
+ struct test_file *file = &test.files[i];
+ int res = emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL);
+
+ if (res)
+ goto failure;
+
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
+ }
+
+ /* Validate contents of the FS */
+ for (i = 0; i < file_num_stage2; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_test_file_content(mount_dir, file) < 0)
+ goto failure;
+ }
+
+ /* Delete all files */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *filename = concat_file_name(mount_dir, file->name);
+ char *filename_in_index = get_index_filename(mount_dir,
+ file->id);
+
+ if (access(filename, F_OK) != 0) {
+ ksft_print_msg("File %s is not visible.\n", filename);
+ goto failure;
+ }
+
+ if (access(filename_in_index, F_OK) != 0) {
+ ksft_print_msg("File %s is not visible.\n",
+ filename_in_index);
+ goto failure;
+ }
+
+ unlink(filename);
+
+ if (access(filename, F_OK) != -1) {
+ ksft_print_msg("File %s is still present.\n", filename);
+ goto failure;
+ }
+
+ if (access(filename_in_index, F_OK) != -1) {
+ ksft_print_msg("File %s is still present.\n",
+ filename_in_index);
+ goto failure;
+ }
+ free(filename);
+ free(filename_in_index);
+ }
+
+ /* Unmount and mount again, to see that deleted files stay deleted. */
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Validate all deleted files are still deleted. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *filename = concat_file_name(mount_dir, file->name);
+
+ if (access(filename, F_OK) != -1) {
+ ksft_print_msg("File %s is still visible.\n", filename);
+ goto failure;
+ }
+ free(filename);
+ }
+
+ /* Final unmount */
+ close(cmd_fd);
+ free(backing_dir);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int attribute_test(const char *mount_dir)
+{
+ char file_attr[] = "metadata123123";
+ char attr_buf[INCFS_MAX_FILE_ATTR_SIZE] = {};
+ int cmd_fd = -1;
+ incfs_uuid_t file_id;
+ int attr_res = 0;
+ char *backing_dir;
+
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ if (emit_file(cmd_fd, NULL, "file", &file_id, 12, file_attr))
+ goto failure;
+
+ /* Test attribute values */
+ attr_res = get_file_attr(mount_dir, file_id, attr_buf,
+ ARRAY_SIZE(attr_buf));
+ if (attr_res != strlen(file_attr)) {
+ ksft_print_msg("Get file attr error: %d\n", attr_res);
+ goto failure;
+ }
+ if (strcmp(attr_buf, file_attr) != 0) {
+ ksft_print_msg("Incorrect file attr value: '%s'", attr_buf);
+ goto failure;
+ }
+
+ /* Unmount and mount again, to see that attributes are persistent. */
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Test attribute values again after remount*/
+ attr_res = get_file_attr(mount_dir, file_id, attr_buf,
+ ARRAY_SIZE(attr_buf));
+ if (attr_res != strlen(file_attr)) {
+ ksft_print_msg("Get dir attr error: %d\n", attr_res);
+ goto failure;
+ }
+ if (strcmp(attr_buf, file_attr) != 0) {
+ ksft_print_msg("Incorrect file attr value: '%s'", attr_buf);
+ goto failure;
+ }
+
+ /* Final unmount */
+ close(cmd_fd);
+ free(backing_dir);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int child_procs_waiting_for_data_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ int cmd_fd = -1;
+ int i;
+ pid_t *child_pids = alloca(file_num * sizeof(pid_t));
+ char *backing_dir;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. (10s wait time) */
+ if (mount_fs(mount_dir, backing_dir, 10000) != 0)
+ goto failure;
+
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Tell FS about the files, without actually providing the data. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL);
+ }
+
+ /* Start child processes acessing data in the files */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ pid_t child_pid = flush_and_fork();
+
+ if (child_pid == 0) {
+ /* This is a child process, do the data validation. */
+ int ret = validate_test_file_content(mount_dir, file);
+
+ if (ret >= 0) {
+ /* Zero exit status if data is valid. */
+ exit(0);
+ }
+
+ /* Positive status if validation error found. */
+ exit(-ret);
+ } else if (child_pid > 0) {
+ child_pids[i] = child_pid;
+ } else {
+ print_error("Fork error");
+ goto failure;
+ }
+ }
+
+ /* Write test data into the command file. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
+ }
+
+ /* Check that all children has finished with 0 exit status */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ int status = wait_for_process(child_pids[i]);
+
+ if (status != 0) {
+ ksft_print_msg(
+ "Validation for the file %s failed with code %d (%s)\n",
+ file->name, status, strerror(status));
+ goto failure;
+ }
+ }
+
+ close(cmd_fd);
+ free(backing_dir);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int multiple_providers_test(const char *mount_dir)
+{
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ const int producer_count = 5;
+ int cmd_fd = -1;
+ int status;
+ int i;
+ pid_t *producer_pids = alloca(producer_count * sizeof(pid_t));
+ char *backing_dir;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. (10s wait time) */
+ if (mount_fs_opt(mount_dir, backing_dir,
+ "read_timeout_ms=10000,report_uid", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Tell FS about the files, without actually providing the data. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL) < 0)
+ goto failure;
+ }
+
+ /* Start producer processes */
+ for (i = 0; i < producer_count; i++) {
+ pid_t producer_pid = flush_and_fork();
+
+ if (producer_pid == 0) {
+ int ret;
+ /*
+ * This is a child that should provide data to
+ * pending reads.
+ */
+
+ ret = data_producer2(mount_dir, &test);
+ exit(-ret);
+ } else if (producer_pid > 0) {
+ producer_pids[i] = producer_pid;
+ } else {
+ print_error("Fork error");
+ goto failure;
+ }
+ }
+
+ /* Validate FS content */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ char *filename = concat_file_name(mount_dir, file->name);
+ loff_t read_result = read_whole_file(filename);
+
+ free(filename);
+ if (read_result != file->size) {
+ ksft_print_msg(
+ "Error validating file %s. Result: %ld\n",
+ file->name, read_result);
+ goto failure;
+ }
+ }
+
+ /* Check that all producers has finished with 0 exit status */
+ for (i = 0; i < producer_count; i++) {
+ status = wait_for_process(producer_pids[i]);
+ if (status != 0) {
+ ksft_print_msg("Producer %d failed with code (%s)\n", i,
+ strerror(status));
+ goto failure;
+ }
+ }
+
+ close(cmd_fd);
+ free(backing_dir);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return TEST_FAILURE;
+}
+
+static int validate_hash_tree(const char *mount_dir, struct test_file *file)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ int fd = -1;
+ unsigned char *buf;
+ int i, err;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TEST(buf = malloc(INCFS_DATA_FILE_BLOCK_SIZE * 8), buf);
+
+ for (i = 0; i < file->mtree_block_count; ) {
+ int blocks_to_read = i % 7 + 1;
+ struct fsverity_read_metadata_arg args = {
+ .metadata_type = FS_VERITY_METADATA_TYPE_MERKLE_TREE,
+ .offset = i * INCFS_DATA_FILE_BLOCK_SIZE,
+ .length = blocks_to_read * INCFS_DATA_FILE_BLOCK_SIZE,
+ .buf_ptr = ptr_to_u64(buf),
+ };
+
+ TEST(err = ioctl(fd, FS_IOC_READ_VERITY_METADATA, &args),
+ err == min(args.length, (file->mtree_block_count - i) *
+ INCFS_DATA_FILE_BLOCK_SIZE));
+ TESTEQUAL(memcmp(buf, file->mtree[i].data, err), 0);
+
+ i += blocks_to_read;
+ }
+
+ result = TEST_SUCCESS;
+
+out:
+ free(buf);
+ close(fd);
+ free(filename);
+ return result;
+}
+
+static int hash_tree_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ const int corrupted_file_idx = 5;
+ int i = 0;
+ int cmd_fd = -1;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ /* Mount FS and release the backing file. */
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Write hashes and data. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+ int res;
+
+ build_mtree(file);
+ res = crypto_emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, file->root_hash,
+ file->sig.add_data);
+
+ if (i == corrupted_file_idx) {
+ /* Corrupt third blocks hash */
+ file->mtree[0].data[2 * SHA256_DIGEST_SIZE] ^= 0xff;
+ }
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
+
+ res = load_hash_tree(mount_dir, file);
+ if (res) {
+ ksft_print_msg("Can't load hashes for %s. error: %s\n",
+ file->name, strerror(-res));
+ goto failure;
+ }
+ }
+
+ /* Validate data */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (i == corrupted_file_idx) {
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+ char *filename =
+ concat_file_name(mount_dir, file->name);
+ int res;
+
+ res = read_test_file(data, INCFS_DATA_FILE_BLOCK_SIZE,
+ filename, 2);
+ free(filename);
+ if (res != -EBADMSG) {
+ ksft_print_msg("Hash violation missed1. %d\n",
+ res);
+ goto failure;
+ }
+ } else if (validate_test_file_content(mount_dir, file) < 0)
+ goto failure;
+ else if (validate_hash_tree(mount_dir, file) < 0)
+ goto failure;
+ }
+
+ /* Unmount and mount again, to that hashes are persistent. */
+ close(cmd_fd);
+ cmd_fd = -1;
+ if (umount(mount_dir) != 0) {
+ print_error("Can't unmout FS");
+ goto failure;
+ }
+ if (mount_fs(mount_dir, backing_dir, 50) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Validate data again */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (i == corrupted_file_idx) {
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+ char *filename =
+ concat_file_name(mount_dir, file->name);
+ int res;
+
+ res = read_test_file(data, INCFS_DATA_FILE_BLOCK_SIZE,
+ filename, 2);
+ free(filename);
+ if (res != -EBADMSG) {
+ ksft_print_msg("Hash violation missed2. %d\n",
+ res);
+ goto failure;
+ }
+ } else if (validate_test_file_content(mount_dir, file) < 0)
+ goto failure;
+ }
+ result = TEST_SUCCESS;
+
+failure:
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ free(file->mtree);
+ }
+
+ close(cmd_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return result;
+}
+
+enum expected_log { FULL_LOG, NO_LOG, PARTIAL_LOG };
+
+static int validate_logs(const char *mount_dir, int log_fd,
+ struct test_file *file,
+ enum expected_log expected_log,
+ bool report_uid, bool expect_data)
+{
+ int result = TEST_FAILURE;
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+ struct incfs_pending_read_info prs[2048] = {};
+ struct incfs_pending_read_info2 prs2[2048] = {};
+ struct incfs_pending_read_info *previous_record = NULL;
+ int prs_size = ARRAY_SIZE(prs);
+ int block_count = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ int expected_read_count, read_count, block_index, read_index;
+ char *filename = NULL;
+ int fd = -1;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+
+ if (block_count > prs_size)
+ block_count = prs_size;
+ expected_read_count = block_count;
+
+ for (block_index = 0; block_index < block_count; block_index++) {
+ int result = pread(fd, data, sizeof(data),
+ INCFS_DATA_FILE_BLOCK_SIZE * block_index);
+
+ /* Make some read logs of type SAME_FILE_NEXT_BLOCK */
+ if (block_index % 100 == 10)
+ usleep(20000);
+
+ /* Skip some blocks to make logs of type SAME_FILE */
+ if (block_index % 10 == 5) {
+ ++block_index;
+ --expected_read_count;
+ }
+
+ if (expect_data)
+ TESTCOND(result > 0);
+
+ if (!expect_data)
+ TESTEQUAL(result, -1);
+ }
+
+ if (report_uid)
+ read_count = wait_for_pending_reads2(log_fd,
+ expected_log == NO_LOG ? 10 : 0,
+ prs2, prs_size);
+ else
+ read_count = wait_for_pending_reads(log_fd,
+ expected_log == NO_LOG ? 10 : 0,
+ prs, prs_size);
+
+ if (expected_log == NO_LOG)
+ TESTEQUAL(read_count, 0);
+
+ if (expected_log == PARTIAL_LOG)
+ TESTCOND(read_count > 0 &&
+ read_count <= expected_read_count);
+
+ if (expected_log == FULL_LOG)
+ TESTEQUAL(read_count, expected_read_count);
+
+ /* If read less than expected, advance block_index appropriately */
+ for (block_index = 0, read_index = 0;
+ read_index < expected_read_count - read_count;
+ block_index++, read_index++)
+ if (block_index % 10 == 5)
+ ++block_index;
+
+ for (read_index = 0; read_index < read_count;
+ block_index++, read_index++) {
+ struct incfs_pending_read_info *record = report_uid ?
+ (struct incfs_pending_read_info *) &prs2[read_index] :
+ &prs[read_index];
+
+ TESTCOND(same_id(&record->file_id, &file->id));
+ TESTEQUAL(record->block_index, block_index);
+ TESTNE(record->timestamp_us, 0);
+ if (previous_record)
+ TESTEQUAL(record->serial_number,
+ previous_record->serial_number + 1);
+
+ previous_record = record;
+ if (block_index % 10 == 5)
+ ++block_index;
+ }
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ free(filename);
+ return result;
+}
+
+static int read_log_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ int i = 0;
+ int cmd_fd = -1, log_fd = -1;
+ char *backing_dir = NULL;
+
+ /* Create files */
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "readahead=0,report_uid,read_timeout_ms=0",
+ false), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL), 0);
+ }
+ close(cmd_fd);
+ cmd_fd = -1;
+
+ /* Validate logs */
+ TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ FULL_LOG, true, false), 0);
+
+ /* Unmount and mount again without report_uid */
+ close(log_fd);
+ log_fd = -1;
+ TESTEQUAL(umount(mount_dir), 0);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "readahead=0,read_timeout_ms=0", false), 0);
+
+ TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ FULL_LOG, false, false), 0);
+
+ /* No read log to make sure poll doesn't crash */
+ close(log_fd);
+ log_fd = -1;
+ TESTEQUAL(umount(mount_dir), 0);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "readahead=0,rlog_pages=0,read_timeout_ms=0",
+ false), 0);
+
+ TEST(log_fd = open_log_file(mount_dir), log_fd != -1);
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ NO_LOG, false, false), 0);
+
+ /* Remount and check that logs start working again */
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "readahead=0,rlog_pages=1,read_timeout_ms=0",
+ true), 0);
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ PARTIAL_LOG, false, false), 0);
+
+ /* Remount and check that logs continue working */
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "readahead=0,rlog_pages=4,read_timeout_ms=0",
+ true), 0);
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ FULL_LOG, false, false), 0);
+
+ /* Check logs work with data */
+ for (i = 0; i < file_num; i++) {
+ TESTEQUAL(emit_test_file_data(mount_dir, &test.files[i]), 0);
+ TESTEQUAL(validate_logs(mount_dir, log_fd, &test.files[i],
+ FULL_LOG, false, true), 0);
+ }
+
+ /* Final unmount */
+ close(log_fd);
+ log_fd = -1;
+ TESTEQUAL(umount(mount_dir), 0);
+
+ result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ close(log_fd);
+ free(backing_dir);
+ umount(mount_dir);
+ return result;
+}
+
+static int emit_partial_test_file_data(const char *mount_dir,
+ struct test_file *file)
+{
+ int i, j;
+ int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ int *block_indexes = NULL;
+ int result = 0;
+ int blocks_written = 0;
+ int bw_fd = -1;
+ char buffer[20];
+ struct pollfd pollfd;
+ long blocks_written_total, blocks_written_new_total;
+
+ if (file->size == 0)
+ return 0;
+
+ bw_fd = open_blocks_written_file(mount_dir);
+ if (bw_fd == -1)
+ return -errno;
+
+ result = read(bw_fd, buffer, sizeof(buffer));
+ if (result <= 0) {
+ result = -EIO;
+ goto out;
+ }
+
+ buffer[result] = 0;
+ blocks_written_total = strtol(buffer, NULL, 10);
+ result = 0;
+
+ pollfd = (struct pollfd) {
+ .fd = bw_fd,
+ .events = POLLIN,
+ };
+
+ result = poll(&pollfd, 1, 0);
+ if (result) {
+ result = -EIO;
+ goto out;
+ }
+
+ /* Emit 2 blocks, skip 2 blocks etc*/
+ block_indexes = calloc(block_cnt, sizeof(*block_indexes));
+ for (i = 0, j = 0; i < block_cnt; ++i)
+ if ((i & 2) == 0) {
+ block_indexes[j] = i;
+ ++j;
+ }
+
+ for (i = 0; i < j; i += blocks_written) {
+ blocks_written = emit_test_blocks(mount_dir, file,
+ block_indexes + i, j - i);
+ if (blocks_written < 0) {
+ result = blocks_written;
+ goto out;
+ }
+ if (blocks_written == 0) {
+ result = -EIO;
+ goto out;
+ }
+
+ result = poll(&pollfd, 1, 0);
+ if (result != 1 || pollfd.revents != POLLIN) {
+ result = -EIO;
+ goto out;
+ }
+
+ result = read(bw_fd, buffer, sizeof(buffer));
+ buffer[result] = 0;
+ blocks_written_new_total = strtol(buffer, NULL, 10);
+
+ if (blocks_written_new_total - blocks_written_total
+ != blocks_written) {
+ result = -EIO;
+ goto out;
+ }
+
+ blocks_written_total = blocks_written_new_total;
+ result = 0;
+ }
+out:
+ free(block_indexes);
+ close(bw_fd);
+ return result;
+}
+
+static int validate_ranges(const char *mount_dir, struct test_file *file)
+{
+ int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ char *filename = concat_file_name(mount_dir, file->name);
+ int fd;
+ struct incfs_filled_range ranges[128];
+ struct incfs_get_filled_blocks_args fba = {
+ .range_buffer = ptr_to_u64(ranges),
+ .range_buffer_size = sizeof(ranges),
+ };
+ int error = TEST_SUCCESS;
+ int i;
+ int range_cnt;
+ int cmd_fd = -1;
+ struct incfs_permit_fill permit_fill;
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ free(filename);
+ if (fd <= 0)
+ return TEST_FAILURE;
+
+ error = ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ if (error != -1 || errno != EPERM) {
+ ksft_print_msg("INCFS_IOC_GET_FILLED_BLOCKS not blocked\n");
+ error = -EPERM;
+ goto out;
+ }
+
+ cmd_fd = open_commands_file(mount_dir);
+ permit_fill.file_descriptor = fd;
+ if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ print_error("INCFS_IOC_PERMIT_FILL failed");
+ return -EPERM;
+ goto out;
+ }
+
+ error = ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ if (error && errno != ERANGE)
+ goto out;
+
+ if (error && errno == ERANGE && block_cnt < 509)
+ goto out;
+
+ if (!error && block_cnt >= 509) {
+ error = -ERANGE;
+ goto out;
+ }
+
+ if (fba.total_blocks_out != block_cnt) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (fba.data_blocks_out != block_cnt) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ range_cnt = (block_cnt + 3) / 4;
+ if (range_cnt > 128)
+ range_cnt = 128;
+ if (range_cnt != fba.range_buffer_size_out / sizeof(*ranges)) {
+ error = -ERANGE;
+ goto out;
+ }
+
+ error = TEST_SUCCESS;
+ for (i = 0; i < fba.range_buffer_size_out / sizeof(*ranges) - 1; ++i)
+ if (ranges[i].begin != i * 4 || ranges[i].end != i * 4 + 2) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (ranges[i].begin != i * 4 ||
+ (ranges[i].end != i * 4 + 1 && ranges[i].end != i * 4 + 2)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ for (i = 0; i < 64; ++i) {
+ fba.start_index = i * 2;
+ fba.end_index = i * 2 + 2;
+ error = ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ if (error)
+ goto out;
+
+ if (fba.total_blocks_out != block_cnt) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (fba.start_index >= block_cnt) {
+ if (fba.index_out != fba.start_index) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ break;
+ }
+
+ if (i % 2) {
+ if (fba.range_buffer_size_out != 0) {
+ error = -EINVAL;
+ goto out;
+ }
+ } else {
+ if (fba.range_buffer_size_out != sizeof(*ranges)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (ranges[0].begin != i * 2) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (ranges[0].end != i * 2 + 1 &&
+ ranges[0].end != i * 2 + 2) {
+ error = -EINVAL;
+ goto out;
+ }
+ }
+ }
+
+out:
+ close(fd);
+ close(cmd_fd);
+ return error;
+}
+
+static int get_blocks_test(const char *mount_dir)
+{
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ /* Write data. */
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (emit_file(cmd_fd, NULL, file->name, &file->id, file->size,
+ NULL))
+ goto failure;
+
+ if (emit_partial_test_file_data(mount_dir, file))
+ goto failure;
+ }
+
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_ranges(mount_dir, file))
+ goto failure;
+
+ /*
+ * The smallest files are filled completely, so this checks that
+ * the fast get_filled_blocks path is not causing issues
+ */
+ if (validate_ranges(mount_dir, file))
+ goto failure;
+ }
+
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return TEST_FAILURE;
+}
+
+static int emit_partial_test_file_hash(const char *mount_dir,
+ struct test_file *file)
+{
+ int err;
+ int fd;
+ struct incfs_fill_blocks fill_blocks = {
+ .count = 1,
+ };
+ struct incfs_fill_block *fill_block_array =
+ calloc(fill_blocks.count, sizeof(struct incfs_fill_block));
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+
+ if (file->size <= 4096 / 32 * 4096)
+ return 0;
+
+ if (!fill_block_array)
+ return -ENOMEM;
+ fill_blocks.fill_blocks = ptr_to_u64(fill_block_array);
+
+ rnd_buf(data, sizeof(data), 0);
+
+ fill_block_array[0] =
+ (struct incfs_fill_block){ .block_index = 1,
+ .data_len =
+ INCFS_DATA_FILE_BLOCK_SIZE,
+ .data = ptr_to_u64(data),
+ .flags = INCFS_BLOCK_FLAGS_HASH };
+
+ fd = open_file_by_id(mount_dir, file->id, true);
+ if (fd < 0) {
+ err = errno;
+ goto failure;
+ }
+
+ err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
+ close(fd);
+ if (err < fill_blocks.count)
+ err = errno;
+ else
+ err = 0;
+
+failure:
+ free(fill_block_array);
+ return err;
+}
+
+static int validate_hash_ranges(const char *mount_dir, struct test_file *file)
+{
+ int block_cnt = 1 + (file->size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
+ char *filename = concat_file_name(mount_dir, file->name);
+ int fd;
+ struct incfs_filled_range ranges[128];
+ struct incfs_get_filled_blocks_args fba = {
+ .range_buffer = ptr_to_u64(ranges),
+ .range_buffer_size = sizeof(ranges),
+ };
+ int error = TEST_SUCCESS;
+ int file_blocks = (file->size + INCFS_DATA_FILE_BLOCK_SIZE - 1) /
+ INCFS_DATA_FILE_BLOCK_SIZE;
+ int cmd_fd = -1;
+ struct incfs_permit_fill permit_fill;
+
+ if (file->size <= 4096 / 32 * 4096)
+ return 0;
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ free(filename);
+ if (fd <= 0)
+ return TEST_FAILURE;
+
+ error = ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ if (error != -1 || errno != EPERM) {
+ ksft_print_msg("INCFS_IOC_GET_FILLED_BLOCKS not blocked\n");
+ error = -EPERM;
+ goto out;
+ }
+
+ cmd_fd = open_commands_file(mount_dir);
+ permit_fill.file_descriptor = fd;
+ if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
+ print_error("INCFS_IOC_PERMIT_FILL failed");
+ return -EPERM;
+ goto out;
+ }
+
+ error = ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ if (error)
+ goto out;
+
+ if (fba.total_blocks_out <= block_cnt) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (fba.data_blocks_out != block_cnt) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (fba.range_buffer_size_out != sizeof(struct incfs_filled_range)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ if (ranges[0].begin != file_blocks + 1 ||
+ ranges[0].end != file_blocks + 2) {
+ error = -EINVAL;
+ goto out;
+ }
+
+out:
+ close(cmd_fd);
+ close(fd);
+ return error;
+}
+
+static int get_hash_blocks_test(const char *mount_dir)
+{
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (crypto_emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, file->root_hash,
+ file->sig.add_data))
+ goto failure;
+
+ if (emit_partial_test_file_hash(mount_dir, file))
+ goto failure;
+ }
+
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ if (validate_hash_ranges(mount_dir, file))
+ goto failure;
+ }
+
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return TEST_SUCCESS;
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return TEST_FAILURE;
+}
+
+#define THREE_GB (3LL * 1024 * 1024 * 1024)
+#define FOUR_GB (4LL * 1024 * 1024 * 1024) /* Have 1GB of margin */
+static int large_file_test(const char *mount_dir)
+{
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i;
+ int result = TEST_FAILURE, ret;
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
+ int block_count = THREE_GB / INCFS_DATA_FILE_BLOCK_SIZE;
+ struct incfs_fill_block *block_buf =
+ calloc(block_count, sizeof(struct incfs_fill_block));
+ struct incfs_fill_blocks fill_blocks = {
+ .count = block_count,
+ .fill_blocks = ptr_to_u64(block_buf),
+ };
+ incfs_uuid_t id;
+ int fd = -1;
+ struct statvfs svfs;
+ unsigned long long free_disksz;
+
+ ret = statvfs(mount_dir, &svfs);
+ if (ret) {
+ ksft_print_msg("Can't get disk size. Skipping %s...\n", __func__);
+ return TEST_SKIP;
+ }
+
+ free_disksz = (unsigned long long)svfs.f_bavail * svfs.f_bsize;
+
+ if (FOUR_GB > free_disksz) {
+ ksft_print_msg("Not enough free disk space (%lldMB). Skipping %s...\n",
+ free_disksz >> 20, __func__);
+ return TEST_SKIP;
+ }
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ if (emit_file(cmd_fd, NULL, "very_large_file", &id,
+ (uint64_t)block_count * INCFS_DATA_FILE_BLOCK_SIZE,
+ NULL) < 0)
+ goto failure;
+
+ for (i = 0; i < block_count; i++) {
+ block_buf[i].compression = COMPRESSION_NONE;
+ block_buf[i].block_index = i;
+ block_buf[i].data_len = INCFS_DATA_FILE_BLOCK_SIZE;
+ block_buf[i].data = ptr_to_u64(data);
+ }
+
+ fd = open_file_by_id(mount_dir, id, true);
+ if (fd < 0)
+ goto failure;
+
+ if (ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks) != block_count)
+ goto failure;
+
+ if (emit_file(cmd_fd, NULL, "very_very_large_file", &id, 1LL << 40,
+ NULL) < 0)
+ goto failure;
+
+ result = TEST_SUCCESS;
+
+failure:
+ close(fd);
+ close(cmd_fd);
+ unlink("very_large_file");
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int validate_mapped_file(const char *orig_name, const char *name,
+ size_t size, size_t offset)
+{
+ struct stat st;
+ int orig_fd = -1, fd = -1;
+ size_t block;
+ int result = TEST_FAILURE;
+
+ if (stat(name, &st)) {
+ ksft_print_msg("Failed to stat %s with error %s\n",
+ name, strerror(errno));
+ goto failure;
+ }
+
+ if (size != st.st_size) {
+ ksft_print_msg("Mismatched file sizes for file %s - expected %llu, got %llu\n",
+ name, size, st.st_size);
+ goto failure;
+ }
+
+ fd = open(name, O_RDONLY | O_CLOEXEC);
+ if (fd == -1) {
+ ksft_print_msg("Failed to open %s with error %s\n", name,
+ strerror(errno));
+ goto failure;
+ }
+
+ orig_fd = open(orig_name, O_RDONLY | O_CLOEXEC);
+ if (orig_fd == -1) {
+ ksft_print_msg("Failed to open %s with error %s\n", orig_name,
+ strerror(errno));
+ goto failure;
+ }
+
+ for (block = 0; block < size; block += INCFS_DATA_FILE_BLOCK_SIZE) {
+ uint8_t orig_data[INCFS_DATA_FILE_BLOCK_SIZE];
+ uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE];
+ ssize_t orig_read, mapped_read;
+
+ orig_read = pread(orig_fd, orig_data,
+ INCFS_DATA_FILE_BLOCK_SIZE, block + offset);
+ mapped_read = pread(fd, data, INCFS_DATA_FILE_BLOCK_SIZE,
+ block);
+
+ if (orig_read < mapped_read ||
+ mapped_read != min(size - block,
+ INCFS_DATA_FILE_BLOCK_SIZE)) {
+ ksft_print_msg("Failed to read enough data: %llu %llu %llu %lld %lld\n",
+ block, size, offset, orig_read,
+ mapped_read);
+ goto failure;
+ }
+
+ if (memcmp(orig_data, data, mapped_read)) {
+ ksft_print_msg("Data doesn't match: %llu %llu %llu %lld %lld\n",
+ block, size, offset, orig_read,
+ mapped_read);
+ goto failure;
+ }
+ }
+
+ result = TEST_SUCCESS;
+
+failure:
+ close(orig_fd);
+ close(fd);
+ return result;
+}
+
+static int mapped_file_test(const char *mount_dir)
+{
+ char *backing_dir;
+ int result = TEST_FAILURE;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+
+ backing_dir = create_backing_dir(mount_dir);
+ if (!backing_dir)
+ goto failure;
+
+ if (mount_fs_opt(mount_dir, backing_dir, "readahead=0", false) != 0)
+ goto failure;
+
+ cmd_fd = open_commands_file(mount_dir);
+ if (cmd_fd < 0)
+ goto failure;
+
+ for (i = 0; i < file_num; ++i) {
+ struct test_file *file = &test.files[i];
+ size_t blocks = file->size / INCFS_DATA_FILE_BLOCK_SIZE;
+ size_t mapped_offset = blocks / 4 *
+ INCFS_DATA_FILE_BLOCK_SIZE;
+ size_t mapped_size = file->size / 4 * 3 - mapped_offset;
+ struct incfs_create_mapped_file_args mfa;
+ char mapped_file_name[FILENAME_MAX];
+ char orig_file_path[PATH_MAX];
+ char mapped_file_path[PATH_MAX];
+
+ if (emit_file(cmd_fd, NULL, file->name, &file->id, file->size,
+ NULL) < 0)
+ goto failure;
+
+ if (emit_test_file_data(mount_dir, file))
+ goto failure;
+
+ if (snprintf(mapped_file_name, ARRAY_SIZE(mapped_file_name),
+ "%s.mapped", file->name) < 0)
+ goto failure;
+
+ mfa = (struct incfs_create_mapped_file_args) {
+ .size = mapped_size,
+ .mode = 0664,
+ .file_name = ptr_to_u64(mapped_file_name),
+ .source_file_id = file->id,
+ .source_offset = mapped_offset,
+ };
+
+ result = ioctl(cmd_fd, INCFS_IOC_CREATE_MAPPED_FILE, &mfa);
+ if (result) {
+ ksft_print_msg(
+ "Failed to create mapped file with error %d\n",
+ result);
+ goto failure;
+ }
+
+ result = snprintf(orig_file_path,
+ ARRAY_SIZE(orig_file_path), "%s/%s",
+ mount_dir, file->name);
+
+ if (result < 0 || result >= ARRAY_SIZE(mapped_file_path)) {
+ result = TEST_FAILURE;
+ goto failure;
+ }
+
+ result = snprintf(mapped_file_path,
+ ARRAY_SIZE(mapped_file_path), "%s/%s",
+ mount_dir, mapped_file_name);
+
+ if (result < 0 || result >= ARRAY_SIZE(mapped_file_path)) {
+ result = TEST_FAILURE;
+ goto failure;
+ }
+
+ result = validate_mapped_file(orig_file_path, mapped_file_path,
+ mapped_size, mapped_offset);
+ if (result)
+ goto failure;
+ }
+
+failure:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static const char v1_file[] = {
+ /* Header */
+ /* 0x00: Magic number */
+ 0x49, 0x4e, 0x43, 0x46, 0x53, 0x00, 0x00, 0x00,
+ /* 0x08: Version */
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x10: Header size */
+ 0x38, 0x00,
+ /* 0x12: Block size */
+ 0x00, 0x10,
+ /* 0x14: Flags */
+ 0x00, 0x00, 0x00, 0x00,
+ /* 0x18: First md offset */
+ 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x20: File size */
+ 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x28: UUID */
+ 0x8c, 0x7d, 0xd9, 0x22, 0xad, 0x47, 0x49, 0x4f,
+ 0xc0, 0x2c, 0x38, 0x8e, 0x12, 0xc0, 0x0e, 0xac,
+
+ /* 0x38: Attribute */
+ 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+ 0x31, 0x32, 0x33, 0x31, 0x32, 0x33,
+
+ /* Attribute md record */
+ /* 0x46: Type */
+ 0x02,
+ /* 0x47: Size */
+ 0x25, 0x00,
+ /* 0x49: CRC */
+ 0x9a, 0xef, 0xef, 0x72,
+ /* 0x4d: Next md offset */
+ 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x55: Prev md offset */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x5d: fa_offset */
+ 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x65: fa_size */
+ 0x0e, 0x00,
+ /* 0x67: fa_crc */
+ 0xfb, 0x5e, 0x72, 0x89,
+
+ /* Blockmap table */
+ /* 0x6b: First 10-byte entry */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+ /* Blockmap md record */
+ /* 0x75: Type */
+ 0x01,
+ /* 0x76: Size */
+ 0x23, 0x00,
+ /* 0x78: CRC */
+ 0x74, 0x45, 0xd3, 0xb9,
+ /* 0x7c: Next md offset */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x84: Prev md offset */
+ 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x8c: blockmap offset */
+ 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ /* 0x94: blockmap count */
+ 0x01, 0x00, 0x00, 0x00,
+};
+
+static int compatibility_test(const char *mount_dir)
+{
+ static const char *name = "file";
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ char *filename = NULL;
+ int fd = -1;
+ uint64_t size = 0x0c;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TEST(filename = concat_file_name(backing_dir, name), filename);
+ TEST(fd = open(filename, O_CREAT | O_WRONLY | O_CLOEXEC, 0777),
+ fd != -1);
+ TESTEQUAL(write(fd, v1_file, sizeof(v1_file)), sizeof(v1_file));
+ TESTEQUAL(fsetxattr(fd, INCFS_XATTR_SIZE_NAME, &size, sizeof(size), 0),
+ 0);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 50), 0);
+ free(filename);
+ TEST(filename = concat_file_name(mount_dir, name), filename);
+ close(fd);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ umount(mount_dir);
+ free(backing_dir);
+ free(filename);
+ return result;
+}
+
+static int zero_blocks_written_count(int fd, uint32_t data_blocks_written,
+ uint32_t hash_blocks_written)
+{
+ int test_result = TEST_FAILURE;
+ uint64_t offset;
+ uint8_t type;
+ uint32_t bw;
+
+ /* Get first md record */
+ TESTEQUAL(pread(fd, &offset, sizeof(offset), 24), sizeof(offset));
+
+ /* Find status md record */
+ for (;;) {
+ TESTNE(offset, 0);
+ TESTEQUAL(pread(fd, &type, sizeof(type), le64_to_cpu(offset)),
+ sizeof(type));
+ if (type == 4)
+ break;
+ TESTEQUAL(pread(fd, &offset, sizeof(offset),
+ le64_to_cpu(offset) + 7),
+ sizeof(offset));
+ }
+
+ /* Read blocks_written */
+ offset = le64_to_cpu(offset);
+ TESTEQUAL(pread(fd, &bw, sizeof(bw), offset + 23), sizeof(bw));
+ TESTEQUAL(le32_to_cpu(bw), data_blocks_written);
+ TESTEQUAL(pread(fd, &bw, sizeof(bw), offset + 27), sizeof(bw));
+ TESTEQUAL(le32_to_cpu(bw), hash_blocks_written);
+
+ /* Write out zero */
+ bw = 0;
+ TESTEQUAL(pwrite(fd, &bw, sizeof(bw), offset + 23), sizeof(bw));
+ TESTEQUAL(pwrite(fd, &bw, sizeof(bw), offset + 27), sizeof(bw));
+
+ test_result = TEST_SUCCESS;
+out:
+ return test_result;
+}
+
+static int validate_block_count(const char *mount_dir, const char *backing_dir,
+ struct test_file *file,
+ int total_data_blocks, int filled_data_blocks,
+ int total_hash_blocks, int filled_hash_blocks)
+{
+ char *filename = NULL;
+ char *backing_filename = NULL;
+ int fd = -1;
+ struct incfs_get_block_count_args bca = {};
+ int test_result = TEST_FAILURE;
+ struct incfs_filled_range ranges[128];
+ struct incfs_get_filled_blocks_args fba = {
+ .range_buffer = ptr_to_u64(ranges),
+ .range_buffer_size = sizeof(ranges),
+ };
+ int cmd_fd = -1;
+ struct incfs_permit_fill permit_fill;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(backing_filename = concat_file_name(backing_dir, file->name),
+ backing_filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
+ TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
+ TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
+
+ close(fd);
+ TESTEQUAL(umount(mount_dir), 0);
+ TEST(fd = open(backing_filename, O_RDWR | O_CLOEXEC), fd != -1);
+ TESTEQUAL(zero_blocks_written_count(fd, filled_data_blocks,
+ filled_hash_blocks),
+ TEST_SUCCESS);
+ close(fd);
+ fd = -1;
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
+ 0);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, 0);
+ TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
+ TESTEQUAL(bca.filled_hash_blocks_out, 0);
+
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ permit_fill.file_descriptor = fd;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill), 0);
+ do {
+ ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &fba);
+ fba.start_index = fba.index_out + 1;
+ } while (fba.index_out < fba.total_blocks_out);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
+ TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
+ TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
+
+ test_result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ close(fd);
+ free(filename);
+ free(backing_filename);
+ return test_result;
+}
+
+
+
+static int validate_data_block_count(const char *mount_dir,
+ const char *backing_dir,
+ struct test_file *file)
+{
+ const int total_data_blocks = 1 + (file->size - 1) /
+ INCFS_DATA_FILE_BLOCK_SIZE;
+ const int filled_data_blocks = (total_data_blocks + 1) / 2;
+
+ int test_result = TEST_FAILURE;
+ char *filename = NULL;
+ char *incomplete_filename = NULL;
+ struct stat stat_buf_incomplete, stat_buf_file;
+ int fd = -1;
+ struct incfs_get_block_count_args bca = {};
+ int i;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(incomplete_filename = get_incomplete_filename(mount_dir, file->id),
+ incomplete_filename);
+
+ TESTEQUAL(stat(filename, &stat_buf_file), 0);
+ TESTEQUAL(stat(incomplete_filename, &stat_buf_incomplete), 0);
+ TESTEQUAL(stat_buf_file.st_ino, stat_buf_incomplete.st_ino);
+ TESTEQUAL(stat_buf_file.st_nlink, 3);
+
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, 0);
+ TESTEQUAL(bca.total_hash_blocks_out, 0);
+ TESTEQUAL(bca.filled_hash_blocks_out, 0);
+
+ for (i = 0; i < total_data_blocks; i += 2)
+ TESTEQUAL(emit_test_block(mount_dir, file, i), 0);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, filled_data_blocks);
+ TESTEQUAL(bca.total_hash_blocks_out, 0);
+ TESTEQUAL(bca.filled_hash_blocks_out, 0);
+ close(fd);
+ fd = -1;
+
+ TESTEQUAL(validate_block_count(mount_dir, backing_dir, file,
+ total_data_blocks, filled_data_blocks,
+ 0, 0),
+ 0);
+
+ for (i = 1; i < total_data_blocks; i += 2)
+ TESTEQUAL(emit_test_block(mount_dir, file, i), 0);
+
+ TESTEQUAL(stat(incomplete_filename, &stat_buf_incomplete), -1);
+ TESTEQUAL(errno, ENOENT);
+
+ test_result = TEST_SUCCESS;
+out:
+ close(fd);
+ free(incomplete_filename);
+ free(filename);
+ return test_result;
+}
+
+static int data_block_count_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
+ 0);
+
+ for (i = 0; i < test.files_count; ++i) {
+ struct test_file *file = &test.files[i];
+
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL),
+ 0);
+ close(cmd_fd);
+ cmd_fd = -1;
+
+ TESTEQUAL(validate_data_block_count(mount_dir, backing_dir,
+ file),
+ 0);
+ }
+
+ result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int validate_hash_block_count(const char *mount_dir,
+ const char *backing_dir,
+ struct test_file *file)
+{
+ const int digest_size = SHA256_DIGEST_SIZE;
+ const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
+ const int total_data_blocks = 1 + (file->size - 1) /
+ INCFS_DATA_FILE_BLOCK_SIZE;
+
+ int result = TEST_FAILURE;
+ int hash_layer = total_data_blocks;
+ int total_hash_blocks = 0;
+ int filled_hash_blocks;
+ char *filename = NULL;
+ int fd = -1;
+ struct incfs_get_block_count_args bca = {};
+
+ while (hash_layer > 1) {
+ hash_layer = (hash_layer + hash_per_block - 1) / hash_per_block;
+ total_hash_blocks += hash_layer;
+ }
+ filled_hash_blocks = total_hash_blocks > 1 ? 1 : 0;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, 0);
+ TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
+ TESTEQUAL(bca.filled_hash_blocks_out, 0);
+
+ TESTEQUAL(emit_partial_test_file_hash(mount_dir, file), 0);
+
+ TESTEQUAL(ioctl(fd, INCFS_IOC_GET_BLOCK_COUNT, &bca), 0);
+ TESTEQUAL(bca.total_data_blocks_out, total_data_blocks);
+ TESTEQUAL(bca.filled_data_blocks_out, 0);
+ TESTEQUAL(bca.total_hash_blocks_out, total_hash_blocks);
+ TESTEQUAL(bca.filled_hash_blocks_out, filled_hash_blocks);
+ close(fd);
+ fd = -1;
+
+ if (filled_hash_blocks)
+ TESTEQUAL(validate_block_count(mount_dir, backing_dir, file,
+ total_data_blocks, 0,
+ total_hash_blocks, filled_hash_blocks),
+ 0);
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ free(filename);
+ return result;
+}
+
+static int hash_block_count_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
+ 0);
+
+ for (i = 0; i < test.files_count; i++) {
+ struct test_file *file = &test.files[i];
+
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(crypto_emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, file->root_hash,
+ file->sig.add_data),
+ 0);
+ close(cmd_fd);
+ cmd_fd = -1;
+
+ TESTEQUAL(validate_hash_block_count(mount_dir, backing_dir,
+ &test.files[i]),
+ 0);
+ }
+
+ result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int is_close(struct timespec *start, int expected_ms)
+{
+ const int allowed_variance = 100;
+ int result = TEST_FAILURE;
+ struct timespec finish;
+ int diff;
+
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &finish), 0);
+ diff = (finish.tv_sec - start->tv_sec) * 1000 +
+ (finish.tv_nsec - start->tv_nsec) / 1000000;
+
+ TESTCOND(diff >= expected_ms - allowed_variance);
+ TESTCOND(diff <= expected_ms + allowed_variance);
+ result = TEST_SUCCESS;
+out:
+ return result;
+}
+
+static int per_uid_read_timeouts_test(const char *mount_dir)
+{
+ struct test_file file = {
+ .name = "file",
+ .size = 16 * INCFS_DATA_FILE_BLOCK_SIZE
+ };
+
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ int pid = -1;
+ int cmd_fd = -1;
+ char *filename = NULL;
+ int fd = -1;
+ struct timespec start;
+ char buffer[4096];
+ struct incfs_per_uid_read_timeouts purt_get[1];
+ struct incfs_get_read_timeouts_args grt = {
+ ptr_to_u64(purt_get),
+ sizeof(purt_get)
+ };
+ struct incfs_per_uid_read_timeouts purt_set[] = {
+ {
+ .uid = 0,
+ .min_time_us = 1000000,
+ .min_pending_time_us = 2000000,
+ .max_pending_time_us = 3000000,
+ },
+ };
+ struct incfs_set_read_timeouts_args srt = {
+ ptr_to_u64(purt_set),
+ sizeof(purt_set)
+ };
+ int status;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "read_timeout_ms=1000,readahead=0", false), 0);
+
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size,
+ NULL), 0);
+
+ TEST(filename = concat_file_name(mount_dir, file.name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC), 0);
+
+ /* Default mount options read failure is 1000 */
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
+ TESTEQUAL(is_close(&start, 1000), 0);
+
+ grt.timeouts_array_size = 0;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
+ TESTEQUAL(grt.timeouts_array_size_out, 0);
+
+ /* Set it to 3000 */
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
+ TESTEQUAL(is_close(&start, 3000), 0);
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), -1);
+ TESTEQUAL(errno, E2BIG);
+ TESTEQUAL(grt.timeouts_array_size_out, sizeof(purt_get));
+ grt.timeouts_array_size = sizeof(purt_get);
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
+ TESTEQUAL(grt.timeouts_array_size_out, sizeof(purt_get));
+ TESTEQUAL(purt_get[0].uid, purt_set[0].uid);
+ TESTEQUAL(purt_get[0].min_time_us, purt_set[0].min_time_us);
+ TESTEQUAL(purt_get[0].min_pending_time_us,
+ purt_set[0].min_pending_time_us);
+ TESTEQUAL(purt_get[0].max_pending_time_us,
+ purt_set[0].max_pending_time_us);
+
+ /* Still 1000 in UID 2 */
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TEST(pid = fork(), pid != -1);
+ if (pid == 0) {
+ TESTEQUAL(setuid(2), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
+ exit(0);
+ }
+ TESTNE(wait(&status), -1);
+ TESTEQUAL(WEXITSTATUS(status), 0);
+ TESTEQUAL(is_close(&start, 1000), 0);
+
+ /* Set it to default */
+ purt_set[0].max_pending_time_us = UINT32_MAX;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), -1);
+ TESTEQUAL(is_close(&start, 1000), 0);
+
+ /* Test min read time */
+ TESTEQUAL(emit_test_block(mount_dir, &file, 0), 0);
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), 0), sizeof(buffer));
+ TESTEQUAL(is_close(&start, 1000), 0);
+
+ /* Test min pending time */
+ purt_set[0].uid = 2;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
+ TESTEQUAL(clock_gettime(CLOCK_MONOTONIC, &start), 0);
+ TEST(pid = fork(), pid != -1);
+ if (pid == 0) {
+ TESTEQUAL(setuid(2), 0);
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), sizeof(buffer)),
+ sizeof(buffer));
+ exit(0);
+ }
+ sleep(1);
+ TESTEQUAL(emit_test_block(mount_dir, &file, 1), 0);
+ TESTNE(wait(&status), -1);
+ TESTEQUAL(WEXITSTATUS(status), 0);
+ TESTEQUAL(is_close(&start, 2000), 0);
+
+ /* Clear timeouts */
+ srt.timeouts_array_size = 0;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
+ grt.timeouts_array_size = 0;
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_READ_TIMEOUTS, &grt), 0);
+ TESTEQUAL(grt.timeouts_array_size_out, 0);
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+
+ if (pid == 0)
+ exit(result);
+
+ free(filename);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+#define DIRS 3
+static int inotify_test(const char *mount_dir)
+{
+ const char *mapped_file_name = "mapped_name";
+ struct test_file file = {
+ .name = "file",
+ .size = 16 * INCFS_DATA_FILE_BLOCK_SIZE
+ };
+
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL, *index_dir = NULL, *incomplete_dir = NULL;
+ char *file_name = NULL;
+ int cmd_fd = -1;
+ int notify_fd = -1;
+ int wds[DIRS];
+ char buffer[DIRS * (sizeof(struct inotify_event) + NAME_MAX + 1)];
+ char *ptr = buffer;
+ struct inotify_event *event;
+ struct inotify_event *events[DIRS] = {};
+ const char *names[DIRS] = {};
+ int res;
+ char id[sizeof(incfs_uuid_t) * 2 + 1];
+ struct incfs_create_mapped_file_args mfa;
+
+ /* File creation triggers inotify events in .index and .incomplete? */
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TEST(index_dir = concat_file_name(mount_dir, ".index"), index_dir);
+ TEST(incomplete_dir = concat_file_name(mount_dir, ".incomplete"),
+ incomplete_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 50), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TEST(notify_fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC),
+ notify_fd != -1);
+ TEST(wds[0] = inotify_add_watch(notify_fd, mount_dir,
+ IN_CREATE | IN_DELETE),
+ wds[0] != -1);
+ TEST(wds[1] = inotify_add_watch(notify_fd, index_dir,
+ IN_CREATE | IN_DELETE),
+ wds[1] != -1);
+ TEST(wds[2] = inotify_add_watch(notify_fd, incomplete_dir,
+ IN_CREATE | IN_DELETE),
+ wds[2] != -1);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size,
+ NULL), 0);
+ TEST(res = read(notify_fd, buffer, sizeof(buffer)), res != -1);
+
+ while (ptr < buffer + res) {
+ int i;
+
+ event = (struct inotify_event *) ptr;
+ TESTCOND(ptr + sizeof(*event) <= buffer + res);
+ for (i = 0; i < DIRS; ++i)
+ if (event->wd == wds[i]) {
+ TESTEQUAL(events[i], NULL);
+ events[i] = event;
+ ptr += sizeof(*event);
+ names[i] = ptr;
+ ptr += events[i]->len;
+ TESTCOND(ptr <= buffer + res);
+ break;
+ }
+ TESTCOND(i < DIRS);
+ }
+
+ TESTNE(events[0], NULL);
+ TESTNE(events[1], NULL);
+ TESTNE(events[2], NULL);
+
+ bin2hex(id, file.id.bytes, sizeof(incfs_uuid_t));
+
+ TESTEQUAL(events[0]->mask, IN_CREATE);
+ TESTEQUAL(events[1]->mask, IN_CREATE);
+ TESTEQUAL(events[2]->mask, IN_CREATE);
+ TESTEQUAL(strcmp(names[0], file.name), 0);
+ TESTEQUAL(strcmp(names[1], id), 0);
+ TESTEQUAL(strcmp(names[2], id), 0);
+
+ /* Creating a mapped file triggers inotify event */
+ mfa = (struct incfs_create_mapped_file_args) {
+ .size = INCFS_DATA_FILE_BLOCK_SIZE,
+ .mode = 0664,
+ .file_name = ptr_to_u64(mapped_file_name),
+ .source_file_id = file.id,
+ .source_offset = INCFS_DATA_FILE_BLOCK_SIZE,
+ };
+
+ TEST(res = ioctl(cmd_fd, INCFS_IOC_CREATE_MAPPED_FILE, &mfa),
+ res != -1);
+ TEST(res = read(notify_fd, buffer, sizeof(buffer)), res != -1);
+ event = (struct inotify_event *) buffer;
+ TESTEQUAL(event->wd, wds[0]);
+ TESTEQUAL(event->mask, IN_CREATE);
+ TESTEQUAL(strcmp(event->name, mapped_file_name), 0);
+
+ /* File completion triggers inotify event in .incomplete? */
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TEST(res = read(notify_fd, buffer, sizeof(buffer)), res != -1);
+ event = (struct inotify_event *) buffer;
+ TESTEQUAL(event->wd, wds[2]);
+ TESTEQUAL(event->mask, IN_DELETE);
+ TESTEQUAL(strcmp(event->name, id), 0);
+
+ /* File unlinking triggers inotify event in .index? */
+ TEST(file_name = concat_file_name(mount_dir, file.name), file_name);
+ TESTEQUAL(unlink(file_name), 0);
+ TEST(res = read(notify_fd, buffer, sizeof(buffer)), res != -1);
+ memset(events, 0, sizeof(events));
+ memset(names, 0, sizeof(names));
+ for (ptr = buffer; ptr < buffer + res;) {
+ event = (struct inotify_event *) ptr;
+ int i;
+
+ TESTCOND(ptr + sizeof(*event) <= buffer + res);
+ for (i = 0; i < DIRS; ++i)
+ if (event->wd == wds[i]) {
+ TESTEQUAL(events[i], NULL);
+ events[i] = event;
+ ptr += sizeof(*event);
+ names[i] = ptr;
+ ptr += events[i]->len;
+ TESTCOND(ptr <= buffer + res);
+ break;
+ }
+ TESTCOND(i < DIRS);
+ }
+
+ TESTNE(events[0], NULL);
+ TESTNE(events[1], NULL);
+ TESTEQUAL(events[2], NULL);
+
+ TESTEQUAL(events[0]->mask, IN_DELETE);
+ TESTEQUAL(events[1]->mask, IN_DELETE);
+ TESTEQUAL(strcmp(names[0], file.name), 0);
+ TESTEQUAL(strcmp(names[1], id), 0);
+
+ result = TEST_SUCCESS;
+out:
+ free(file_name);
+ close(notify_fd);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ free(index_dir);
+ free(incomplete_dir);
+ return result;
+}
+
+static EVP_PKEY *create_key(void)
+{
+ EVP_PKEY *pkey = NULL;
+ RSA *rsa = NULL;
+ BIGNUM *bn = NULL;
+
+ pkey = EVP_PKEY_new();
+ if (!pkey)
+ goto fail;
+
+ bn = BN_new();
+ BN_set_word(bn, RSA_F4);
+
+ rsa = RSA_new();
+ if (!rsa)
+ goto fail;
+
+ RSA_generate_key_ex(rsa, 4096, bn, NULL);
+ EVP_PKEY_assign_RSA(pkey, rsa);
+
+ BN_free(bn);
+ return pkey;
+
+fail:
+ BN_free(bn);
+ EVP_PKEY_free(pkey);
+ return NULL;
+}
+
+static X509 *get_cert(EVP_PKEY *key)
+{
+ X509 *x509 = NULL;
+ X509_NAME *name = NULL;
+
+ x509 = X509_new();
+ if (!x509)
+ return NULL;
+
+ ASN1_INTEGER_set(X509_get_serialNumber(x509), 1);
+ X509_gmtime_adj(X509_get_notBefore(x509), 0);
+ X509_gmtime_adj(X509_get_notAfter(x509), 31536000L);
+ X509_set_pubkey(x509, key);
+
+ name = X509_get_subject_name(x509);
+ X509_NAME_add_entry_by_txt(name, "C", MBSTRING_ASC,
+ (const unsigned char *)"US", -1, -1, 0);
+ X509_NAME_add_entry_by_txt(name, "ST", MBSTRING_ASC,
+ (const unsigned char *)"CA", -1, -1, 0);
+ X509_NAME_add_entry_by_txt(name, "L", MBSTRING_ASC,
+ (const unsigned char *)"San Jose", -1, -1, 0);
+ X509_NAME_add_entry_by_txt(name, "O", MBSTRING_ASC,
+ (const unsigned char *)"Example", -1, -1, 0);
+ X509_NAME_add_entry_by_txt(name, "OU", MBSTRING_ASC,
+ (const unsigned char *)"Org", -1, -1, 0);
+ X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC,
+ (const unsigned char *)"www.example.com", -1, -1, 0);
+ X509_set_issuer_name(x509, name);
+
+ if (!X509_sign(x509, key, EVP_sha256()))
+ return NULL;
+
+ return x509;
+}
+
+static int sign(EVP_PKEY *key, X509 *cert, const char *data, size_t len,
+ unsigned char **sig, size_t *sig_len)
+{
+ const int pkcs7_flags = PKCS7_BINARY | PKCS7_NOATTR | PKCS7_PARTIAL |
+ PKCS7_DETACHED;
+ const EVP_MD *md = EVP_sha256();
+
+ int result = TEST_FAILURE;
+
+ BIO *bio = NULL;
+ PKCS7 *p7 = NULL;
+ unsigned char *bio_buffer;
+
+ TEST(bio = BIO_new_mem_buf(data, len), bio);
+ TEST(p7 = PKCS7_sign(NULL, NULL, NULL, bio, pkcs7_flags), p7);
+ TESTNE(PKCS7_sign_add_signer(p7, cert, key, md, pkcs7_flags), 0);
+ TESTEQUAL(PKCS7_final(p7, bio, pkcs7_flags), 1);
+ TEST(*sig_len = i2d_PKCS7(p7, NULL), *sig_len);
+ TEST(bio_buffer = malloc(*sig_len), bio_buffer);
+ *sig = bio_buffer;
+ TEST(*sig_len = i2d_PKCS7(p7, &bio_buffer), *sig_len);
+ TESTEQUAL(PKCS7_verify(p7, NULL, NULL, bio, NULL,
+ pkcs7_flags | PKCS7_NOVERIFY | PKCS7_NOSIGS), 1);
+
+ result = TEST_SUCCESS;
+out:
+ PKCS7_free(p7);
+ BIO_free(bio);
+ return result;
+}
+
+static int verity_installed(const char *mount_dir, int cmd_fd, bool *installed)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ int fd = -1;
+ struct test_file *file = &get_test_files_set().files[0];
+
+ TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id, file->size,
+ NULL), 0);
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl(fd, FS_IOC_ENABLE_VERITY, NULL), -1);
+ *installed = errno != EOPNOTSUPP;
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ if (filename)
+ remove(filename);
+ free(filename);
+ return result;
+}
+
+static int enable_verity(const char *mount_dir, struct test_file *file,
+ EVP_PKEY *key, X509 *cert, bool use_signatures)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ int fd = -1;
+ struct fsverity_enable_arg fear = {
+ .version = 1,
+ .hash_algorithm = FS_VERITY_HASH_ALG_SHA256,
+ .block_size = INCFS_DATA_FILE_BLOCK_SIZE,
+ .sig_size = 0,
+ .sig_ptr = 0,
+ };
+ struct {
+ __u8 version; /* must be 1 */
+ __u8 hash_algorithm; /* Merkle tree hash algorithm */
+ __u8 log_blocksize; /* log2 of size of data and tree blocks */
+ __u8 salt_size; /* size of salt in bytes; 0 if none */
+ __le32 sig_size; /* must be 0 */
+ __le64 data_size; /* size of file the Merkle tree is built over */
+ __u8 root_hash[64]; /* Merkle tree root hash */
+ __u8 salt[32]; /* salt prepended to each hashed block */
+ __u8 __reserved[144]; /* must be 0's */
+ } __packed fsverity_descriptor = {
+ .version = 1,
+ .hash_algorithm = 1,
+ .log_blocksize = 12,
+ .data_size = file->size,
+ };
+ struct {
+ char magic[8]; /* must be "FSVerity" */
+ __le16 digest_algorithm;
+ __le16 digest_size;
+ __u8 digest[32];
+ } __packed fsverity_signed_digest = {
+ .digest_algorithm = 1,
+ .digest_size = 32
+ };
+ unsigned char *sig = NULL;
+ size_t sig_size = 0;
+ uint64_t flags;
+ struct statx statxbuf = {};
+
+ memcpy(fsverity_signed_digest.magic, "FSVerity", 8);
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TESTEQUAL(syscall(__NR_statx, AT_FDCWD, filename, 0, STATX_ALL,
+ &statxbuf), 0);
+ TESTEQUAL(statxbuf.stx_attributes_mask & STATX_ATTR_VERITY,
+ STATX_ATTR_VERITY);
+ TESTEQUAL(statxbuf.stx_attributes & STATX_ATTR_VERITY, 0);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl(fd, FS_IOC_GETFLAGS, &flags), 0);
+ TESTEQUAL(flags & FS_VERITY_FL, 0);
+
+ /* First try to enable verity with random digest */
+ if (key) {
+ TESTEQUAL(sign(key, cert, (void *)&fsverity_signed_digest,
+ sizeof(fsverity_signed_digest), &sig, &sig_size),
+ 0);
+
+ fear.sig_size = sig_size;
+ fear.sig_ptr = ptr_to_u64(sig);
+ TESTEQUAL(ioctl(fd, FS_IOC_ENABLE_VERITY, &fear), -1);
+ }
+
+ /* Now try with correct digest */
+ memcpy(fsverity_descriptor.root_hash, file->root_hash, 32);
+ sha256((char *)&fsverity_descriptor, sizeof(fsverity_descriptor),
+ (char *)fsverity_signed_digest.digest);
+
+ if (ioctl(fd, FS_IOC_ENABLE_VERITY, NULL) == -1 &&
+ errno == EOPNOTSUPP) {
+ result = TEST_SUCCESS;
+ goto out;
+ }
+
+ free(sig);
+ sig = NULL;
+
+ if (key)
+ TESTEQUAL(sign(key, cert, (void *)&fsverity_signed_digest,
+ sizeof(fsverity_signed_digest),
+ &sig, &sig_size),
+ 0);
+
+ if (use_signatures) {
+ fear.sig_size = sig_size;
+ file->verity_sig_size = sig_size;
+ fear.sig_ptr = ptr_to_u64(sig);
+ file->verity_sig = sig;
+ sig = NULL;
+ } else {
+ fear.sig_size = 0;
+ fear.sig_ptr = 0;
+ }
+ TESTEQUAL(ioctl(fd, FS_IOC_ENABLE_VERITY, &fear), 0);
+
+ result = TEST_SUCCESS;
+out:
+ free(sig);
+ close(fd);
+ free(filename);
+ return result;
+}
+
+static int memzero(const unsigned char *buf, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; ++i)
+ if (buf[i])
+ return -1;
+ return 0;
+}
+
+static int validate_verity(const char *mount_dir, struct test_file *file)
+{
+ int result = TEST_FAILURE;
+ char *filename = concat_file_name(mount_dir, file->name);
+ int fd = -1;
+ uint64_t flags;
+ struct fsverity_digest *digest;
+ struct statx statxbuf = {};
+ struct fsverity_read_metadata_arg frma = {};
+ uint8_t *buf = NULL;
+ struct fsverity_descriptor desc;
+
+ TEST(digest = malloc(sizeof(struct fsverity_digest) +
+ INCFS_MAX_HASH_SIZE), digest != NULL);
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TESTEQUAL(syscall(__NR_statx, AT_FDCWD, filename, 0, STATX_ALL,
+ &statxbuf), 0);
+ TESTEQUAL(statxbuf.stx_attributes & STATX_ATTR_VERITY,
+ STATX_ATTR_VERITY);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl(fd, FS_IOC_GETFLAGS, &flags), 0);
+ TESTEQUAL(flags & FS_VERITY_FL, FS_VERITY_FL);
+ digest->digest_size = INCFS_MAX_HASH_SIZE;
+ TESTEQUAL(ioctl(fd, FS_IOC_MEASURE_VERITY, digest), 0);
+ TESTEQUAL(digest->digest_algorithm, FS_VERITY_HASH_ALG_SHA256);
+ TESTEQUAL(digest->digest_size, 32);
+
+ if (file->verity_sig) {
+ TEST(buf = malloc(file->verity_sig_size), buf);
+ frma = (struct fsverity_read_metadata_arg) {
+ .metadata_type = FS_VERITY_METADATA_TYPE_SIGNATURE,
+ .length = file->verity_sig_size,
+ .buf_ptr = ptr_to_u64(buf),
+ };
+ TESTEQUAL(ioctl(fd, FS_IOC_READ_VERITY_METADATA, &frma),
+ file->verity_sig_size);
+ TESTEQUAL(memcmp(buf, file->verity_sig, file->verity_sig_size),
+ 0);
+ } else {
+ frma = (struct fsverity_read_metadata_arg) {
+ .metadata_type = FS_VERITY_METADATA_TYPE_SIGNATURE,
+ };
+ TESTEQUAL(ioctl(fd, FS_IOC_READ_VERITY_METADATA, &frma), -1);
+ TESTEQUAL(errno, ENODATA);
+ }
+
+ frma = (struct fsverity_read_metadata_arg) {
+ .metadata_type = FS_VERITY_METADATA_TYPE_DESCRIPTOR,
+ .length = sizeof(desc),
+ .buf_ptr = ptr_to_u64(&desc),
+ };
+ TESTEQUAL(ioctl(fd, FS_IOC_READ_VERITY_METADATA, &frma),
+ sizeof(desc));
+ TESTEQUAL(desc.version, 1);
+ TESTEQUAL(desc.hash_algorithm, FS_VERITY_HASH_ALG_SHA256);
+ TESTEQUAL(desc.log_blocksize, ilog2(INCFS_DATA_FILE_BLOCK_SIZE));
+ TESTEQUAL(desc.salt_size, 0);
+ TESTEQUAL(desc.__reserved_0x04, 0);
+ TESTEQUAL(desc.data_size, file->size);
+ TESTEQUAL(memcmp(desc.root_hash, file->root_hash, SHA256_DIGEST_SIZE),
+ 0);
+ TESTEQUAL(memzero(desc.root_hash + SHA256_DIGEST_SIZE,
+ sizeof(desc.root_hash) - SHA256_DIGEST_SIZE), 0);
+ TESTEQUAL(memzero(desc.salt, sizeof(desc.salt)), 0);
+ TESTEQUAL(memzero(desc.__reserved, sizeof(desc.__reserved)), 0);
+
+ result = TEST_SUCCESS;
+out:
+ free(buf);
+ close(fd);
+ free(filename);
+ free(digest);
+ return result;
+}
+
+static int verity_test_optional_sigs(const char *mount_dir, bool use_signatures)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ bool installed;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+ EVP_PKEY *key = NULL;
+ X509 *cert = NULL;
+ BIO *mem = NULL;
+ long len;
+ void *ptr;
+ FILE *proc_key_fd = NULL;
+ char *line = NULL;
+ size_t read = 0;
+ int key_id = -1;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
+ 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(verity_installed(mount_dir, cmd_fd, &installed), 0);
+ if (!installed) {
+ result = TEST_SUCCESS;
+ goto out;
+ }
+ TEST(key = create_key(), key);
+ TEST(cert = get_cert(key), cert);
+
+ TEST(proc_key_fd = fopen("/proc/keys", "r"), proc_key_fd != NULL);
+ while (getline(&line, &read, proc_key_fd) != -1)
+ if (strstr(line, ".fs-verity"))
+ key_id = strtol(line, NULL, 16);
+
+ TEST(mem = BIO_new(BIO_s_mem()), mem != NULL);
+ TESTEQUAL(i2d_X509_bio(mem, cert), 1);
+ TEST(len = BIO_get_mem_data(mem, &ptr), len != 0);
+ TESTCOND(key_id == -1
+ || syscall(__NR_add_key, "asymmetric", "test:key", ptr, len,
+ key_id) != -1);
+
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ build_mtree(file);
+ TESTEQUAL(crypto_emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, file->root_hash,
+ file->sig.add_data), 0);
+
+ TESTEQUAL(load_hash_tree(mount_dir, file), 0);
+ TESTEQUAL(enable_verity(mount_dir, file, key, cert,
+ use_signatures),
+ 0);
+ }
+
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_verity(mount_dir, &test.files[i]), 0);
+
+ close(cmd_fd);
+ cmd_fd = -1;
+ TESTEQUAL(umount(mount_dir), 0);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "readahead=0", false),
+ 0);
+
+ for (i = 0; i < file_num; i++)
+ TESTEQUAL(validate_verity(mount_dir, &test.files[i]), 0);
+
+ result = TEST_SUCCESS;
+out:
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ free(file->mtree);
+ free(file->verity_sig);
+
+ file->mtree = NULL;
+ file->verity_sig = NULL;
+ }
+
+ free(line);
+ BIO_free(mem);
+ X509_free(cert);
+ EVP_PKEY_free(key);
+ fclose(proc_key_fd);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int verity_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+
+ TESTEQUAL(verity_test_optional_sigs(mount_dir, true), TEST_SUCCESS);
+ TESTEQUAL(verity_test_optional_sigs(mount_dir, false), TEST_SUCCESS);
+ result = TEST_SUCCESS;
+out:
+ return result;
+}
+
+static int verity_file_valid(const char *mount_dir, struct test_file *file)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ int fd = -1;
+ uint8_t buffer[INCFS_DATA_FILE_BLOCK_SIZE];
+ struct incfs_get_file_sig_args gfsa = {
+ .file_signature = ptr_to_u64(buffer),
+ .file_signature_buf_size = sizeof(buffer),
+ };
+ int i;
+
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl(fd, INCFS_IOC_READ_FILE_SIGNATURE, &gfsa), 0);
+ for (i = 0; i < file->size; i += sizeof(buffer))
+ TESTEQUAL(pread(fd, buffer, sizeof(buffer), i),
+ file->size - i > sizeof(buffer) ?
+ sizeof(buffer) : file->size - i);
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ free(filename);
+ return result;
+}
+
+static int enable_verity_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ bool installed;
+ int cmd_fd = -1;
+ struct test_files_set test = get_test_files_set();
+ int i;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(verity_installed(mount_dir, cmd_fd, &installed), 0);
+ if (!installed) {
+ result = TEST_SUCCESS;
+ goto out;
+ }
+ for (i = 0; i < test.files_count; ++i) {
+ struct test_file *file = &test.files[i];
+
+ TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL), 0);
+ TESTEQUAL(emit_test_file_data(mount_dir, file), 0);
+ TESTEQUAL(enable_verity(mount_dir, file, NULL, NULL, false), 0);
+ }
+
+ /* Check files are valid on disk */
+ close(cmd_fd);
+ cmd_fd = -1;
+ TESTEQUAL(umount(mount_dir), 0);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ for (i = 0; i < test.files_count; ++i)
+ TESTEQUAL(verity_file_valid(mount_dir, &test.files[i]), 0);
+
+ result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int mmap_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ int cmd_fd = -1;
+ /*
+ * File is big enough to have a two layer tree with two hashes in the
+ * higher level, so we can corrupt the second one
+ */
+ int shas_per_block = INCFS_DATA_FILE_BLOCK_SIZE / SHA256_DIGEST_SIZE;
+ struct test_file file = {
+ .name = "file",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE * shas_per_block * 2,
+ };
+ char *filename = NULL;
+ int fd = -1;
+ char *addr = (void *)-1;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+
+ TESTEQUAL(build_mtree(&file), 0);
+ file.mtree[1].data[INCFS_DATA_FILE_BLOCK_SIZE] ^= 0xff;
+ TESTEQUAL(crypto_emit_file(cmd_fd, NULL, file.name, &file.id,
+ file.size, file.root_hash,
+ file.sig.add_data), 0);
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTEQUAL(load_hash_tree(mount_dir, &file), 0);
+ TEST(filename = concat_file_name(mount_dir, file.name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TEST(addr = mmap(NULL, file.size, PROT_READ, MAP_PRIVATE, fd, 0),
+ addr != (void *) -1);
+ TESTEQUAL(mlock(addr, INCFS_DATA_FILE_BLOCK_SIZE), 0);
+ TESTEQUAL(munlock(addr, INCFS_DATA_FILE_BLOCK_SIZE), 0);
+ TESTEQUAL(mlock(addr + shas_per_block * INCFS_DATA_FILE_BLOCK_SIZE,
+ INCFS_DATA_FILE_BLOCK_SIZE), -1);
+ TESTEQUAL(mlock(addr + (shas_per_block - 1) *
+ INCFS_DATA_FILE_BLOCK_SIZE,
+ INCFS_DATA_FILE_BLOCK_SIZE), 0);
+ TESTEQUAL(munlock(addr + (shas_per_block - 1) *
+ INCFS_DATA_FILE_BLOCK_SIZE,
+ INCFS_DATA_FILE_BLOCK_SIZE), 0);
+ TESTEQUAL(mlock(addr + (shas_per_block - 1) *
+ INCFS_DATA_FILE_BLOCK_SIZE,
+ INCFS_DATA_FILE_BLOCK_SIZE * 2), -1);
+ TESTEQUAL(munmap(addr, file.size), 0);
+
+ result = TEST_SUCCESS;
+out:
+ free(file.mtree);
+ close(fd);
+ free(filename);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int truncate_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ int cmd_fd = -1;
+ struct test_file file = {
+ .name = "file",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE,
+ };
+ char *backing_file = NULL;
+ int fd = -1;
+ struct stat st;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size, NULL),
+ 0);
+ TEST(backing_file = concat_file_name(backing_dir, file.name),
+ backing_file);
+ TEST(fd = open(backing_file, O_RDWR | O_CLOEXEC), fd != -1);
+ TESTEQUAL(stat(backing_file, &st), 0);
+ TESTCOND(st.st_blocks < 128);
+ TESTEQUAL(fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1 << 24), 0);
+ TESTEQUAL(stat(backing_file, &st), 0);
+ TESTCOND(st.st_blocks > 32768);
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTEQUAL(stat(backing_file, &st), 0);
+ TESTCOND(st.st_blocks < 128);
+
+ result = TEST_SUCCESS;
+out:
+ close(fd);
+ free(backing_file);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int stat_file_test(const char *mount_dir, int cmd_fd,
+ struct test_file *file)
+{
+ int result = TEST_FAILURE;
+ struct stat st;
+ char *filename = NULL;
+
+ TESTEQUAL(emit_file(cmd_fd, NULL, file->name, &file->id,
+ file->size, NULL), 0);
+ TEST(filename = concat_file_name(mount_dir, file->name), filename);
+ TESTEQUAL(stat(filename, &st), 0);
+ TESTCOND(st.st_blocks < 32);
+ TESTEQUAL(emit_test_file_data(mount_dir, file), 0);
+ TESTEQUAL(stat(filename, &st), 0);
+ TESTCOND(st.st_blocks > file->size / 512);
+
+ result = TEST_SUCCESS;
+out:
+ free(filename);
+ return result;
+}
+
+static int stat_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ int cmd_fd = -1;
+ int i;
+ struct test_files_set test = get_test_files_set();
+ const int file_num = test.files_count;
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+
+ for (i = 0; i < file_num; i++) {
+ struct test_file *file = &test.files[i];
+
+ TESTEQUAL(stat_file_test(mount_dir, cmd_fd, file), 0);
+ }
+
+ result = TEST_SUCCESS;
+out:
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+#define SYSFS_DIR "/sys/fs/incremental-fs/instances/test_node/"
+
+static int sysfs_test_value(const char *name, uint64_t value)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ FILE *file = NULL;
+ uint64_t res;
+
+ TEST(filename = concat_file_name(SYSFS_DIR, name), filename);
+ TEST(file = fopen(filename, "re"), file);
+ TESTEQUAL(fscanf(file, "%lu", &res), 1);
+ TESTEQUAL(res, value);
+
+ result = TEST_SUCCESS;
+out:
+ if (file)
+ fclose(file);
+ free(filename);
+ return result;
+}
+
+static int sysfs_test_value_range(const char *name, uint64_t low, uint64_t high)
+{
+ int result = TEST_FAILURE;
+ char *filename = NULL;
+ FILE *file = NULL;
+ uint64_t res;
+
+ TEST(filename = concat_file_name(SYSFS_DIR, name), filename);
+ TEST(file = fopen(filename, "re"), file);
+ TESTEQUAL(fscanf(file, "%lu", &res), 1);
+ TESTCOND(res >= low && res <= high);
+
+ result = TEST_SUCCESS;
+out:
+ if (file)
+ fclose(file);
+ free(filename);
+ return result;
+}
+
+static int ioctl_test_last_error(int cmd_fd, const incfs_uuid_t *file_id,
+ int page, int error)
+{
+ int result = TEST_FAILURE;
+ struct incfs_get_last_read_error_args glre;
+
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_GET_LAST_READ_ERROR, &glre), 0);
+ if (file_id)
+ TESTEQUAL(memcmp(&glre.file_id_out, file_id, sizeof(*file_id)),
+ 0);
+
+ TESTEQUAL(glre.page_out, page);
+ TESTEQUAL(glre.errno_out, error);
+ result = TEST_SUCCESS;
+out:
+ return result;
+}
+
+static int sysfs_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ int cmd_fd = -1;
+ struct test_file file = {
+ .name = "file",
+ .size = INCFS_DATA_FILE_BLOCK_SIZE,
+ };
+ char *filename = NULL;
+ int fd = -1;
+ int pid = -1;
+ char buffer[32];
+ char *null_buf = NULL;
+ int status;
+ struct incfs_per_uid_read_timeouts purt_set[] = {
+ {
+ .uid = 0,
+ .min_time_us = 1000000,
+ .min_pending_time_us = 1000000,
+ .max_pending_time_us = 2000000,
+ },
+ };
+ struct incfs_set_read_timeouts_args srt = {
+ ptr_to_u64(purt_set),
+ sizeof(purt_set)
+ };
+
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "sysfs_name=test_node",
+ false),
+ 0);
+ TEST(cmd_fd = open_commands_file(mount_dir), cmd_fd != -1);
+ TESTEQUAL(build_mtree(&file), 0);
+ file.root_hash[0] ^= 0xff;
+ TESTEQUAL(crypto_emit_file(cmd_fd, NULL, file.name, &file.id, file.size,
+ file.root_hash, file.sig.add_data),
+ 0);
+ TEST(filename = concat_file_name(mount_dir, file.name), filename);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(ioctl_test_last_error(cmd_fd, NULL, 0, 0), 0);
+ TESTEQUAL(sysfs_test_value("reads_failed_timed_out", 0), 0);
+ TESTEQUAL(read(fd, null_buf, 1), -1);
+ TESTEQUAL(ioctl_test_last_error(cmd_fd, &file.id, 0, -ETIME), 0);
+ TESTEQUAL(sysfs_test_value("reads_failed_timed_out", 2), 0);
+
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTEQUAL(sysfs_test_value("reads_failed_hash_verification", 0), 0);
+ TESTEQUAL(read(fd, null_buf, 1), -1);
+ TESTEQUAL(sysfs_test_value("reads_failed_hash_verification", 1), 0);
+ TESTSYSCALL(close(fd));
+ fd = -1;
+
+ TESTSYSCALL(unlink(filename));
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir,
+ "read_timeout_ms=10000,sysfs_name=test_node",
+ true),
+ 0);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size, NULL),
+ 0);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTSYSCALL(fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC));
+ TEST(pid = fork(), pid != -1);
+ if (pid == 0) {
+ TESTEQUAL(read(fd, buffer, sizeof(buffer)), sizeof(buffer));
+ exit(0);
+ }
+ sleep(1);
+ TESTEQUAL(sysfs_test_value("reads_delayed_pending", 0), 0);
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTNE(wait(&status), -1);
+ TESTEQUAL(status, 0);
+ TESTEQUAL(sysfs_test_value("reads_delayed_pending", 1), 0);
+ /* Allow +/- 10% */
+ TESTEQUAL(sysfs_test_value_range("reads_delayed_pending_us", 900000, 1100000),
+ 0);
+
+ TESTSYSCALL(close(fd));
+ fd = -1;
+
+ TESTSYSCALL(unlink(filename));
+ TESTEQUAL(ioctl(cmd_fd, INCFS_IOC_SET_READ_TIMEOUTS, &srt), 0);
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size, NULL),
+ 0);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(sysfs_test_value("reads_delayed_min", 0), 0);
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTEQUAL(read(fd, buffer, sizeof(buffer)), sizeof(buffer));
+ TESTEQUAL(sysfs_test_value("reads_delayed_min", 1), 0);
+ /* This should be exact */
+ TESTEQUAL(sysfs_test_value("reads_delayed_min_us", 1000000), 0);
+
+ TESTSYSCALL(close(fd));
+ fd = -1;
+
+ TESTSYSCALL(unlink(filename));
+ TESTEQUAL(emit_file(cmd_fd, NULL, file.name, &file.id, file.size, NULL),
+ 0);
+ TEST(fd = open(filename, O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTSYSCALL(fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC));
+ TEST(pid = fork(), pid != -1);
+ if (pid == 0) {
+ TESTEQUAL(read(fd, buffer, sizeof(buffer)), sizeof(buffer));
+ exit(0);
+ }
+ usleep(500000);
+ TESTEQUAL(sysfs_test_value("reads_delayed_pending", 1), 0);
+ TESTEQUAL(sysfs_test_value("reads_delayed_min", 1), 0);
+ TESTEQUAL(emit_test_file_data(mount_dir, &file), 0);
+ TESTNE(wait(&status), -1);
+ TESTEQUAL(status, 0);
+ TESTEQUAL(sysfs_test_value("reads_delayed_pending", 2), 0);
+ TESTEQUAL(sysfs_test_value("reads_delayed_min", 2), 0);
+ /* Exact 1000000 plus 500000 +/- 10% */
+ TESTEQUAL(sysfs_test_value_range("reads_delayed_min_us", 1450000, 1550000), 0);
+ /* Allow +/- 10% */
+ TESTEQUAL(sysfs_test_value_range("reads_delayed_pending_us", 1350000, 1650000),
+ 0);
+
+ result = TEST_SUCCESS;
+out:
+ if (pid == 0)
+ exit(result);
+ free(file.mtree);
+ free(filename);
+ close(fd);
+ close(cmd_fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int sysfs_test_directories(bool one_present, bool two_present)
+{
+ int result = TEST_FAILURE;
+ struct stat st;
+
+ TESTEQUAL(stat("/sys/fs/incremental-fs/instances/1", &st),
+ one_present ? 0 : -1);
+ if (one_present)
+ TESTCOND(S_ISDIR(st.st_mode));
+ else
+ TESTEQUAL(errno, ENOENT);
+ TESTEQUAL(stat("/sys/fs/incremental-fs/instances/2", &st),
+ two_present ? 0 : -1);
+ if (two_present)
+ TESTCOND(S_ISDIR(st.st_mode));
+ else
+ TESTEQUAL(errno, ENOENT);
+
+ result = TEST_SUCCESS;
+out:
+ return result;
+}
+
+static int sysfs_rename_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+ char *mount_dir2 = NULL;
+ int fd = -1;
+ char c;
+
+ /* Mount with no node */
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ TESTEQUAL(sysfs_test_directories(false, false), 0);
+
+ /* Remount with node */
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "sysfs_name=1", true),
+ 0);
+ TESTEQUAL(sysfs_test_directories(true, false), 0);
+ TEST(fd = open("/sys/fs/incremental-fs/instances/1/reads_delayed_min",
+ O_RDONLY | O_CLOEXEC), fd != -1);
+ TESTEQUAL(pread(fd, &c, 1, 0), 1);
+ TESTEQUAL(c, '0');
+ TESTEQUAL(pread(fd, &c, 1, 0), 1);
+ TESTEQUAL(c, '0');
+
+ /* Rename node */
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "sysfs_name=2", true),
+ 0);
+ TESTEQUAL(sysfs_test_directories(false, true), 0);
+ TESTEQUAL(pread(fd, &c, 1, 0), -1);
+
+ /* Try mounting another instance with same node name */
+ TEST(mount_dir2 = concat_file_name(backing_dir, "incfs-mount-dir2"),
+ mount_dir2);
+ rmdir(mount_dir2); /* In case we crashed before */
+ TESTSYSCALL(mkdir(mount_dir2, 0777));
+ TEST(mount_fs_opt(mount_dir2, backing_dir, "sysfs_name=2", false),
+ -1);
+
+ /* Try mounting another instance then remounting with existing name */
+ TESTEQUAL(mount_fs(mount_dir2, backing_dir, 0), 0);
+ TESTEQUAL(mount_fs_opt(mount_dir2, backing_dir, "sysfs_name=2", true),
+ -1);
+
+ /* Remount with no node */
+ TESTEQUAL(mount_fs_opt(mount_dir, backing_dir, "", true),
+ 0);
+ TESTEQUAL(sysfs_test_directories(false, false), 0);
+
+ result = TEST_SUCCESS;
+out:
+ umount(mount_dir2);
+ rmdir(mount_dir2);
+ free(mount_dir2);
+ close(fd);
+ umount(mount_dir);
+ free(backing_dir);
+ return result;
+}
+
+static int stacked_mount_test(const char *mount_dir)
+{
+ int result = TEST_FAILURE;
+ char *backing_dir = NULL;
+
+ /* Mount with no node */
+ TEST(backing_dir = create_backing_dir(mount_dir), backing_dir);
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ /* Try mounting another instance with same name */
+ TESTEQUAL(mount_fs(mount_dir, backing_dir, 0), 0);
+ /* Try unmounting the first instance */
+ TESTEQUAL(umount_fs(mount_dir), 0);
+ /* Try unmounting the second instance */
+ TESTEQUAL(umount_fs(mount_dir), 0);
+ result = TEST_SUCCESS;
+out:
+ /* Cleanup */
+ rmdir(mount_dir);
+ rmdir(backing_dir);
+ free(backing_dir);
+ return result;
+}
+
+static char *setup_mount_dir()
+{
+ struct stat st;
+ char *current_dir = getcwd(NULL, 0);
+ char *mount_dir = concat_file_name(current_dir, "incfs-mount-dir");
+
+ free(current_dir);
+ if (stat(mount_dir, &st) == 0) {
+ if (S_ISDIR(st.st_mode))
+ return mount_dir;
+
+ ksft_print_msg("%s is a file, not a dir.\n", mount_dir);
+ return NULL;
+ }
+
+ if (mkdir(mount_dir, 0777)) {
+ print_error("Can't create mount dir.");
+ return NULL;
+ }
+
+ return mount_dir;
+}
+
+int parse_options(int argc, char *const *argv)
+{
+ signed char c;
+
+ while ((c = getopt(argc, argv, "f:t:v")) != -1)
+ switch (c) {
+ case 'f':
+ options.file = strtol(optarg, NULL, 10);
+ break;
+
+ case 't':
+ options.test = strtol(optarg, NULL, 10);
+ break;
+
+ case 'v':
+ options.verbose = true;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+struct test_case {
+ int (*pfunc)(const char *dir);
+ const char *name;
+};
+
+void run_one_test(const char *mount_dir, struct test_case *test_case)
+{
+ int ret;
+
+ ksft_print_msg("Running %s\n", test_case->name);
+ ret = test_case->pfunc(mount_dir);
+
+ if (ret == TEST_SUCCESS)
+ ksft_test_result_pass("%s\n", test_case->name);
+ else if (ret == TEST_SKIP)
+ ksft_test_result_skip("%s\n", test_case->name);
+ else
+ ksft_test_result_fail("%s\n", test_case->name);
+}
+
+int main(int argc, char *argv[])
+{
+ char *mount_dir = NULL;
+ int i;
+ int fd, count;
+
+ if (parse_options(argc, argv))
+ ksft_exit_fail_msg("Bad options\n");
+
+ // Seed randomness pool for testing on QEMU
+ // NOTE - this abuses the concept of randomness - do *not* ever do this
+ // on a machine for production use - the device will think it has good
+ // randomness when it does not.
+ fd = open("/dev/urandom", O_WRONLY | O_CLOEXEC);
+ count = 4096;
+ for (int i = 0; i < 128; ++i)
+ ioctl(fd, RNDADDTOENTCNT, &count);
+ close(fd);
+
+ ksft_print_header();
+
+ if (geteuid() != 0)
+ ksft_print_msg("Not a root, might fail to mount.\n");
+
+ mount_dir = setup_mount_dir();
+ if (mount_dir == NULL)
+ ksft_exit_fail_msg("Can't create a mount dir\n");
+
+#define MAKE_TEST(test) \
+ { \
+ test, #test \
+ }
+ struct test_case cases[] = {
+ MAKE_TEST(basic_file_ops_test),
+ MAKE_TEST(cant_touch_index_test),
+ MAKE_TEST(dynamic_files_and_data_test),
+ MAKE_TEST(concurrent_reads_and_writes_test),
+ MAKE_TEST(attribute_test),
+ MAKE_TEST(work_after_remount_test),
+ MAKE_TEST(child_procs_waiting_for_data_test),
+ MAKE_TEST(multiple_providers_test),
+ MAKE_TEST(hash_tree_test),
+ MAKE_TEST(read_log_test),
+ MAKE_TEST(get_blocks_test),
+ MAKE_TEST(get_hash_blocks_test),
+ MAKE_TEST(large_file_test),
+ MAKE_TEST(mapped_file_test),
+ MAKE_TEST(compatibility_test),
+ MAKE_TEST(data_block_count_test),
+ MAKE_TEST(hash_block_count_test),
+ MAKE_TEST(per_uid_read_timeouts_test),
+ MAKE_TEST(inotify_test),
+ MAKE_TEST(verity_test),
+ MAKE_TEST(enable_verity_test),
+ MAKE_TEST(mmap_test),
+ MAKE_TEST(truncate_test),
+ MAKE_TEST(stat_test),
+ MAKE_TEST(sysfs_test),
+ MAKE_TEST(sysfs_rename_test),
+ MAKE_TEST(stacked_mount_test),
+ };
+#undef MAKE_TEST
+
+ if (options.test) {
+ if (options.test <= 0 || options.test > ARRAY_SIZE(cases))
+ ksft_exit_fail_msg("Invalid test\n");
+
+ ksft_set_plan(1);
+ run_one_test(mount_dir, &cases[options.test - 1]);
+ } else {
+ ksft_set_plan(ARRAY_SIZE(cases));
+ for (i = 0; i < ARRAY_SIZE(cases); ++i)
+ run_one_test(mount_dir, &cases[i]);
+ }
+
+ umount2(mount_dir, MNT_FORCE);
+ rmdir(mount_dir);
+ return !ksft_get_fail_cnt() ? ksft_exit_pass() : ksft_exit_fail();
+}
diff --git a/tools/testing/selftests/filesystems/incfs/utils.c b/tools/testing/selftests/filesystems/incfs/utils.c
new file mode 100644
index 0000000..d7deb53
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/utils.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <sys/ioctl.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <openssl/sha.h>
+#include <openssl/md5.h>
+
+#include "utils.h"
+
+#ifndef __S_IFREG
+#define __S_IFREG S_IFREG
+#endif
+
+unsigned int rnd(unsigned int max, unsigned int *seed)
+{
+ return rand_r(seed) * ((uint64_t)max + 1) / RAND_MAX;
+}
+
+int remove_dir(const char *dir)
+{
+ int err = rmdir(dir);
+
+ if (err && errno == ENOTEMPTY) {
+ err = delete_dir_tree(dir);
+ if (err)
+ return err;
+ return 0;
+ }
+
+ if (err && errno != ENOENT)
+ return -errno;
+
+ return 0;
+}
+
+int drop_caches(void)
+{
+ int drop_caches =
+ open("/proc/sys/vm/drop_caches", O_WRONLY | O_CLOEXEC);
+ int i;
+
+ if (drop_caches == -1)
+ return -errno;
+ i = write(drop_caches, "3", 1);
+ close(drop_caches);
+
+ if (i != 1)
+ return -errno;
+
+ return 0;
+}
+
+int mount_fs(const char *mount_dir, const char *backing_dir,
+ int read_timeout_ms)
+{
+ static const char fs_name[] = INCFS_NAME;
+ char mount_options[512];
+ int result;
+
+ snprintf(mount_options, ARRAY_SIZE(mount_options),
+ "read_timeout_ms=%u",
+ read_timeout_ms);
+
+ result = mount(backing_dir, mount_dir, fs_name, 0, mount_options);
+ if (result != 0)
+ perror("Error mounting fs.");
+ return result;
+}
+
+int umount_fs(const char *mount_dir)
+{
+ int result;
+
+ result = umount(mount_dir);
+ if (result != 0)
+ perror("Error unmounting fs.");
+ return result;
+}
+
+int mount_fs_opt(const char *mount_dir, const char *backing_dir,
+ const char *opt, bool remount)
+{
+ static const char fs_name[] = INCFS_NAME;
+ int result;
+
+ result = mount(backing_dir, mount_dir, fs_name,
+ remount ? MS_REMOUNT : 0, opt);
+ if (result != 0)
+ perror("Error mounting fs.");
+ return result;
+}
+
+struct hash_section {
+ uint32_t algorithm;
+ uint8_t log2_blocksize;
+ uint32_t salt_size;
+ /* no salt */
+ uint32_t hash_size;
+ uint8_t hash[SHA256_DIGEST_SIZE];
+} __packed;
+
+struct signature_blob {
+ uint32_t version;
+ uint32_t hash_section_size;
+ struct hash_section hash_section;
+ uint32_t signing_section_size;
+ uint8_t signing_section[];
+} __packed;
+
+size_t format_signature(void **buf, const char *root_hash, const char *add_data)
+{
+ size_t size = sizeof(struct signature_blob) + strlen(add_data) + 1;
+ struct signature_blob *sb = malloc(size);
+
+ if (!sb)
+ return 0;
+
+ *sb = (struct signature_blob){
+ .version = INCFS_SIGNATURE_VERSION,
+ .hash_section_size = sizeof(struct hash_section),
+ .hash_section =
+ (struct hash_section){
+ .algorithm = INCFS_HASH_TREE_SHA256,
+ .log2_blocksize = 12,
+ .salt_size = 0,
+ .hash_size = SHA256_DIGEST_SIZE,
+ },
+ .signing_section_size = strlen(add_data) + 1,
+ };
+
+ memcpy(sb->hash_section.hash, root_hash, SHA256_DIGEST_SIZE);
+ memcpy((char *)sb->signing_section, add_data, strlen(add_data) + 1);
+ *buf = sb;
+ return size;
+}
+
+int crypto_emit_file(int fd, const char *dir, const char *filename,
+ incfs_uuid_t *id_out, size_t size, const char *root_hash,
+ const char *add_data)
+{
+ int mode = __S_IFREG | 0555;
+ void *signature;
+ int error = 0;
+
+ struct incfs_new_file_args args = {
+ .size = size,
+ .mode = mode,
+ .file_name = ptr_to_u64(filename),
+ .directory_path = ptr_to_u64(dir),
+ .file_attr = 0,
+ .file_attr_len = 0
+ };
+
+ args.signature_size = format_signature(&signature, root_hash, add_data);
+ args.signature_info = ptr_to_u64(signature);
+
+ md5(filename, strlen(filename), (char *)args.file_id.bytes);
+
+ if (ioctl(fd, INCFS_IOC_CREATE_FILE, &args) != 0) {
+ error = -errno;
+ goto out;
+ }
+
+ *id_out = args.file_id;
+
+out:
+ free(signature);
+ return error;
+}
+
+int emit_file(int fd, const char *dir, const char *filename,
+ incfs_uuid_t *id_out, size_t size, const char *attr)
+{
+ int mode = __S_IFREG | 0555;
+ struct incfs_new_file_args args = { .size = size,
+ .mode = mode,
+ .file_name = ptr_to_u64(filename),
+ .directory_path = ptr_to_u64(dir),
+ .signature_info = ptr_to_u64(NULL),
+ .signature_size = 0,
+ .file_attr = ptr_to_u64(attr),
+ .file_attr_len =
+ attr ? strlen(attr) : 0 };
+
+ md5(filename, strlen(filename), (char *)args.file_id.bytes);
+
+ if (ioctl(fd, INCFS_IOC_CREATE_FILE, &args) != 0)
+ return -errno;
+
+ *id_out = args.file_id;
+ return 0;
+}
+
+int get_file_bmap(int cmd_fd, int ino, unsigned char *buf, int buf_size)
+{
+ return 0;
+}
+
+int get_file_signature(int fd, unsigned char *buf, int buf_size)
+{
+ struct incfs_get_file_sig_args args = {
+ .file_signature = ptr_to_u64(buf),
+ .file_signature_buf_size = buf_size
+ };
+
+ if (ioctl(fd, INCFS_IOC_READ_FILE_SIGNATURE, &args) == 0)
+ return args.file_signature_len_out;
+ return -errno;
+}
+
+loff_t get_file_size(const char *name)
+{
+ struct stat st;
+
+ if (stat(name, &st) == 0)
+ return st.st_size;
+ return -ENOENT;
+}
+
+int open_commands_file(const char *mount_dir)
+{
+ char cmd_file[255];
+ int cmd_fd;
+
+ snprintf(cmd_file, ARRAY_SIZE(cmd_file),
+ "%s/%s", mount_dir, INCFS_PENDING_READS_FILENAME);
+ cmd_fd = open(cmd_file, O_RDONLY | O_CLOEXEC);
+
+ if (cmd_fd < 0)
+ perror("Can't open commands file");
+ return cmd_fd;
+}
+
+int open_log_file(const char *mount_dir)
+{
+ char file[255];
+ int fd;
+
+ snprintf(file, ARRAY_SIZE(file), "%s/.log", mount_dir);
+ fd = open(file, O_RDWR | O_CLOEXEC);
+ if (fd < 0)
+ perror("Can't open log file");
+ return fd;
+}
+
+int open_blocks_written_file(const char *mount_dir)
+{
+ char file[255];
+ int fd;
+
+ snprintf(file, ARRAY_SIZE(file),
+ "%s/%s", mount_dir, INCFS_BLOCKS_WRITTEN_FILENAME);
+ fd = open(file, O_RDONLY | O_CLOEXEC);
+
+ if (fd < 0)
+ perror("Can't open blocks_written file");
+ return fd;
+}
+
+int wait_for_pending_reads(int fd, int timeout_ms,
+ struct incfs_pending_read_info *prs, int prs_count)
+{
+ ssize_t read_res = 0;
+
+ if (timeout_ms > 0) {
+ int poll_res = 0;
+ struct pollfd pollfd = {
+ .fd = fd,
+ .events = POLLIN
+ };
+
+ poll_res = poll(&pollfd, 1, timeout_ms);
+ if (poll_res < 0)
+ return -errno;
+ if (poll_res == 0)
+ return 0;
+ if (!(pollfd.revents | POLLIN))
+ return 0;
+ }
+
+ read_res = read(fd, prs, prs_count * sizeof(*prs));
+ if (read_res < 0)
+ return -errno;
+
+ return read_res / sizeof(*prs);
+}
+
+int wait_for_pending_reads2(int fd, int timeout_ms,
+ struct incfs_pending_read_info2 *prs, int prs_count)
+{
+ ssize_t read_res = 0;
+
+ if (timeout_ms > 0) {
+ int poll_res = 0;
+ struct pollfd pollfd = {
+ .fd = fd,
+ .events = POLLIN
+ };
+
+ poll_res = poll(&pollfd, 1, timeout_ms);
+ if (poll_res < 0)
+ return -errno;
+ if (poll_res == 0)
+ return 0;
+ if (!(pollfd.revents | POLLIN))
+ return 0;
+ }
+
+ read_res = read(fd, prs, prs_count * sizeof(*prs));
+ if (read_res < 0)
+ return -errno;
+
+ return read_res / sizeof(*prs);
+}
+
+char *concat_file_name(const char *dir, const char *file)
+{
+ char full_name[FILENAME_MAX] = "";
+
+ if (snprintf(full_name, ARRAY_SIZE(full_name), "%s/%s", dir, file) < 0)
+ return NULL;
+ return strdup(full_name);
+}
+
+int delete_dir_tree(const char *dir_path)
+{
+ DIR *dir = NULL;
+ struct dirent *dp;
+ int result = 0;
+
+ dir = opendir(dir_path);
+ if (!dir) {
+ result = -errno;
+ goto out;
+ }
+
+ while ((dp = readdir(dir))) {
+ char *full_path;
+
+ if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, ".."))
+ continue;
+
+ full_path = concat_file_name(dir_path, dp->d_name);
+ if (dp->d_type == DT_DIR)
+ result = delete_dir_tree(full_path);
+ else
+ result = unlink(full_path);
+ free(full_path);
+ if (result)
+ goto out;
+ }
+
+out:
+ if (dir)
+ closedir(dir);
+ if (!result)
+ rmdir(dir_path);
+ return result;
+}
+
+void sha256(const char *data, size_t dsize, char *hash)
+{
+ SHA256_CTX ctx;
+
+ SHA256_Init(&ctx);
+ SHA256_Update(&ctx, data, dsize);
+ SHA256_Final((unsigned char *)hash, &ctx);
+}
+
+void md5(const char *data, size_t dsize, char *hash)
+{
+ MD5_CTX ctx;
+
+ MD5_Init(&ctx);
+ MD5_Update(&ctx, data, dsize);
+ MD5_Final((unsigned char *)hash, &ctx);
+}
diff --git a/tools/testing/selftests/filesystems/incfs/utils.h b/tools/testing/selftests/filesystems/incfs/utils.h
new file mode 100644
index 0000000..17a1ac5
--- /dev/null
+++ b/tools/testing/selftests/filesystems/incfs/utils.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2019 Google LLC
+ */
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#include <include/uapi/linux/incrementalfs.h>
+
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
+
+#define __packed __attribute__((__packed__))
+
+#ifdef __LP64__
+#define ptr_to_u64(p) ((__u64)p)
+#else
+#define ptr_to_u64(p) ((__u64)(__u32)p)
+#endif
+
+#define SHA256_DIGEST_SIZE 32
+#define INCFS_MAX_MTREE_LEVELS 8
+
+unsigned int rnd(unsigned int max, unsigned int *seed);
+
+int remove_dir(const char *dir);
+
+int drop_caches(void);
+
+int mount_fs(const char *mount_dir, const char *backing_dir,
+ int read_timeout_ms);
+
+int umount_fs(const char *mount_dir);
+
+int mount_fs_opt(const char *mount_dir, const char *backing_dir,
+ const char *opt, bool remount);
+
+int get_file_bmap(int cmd_fd, int ino, unsigned char *buf, int buf_size);
+
+int get_file_signature(int fd, unsigned char *buf, int buf_size);
+
+int emit_node(int fd, char *filename, int *ino_out, int parent_ino,
+ size_t size, mode_t mode, char *attr);
+
+int emit_file(int fd, const char *dir, const char *filename,
+ incfs_uuid_t *id_out, size_t size, const char *attr);
+
+int crypto_emit_file(int fd, const char *dir, const char *filename,
+ incfs_uuid_t *id_out, size_t size, const char *root_hash,
+ const char *add_data);
+
+loff_t get_file_size(const char *name);
+
+int open_commands_file(const char *mount_dir);
+
+int open_log_file(const char *mount_dir);
+
+int open_blocks_written_file(const char *mount_dir);
+
+int wait_for_pending_reads(int fd, int timeout_ms,
+ struct incfs_pending_read_info *prs, int prs_count);
+
+int wait_for_pending_reads2(int fd, int timeout_ms,
+ struct incfs_pending_read_info2 *prs, int prs_count);
+
+char *concat_file_name(const char *dir, const char *file);
+
+void sha256(const char *data, size_t dsize, char *hash);
+
+void md5(const char *data, size_t dsize, char *hash);
+
+int delete_dir_tree(const char *path);
